Innovenergy_trunk/csharp/App/S3Explorer/Program.cs

using Amazon.IdentityManagement.Model;
using Amazon.S3;
using Amazon.S3.Model;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils;
using S3Cfg = InnovEnergy.Lib.S3Utils.S3Cfg;

namespace InnovEnergy.App.S3Explorer;

public static class Program
{
    private const String BucketSalt = "-3e5b3069-214a-43ee-8d85-57d72000c19d";

    public static async Task<Int32> Main(String[] args)
    {
        var region = S3Cfg.GetDefaultRegionAndCredentials();
        var x = Iam.GetIamClient(region!);

        var c = await x.ListRolesAsync(new ListRolesRequest());

        
            
        
        
        
        // Todo refactor S3Access into Lib
        
        // Sssssecret
        if (args.Contains("-s"))
            await SnakeGameSs.PlaySnake();

        // Help message
        if (args.Length < 1 || args.Contains("-h"))
        {
            Console.WriteLine("Usage: S3Explorer installation-id [from-unix-time] [to-unix-time] [nb-data-points]");
            Console.WriteLine("-h Shows this message.");
            Console.WriteLine("-s 🐍");
            return 0;
        }
        
        // Parsing Arguments
        var bucketName = args[0] + BucketSalt;
        var now = DateTime.Now;
        
        var startTime   = Int64.Parse(args.ElementAtOr(1, (now - TimeSpan.FromSeconds(20)).ToString()));
        var endTime     = Int64.Parse(args.ElementAtOr(2, now.ToString()));
        var nDataPoints = Int64.Parse(args.ElementAtOr(3, "10"));
        
        var timestampList = GetDataTimestamps(startTime, endTime, nDataPoints);

        await PrintFiles(bucketName, timestampList);
        
        // Success
        return 0;
    }

    public static async Task ListingObjectsAsync(IAmazonS3 client, String bucketName)
    {
        var request = new ListObjectsV2Request { BucketName = bucketName , Prefix = "1689236"};

        var listObjectsV2Paginator = client.Paginators.ListObjectsV2(request);

        await foreach (var response in listObjectsV2Paginator.Responses)
        {
            Console.WriteLine($"HttpStatusCode: {response.HttpStatusCode}");
            Console.WriteLine($"Number of Keys: {response.KeyCount}");
            
            foreach (var entry in response.S3Objects)
            {
                Console.WriteLine($"Key = {entry.Key} Size = {entry.Size}");
            }
        }
    }
    

    private static IEnumerable<Int64> GetDataTimestamps(Int64 startTime, Int64 endTime, Int64 nDataPoints)
    {
        // Calculating temporal distance of data files from the number of requested points. (rounding for int division)
        var timeSpan = endTime - startTime;
        var timeBetweenDataPoints = (Double)timeSpan / nDataPoints;
        // We only upload data every second second so sampling more is impossible.
        // If this ever changes we might have to change this as well.
        
        // Building a List of the timestamps we want to grab the files for.
        for (Double i = startTime; i <= endTime; i += timeBetweenDataPoints)
        {
            //Rounding to even numbers only (we only save every second second)
            var integer = (Int64) Math.Round(i);
            yield return integer / 2 * 2;
        }
    }
    
    private static async Task PrintFiles(String bucketName, IEnumerable<Int64> timestamps)
    {
        var columns = new Dictionary<String, List<String?>>
        {
            ["timestamp"] = new()
        };
        var index = 0;
        
        foreach (var timestamp in timestamps)
        {
            var csvFileText = await GetFileText(bucketName, timestamp);

            columns["timestamp"].Add(timestamp.ToString());
            
            var dict = csvFileText is null
                     ? new Dictionary<String, String>()
                     : csvFileText
                      .Select(l => l.Split(";"))
                      .ToDictionary(kv => kv[0], kv => kv[1]);
            
            foreach (var key in dict.Keys)
            {
                // if a key is not yet present in columns we need to backfill it with nulls
                if (!columns.ContainsKey(key))
                    columns[key] = Enumerable.Repeat<String?>(null, index).ToList();

                columns[key].Add(dict[key]);
            }

            // if a key in columns is not present in this record (dict) (except the timestamp) we need to set it to null 
            foreach (var key in columns.Keys.Where(key => !dict.ContainsKey(key) && key != "timestamp"))
            {
                columns[key].Add(null);
            }
            index++;
        }

        var headerKeys = columns
                        .Keys
                        .OrderBy(k => k)
                        .Where(k => k != "timestamp")
                        .Prepend("timestamp")
                        .ToList();
        
        String.Join(';', headerKeys).WriteLine();

        Enumerable.Range(0, index)
                  .Select(i => headerKeys.Select(hk => columns[hk][i]).JoinWith(";"))
                  .ForEach(Console.WriteLine);
    }

    // This Method extracts the Text from a given csv file on the s3 bucket
    private static async Task<IReadOnlyList<String>?> GetFileText(String bucketName, Int64 timestamp)
    {
        var csv = await S3Cfg
                       .GetDefaultRegionAndCredentials()!
                       .Bucket(bucketName)
                       .Path($"{timestamp}.csv")
                       .GetObjectAsString();
        
        return csv.Split(Environment.NewLine);
    }
}