using Amazon.IdentityManagement.Model; using Amazon.S3; using Amazon.S3.Model; using InnovEnergy.Lib.S3Utils; using InnovEnergy.Lib.S3Utils.DataTypes; using InnovEnergy.Lib.Utils; using S3Cfg = InnovEnergy.Lib.S3Utils.S3Cfg; namespace InnovEnergy.App.S3Explorer; public static class Program { private const String BucketSalt = "-3e5b3069-214a-43ee-8d85-57d72000c19d"; public static async Task Main(String[] args) { var region = S3Cfg.GetDefaultRegionAndCredentials(); var x = Iam.GetIamClient(region!); var c = await x.ListRolesAsync(new ListRolesRequest()); // Todo refactor S3Access into Lib // Sssssecret if (args.Contains("-s")) await SnakeGameSs.PlaySnake(); // Help message if (args.Length < 1 || args.Contains("-h")) { Console.WriteLine("Usage: S3Explorer installation-id [from-unix-time] [to-unix-time] [nb-data-points]"); Console.WriteLine("-h Shows this message."); Console.WriteLine("-s 🐍"); return 0; } // Parsing Arguments var bucketName = args[0] + BucketSalt; var now = DateTime.Now; var startTime = Int64.Parse(args.ElementAtOr(1, (now - TimeSpan.FromSeconds(20)).ToString())); var endTime = Int64.Parse(args.ElementAtOr(2, now.ToString())); var nDataPoints = Int64.Parse(args.ElementAtOr(3, "10")); var timestampList = GetDataTimestamps(startTime, endTime, nDataPoints); await PrintFiles(bucketName, timestampList); // Success return 0; } public static async Task ListingObjectsAsync(IAmazonS3 client, String bucketName) { var request = new ListObjectsV2Request { BucketName = bucketName , Prefix = "1689236"}; var listObjectsV2Paginator = client.Paginators.ListObjectsV2(request); await foreach (var response in listObjectsV2Paginator.Responses) { Console.WriteLine($"HttpStatusCode: {response.HttpStatusCode}"); Console.WriteLine($"Number of Keys: {response.KeyCount}"); foreach (var entry in response.S3Objects) { Console.WriteLine($"Key = {entry.Key} Size = {entry.Size}"); } } } private static IEnumerable GetDataTimestamps(Int64 startTime, Int64 endTime, Int64 nDataPoints) { // Calculating temporal distance of data files from the number of requested points. (rounding for int division) var timeSpan = endTime - startTime; var timeBetweenDataPoints = (Double)timeSpan / nDataPoints; // We only upload data every second second so sampling more is impossible. // If this ever changes we might have to change this as well. // Building a List of the timestamps we want to grab the files for. for (Double i = startTime; i <= endTime; i += timeBetweenDataPoints) { //Rounding to even numbers only (we only save every second second) var integer = (Int64) Math.Round(i); yield return integer / 2 * 2; } } private static async Task PrintFiles(String bucketName, IEnumerable timestamps) { var columns = new Dictionary> { ["timestamp"] = new() }; var index = 0; foreach (var timestamp in timestamps) { var csvFileText = await GetFileText(bucketName, timestamp); columns["timestamp"].Add(timestamp.ToString()); var dict = csvFileText is null ? new Dictionary() : csvFileText .Select(l => l.Split(";")) .ToDictionary(kv => kv[0], kv => kv[1]); foreach (var key in dict.Keys) { // if a key is not yet present in columns we need to backfill it with nulls if (!columns.ContainsKey(key)) columns[key] = Enumerable.Repeat(null, index).ToList(); columns[key].Add(dict[key]); } // if a key in columns is not present in this record (dict) (except the timestamp) we need to set it to null foreach (var key in columns.Keys.Where(key => !dict.ContainsKey(key) && key != "timestamp")) { columns[key].Add(null); } index++; } var headerKeys = columns .Keys .OrderBy(k => k) .Where(k => k != "timestamp") .Prepend("timestamp") .ToList(); String.Join(';', headerKeys).WriteLine(); Enumerable.Range(0, index) .Select(i => headerKeys.Select(hk => columns[hk][i]).JoinWith(";")) .ForEach(Console.WriteLine); } // This Method extracts the Text from a given csv file on the s3 bucket private static async Task?> GetFileText(String bucketName, Int64 timestamp) { var csv = await S3Cfg .GetDefaultRegionAndCredentials()! .Bucket(bucketName) .Path($"{timestamp}.csv") .GetObjectAsString(); return csv.Split(Environment.NewLine); } }