Innovenergy_trunk/csharp/App/S3Explorer/Program.cs

156 lines
5.5 KiB
C#
Raw Normal View History

2023-10-02 13:37:27 +00:00
using Amazon.IdentityManagement.Model;
using Amazon.S3;
using Amazon.S3.Model;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils;
using S3Cfg = InnovEnergy.Lib.S3Utils.S3Cfg;
2023-10-02 13:37:27 +00:00
namespace InnovEnergy.App.S3Explorer;
public static class Program
{
2023-10-02 13:37:27 +00:00
private const String BucketSalt = "-3e5b3069-214a-43ee-8d85-57d72000c19d";
public static async Task<Int32> Main(String[] args)
{
2023-10-02 13:37:27 +00:00
var region = S3Cfg.GetDefaultRegionAndCredentials();
var x = Iam.GetIamClient(region!);
var c = await x.ListRolesAsync(new ListRolesRequest());
2023-07-13 14:37:52 +00:00
// Todo refactor S3Access into Lib
// Sssssecret
if (args.Contains("-s"))
await SnakeGameSs.PlaySnake();
2023-10-02 13:37:27 +00:00
2023-07-13 14:37:52 +00:00
// Help message
2023-10-02 13:37:27 +00:00
if (args.Length < 1 || args.Contains("-h"))
{
2023-10-02 13:37:27 +00:00
Console.WriteLine("Usage: S3Explorer installation-id [from-unix-time] [to-unix-time] [nb-data-points]");
2023-07-13 14:37:52 +00:00
Console.WriteLine("-h Shows this message.");
Console.WriteLine("-s 🐍");
return 0;
}
2023-07-13 14:37:52 +00:00
// Parsing Arguments
2023-10-02 13:37:27 +00:00
var bucketName = args[0] + BucketSalt;
var now = DateTime.Now;
2023-10-02 13:37:27 +00:00
var startTime = Int64.Parse(args.ElementAtOr(1, (now - TimeSpan.FromSeconds(20)).ToString()));
var endTime = Int64.Parse(args.ElementAtOr(2, now.ToString()));
var nDataPoints = Int64.Parse(args.ElementAtOr(3, "10"));
var timestampList = GetDataTimestamps(startTime, endTime, nDataPoints);
2023-07-13 14:37:52 +00:00
2023-10-02 13:37:27 +00:00
await PrintFiles(bucketName, timestampList);
2023-07-13 14:37:52 +00:00
// Success
return 0;
}
2023-10-02 13:37:27 +00:00
public static async Task ListingObjectsAsync(IAmazonS3 client, String bucketName)
{
2023-10-02 13:37:27 +00:00
var request = new ListObjectsV2Request { BucketName = bucketName , Prefix = "1689236"};
var listObjectsV2Paginator = client.Paginators.ListObjectsV2(request);
await foreach (var response in listObjectsV2Paginator.Responses)
{
2023-10-02 13:37:27 +00:00
Console.WriteLine($"HttpStatusCode: {response.HttpStatusCode}");
Console.WriteLine($"Number of Keys: {response.KeyCount}");
2023-07-13 14:37:52 +00:00
2023-10-02 13:37:27 +00:00
foreach (var entry in response.S3Objects)
{
Console.WriteLine($"Key = {entry.Key} Size = {entry.Size}");
}
}
}
2023-07-13 14:37:52 +00:00
2023-10-02 13:37:27 +00:00
private static IEnumerable<Int64> GetDataTimestamps(Int64 startTime, Int64 endTime, Int64 nDataPoints)
2023-07-13 14:37:52 +00:00
{
2023-10-02 13:37:27 +00:00
// Calculating temporal distance of data files from the number of requested points. (rounding for int division)
2023-07-13 14:37:52 +00:00
var timeSpan = endTime - startTime;
2023-10-02 13:37:27 +00:00
var timeBetweenDataPoints = (Double)timeSpan / nDataPoints;
2023-07-13 14:37:52 +00:00
// We only upload data every second second so sampling more is impossible.
// If this ever changes we might have to change this as well.
2023-10-02 13:37:27 +00:00
// Building a List of the timestamps we want to grab the files for.
for (Double i = startTime; i <= endTime; i += timeBetweenDataPoints)
{
//Rounding to even numbers only (we only save every second second)
var integer = (Int64) Math.Round(i);
yield return integer / 2 * 2;
}
2023-07-13 14:37:52 +00:00
}
2023-10-02 13:37:27 +00:00
private static async Task PrintFiles(String bucketName, IEnumerable<Int64> timestamps)
{
var columns = new Dictionary<String, List<String?>>
{
["timestamp"] = new()
};
var index = 0;
foreach (var timestamp in timestamps)
{
var csvFileText = await GetFileText(bucketName, timestamp);
columns["timestamp"].Add(timestamp.ToString());
var dict = csvFileText is null
? new Dictionary<String, String>()
: csvFileText
.Select(l => l.Split(";"))
.ToDictionary(kv => kv[0], kv => kv[1]);
foreach (var key in dict.Keys)
{
// if a key is not yet present in columns we need to backfill it with nulls
if (!columns.ContainsKey(key))
columns[key] = Enumerable.Repeat<String?>(null, index).ToList();
columns[key].Add(dict[key]);
}
// if a key in columns is not present in this record (dict) (except the timestamp) we need to set it to null
foreach (var key in columns.Keys.Where(key => !dict.ContainsKey(key) && key != "timestamp"))
{
columns[key].Add(null);
}
index++;
}
var headerKeys = columns
.Keys
.OrderBy(k => k)
.Where(k => k != "timestamp")
.Prepend("timestamp")
.ToList();
String.Join(';', headerKeys).WriteLine();
Enumerable.Range(0, index)
.Select(i => headerKeys.Select(hk => columns[hk][i]).JoinWith(";"))
.ForEach(Console.WriteLine);
}
2023-07-13 14:37:52 +00:00
// This Method extracts the Text from a given csv file on the s3 bucket
2023-10-02 13:37:27 +00:00
private static async Task<IReadOnlyList<String>?> GetFileText(String bucketName, Int64 timestamp)
{
2023-10-02 13:37:27 +00:00
var csv = await S3Cfg
.GetDefaultRegionAndCredentials()!
.Bucket(bucketName)
.Path($"{timestamp}.csv")
.GetObjectAsString();
return csv.Split(Environment.NewLine);
}
}