Innovenergy
/
Innovenergy_trunk
2
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

S3 Utils

This commit is contained in:
ig 2023-10-02 15:37:27 +02:00
parent 7b7a17ca49
commit 2f5622e26c
20 changed files with 551 additions and 404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
csharp/App/Backend/DataTypes/Methods/Installation.cs
View File

@ -1,6 +1,4 @@
using InnovEnergy.App.Backend.Database; using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.Relations;
using InnovEnergy.App.Backend.S3;
using InnovEnergy.Lib.Utils; using InnovEnergy.Lib.Utils;
namespace InnovEnergy.App.Backend.DataTypes.Methods; namespace InnovEnergy.App.Backend.DataTypes.Methods;
@ -31,27 +29,35 @@ public static class InstallationMethods
installation.S3Key = key; installation.S3Key = key;
installation.S3Secret = secret; installation.S3Secret = secret;
return Db.Update(installation); return Db.Update(installation);
} }
public static Task<Boolean> CreateBucket(this Installation installation) public static Task<Boolean> CreateBucket(this Installation installation)
{ {
return S3Access // TODO
.Admin
.CreateBucket(installation.BucketName()); throw new NotImplementedException();
// return S3Access
// .Admin
// .CreateBucket(installation.BucketName());
} }
public static Task<Boolean> DeleteBucket(this Installation installation) public static Task<Boolean> DeleteBucket(this Installation installation)
{ {
return S3Access // TODO
.ReadWrite
.DeleteBucket(installation.BucketName()); throw new NotImplementedException();
// return S3Access
// .ReadWrite
// .DeleteBucket(installation.BucketName());
} }
public static IEnumerable<User> UsersWithAccess(this Installation installation) public static IEnumerable<User> UsersWithAccess(this Installation installation)
{ {
return installation.UsersWithDirectAccess() return installation
.UsersWithDirectAccess()
.Concat(installation.UsersWithInheritedAccess()); .Concat(installation.UsersWithInheritedAccess());
} }
@ -100,7 +106,9 @@ public static class InstallationMethods
public static Installation HideWriteKeyIfUserIsNotAdmin(this Installation installation, Boolean userIsAdmin) public static Installation HideWriteKeyIfUserIsNotAdmin(this Installation installation, Boolean userIsAdmin)
{ {
if(userIsAdmin) return installation; if(userIsAdmin)
return installation;
installation.S3WriteKey = ""; installation.S3WriteKey = "";
installation.S3WriteSecret = ""; installation.S3WriteSecret = "";
@ -120,28 +128,12 @@ public static class InstallationMethods
return Db.Installations.Any(i => i.Id == installation.Id); return Db.Installations.Any(i => i.Id == installation.Id);
} }
public static Boolean SetOrderNumbers(this Installation installation) public static IReadOnlyList<String> GetOrderNumbers(this Installation installation)
{ {
foreach (var orderNumber in installation.OrderNumbers.Split(',')) return Db.OrderNumber2Installation
{
var o2I = new OrderNumber2Installation
{
OrderNumber = orderNumber,
InstallationId = installation.Id
};
Db.Create(o2I);
}
return true;
}
public static String GetOrderNumbers(this Installation installation)
{
return String.Join(", ", Db.OrderNumber2Installation
.Where(i => i.InstallationId == installation.Id) .Where(i => i.InstallationId == installation.Id)
.Select(i => i.OrderNumber) .Select(i => i.OrderNumber)
.ToReadOnlyList()); .ToReadOnlyList();
} }
public static Installation FillOrderNumbers(this Installation installation) public static Installation FillOrderNumbers(this Installation installation)

33
csharp/App/Backend/S3/S3Access.cs
View File

@ -1,13 +1,20 @@
using System.Diagnostics.CodeAnalysis; // using System.Diagnostics.CodeAnalysis;
using static System.IO.File; // using static System.IO.File;
using static System.Text.Json.JsonSerializer; // using static System.Text.Json.JsonSerializer;
//
namespace InnovEnergy.App.Backend.S3; // namespace InnovEnergy.App.Backend.S3;
//
[SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")] // [SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")]
public static class S3Access // public static class S3Access
{ // {
public static S3Cmd ReadOnly => Deserialize<S3Cmd>(OpenRead("./Resources/s3ReadOnlyKey.json"))!; // public static S3Cmd ReadOnly => ParseJsonFile<S3Cmd>("./Resources/s3ReadOnlyKey.json")!;
public static S3Cmd ReadWrite => Deserialize<S3Cmd>(OpenRead("./Resources/s3ReadWriteKey.json"))!; //
public static S3Cmd Admin => Deserialize<S3Cmd>(OpenRead("./Resources/s3AdminKey.json"))!; // private static T? ParseJsonFile<T>(String file)
} // {
// var fileStream = OpenRead(file);
// return Deserialize<T>(fileStream);
// }
//
// public static S3Cmd ReadWrite => Deserialize<S3Cmd>(OpenRead("./Resources/s3ReadWriteKey.json"))!;
// public static S3Cmd Admin => Deserialize<S3Cmd>(OpenRead("./Resources/s3AdminKey.json"))!;
// }

186
csharp/App/Backend/S3/S3Cmd.cs
View File

@ -1,106 +1,90 @@
using System.Diagnostics.CodeAnalysis; // using System.Diagnostics.CodeAnalysis;
using CliWrap; // using CliWrap;
using CliWrap.Buffered; // using CliWrap.Buffered;
using InnovEnergy.Lib.Utils; // using InnovEnergy.Lib.Utils;
//
namespace InnovEnergy.App.Backend.S3; // #pragma warning disable CS8618
//
[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] // namespace InnovEnergy.App.Backend.S3;
public class S3Cmd //
{ // [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")]
private static readonly Command Python = Cli.Wrap("python3"); // public class S3Cmd
private const String? S3CmdPath = "Resources/s3cmd.py";
private const String S3Prefix = "s3://";
public String Key { get; init;}
public String Secret { get; init;}
public String Region { get; init; } = "sos-ch-dk-2.exo.io";
// private String?[] DefaultArgs { get; }
// ReSharper disable StringLiteralTypo
// ReSharper enable StringLiteralTypo
[Obsolete("Only to be used by Json-Deserializer")] public S3Cmd()
{
}
// public S3Cmd(String? key, String? secret)
// { // {
// DefaultArgs = new[] // private static readonly Command Python = Cli.Wrap("python3");
//
// private const String S3CmdPath = "Resources/s3cmd.py";
// private const String S3Prefix = "s3://";
//
// public String Key { get; init; }
// public String Secret { get; init; }
// public String Region { get; init; } = "sos-ch-dk-2.exo.io";
//
// [Obsolete("Only to be used by Json-Deserializer")] public S3Cmd()
// {}
//
// public async Task<Boolean> CreateBucket(String bucketName)
// {
// const String cors = "./Resources/CORS";
//
// var makeBucket = await Run(bucketName, "mb");
//
// if (makeBucket.ExitCode != 0)
// return false;
//
// var setCors = await Run(bucketName, "setcors", cors);
//
// return setCors.ExitCode == 0;
// }
//
// public async Task<String> ListFilesInBucket(String bucketName)
// {
// var result = await Run(bucketName, "ls");
// return result.StandardOutput;
// }
//
// public async Task<IReadOnlyList<String>?> GetFileLines(String bucketName, String filename)
// {
// try
// {
// await Run(bucketName + "/" + filename, "get", "--force");
// }
// catch
// {
// return null;
// }
//
// var lines = File.ReadAllLines($"./{filename}");
// File.Delete(filename);
// return lines;
// }
//
// public async Task<Boolean> DeleteBucket(String bucketName)
// {
// var result = await Run(bucketName, "rb");
// return result.ExitCode == 0;
// }
//
// private Task<BufferedCommandResult> Run(String s3Path, String operation, params String[] optionalArgs)
// {
// var credentials = new[]
// { // {
// S3CmdPath, // S3CmdPath,
// "--access_key", key, // "--access_key", Key,
// "--secret_key", secret, // "--secret_key", Secret,
// "--host" , Region
// }; // };
//
// var args = credentials
// .Append(operation)
// .Concat(optionalArgs)
// .Append(s3Path.EnsureStartsWith(S3Prefix));
//
// var withArguments = Python
// .WithArguments(args);
//
// return withArguments
// .WithValidation(CommandResultValidation.None)
// .ExecuteBufferedAsync();
// }
//
// } // }
public async Task<String> SignUrl(String bucketName, TimeSpan validity)
{
var result = await Run(bucketName, "signurl", $"+{validity.TotalSeconds}");
return result
.StandardOutput
.Replace("\n", "")
.Replace(" ", "");
}
public async Task<Boolean> CreateBucket(String bucketName)
{
const String cors = "./Resources/CORS";
var result = await Run(bucketName, "mb");
var setCors = await Run(bucketName, "setcors", cors);
return result.ExitCode == 0 && setCors.ExitCode == 0;
}
public async Task<String> ListFilesInBucket(String bucketName)
{
var result = await Run(bucketName, "ls");
return result.StandardOutput;
}
public async Task<IReadOnlyList<String>?> GetFileLines(String bucketName, String filename)
{
try
{
await Run(bucketName + "/" + filename, "get", "--force");
}
catch
{
return null;
}
var lines = File.ReadAllLines($"./{filename}");
File.Delete(filename);
return lines;
}
public async Task<Boolean> DeleteBucket(String bucketName)
{
var result = await Run(bucketName, "rb");
return result.ExitCode == 0;
}
private Task<BufferedCommandResult> Run(String bucketName, String operation, params String[] optionalArgs)
{
var credentials = new[]
{
S3CmdPath,
"--access_key", Key,
"--secret_key", Secret,
"--host", Region
};
var args = credentials
.Append(operation)
.Concat(optionalArgs)
.Append(bucketName.EnsureStartsWith(S3Prefix));
return Python
.WithArguments(args)
.ExecuteBufferedAsync();
}
}

152
csharp/App/S3Explorer/Program.cs
View File

@ -1,46 +1,52 @@
using System.ComponentModel; using Amazon.IdentityManagement.Model;
using InnovEnergy.App.Backend.S3; using Amazon.S3;
using Amazon.S3.Model;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils; using InnovEnergy.Lib.Utils;
namespace S3Explorer; namespace InnovEnergy.App.S3Explorer;
public static class Program public static class Program
{ {
private const String BucketSalt = "-3e5b3069-214a-43ee-8d85-57d72000c19d";
public static async Task<Int32> Main(String[] args) public static async Task<Int32> Main(String[] args)
{ {
var region = S3Cfg.GetDefaultRegionAndCredentials();
var x = Iam.GetIamClient(region!);
var c = await x.ListRolesAsync(new ListRolesRequest());
// Todo refactor S3Access into Lib // Todo refactor S3Access into Lib
// Sssssecret // Sssssecret
if (args.Contains("-s")) if (args.Contains("-s"))
{
await SnakeGameSs.PlaySnake(); await SnakeGameSs.PlaySnake();
}
// Help message // Help message
if (args.Length < 4 || args.Contains("-h")) if (args.Length < 1 || args.Contains("-h"))
{ {
Console.WriteLine("Usage: S3Explorer [BucketId] [from:Unix-time] [to:Unix-time] [#Data-points]"); Console.WriteLine("Usage: S3Explorer installation-id [from-unix-time] [to-unix-time] [nb-data-points]");
Console.WriteLine("-h Shows this message."); Console.WriteLine("-h Shows this message.");
Console.WriteLine("-s 🐍"); Console.WriteLine("-s 🐍");
return 0; return 0;
} }
// Parsing Arguments // Parsing Arguments
var bucketName = args[0] + "-3e5b3069-214a-43ee-8d85-57d72000c19d"; var bucketName = args[0] + BucketSalt;
var startTime = Int64.Parse(args[1]); var now = DateTime.Now;
var endTime = Int64.Parse(args[2]);
var numberOfDataPoints = Int64.Parse(args[3]);
var timeBetweenDataPoints = TimeBetweenDataPoints(startTime, endTime, numberOfDataPoints); var startTime = Int64.Parse(args.ElementAtOr(1, (now - TimeSpan.FromSeconds(20)).ToString()));
var endTime = Int64.Parse(args.ElementAtOr(2, now.ToString()));
var nDataPoints = Int64.Parse(args.ElementAtOr(3, "10"));
// Building a List of the timestamps we want to grab the files for. var timestampList = GetDataTimestamps(startTime, endTime, nDataPoints);
var timestampList = new List<String> { };
for (var i = startTime; i <= endTime; i += timeBetweenDataPoints)
{
//Rounding to even numbers only (we only save every second second)
timestampList.Add((i/2 *2).ToString());
}
await PrintFiles(bucketName, timestampList); await PrintFiles(bucketName, timestampList);
@ -48,48 +54,102 @@ public static class Program
return 0; return 0;
} }
private static async Task PrintFiles(String bucketName, List<String> timestampList) public static async Task ListingObjectsAsync(IAmazonS3 client, String bucketName)
{ {
var newestDataFilename = timestampList.Last(); var request = new ListObjectsV2Request { BucketName = bucketName , Prefix = "1689236"};
var csvFileText = await GetFileText(bucketName, newestDataFilename);
// Building Header-Row from the newest data var listObjectsV2Paginator = client.Paginators.ListObjectsV2(request);
csvFileText
.Select(l => l.Split(";"))
.Select(l => l[0])
.Prepend("Timestamp")
.JoinWith(";")
.WriteLine();
foreach (var timestamp in timestampList) await foreach (var response in listObjectsV2Paginator.Responses)
{ {
csvFileText = await GetFileText(bucketName, timestamp); Console.WriteLine($"HttpStatusCode: {response.HttpStatusCode}");
Console.WriteLine($"Number of Keys: {response.KeyCount}");
// Writing Data below data-keys in a timestamped row foreach (var entry in response.S3Objects)
csvFileText.Select(l => l.Split(";")) {
.Select(l => l[1]) Console.WriteLine($"Key = {entry.Key} Size = {entry.Size}");
.Prepend(timestamp) }
.JoinWith(";") }
.WriteLine();
} }
}
private static Int64 TimeBetweenDataPoints(Int64 startTime, Int64 endTime, Int64 numberOfDataPoints) private static IEnumerable<Int64> GetDataTimestamps(Int64 startTime, Int64 endTime, Int64 nDataPoints)
{ {
// Calculating temporal distance of data files from the number of requested points. // Calculating temporal distance of data files from the number of requested points. (rounding for int division)
var timeSpan = endTime - startTime; var timeSpan = endTime - startTime;
var timeBetweenDataPoints = timeSpan / numberOfDataPoints; var timeBetweenDataPoints = (Double)timeSpan / nDataPoints;
// We only upload data every second second so sampling more is impossible. // We only upload data every second second so sampling more is impossible.
// If this ever changes we might have to change this as well. // If this ever changes we might have to change this as well.
timeBetweenDataPoints = Math.Max(timeBetweenDataPoints, 2);
return timeBetweenDataPoints; // Building a List of the timestamps we want to grab the files for.
for (Double i = startTime; i <= endTime; i += timeBetweenDataPoints)
{
//Rounding to even numbers only (we only save every second second)
var integer = (Int64) Math.Round(i);
yield return integer / 2 * 2;
}
}
private static async Task PrintFiles(String bucketName, IEnumerable<Int64> timestamps)
{
var columns = new Dictionary<String, List<String?>>
{
["timestamp"] = new()
};
var index = 0;
foreach (var timestamp in timestamps)
{
var csvFileText = await GetFileText(bucketName, timestamp);
columns["timestamp"].Add(timestamp.ToString());
var dict = csvFileText is null
? new Dictionary<String, String>()
: csvFileText
.Select(l => l.Split(";"))
.ToDictionary(kv => kv[0], kv => kv[1]);
foreach (var key in dict.Keys)
{
// if a key is not yet present in columns we need to backfill it with nulls
if (!columns.ContainsKey(key))
columns[key] = Enumerable.Repeat<String?>(null, index).ToList();
columns[key].Add(dict[key]);
}
// if a key in columns is not present in this record (dict) (except the timestamp) we need to set it to null
foreach (var key in columns.Keys.Where(key => !dict.ContainsKey(key) && key != "timestamp"))
{
columns[key].Add(null);
}
index++;
}
var headerKeys = columns
.Keys
.OrderBy(k => k)
.Where(k => k != "timestamp")
.Prepend("timestamp")
.ToList();
String.Join(';', headerKeys).WriteLine();
Enumerable.Range(0, index)
.Select(i => headerKeys.Select(hk => columns[hk][i]).JoinWith(";"))
.ForEach(Console.WriteLine);
} }
// This Method extracts the Text from a given csv file on the s3 bucket // This Method extracts the Text from a given csv file on the s3 bucket
private static async Task<String[]> GetFileText(String bucketName, String filename) private static async Task<IReadOnlyList<String>?> GetFileText(String bucketName, Int64 timestamp)
{ {
return await S3Access.Admin.GetFileText(bucketName, filename + ".csv"); var csv = await S3Cfg
.GetDefaultRegionAndCredentials()!
.Bucket(bucketName)
.Path($"{timestamp}.csv")
.GetObjectAsString();
return csv.Split(Environment.NewLine);
} }
} }

12
csharp/App/S3Explorer/S3Explorer.csproj
View File

@ -1,15 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<Import Project="../InnovEnergy.App.props" />
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <RootNamespace>InnovEnergy.App.S3Explorer</RootNamespace>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\Lib\Time\Time.csproj" /> <ProjectReference Include="../../Lib/Time/Time.csproj" />
<ProjectReference Include="..\Backend\Backend.csproj" /> <ProjectReference Include="../Backend/Backend.csproj" />
<ProjectReference Include="../../Lib/S3Utils/S3Utils.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

2
csharp/App/S3Explorer/SnakeGameSS.cs
View File

@ -1,4 +1,4 @@
namespace S3Explorer; namespace InnovEnergy.App.S3Explorer;
public static class SnakeGameSs public static class SnakeGameSs
{ {

6
csharp/Lib/S3Utils/Data/S3Bucket.cs
View File

@ -1,6 +0,0 @@
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3Bucket : S3RegionCredentials
{
public required String Bucket { get; init; }
}

47
csharp/Lib/S3Utils/Data/S3Credentials.cs
View File

@ -1,47 +0,0 @@
using static System.Environment;
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3Credentials
{
protected static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile).TrimEnd('\\', '/') + "/.s3cfg";
public required String Key { get; init; }
public required String Secret { get; init; }
public static S3Credentials? FromS3Cfg() => FromFile(S3CfgFile);
public static S3Credentials? FromFile(String file)
{
// [default]
// host_base = sos-ch-dk-2.exo.io
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
// access_key = xxxxxxxxxxxxxxx
// secret_key = xxxxxxxxxxxxxxx
// use_https = True
try
{
var cfg = ParseFile(file);
return new S3Credentials
{
Key = cfg["access_key"],
Secret = cfg["secret_key"]
};
}
catch
{
return null;
}
}
protected static Dictionary<String, String> ParseFile(String cfgFile)
{
return File
.ReadAllLines(cfgFile)
.Where(l => l.Contains("="))
.Select(l => l.Split("="))
.ToDictionary(l => l[0].Trim(), l => l[1].Trim());
}
}

6
csharp/Lib/S3Utils/Data/S3Path.cs
View File

@ -1,6 +0,0 @@
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3Path : S3Bucket
{
public required String Path { get; init; }
}

49
csharp/Lib/S3Utils/Data/S3RegionCredentials.cs
View File

@ -1,49 +0,0 @@
using Amazon.Runtime;
using Amazon.S3;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3RegionCredentials : S3Credentials
{
public required String Region { get; init; }
private AmazonS3Client? _Client;
internal AmazonS3Client Client => _Client ??= new AmazonS3Client
(
credentials: new BasicAWSCredentials(Key, Secret),
clientConfig: new()
{
ServiceURL = Region.EnsureStartsWith("https://"),
ForcePathStyle = true,
}
);
public new static S3RegionCredentials? FromS3Cfg() => FromFile(S3CfgFile);
public new static S3RegionCredentials? FromFile(String file)
{
// [default]
// host_base = sos-ch-dk-2.exo.io
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
// access_key = xxxxxxxxxxxxxxx
// secret_key = xxxxxxxxxxxxxxx
// use_https = True
try
{
var cfg = ParseFile(file);
return new S3RegionCredentials
{
Key = cfg["access_key"],
Secret = cfg["secret_key"],
Region = cfg["host_base"],
};
}
catch
{
return null;
}
}
}

7
csharp/Lib/S3Utils/DataTypes/S3Bucket.cs Normal file
View File

@ -0,0 +1,7 @@
namespace InnovEnergy.Lib.S3Utils.DataTypes;
public record S3Bucket
(
String Name,
S3Region Region
);

67
csharp/Lib/S3Utils/DataTypes/S3Cfg.cs Normal file
View File

@ -0,0 +1,67 @@
using static System.Environment;
namespace InnovEnergy.Lib.S3Utils.DataTypes;
public static class S3Cfg
{
internal static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile)
.TrimEnd('\\', '/') + "/.s3cfg";
internal static Dictionary<String, String> FromFile(String cfgFilePath)
{
return File
.ReadAllLines(cfgFilePath)
.Where(l => l.Contains("="))
.Select(l => l.Split("="))
.ToDictionary(l => l[0].Trim(), l => l[1].Trim());
}
public static S3Credentials? GetDefaultUserCredentials() => GetCredentialsFromFile(S3CfgFile);
public static S3Credentials? GetCredentialsFromFile(String file)
{
// [default]
// host_base = sos-ch-dk-2.exo.io
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
// access_key = xxxxxxxxxxxxxxx
// secret_key = xxxxxxxxxxxxxxx
// use_https = True
try
{
var cfg = FromFile(file);
return new S3Credentials
(
Key : cfg["access_key"],
Secret: cfg["secret_key"]
);
}
catch
{
return null;
}
}
public static S3Region? GetDefaultRegionAndCredentials() => GetRegionAndCredentialsFromFile(S3CfgFile);
public static S3Region? GetRegionAndCredentialsFromFile(String file)
{
try
{
var cfg = FromFile(file);
var credentials = new S3Credentials
(
Key : cfg["access_key"],
Secret: cfg["secret_key"]
);
return new S3Region(Name: cfg["host_base"], Credentials: credentials);
}
catch
{
return null;
}
}
}

7
csharp/Lib/S3Utils/DataTypes/S3Credentials.cs Normal file
View File

@ -0,0 +1,7 @@
namespace InnovEnergy.Lib.S3Utils.DataTypes;
public record S3Credentials
(
String Key,
String Secret
);

7
csharp/Lib/S3Utils/DataTypes/S3Region.cs Normal file
View File

@ -0,0 +1,7 @@
namespace InnovEnergy.Lib.S3Utils.DataTypes;
public record S3Region
(
String Name,
S3Credentials Credentials
);

7
csharp/Lib/S3Utils/DataTypes/S3Url.cs Normal file
View File

@ -0,0 +1,7 @@
namespace InnovEnergy.Lib.S3Utils.DataTypes;
public record S3Url
(
String Path,
S3Bucket Bucket
);

21
csharp/Lib/S3Utils/ExoScale/DefaultCredentials.cs
View File

@ -1,21 +0,0 @@
using InnovEnergy.Lib.S3Utils.Data;
namespace InnovEnergy.Lib.S3Utils.ExoScale;
public static class DefaultCredentials
{
public static S3Credentials ReadOnly => new S3Credentials
{
Key = "EXOb6d6dc1880cdd51f1ebc6692",
Secret = "kpIey4QJlQFuWG_WoTazcY7kBEjN2f_ll2cDBeg64m4",
};
public static S3Credentials ReadWrite => new S3Credentials
{
Key = "EXO87ca85e29dd412f1238f1cf0",
Secret = "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU",
};
}

6
csharp/Lib/S3Utils/ExoScale/Regions.cs
View File

@ -1,6 +0,0 @@
namespace InnovEnergy.Lib.S3Utils.ExoScale;
public static class Regions
{
public static String Default => "sos-ch-dk-2.exo.io";
}

32
csharp/Lib/S3Utils/Iam.cs Normal file
View File

@ -0,0 +1,32 @@
using System.Collections.Concurrent;
using Amazon.IdentityManagement;
using Amazon.Runtime;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3Utils;
public static class Iam
{
// TODO
private static readonly ConcurrentDictionary<S3Region, AmazonIdentityManagementServiceClient> AimClientCache = new();
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Url url ) => url.Bucket.GetIamClient();
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Bucket bucket) => bucket.Region.GetIamClient();
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Region region)
{
return AimClientCache.GetOrAdd(region, CreateIamClient); // Memoize
}
private static AmazonIdentityManagementServiceClient CreateIamClient(S3Region region) => new
(
credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret),
clientConfig: new() { ServiceURL = StringUtils.EnsureStartsWith(region.Name, "https://") }
);
}

195
csharp/Lib/S3Utils/S3.cs
View File

@ -1,81 +1,119 @@
using System.Collections.Concurrent;
using System.Net;
using System.Text;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model; using Amazon.S3.Model;
using InnovEnergy.Lib.S3Utils.Data; using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils; using InnovEnergy.Lib.Utils;
using S3Bucket = InnovEnergy.Lib.S3Utils.Data.S3Bucket; using S3Bucket = InnovEnergy.Lib.S3Utils.DataTypes.S3Bucket;
using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
namespace InnovEnergy.Lib.S3Utils; namespace InnovEnergy.Lib.S3Utils;
public static class S3 public static class S3
{ {
public static S3RegionCredentials Region(this S3Credentials credentials, String region) => new() private static readonly ConcurrentDictionary<S3Region, AmazonS3Client> S3ClientCache = new();
{
Secret = credentials.Secret,
Key = credentials.Key,
Region = region,
};
public static S3Bucket Bucket(this S3RegionCredentials region, String bucket) => new() // QOL method
{ public static S3Region Region(this S3Credentials credentials, String name) => new
Region = region.Region, (
Secret = region.Secret, Name: name,
Key = region.Key, Credentials: credentials
Bucket = bucket, );
};
public static S3Path Path(this S3Bucket bucket, String path) => new() // QOL method
{ public static S3Bucket Bucket(this S3Region region, String name) => new
Bucket = bucket.Bucket, (
Region = bucket.Region, Name: name,
Secret = bucket.Secret, Region: region
Key = bucket.Key, );
Path = path
};
public static IAsyncEnumerable<S3Path> ListObjects(this S3Bucket bucketOrPrefixPath) // QOL method
{ public static S3Url Path(this S3Bucket bucket, String path) => new
var path = bucketOrPrefixPath as S3Path ?? bucketOrPrefixPath.Path(null!); (
Bucket: bucket,
Path: path
);
var request = new ListObjectsV2Request public static IAsyncEnumerable<S3Url> ListObjects(this S3Bucket bucket) => ListObjects(bucket, null);
{
BucketName = path.Bucket,
Prefix = path.Path
};
return bucketOrPrefixPath public static IAsyncEnumerable<S3Url> ListObjects(this S3Bucket bucket, String? pathPrefix)
.Client {
return bucket
.Region
.GetS3Client()
.Paginators .Paginators
.ListObjectsV2(request) .ListObjectsV2(new() { BucketName = bucket.Name, Prefix = pathPrefix })
.Responses .S3Objects
.SelectMany(r => r.S3Objects) .Select(o => new S3Url(o.Key, bucket));
.Select(o => path with { Path = o.Key });
} }
public static async Task<String> GetObject(this S3Path path) public static Task<Boolean> PutObject(this S3Url path, String data, Encoding encoding) => path.PutObject(encoding.GetBytes(data));
public static Task<Boolean> PutObject(this S3Url path, String data) => path.PutObject(data, Encoding.UTF8);
public static Task<Boolean> PutObject(this S3Url path, Byte[] data) => path.PutObject(new MemoryStream(data));
public static async Task<Boolean> PutObject(this S3Url path, Stream data)
{ {
var request = new GetObjectRequest var request = new PutObjectRequest
{ {
BucketName = path.Bucket, BucketName = path.Bucket.Name,
Key = path.Path Key = path.Path,
InputStream = data
}; };
using var response = await path.Client.GetObjectAsync(request); var response = await path
await using var responseStream = response.ResponseStream; .Bucket
using var reader = new StreamReader(responseStream); .Region
.GetS3Client()
.PutObjectAsync(request);
return response.HttpStatusCode == HttpStatusCode.OK;
}
public static Task<String> GetObjectAsString(this S3Url path) => GetObjectAsString(path, Encoding.UTF8);
public static async Task<String> GetObjectAsString(this S3Url path, Encoding encoding)
{
await using var stream = await GetObjectAsStream(path);
using var reader = new StreamReader(stream, encoding);
return await reader.ReadToEndAsync(); return await reader.ReadToEndAsync();
} }
public static async IAsyncEnumerable<String> GetObjectLineByLine(this S3Path path) public static async Task<Stream> GetObjectAsStream(this S3Url path)
{ {
var request = new GetObjectRequest var request = new GetObjectRequest
{ {
BucketName = path.Bucket, BucketName = path.Bucket.Name,
Key = path.Path Key = path.Path
}; };
using var response = await path.Client.GetObjectAsync(request); var response = await path
await using var responseStream = response.ResponseStream; .Bucket
using var reader = new StreamReader(responseStream); .Region
.GetS3Client()
.GetObjectAsync(request);
return response.ResponseStream;
}
public static async Task<IReadOnlyList<Byte>> GetObject(this S3Url url)
{
// beautiful await using stream soup...
await using var stream = await url.GetObjectAsStream();
using var memoryStream = new MemoryStream();
await stream.CopyToAsync(memoryStream);
return memoryStream.ToArray();
}
public static IAsyncEnumerable<String> GetObjectLineByLine(this S3Url url) => GetObjectLineByLine(url, Encoding.UTF8);
public static async IAsyncEnumerable<String> GetObjectLineByLine(this S3Url url, Encoding encoding)
{
await using var stream = await url.GetObjectAsStream();
using var reader = new StreamReader(stream, encoding);
while (true) while (true)
{ {
@ -87,4 +125,63 @@ public static class S3
yield break; yield break;
} }
} }
public static async Task<S3Bucket?> PutBucket(this S3Region region, String name)
{
var request = new PutBucketRequest { BucketName = name };
var response = await region
.GetS3Client()
.PutBucketAsync(request);
return response.HttpStatusCode switch
{
HttpStatusCode.OK => region.Bucket(name),
_ => null
};
}
public static async Task<Boolean> PutCors(this S3Bucket bucket, CORSConfiguration corsConfiguration)
{
var request = new PutCORSConfigurationRequest
{
BucketName = bucket.Name,
Configuration = corsConfiguration
};
var response = await bucket
.GetS3Client()
.PutCORSConfigurationAsync(request);
return response.HttpStatusCode == HttpStatusCode.OK;
}
public static async Task<Boolean> DeleteBucket(this S3Bucket bucket)
{
var request = new DeleteBucketRequest { BucketName = bucket.Name };
var response = await bucket
.GetS3Client()
.DeleteBucketAsync(request);
return response.HttpStatusCode == HttpStatusCode.OK;
}
private static AmazonS3Client GetS3Client(this S3Url url ) => url.Bucket.GetS3Client();
private static AmazonS3Client GetS3Client(this S3Bucket bucket) => bucket.Region.GetS3Client();
private static AmazonS3Client GetS3Client(this S3Region region)
{
return S3ClientCache.GetOrAdd(region, CreateS3Client); // Memoize
}
private static AmazonS3Client CreateS3Client(S3Region region) => new
(
credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret),
clientConfig: new()
{
ServiceURL = StringUtils.EnsureStartsWith(region.Name, "https://"),
ForcePathStyle = true,
}
);
} }

15
csharp/Lib/S3Utils/S3Utils.csproj
View File

@ -11,8 +11,23 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.200.39" />
<PackageReference Include="AWSSDK.S3" Version="3.7.203.12" /> <PackageReference Include="AWSSDK.S3" Version="3.7.203.12" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" /> <PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<Compile Remove="ExoScale\DefaultCredentials.cs" />
<Compile Remove="Data\S3Credentials.cs" />
<Compile Remove="Data\S3Path.cs" />
<Compile Remove="Data\S3Bucket.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="Data\" />
<Folder Include="ExoScale\" />
</ItemGroup>
</Project> </Project>