Merge branch 'new_S3_API'
# Conflicts: # csharp/App/VrmGrabber/Controller.cs
This commit is contained in:
commit
4eedb913c5
|
@ -34,6 +34,7 @@
|
|||
<ItemGroup>
|
||||
<ProjectReference Include="../../Lib/Utils/Utils.csproj" />
|
||||
<ProjectReference Include="../../Lib/Mailer/Mailer.csproj" />
|
||||
<ProjectReference Include="..\..\Lib\S3Utils\S3Utils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -10,7 +10,7 @@ public class Installation : TreeNode
|
|||
|
||||
// TODO: make relation
|
||||
//public IReadOnlyList<String> OrderNumbers { get; set; } = Array.Empty<String>();
|
||||
public String OrderNumbers { get; set; } = "";
|
||||
public String? OrderNumbers { get; set; } = "";
|
||||
|
||||
public Double Lat { get; set; }
|
||||
public Double Long { get; set; }
|
||||
|
|
|
@ -1,71 +1,63 @@
|
|||
using CliWrap;
|
||||
using CliWrap.Buffered;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
using System.Text.Json;
|
||||
using InnovEnergy.Lib.S3Utils;
|
||||
using InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace InnovEnergy.App.Backend.DataTypes.Methods;
|
||||
|
||||
public static class ExoCmd
|
||||
{
|
||||
private static readonly Command Exo = Cli.Wrap("exo");
|
||||
private const String ConfigFile = "./exoscale.toml";
|
||||
[UnconditionalSuppressMessage("Trimming", "IL2026:Members annotated with 'RequiresUnreferencedCodeAttribute' require dynamic access otherwise can break functionality when trimming application code", Justification = "<Pending>")]
|
||||
private static readonly S3Credentials? S3Creds = JsonSerializer.Deserialize<S3Credentials>("./exoscaleS3.json");
|
||||
|
||||
public static async Task<(String key, String secret)> CreateReadKey(this Installation installation)
|
||||
{
|
||||
//if (installation.Id != 1) return "help"; //Todo remove me I am for debugging
|
||||
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
|
||||
if (!await Iam.UserExists(iamService, $"READ{installation.BucketName()}"))
|
||||
{
|
||||
|
||||
var readOnlyPolicy = $"{installation.BucketName()}"; // TODO make me
|
||||
await Iam.CreateUserAsync(iamService, $"READ{installation.BucketName()}");
|
||||
await Iam.PutUserPolicyAsync(iamService, $"READ{installation.BucketName()}", $"READ{installation.BucketName()}",readOnlyPolicy);
|
||||
}
|
||||
|
||||
var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READ{installation.BucketName()}");
|
||||
|
||||
|
||||
return (keySecret.AccessKeyId, keySecret.SecretAccessKey);
|
||||
}
|
||||
|
||||
var preParse = await Exo
|
||||
.WithArguments("iam access-key create " + installation.BucketName()
|
||||
+ " --operation get-sos-object"
|
||||
+ " --resource sos/bucket:" + installation.BucketName()
|
||||
+ " -C " + ConfigFile
|
||||
+ " -O text")
|
||||
.ExecuteBufferedAsync();
|
||||
public static async Task<Boolean> RevokeReadKey(this Installation installation)
|
||||
{
|
||||
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
|
||||
if (!await Iam.UserExists(iamService, $"READ{installation.BucketName()}"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var key = preParse.StandardOutput.Split("\t")[2];
|
||||
var secret = preParse.StandardOutput.Split("\t")[3];
|
||||
|
||||
return (key, secret);
|
||||
|
||||
//return $"{key};{secret}";
|
||||
return await Iam.RevokeAccessKey(iamService, $"READ{installation.BucketName()}");
|
||||
}
|
||||
|
||||
public static async Task<(String key, String secret)> CreateWriteKey(this Installation installation)
|
||||
{
|
||||
//if (installation.Id != 1) return "help"; //Todo remove me I am for debugging
|
||||
|
||||
|
||||
|
||||
var preParse = await Exo
|
||||
.WithArguments("iam access-key create " + installation.BucketName()
|
||||
+ " --resource sos/bucket:" + installation.BucketName()
|
||||
+ " -C " + ConfigFile
|
||||
+ " -O text")
|
||||
.ExecuteBufferedAsync();
|
||||
|
||||
var key = preParse.StandardOutput.Split("\t")[2];
|
||||
var secret = preParse.StandardOutput.Split("\t")[3];
|
||||
|
||||
return (key, secret);
|
||||
|
||||
//return $"{key};{secret}";
|
||||
}
|
||||
|
||||
public static async Task RevokeReadKey(this Installation installation)
|
||||
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
|
||||
if (!await Iam.UserExists(iamService, $"READWRITE{installation.BucketName()}"))
|
||||
{
|
||||
try
|
||||
{
|
||||
await Exo
|
||||
.WithArguments("iam access-key revoke " + installation.S3Key + " -f " + " -C " + ConfigFile)
|
||||
.WithValidation(CommandResultValidation.None)
|
||||
.ExecuteAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// TODO
|
||||
("Failed to revoke key for installation " + installation.Name).WriteLine();
|
||||
}
|
||||
var readWritePolicy = $"{installation.BucketName()}"; // TODO make me
|
||||
await Iam.CreateUserAsync(iamService, $"READWRITE{installation.BucketName()}");
|
||||
await Iam.PutUserPolicyAsync(iamService, $"READWRITE{installation.BucketName()}", $"READWRITE{installation.BucketName()}",readWritePolicy);
|
||||
}
|
||||
|
||||
var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READWRITE{installation.BucketName()}");
|
||||
|
||||
|
||||
return (keySecret.AccessKeyId, keySecret.SecretAccessKey);
|
||||
}
|
||||
|
||||
|
||||
public static async Task<Boolean> CreateBucket(this Installation installation)
|
||||
{
|
||||
var s3Region = new S3Region(installation.Region, S3Creds!);
|
||||
return await s3Region.PutBucket(installation.BucketName()) != null;
|
||||
}
|
||||
}
|
|
@ -1,6 +1,5 @@
|
|||
using InnovEnergy.App.Backend.Database;
|
||||
using InnovEnergy.App.Backend.Relations;
|
||||
using InnovEnergy.App.Backend.S3;
|
||||
using InnovEnergy.Lib.S3Utils;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
|
||||
namespace InnovEnergy.App.Backend.DataTypes.Methods;
|
||||
|
@ -31,27 +30,20 @@ public static class InstallationMethods
|
|||
installation.S3Key = key;
|
||||
installation.S3Secret = secret;
|
||||
|
||||
|
||||
return Db.Update(installation);
|
||||
}
|
||||
|
||||
public static Task<Boolean> CreateBucket(this Installation installation)
|
||||
{
|
||||
return S3Access
|
||||
.Admin
|
||||
.CreateBucket(installation.BucketName());
|
||||
}
|
||||
|
||||
public static Task<Boolean> DeleteBucket(this Installation installation)
|
||||
public static async Task<Boolean> DeleteBucket(this Installation installation)
|
||||
{
|
||||
return S3Access
|
||||
.ReadWrite
|
||||
.DeleteBucket(installation.BucketName());
|
||||
// TODO We dont do this here
|
||||
return true;
|
||||
}
|
||||
|
||||
public static IEnumerable<User> UsersWithAccess(this Installation installation)
|
||||
{
|
||||
return installation.UsersWithDirectAccess()
|
||||
return installation
|
||||
.UsersWithDirectAccess()
|
||||
.Concat(installation.UsersWithInheritedAccess());
|
||||
}
|
||||
|
||||
|
@ -100,7 +92,9 @@ public static class InstallationMethods
|
|||
|
||||
public static Installation HideWriteKeyIfUserIsNotAdmin(this Installation installation, Boolean userIsAdmin)
|
||||
{
|
||||
if(userIsAdmin) return installation;
|
||||
if(userIsAdmin)
|
||||
return installation;
|
||||
|
||||
installation.S3WriteKey = "";
|
||||
installation.S3WriteSecret = "";
|
||||
|
||||
|
@ -120,33 +114,17 @@ public static class InstallationMethods
|
|||
return Db.Installations.Any(i => i.Id == installation.Id);
|
||||
}
|
||||
|
||||
public static Boolean SetOrderNumbers(this Installation installation)
|
||||
public static IReadOnlyList<String> GetOrderNumbers(this Installation installation)
|
||||
{
|
||||
foreach (var orderNumber in installation.OrderNumbers.Split(','))
|
||||
{
|
||||
|
||||
var o2I = new OrderNumber2Installation
|
||||
{
|
||||
OrderNumber = orderNumber,
|
||||
InstallationId = installation.Id
|
||||
};
|
||||
Db.Create(o2I);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public static String GetOrderNumbers(this Installation installation)
|
||||
{
|
||||
return String.Join(", ", Db.OrderNumber2Installation
|
||||
return Db.OrderNumber2Installation
|
||||
.Where(i => i.InstallationId == installation.Id)
|
||||
.Select(i => i.OrderNumber)
|
||||
.ToReadOnlyList());
|
||||
.ToReadOnlyList();
|
||||
}
|
||||
|
||||
public static Installation FillOrderNumbers(this Installation installation)
|
||||
{
|
||||
installation.OrderNumbers = installation.GetOrderNumbers();
|
||||
installation.OrderNumbers = installation.GetOrderNumbers().ToString();
|
||||
return installation;
|
||||
}
|
||||
|
||||
|
|
|
@ -87,7 +87,7 @@ public static class SessionMethods
|
|||
&& user.HasWriteAccess
|
||||
&& user.HasAccessToParentOf(installation)
|
||||
&& Db.Create(installation) // TODO: these two in a transaction
|
||||
&& installation.SetOrderNumbers()
|
||||
// && installation.SetOrderNumbers()
|
||||
&& Db.Create(new InstallationAccess { UserId = user.Id, InstallationId = installation.Id })
|
||||
&& await installation.CreateBucket()
|
||||
&& await installation.RenewS3Credentials(); // generation of access _after_ generation of
|
||||
|
@ -101,7 +101,7 @@ public static class SessionMethods
|
|||
var user = session?.User;
|
||||
|
||||
var original = Db.GetInstallationById(installation?.Id);
|
||||
var originalOrderNumbers = original!.GetOrderNumbers();
|
||||
var originalOrderNumbers = original.OrderNumbers;
|
||||
|
||||
if (!Equals(originalOrderNumbers, installation?.OrderNumbers))
|
||||
{
|
||||
|
|
|
@ -1,10 +0,0 @@
|
|||
defaultaccount = "Kim"
|
||||
|
||||
[[accounts]]
|
||||
account = "innovenergy-ag"
|
||||
defaultZone = "ch-dk-2"
|
||||
endpoint = "https://api.exoscale.com/v1"
|
||||
environment = ""
|
||||
key = "EXOf67c5b528282988503ddab12"
|
||||
name = "Kim"
|
||||
secret = "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"Key": "EXOf67c5b528282988503ddab12",
|
||||
"Secret": "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"
|
||||
}
|
|
@ -9,7 +9,8 @@
|
|||
"launchUrl": "swagger",
|
||||
"applicationUrl": "https://localhost:7087",
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||
"HOME":"~/backend"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,13 +0,0 @@
|
|||
using System.Diagnostics.CodeAnalysis;
|
||||
using static System.IO.File;
|
||||
using static System.Text.Json.JsonSerializer;
|
||||
|
||||
namespace InnovEnergy.App.Backend.S3;
|
||||
|
||||
[SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")]
|
||||
public static class S3Access
|
||||
{
|
||||
public static S3Cmd ReadOnly => Deserialize<S3Cmd>(OpenRead("./Resources/s3ReadOnlyKey.json"))!;
|
||||
public static S3Cmd ReadWrite => Deserialize<S3Cmd>(OpenRead("./Resources/s3ReadWriteKey.json"))!;
|
||||
public static S3Cmd Admin => Deserialize<S3Cmd>(OpenRead("./Resources/s3AdminKey.json"))!;
|
||||
}
|
|
@ -1,106 +0,0 @@
|
|||
using System.Diagnostics.CodeAnalysis;
|
||||
using CliWrap;
|
||||
using CliWrap.Buffered;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
|
||||
namespace InnovEnergy.App.Backend.S3;
|
||||
|
||||
[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")]
|
||||
public class S3Cmd
|
||||
{
|
||||
private static readonly Command Python = Cli.Wrap("python3");
|
||||
|
||||
private const String? S3CmdPath = "Resources/s3cmd.py";
|
||||
private const String S3Prefix = "s3://";
|
||||
|
||||
public String Key { get; init;}
|
||||
public String Secret { get; init;}
|
||||
public String Region { get; init; } = "sos-ch-dk-2.exo.io";
|
||||
|
||||
// private String?[] DefaultArgs { get; }
|
||||
|
||||
// ReSharper disable StringLiteralTypo
|
||||
// ReSharper enable StringLiteralTypo
|
||||
[Obsolete("Only to be used by Json-Deserializer")] public S3Cmd()
|
||||
{
|
||||
}
|
||||
|
||||
// public S3Cmd(String? key, String? secret)
|
||||
// {
|
||||
// DefaultArgs = new[]
|
||||
// {
|
||||
// S3CmdPath,
|
||||
// "--access_key", key,
|
||||
// "--secret_key", secret,
|
||||
// };
|
||||
// }
|
||||
|
||||
public async Task<String> SignUrl(String bucketName, TimeSpan validity)
|
||||
{
|
||||
var result = await Run(bucketName, "signurl", $"+{validity.TotalSeconds}");
|
||||
|
||||
return result
|
||||
.StandardOutput
|
||||
.Replace("\n", "")
|
||||
.Replace(" ", "");
|
||||
}
|
||||
|
||||
public async Task<Boolean> CreateBucket(String bucketName)
|
||||
{
|
||||
const String cors = "./Resources/CORS";
|
||||
|
||||
var result = await Run(bucketName, "mb");
|
||||
var setCors = await Run(bucketName, "setcors", cors);
|
||||
|
||||
return result.ExitCode == 0 && setCors.ExitCode == 0;
|
||||
}
|
||||
|
||||
public async Task<String> ListFilesInBucket(String bucketName)
|
||||
{
|
||||
var result = await Run(bucketName, "ls");
|
||||
return result.StandardOutput;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<String>?> GetFileLines(String bucketName, String filename)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Run(bucketName + "/" + filename, "get", "--force");
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var lines = File.ReadAllLines($"./{filename}");
|
||||
File.Delete(filename);
|
||||
return lines;
|
||||
}
|
||||
|
||||
public async Task<Boolean> DeleteBucket(String bucketName)
|
||||
{
|
||||
var result = await Run(bucketName, "rb");
|
||||
return result.ExitCode == 0;
|
||||
}
|
||||
|
||||
private Task<BufferedCommandResult> Run(String bucketName, String operation, params String[] optionalArgs)
|
||||
{
|
||||
var credentials = new[]
|
||||
{
|
||||
S3CmdPath,
|
||||
"--access_key", Key,
|
||||
"--secret_key", Secret,
|
||||
"--host", Region
|
||||
};
|
||||
|
||||
var args = credentials
|
||||
.Append(operation)
|
||||
.Concat(optionalArgs)
|
||||
.Append(bucketName.EnsureStartsWith(S3Prefix));
|
||||
|
||||
return Python
|
||||
.WithArguments(args)
|
||||
.ExecuteBufferedAsync();
|
||||
}
|
||||
|
||||
}
|
|
@ -1,10 +0,0 @@
|
|||
defaultaccount = "Kim"
|
||||
|
||||
[[accounts]]
|
||||
account = "innovenergy-ag"
|
||||
defaultZone = "ch-dk-2"
|
||||
endpoint = "https://api.exoscale.com/v1"
|
||||
environment = ""
|
||||
key = "EXOf67c5b528282988503ddab12"
|
||||
name = "Kim"
|
||||
secret = "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"
|
|
@ -1,95 +1,156 @@
|
|||
using System.ComponentModel;
|
||||
using InnovEnergy.App.Backend.S3;
|
||||
using Amazon.IdentityManagement.Model;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using InnovEnergy.Lib.S3Utils;
|
||||
using InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
using S3Cfg = InnovEnergy.Lib.S3Utils.S3Cfg;
|
||||
|
||||
namespace S3Explorer;
|
||||
namespace InnovEnergy.App.S3Explorer;
|
||||
|
||||
public static class Program
|
||||
{
|
||||
private const String BucketSalt = "-3e5b3069-214a-43ee-8d85-57d72000c19d";
|
||||
|
||||
public static async Task<Int32> Main(String[] args)
|
||||
{
|
||||
var region = S3Cfg.GetDefaultRegionAndCredentials();
|
||||
var x = Iam.GetIamClient(region!);
|
||||
|
||||
var c = await x.ListRolesAsync(new ListRolesRequest());
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// Todo refactor S3Access into Lib
|
||||
|
||||
// Sssssecret
|
||||
if (args.Contains("-s"))
|
||||
{
|
||||
await SnakeGameSs.PlaySnake();
|
||||
}
|
||||
|
||||
// Help message
|
||||
if (args.Length < 4 || args.Contains("-h"))
|
||||
if (args.Length < 1 || args.Contains("-h"))
|
||||
{
|
||||
Console.WriteLine("Usage: S3Explorer [BucketId] [from:Unix-time] [to:Unix-time] [#Data-points]");
|
||||
Console.WriteLine("Usage: S3Explorer installation-id [from-unix-time] [to-unix-time] [nb-data-points]");
|
||||
Console.WriteLine("-h Shows this message.");
|
||||
Console.WriteLine("-s 🐍");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Parsing Arguments
|
||||
var bucketName = args[0] + "-3e5b3069-214a-43ee-8d85-57d72000c19d";
|
||||
var startTime = Int64.Parse(args[1]);
|
||||
var endTime = Int64.Parse(args[2]);
|
||||
var numberOfDataPoints = Int64.Parse(args[3]);
|
||||
var bucketName = args[0] + BucketSalt;
|
||||
var now = DateTime.Now;
|
||||
|
||||
var timeBetweenDataPoints = TimeBetweenDataPoints(startTime, endTime, numberOfDataPoints);
|
||||
var startTime = Int64.Parse(args.ElementAtOr(1, (now - TimeSpan.FromSeconds(20)).ToString()));
|
||||
var endTime = Int64.Parse(args.ElementAtOr(2, now.ToString()));
|
||||
var nDataPoints = Int64.Parse(args.ElementAtOr(3, "10"));
|
||||
|
||||
// Building a List of the timestamps we want to grab the files for.
|
||||
var timestampList = new List<String> { };
|
||||
for (var i = startTime; i <= endTime; i += timeBetweenDataPoints)
|
||||
{
|
||||
//Rounding to even numbers only (we only save every second second)
|
||||
timestampList.Add((i/2 *2).ToString());
|
||||
}
|
||||
var timestampList = GetDataTimestamps(startTime, endTime, nDataPoints);
|
||||
|
||||
await PrintFiles(bucketName,timestampList);
|
||||
await PrintFiles(bucketName, timestampList);
|
||||
|
||||
// Success
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static async Task PrintFiles(String bucketName, List<String> timestampList)
|
||||
public static async Task ListingObjectsAsync(IAmazonS3 client, String bucketName)
|
||||
{
|
||||
var newestDataFilename = timestampList.Last();
|
||||
var csvFileText = await GetFileText(bucketName, newestDataFilename);
|
||||
var request = new ListObjectsV2Request { BucketName = bucketName , Prefix = "1689236"};
|
||||
|
||||
// Building Header-Row from the newest data
|
||||
csvFileText
|
||||
.Select(l => l.Split(";"))
|
||||
.Select(l => l[0])
|
||||
.Prepend("Timestamp")
|
||||
.JoinWith(";")
|
||||
.WriteLine();
|
||||
var listObjectsV2Paginator = client.Paginators.ListObjectsV2(request);
|
||||
|
||||
foreach (var timestamp in timestampList)
|
||||
await foreach (var response in listObjectsV2Paginator.Responses)
|
||||
{
|
||||
csvFileText = await GetFileText(bucketName, timestamp);
|
||||
Console.WriteLine($"HttpStatusCode: {response.HttpStatusCode}");
|
||||
Console.WriteLine($"Number of Keys: {response.KeyCount}");
|
||||
|
||||
// Writing Data below data-keys in a timestamped row
|
||||
csvFileText.Select(l => l.Split(";"))
|
||||
.Select(l => l[1])
|
||||
.Prepend(timestamp)
|
||||
.JoinWith(";")
|
||||
.WriteLine();
|
||||
foreach (var entry in response.S3Objects)
|
||||
{
|
||||
Console.WriteLine($"Key = {entry.Key} Size = {entry.Size}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static Int64 TimeBetweenDataPoints(Int64 startTime, Int64 endTime, Int64 numberOfDataPoints)
|
||||
private static IEnumerable<Int64> GetDataTimestamps(Int64 startTime, Int64 endTime, Int64 nDataPoints)
|
||||
{
|
||||
// Calculating temporal distance of data files from the number of requested points.
|
||||
// Calculating temporal distance of data files from the number of requested points. (rounding for int division)
|
||||
var timeSpan = endTime - startTime;
|
||||
var timeBetweenDataPoints = timeSpan / numberOfDataPoints;
|
||||
|
||||
var timeBetweenDataPoints = (Double)timeSpan / nDataPoints;
|
||||
// We only upload data every second second so sampling more is impossible.
|
||||
// If this ever changes we might have to change this as well.
|
||||
timeBetweenDataPoints = Math.Max(timeBetweenDataPoints, 2);
|
||||
return timeBetweenDataPoints;
|
||||
|
||||
// Building a List of the timestamps we want to grab the files for.
|
||||
for (Double i = startTime; i <= endTime; i += timeBetweenDataPoints)
|
||||
{
|
||||
//Rounding to even numbers only (we only save every second second)
|
||||
var integer = (Int64) Math.Round(i);
|
||||
yield return integer / 2 * 2;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task PrintFiles(String bucketName, IEnumerable<Int64> timestamps)
|
||||
{
|
||||
var columns = new Dictionary<String, List<String?>>
|
||||
{
|
||||
["timestamp"] = new()
|
||||
};
|
||||
var index = 0;
|
||||
|
||||
foreach (var timestamp in timestamps)
|
||||
{
|
||||
var csvFileText = await GetFileText(bucketName, timestamp);
|
||||
|
||||
columns["timestamp"].Add(timestamp.ToString());
|
||||
|
||||
var dict = csvFileText is null
|
||||
? new Dictionary<String, String>()
|
||||
: csvFileText
|
||||
.Select(l => l.Split(";"))
|
||||
.ToDictionary(kv => kv[0], kv => kv[1]);
|
||||
|
||||
foreach (var key in dict.Keys)
|
||||
{
|
||||
// if a key is not yet present in columns we need to backfill it with nulls
|
||||
if (!columns.ContainsKey(key))
|
||||
columns[key] = Enumerable.Repeat<String?>(null, index).ToList();
|
||||
|
||||
columns[key].Add(dict[key]);
|
||||
}
|
||||
|
||||
// if a key in columns is not present in this record (dict) (except the timestamp) we need to set it to null
|
||||
foreach (var key in columns.Keys.Where(key => !dict.ContainsKey(key) && key != "timestamp"))
|
||||
{
|
||||
columns[key].Add(null);
|
||||
}
|
||||
index++;
|
||||
}
|
||||
|
||||
var headerKeys = columns
|
||||
.Keys
|
||||
.OrderBy(k => k)
|
||||
.Where(k => k != "timestamp")
|
||||
.Prepend("timestamp")
|
||||
.ToList();
|
||||
|
||||
String.Join(';', headerKeys).WriteLine();
|
||||
|
||||
Enumerable.Range(0, index)
|
||||
.Select(i => headerKeys.Select(hk => columns[hk][i]).JoinWith(";"))
|
||||
.ForEach(Console.WriteLine);
|
||||
}
|
||||
|
||||
// This Method extracts the Text from a given csv file on the s3 bucket
|
||||
private static async Task<String[]> GetFileText(String bucketName, String filename)
|
||||
private static async Task<IReadOnlyList<String>?> GetFileText(String bucketName, Int64 timestamp)
|
||||
{
|
||||
return await S3Access.Admin.GetFileText(bucketName, filename + ".csv");
|
||||
var csv = await S3Cfg
|
||||
.GetDefaultRegionAndCredentials()!
|
||||
.Bucket(bucketName)
|
||||
.Path($"{timestamp}.csv")
|
||||
.GetObjectAsString();
|
||||
|
||||
return csv.Split(Environment.NewLine);
|
||||
}
|
||||
}
|
|
@ -1,15 +1,20 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<Import Project="../InnovEnergy.App.props" />
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<RootNamespace>InnovEnergy.App.S3Explorer</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Lib\Time\Time.csproj" />
|
||||
<ProjectReference Include="..\Backend\Backend.csproj" />
|
||||
<ProjectReference Include="../../Lib/Time/Time.csproj" />
|
||||
<ProjectReference Include="../Backend/Backend.csproj" />
|
||||
<ProjectReference Include="../../Lib/S3Utils/S3Utils.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AWSSDK.S3" Version="3.7.203.12" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
namespace S3Explorer;
|
||||
namespace InnovEnergy.App.S3Explorer;
|
||||
|
||||
public static class SnakeGameSs
|
||||
{
|
||||
|
|
|
@ -123,7 +123,7 @@ th { /* header cell */
|
|||
<td>{{Serial}}</td>
|
||||
<td>{{NumBatteries}}</td>
|
||||
<td>{{BatteryVersion}}</td>
|
||||
<td><a target='_blank' href=http://{{ServerIp}}/UpdateBatteryFirmware/{{Ip}}>⬆️{{FirmwareVersion}}</a></td>
|
||||
<td><a target='_blank' href=http://{{ServerIp}}/UpdateBatteryFirmware/{{Ip}}/{{NumBatteries}}>⬆️{{FirmwareVersion}}</a></td>
|
||||
<td>{{BatteryUpdateStatus}}</td>
|
||||
</tr>";
|
||||
|
||||
|
@ -200,12 +200,14 @@ th { /* header cell */
|
|||
#pragma warning disable CS4014
|
||||
Console.WriteLine(localCommand);
|
||||
Db.ExecuteBufferedAsyncCommandOnIp(installationIp, localCommand)
|
||||
.ContinueWith(t =>
|
||||
.ContinueWith(async t =>
|
||||
{
|
||||
Console.WriteLine(t.Result);
|
||||
installation.BatteryUpdateStatus = "Complete";
|
||||
Db.Update(installation: installation);
|
||||
UpdateVrmTagsToNewFirmware(installationIp);
|
||||
var vrmInst = await FindVrmInstallationByIp(installation.Ip!);
|
||||
await UpdateVrmTagsToNewFirmware(installationIp);
|
||||
await Db.UpdateAlarms(vrmInst);
|
||||
});
|
||||
#pragma warning restore CS4014
|
||||
return "Battery update is successfully initiated, it will take around 15 minutes to complete! You can close this page now.";
|
||||
|
@ -234,8 +236,8 @@ th { /* header cell */
|
|||
private static async Task SendNewBatteryFirmware(String installationIp)
|
||||
{
|
||||
await Cli.Wrap("rsync")
|
||||
.WithArguments($@"-r {FirmwareVersion}.bin")
|
||||
.AppendArgument($@"root@{installationIp}:/opt/innovenergy/bms-firmware/{FirmwareVersion}.bin")
|
||||
.WithArguments($@"-r --relative bms-firmware/{FirmwareVersion}.bin")
|
||||
.AppendArgument($@"root@{installationIp}:/opt/innovenergy")
|
||||
.ExecuteAsync();
|
||||
}
|
||||
// [HttpGet(nameof(GetInstallation))]
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO.Enumeration;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Web;
|
||||
|
@ -82,7 +83,8 @@ public static partial class Db
|
|||
Console.WriteLine(installation.Name);
|
||||
sd_notify(0, "WATCHDOG=1");
|
||||
var details = await GetInstallationDetails(installation);
|
||||
await updateAlarms(installation);
|
||||
// Thread.Sleep(1000);
|
||||
await UpdateAlarms(installation);
|
||||
var ip = Ip(details);
|
||||
var updatedInstallation = new Installation(
|
||||
installation.Name,
|
||||
|
@ -113,23 +115,21 @@ public static partial class Db
|
|||
}
|
||||
}
|
||||
|
||||
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToElement<TValue>(TValue, JsonSerializerOptions)")]
|
||||
private static async Task<Boolean> updateAlarms(VrmInstallation installation)
|
||||
[RequiresUnreferencedCode(
|
||||
"Calls System.Text.Json.JsonSerializer.SerializeToElement<TValue>(TValue, JsonSerializerOptions)")]
|
||||
public static async Task<Boolean> UpdateAlarms(VrmInstallation installation)
|
||||
{
|
||||
// installation.GetDevices().batteryMonitor.setTemperatureAlarms(245,250,315,313);
|
||||
var alarmJson = Deserialize<JsonObject>("""
|
||||
{
|
||||
"AlarmEnabled": 1,
|
||||
"NotifyAfterSeconds": 60,
|
||||
"highAlarm": 315,
|
||||
"highAlarmHysteresis": 313,
|
||||
"lowAlarm": 245,
|
||||
"lowAlarmHysteresis": 250
|
||||
}
|
||||
""")!;
|
||||
var alarmJson = JsonNode.Parse(File.ReadAllText("./alarm.json"))!.AsObject();
|
||||
var tags = await installation.GetTags();
|
||||
if (tags.Contains("FM-AF09"))
|
||||
{
|
||||
// Console.WriteLine(installation.IdSite.ToString());
|
||||
// await installation.GetAlarms();
|
||||
return await installation.SetAlarms(alarmJson);
|
||||
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"idDataAttribute":115,
|
||||
"instance":1,
|
||||
"lowAlarm":245,
|
||||
"lowAlarmHysteresis":250,
|
||||
"highAlarm":315,
|
||||
"highAlarmHysteresis":313,
|
||||
"AlarmEnabled":1,
|
||||
"NotifyAfterSeconds":60,
|
||||
"meta_info": {"icon":"device-battery-monitor","name":"Battery Monitor [1]","idDeviceType":2,"dataAttribute":"Battery temperature"}}
|
Binary file not shown.
|
@ -1,6 +0,0 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.Data;
|
||||
|
||||
public record S3Bucket : S3RegionCredentials
|
||||
{
|
||||
public required String Bucket { get; init; }
|
||||
}
|
|
@ -1,47 +0,0 @@
|
|||
using static System.Environment;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils.Data;
|
||||
|
||||
public record S3Credentials
|
||||
{
|
||||
protected static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile).TrimEnd('\\', '/') + "/.s3cfg";
|
||||
|
||||
public required String Key { get; init; }
|
||||
public required String Secret { get; init; }
|
||||
|
||||
public static S3Credentials? FromS3Cfg() => FromFile(S3CfgFile);
|
||||
|
||||
public static S3Credentials? FromFile(String file)
|
||||
{
|
||||
// [default]
|
||||
// host_base = sos-ch-dk-2.exo.io
|
||||
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
|
||||
// access_key = xxxxxxxxxxxxxxx
|
||||
// secret_key = xxxxxxxxxxxxxxx
|
||||
// use_https = True
|
||||
|
||||
try
|
||||
{
|
||||
var cfg = ParseFile(file);
|
||||
|
||||
return new S3Credentials
|
||||
{
|
||||
Key = cfg["access_key"],
|
||||
Secret = cfg["secret_key"]
|
||||
};
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected static Dictionary<String, String> ParseFile(String cfgFile)
|
||||
{
|
||||
return File
|
||||
.ReadAllLines(cfgFile)
|
||||
.Where(l => l.Contains("="))
|
||||
.Select(l => l.Split("="))
|
||||
.ToDictionary(l => l[0].Trim(), l => l[1].Trim());
|
||||
}
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.Data;
|
||||
|
||||
public record S3Path : S3Bucket
|
||||
{
|
||||
public required String Path { get; init; }
|
||||
}
|
|
@ -1,49 +0,0 @@
|
|||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils.Data;
|
||||
|
||||
public record S3RegionCredentials : S3Credentials
|
||||
{
|
||||
public required String Region { get; init; }
|
||||
|
||||
private AmazonS3Client? _Client;
|
||||
internal AmazonS3Client Client => _Client ??= new AmazonS3Client
|
||||
(
|
||||
credentials: new BasicAWSCredentials(Key, Secret),
|
||||
clientConfig: new()
|
||||
{
|
||||
ServiceURL = Region.EnsureStartsWith("https://"),
|
||||
ForcePathStyle = true,
|
||||
}
|
||||
);
|
||||
|
||||
public new static S3RegionCredentials? FromS3Cfg() => FromFile(S3CfgFile);
|
||||
|
||||
public new static S3RegionCredentials? FromFile(String file)
|
||||
{
|
||||
// [default]
|
||||
// host_base = sos-ch-dk-2.exo.io
|
||||
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
|
||||
// access_key = xxxxxxxxxxxxxxx
|
||||
// secret_key = xxxxxxxxxxxxxxx
|
||||
// use_https = True
|
||||
|
||||
try
|
||||
{
|
||||
var cfg = ParseFile(file);
|
||||
|
||||
return new S3RegionCredentials
|
||||
{
|
||||
Key = cfg["access_key"],
|
||||
Secret = cfg["secret_key"],
|
||||
Region = cfg["host_base"],
|
||||
};
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
|
||||
public record S3Bucket
|
||||
(
|
||||
String Name,
|
||||
S3Region Region
|
||||
);
|
|
@ -0,0 +1,67 @@
|
|||
using static System.Environment;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
|
||||
public static class S3Cfg
|
||||
{
|
||||
internal static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile)
|
||||
.TrimEnd('\\', '/') + "/.s3cfg";
|
||||
|
||||
internal static Dictionary<String, String> FromFile(String cfgFilePath)
|
||||
{
|
||||
return File
|
||||
.ReadAllLines(cfgFilePath)
|
||||
.Where(l => l.Contains("="))
|
||||
.Select(l => l.Split("="))
|
||||
.ToDictionary(l => l[0].Trim(), l => l[1].Trim());
|
||||
}
|
||||
|
||||
public static S3Credentials? GetDefaultUserCredentials() => GetCredentialsFromFile(S3CfgFile);
|
||||
|
||||
public static S3Credentials? GetCredentialsFromFile(String file)
|
||||
{
|
||||
// [default]
|
||||
// host_base = sos-ch-dk-2.exo.io
|
||||
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
|
||||
// access_key = xxxxxxxxxxxxxxx
|
||||
// secret_key = xxxxxxxxxxxxxxx
|
||||
// use_https = True
|
||||
|
||||
try
|
||||
{
|
||||
var cfg = FromFile(file);
|
||||
|
||||
return new S3Credentials
|
||||
(
|
||||
Key : cfg["access_key"],
|
||||
Secret: cfg["secret_key"]
|
||||
);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static S3Region? GetDefaultRegionAndCredentials() => GetRegionAndCredentialsFromFile(S3CfgFile);
|
||||
|
||||
public static S3Region? GetRegionAndCredentialsFromFile(String file)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cfg = FromFile(file);
|
||||
|
||||
var credentials = new S3Credentials
|
||||
(
|
||||
Key : cfg["access_key"],
|
||||
Secret: cfg["secret_key"]
|
||||
);
|
||||
|
||||
return new S3Region(cfg["host_base"], credentials);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
|
||||
public record S3Credentials
|
||||
(
|
||||
String Key,
|
||||
String Secret
|
||||
);
|
|
@ -0,0 +1,13 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
|
||||
public record S3Region
|
||||
{
|
||||
public S3Region(String name, S3Credentials creds)
|
||||
{
|
||||
Name = name;
|
||||
Credentials = creds;
|
||||
}
|
||||
public String Name { get; init; }
|
||||
public S3Credentials Credentials { get; init; }
|
||||
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
|
||||
public record S3Url
|
||||
(
|
||||
String Path,
|
||||
S3Bucket Bucket
|
||||
);
|
|
@ -1,21 +0,0 @@
|
|||
|
||||
using InnovEnergy.Lib.S3Utils.Data;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils.ExoScale;
|
||||
|
||||
public static class DefaultCredentials
|
||||
{
|
||||
public static S3Credentials ReadOnly => new S3Credentials
|
||||
{
|
||||
Key = "EXOb6d6dc1880cdd51f1ebc6692",
|
||||
Secret = "kpIey4QJlQFuWG_WoTazcY7kBEjN2f_ll2cDBeg64m4",
|
||||
};
|
||||
|
||||
|
||||
public static S3Credentials ReadWrite => new S3Credentials
|
||||
{
|
||||
Key = "EXO87ca85e29dd412f1238f1cf0",
|
||||
Secret = "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU",
|
||||
};
|
||||
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
namespace InnovEnergy.Lib.S3Utils.ExoScale;
|
||||
|
||||
public static class Regions
|
||||
{
|
||||
public static String Default => "sos-ch-dk-2.exo.io";
|
||||
}
|
|
@ -0,0 +1,74 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using Amazon.IdentityManagement;
|
||||
using Amazon.IdentityManagement.Model;
|
||||
using Amazon.Runtime;
|
||||
using InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils;
|
||||
|
||||
public static class Iam
|
||||
{
|
||||
|
||||
// TODO
|
||||
|
||||
|
||||
|
||||
private static readonly ConcurrentDictionary<S3Region, AmazonIdentityManagementServiceClient> AimClientCache = new();
|
||||
|
||||
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Url url ) => url.Bucket.GetIamClient();
|
||||
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Bucket bucket) => bucket.Region.GetIamClient();
|
||||
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Region region)
|
||||
{
|
||||
return AimClientCache.GetOrAdd(region, CreateIamClient); // Memoize
|
||||
}
|
||||
|
||||
private static AmazonIdentityManagementServiceClient CreateIamClient(S3Region region) => new
|
||||
(
|
||||
credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret),
|
||||
clientConfig: new() { ServiceURL = region.Name.EnsureStartsWith("https://") }
|
||||
);
|
||||
|
||||
public static async Task<User> CreateUserAsync(AmazonIdentityManagementServiceClient iamService,String userName)
|
||||
{
|
||||
var response = await iamService.CreateUserAsync(new CreateUserRequest { UserName = userName });
|
||||
return response.User;
|
||||
}
|
||||
|
||||
public static async Task<Boolean> PutUserPolicyAsync(AmazonIdentityManagementServiceClient iamService, String userName, String policyName, String policyDocument)
|
||||
{
|
||||
var request = new PutUserPolicyRequest()
|
||||
{
|
||||
UserName = userName,
|
||||
PolicyName = policyName,
|
||||
PolicyDocument = policyDocument
|
||||
};
|
||||
|
||||
var response = await iamService.PutUserPolicyAsync(request);
|
||||
return response.HttpStatusCode == System.Net.HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
public static async Task<AccessKey> CreateAccessKeyAsync(AmazonIdentityManagementServiceClient iamService, String userName)
|
||||
{
|
||||
var response = await iamService.CreateAccessKeyAsync(new CreateAccessKeyRequest
|
||||
{
|
||||
UserName = userName,
|
||||
});
|
||||
|
||||
return response.AccessKey;
|
||||
|
||||
}
|
||||
|
||||
public static async Task<Boolean> UserExists(AmazonIdentityManagementServiceClient iamService, String userName)
|
||||
{
|
||||
var response = await iamService.GetUserAsync(new GetUserRequest { UserName = userName });
|
||||
return response.HttpStatusCode == HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
public static async Task<Boolean> RevokeAccessKey(AmazonIdentityManagementServiceClient iamService, String userName)
|
||||
{
|
||||
var response = await iamService.DeleteAccessKeyAsync(new DeleteAccessKeyRequest{ AccessKeyId = userName });
|
||||
return response.HttpStatusCode == HttpStatusCode.OK;
|
||||
}
|
||||
}
|
|
@ -1,9 +0,0 @@
|
|||
<?xml version="1.0" ?>
|
||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<CORSRule>
|
||||
<AllowedOrigin>*</AllowedOrigin>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
<AllowedMethod>HEAD</AllowedMethod>
|
||||
<AllowedHeader>*</AllowedHeader>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>
|
|
@ -1,4 +0,0 @@
|
|||
{
|
||||
"Key": "EXO1abcb772bf43ab72951ba1dc",
|
||||
"Secret": "_ym1KsGBSp90S5dwhZn18XD-u9Y4ghHvyIxg5gv5fHw"
|
||||
}
|
|
@ -1,5 +0,0 @@
|
|||
{
|
||||
"Key": "EXOb6d6dc1880cdd51f1ebc6692",
|
||||
"Secret": "kpIey4QJlQFuWG_WoTazcY7kBEjN2f_ll2cDBeg64m4",
|
||||
"Region": "sos-ch-dk-2.exo.io"
|
||||
}
|
|
@ -1,5 +0,0 @@
|
|||
{
|
||||
"Key": "EXO87ca85e29dd412f1238f1cf0",
|
||||
"Secret": "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU",
|
||||
"Region": "sos-ch-dk-2.exo.io"
|
||||
}
|
File diff suppressed because it is too large
Load Diff
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"Url": "mail.agenturserver.de",
|
||||
"Port": 587,
|
||||
"Username": "p518526p69",
|
||||
"Password": "i;b*xqm4iB5uhl"
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"ReadOnlyS3Key": "EXO44d2979c8e570eae81ead564",
|
||||
"ReadOnlyS3Secret": "55MAqyO_FqUmh7O64VIO0egq50ERn_WIAWuc2QC44QU" ,
|
||||
"ReadWriteS3Key": "EXO87ca85e29dd412f1238f1cf0",
|
||||
"ReadWriteS3Secret": "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU"
|
||||
}
|
|
@ -1,81 +1,112 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using InnovEnergy.Lib.S3Utils.Data;
|
||||
using InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
using S3Bucket = InnovEnergy.Lib.S3Utils.Data.S3Bucket;
|
||||
using S3Bucket = InnovEnergy.Lib.S3Utils.DataTypes.S3Bucket;
|
||||
using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils;
|
||||
|
||||
public static class S3
|
||||
{
|
||||
public static S3RegionCredentials Region(this S3Credentials credentials, String region) => new()
|
||||
{
|
||||
Secret = credentials.Secret,
|
||||
Key = credentials.Key,
|
||||
Region = region,
|
||||
};
|
||||
private static readonly ConcurrentDictionary<S3Region, AmazonS3Client> S3ClientCache = new();
|
||||
|
||||
public static S3Bucket Bucket(this S3RegionCredentials region, String bucket) => new()
|
||||
{
|
||||
Region = region.Region,
|
||||
Secret = region.Secret,
|
||||
Key = region.Key,
|
||||
Bucket = bucket,
|
||||
};
|
||||
// QOL method
|
||||
public static S3Bucket Bucket(this S3Region region, String name) => new
|
||||
(
|
||||
Name: name,
|
||||
Region: region
|
||||
);
|
||||
|
||||
public static S3Path Path(this S3Bucket bucket, String path) => new()
|
||||
{
|
||||
Bucket = bucket.Bucket,
|
||||
Region = bucket.Region,
|
||||
Secret = bucket.Secret,
|
||||
Key = bucket.Key,
|
||||
Path = path
|
||||
};
|
||||
// QOL method
|
||||
public static S3Url Path(this S3Bucket bucket, String path) => new
|
||||
(
|
||||
Bucket: bucket,
|
||||
Path: path
|
||||
);
|
||||
|
||||
public static IAsyncEnumerable<S3Path> ListObjects(this S3Bucket bucketOrPrefixPath)
|
||||
{
|
||||
var path = bucketOrPrefixPath as S3Path ?? bucketOrPrefixPath.Path(null!);
|
||||
public static IAsyncEnumerable<S3Url> ListObjects(this S3Bucket bucket) => ListObjects(bucket, null);
|
||||
|
||||
var request = new ListObjectsV2Request
|
||||
public static IAsyncEnumerable<S3Url> ListObjects(this S3Bucket bucket, String? pathPrefix)
|
||||
{
|
||||
BucketName = path.Bucket,
|
||||
Prefix = path.Path
|
||||
};
|
||||
|
||||
return bucketOrPrefixPath
|
||||
.Client
|
||||
return bucket
|
||||
.Region
|
||||
.GetS3Client()
|
||||
.Paginators
|
||||
.ListObjectsV2(request)
|
||||
.Responses
|
||||
.SelectMany(r => r.S3Objects)
|
||||
.Select(o => path with { Path = o.Key });
|
||||
.ListObjectsV2(new() { BucketName = bucket.Name, Prefix = pathPrefix })
|
||||
.S3Objects
|
||||
.Select(o => new S3Url(o.Key, bucket));
|
||||
}
|
||||
|
||||
public static async Task<String> GetObject(this S3Path path)
|
||||
public static Task<Boolean> PutObject(this S3Url path, String data, Encoding encoding) => path.PutObject(encoding.GetBytes(data));
|
||||
public static Task<Boolean> PutObject(this S3Url path, String data) => path.PutObject(data, Encoding.UTF8);
|
||||
public static Task<Boolean> PutObject(this S3Url path, Byte[] data) => path.PutObject(new MemoryStream(data));
|
||||
|
||||
public static async Task<Boolean> PutObject(this S3Url path, Stream data)
|
||||
{
|
||||
var request = new GetObjectRequest
|
||||
var request = new PutObjectRequest
|
||||
{
|
||||
BucketName = path.Bucket,
|
||||
Key = path.Path
|
||||
BucketName = path.Bucket.Name,
|
||||
Key = path.Path,
|
||||
InputStream = data
|
||||
};
|
||||
|
||||
using var response = await path.Client.GetObjectAsync(request);
|
||||
await using var responseStream = response.ResponseStream;
|
||||
using var reader = new StreamReader(responseStream);
|
||||
var response = await path
|
||||
.Bucket
|
||||
.Region
|
||||
.GetS3Client()
|
||||
.PutObjectAsync(request);
|
||||
|
||||
return response.HttpStatusCode == HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
public static Task<String> GetObjectAsString(this S3Url path) => GetObjectAsString(path, Encoding.UTF8);
|
||||
|
||||
public static async Task<String> GetObjectAsString(this S3Url path, Encoding encoding)
|
||||
{
|
||||
await using var stream = await GetObjectAsStream(path);
|
||||
using var reader = new StreamReader(stream, encoding);
|
||||
return await reader.ReadToEndAsync();
|
||||
}
|
||||
|
||||
public static async IAsyncEnumerable<String> GetObjectLineByLine(this S3Path path)
|
||||
public static async Task<Stream> GetObjectAsStream(this S3Url path)
|
||||
{
|
||||
var request = new GetObjectRequest
|
||||
{
|
||||
BucketName = path.Bucket,
|
||||
BucketName = path.Bucket.Name,
|
||||
Key = path.Path
|
||||
};
|
||||
|
||||
using var response = await path.Client.GetObjectAsync(request);
|
||||
await using var responseStream = response.ResponseStream;
|
||||
using var reader = new StreamReader(responseStream);
|
||||
var response = await path
|
||||
.Bucket
|
||||
.Region
|
||||
.GetS3Client()
|
||||
.GetObjectAsync(request);
|
||||
|
||||
return response.ResponseStream;
|
||||
}
|
||||
|
||||
public static async Task<IReadOnlyList<Byte>> GetObject(this S3Url url)
|
||||
{
|
||||
// beautiful await using stream soup...
|
||||
|
||||
await using var stream = await url.GetObjectAsStream();
|
||||
using var memoryStream = new MemoryStream();
|
||||
await stream.CopyToAsync(memoryStream);
|
||||
return memoryStream.ToArray();
|
||||
}
|
||||
|
||||
public static IAsyncEnumerable<String> GetObjectLineByLine(this S3Url url) => GetObjectLineByLine(url, Encoding.UTF8);
|
||||
|
||||
public static async IAsyncEnumerable<String> GetObjectLineByLine(this S3Url url, Encoding encoding)
|
||||
{
|
||||
await using var stream = await url.GetObjectAsStream();
|
||||
|
||||
using var reader = new StreamReader(stream, encoding);
|
||||
|
||||
while (true)
|
||||
{
|
||||
|
@ -87,4 +118,63 @@ public static class S3
|
|||
yield break;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<S3Bucket?> PutBucket(this S3Region region, String name)
|
||||
{
|
||||
var request = new PutBucketRequest { BucketName = name };
|
||||
|
||||
var response = await region
|
||||
.GetS3Client()
|
||||
.PutBucketAsync(request);
|
||||
|
||||
return response.HttpStatusCode switch
|
||||
{
|
||||
HttpStatusCode.OK => region.Bucket(name),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public static async Task<Boolean> PutCors(this S3Bucket bucket, CORSConfiguration corsConfiguration)
|
||||
{
|
||||
var request = new PutCORSConfigurationRequest
|
||||
{
|
||||
BucketName = bucket.Name,
|
||||
Configuration = corsConfiguration
|
||||
};
|
||||
|
||||
var response = await bucket
|
||||
.GetS3Client()
|
||||
.PutCORSConfigurationAsync(request);
|
||||
|
||||
return response.HttpStatusCode == HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
public static async Task<Boolean> DeleteBucket(this S3Bucket bucket)
|
||||
{
|
||||
var request = new DeleteBucketRequest { BucketName = bucket.Name };
|
||||
var response = await bucket
|
||||
.GetS3Client()
|
||||
.DeleteBucketAsync(request);
|
||||
|
||||
return response.HttpStatusCode == HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
private static AmazonS3Client GetS3Client(this S3Url url ) => url.Bucket.GetS3Client();
|
||||
private static AmazonS3Client GetS3Client(this S3Bucket bucket) => bucket.Region.GetS3Client();
|
||||
private static AmazonS3Client GetS3Client(this S3Region region)
|
||||
{
|
||||
return S3ClientCache.GetOrAdd(region, CreateS3Client); // Memoize
|
||||
}
|
||||
|
||||
private static AmazonS3Client CreateS3Client(S3Region region) => new
|
||||
(
|
||||
credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret),
|
||||
clientConfig: new()
|
||||
{
|
||||
ServiceURL = region.Name.EnsureStartsWith("https://"),
|
||||
ForcePathStyle = true,
|
||||
}
|
||||
);
|
||||
|
||||
}
|
|
@ -0,0 +1,68 @@
|
|||
using InnovEnergy.Lib.S3Utils.DataTypes;
|
||||
using static System.Environment;
|
||||
|
||||
namespace InnovEnergy.Lib.S3Utils;
|
||||
|
||||
public static class S3Cfg
|
||||
{
|
||||
internal static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile)
|
||||
.TrimEnd('\\', '/') + "/.s3cfg";
|
||||
|
||||
internal static Dictionary<String, String> FromFile(String cfgFilePath)
|
||||
{
|
||||
return File
|
||||
.ReadAllLines(cfgFilePath)
|
||||
.Where(l => l.Contains("="))
|
||||
.Select(l => l.Split("="))
|
||||
.ToDictionary(l => l[0].Trim(), l => l[1].Trim());
|
||||
}
|
||||
|
||||
public static S3Credentials? GetDefaultUserCredentials() => GetCredentialsFromFile(S3CfgFile);
|
||||
|
||||
public static S3Credentials? GetCredentialsFromFile(String file)
|
||||
{
|
||||
// [default]
|
||||
// host_base = sos-ch-dk-2.exo.io
|
||||
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
|
||||
// access_key = xxxxxxxxxxxxxxx
|
||||
// secret_key = xxxxxxxxxxxxxxx
|
||||
// use_https = True
|
||||
|
||||
try
|
||||
{
|
||||
var cfg = FromFile(file);
|
||||
|
||||
return new S3Credentials
|
||||
(
|
||||
Key : cfg["access_key"],
|
||||
Secret: cfg["secret_key"]
|
||||
);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static S3Region? GetDefaultRegionAndCredentials() => GetRegionAndCredentialsFromFile(S3CfgFile);
|
||||
|
||||
public static S3Region? GetRegionAndCredentialsFromFile(String file)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cfg = FromFile(file);
|
||||
|
||||
var credentials = new S3Credentials
|
||||
(
|
||||
Key : cfg["access_key"],
|
||||
Secret: cfg["secret_key"]
|
||||
);
|
||||
|
||||
return new S3Region(cfg["host_base"], credentials);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -11,6 +11,7 @@
|
|||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.200.39" />
|
||||
<PackageReference Include="AWSSDK.S3" Version="3.7.203.12" />
|
||||
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
|
||||
</ItemGroup>
|
||||
|
|
|
@ -73,7 +73,13 @@ public static class AsyncEnumerableEx
|
|||
}
|
||||
}
|
||||
|
||||
#pragma warning disable 1998
|
||||
public static async void ForEach<T>(this IAsyncEnumerable<T> ts, Action<T> action)
|
||||
{
|
||||
await foreach (var t in ts)
|
||||
action(t);
|
||||
}
|
||||
|
||||
#pragma warning disable 1998
|
||||
public static async IAsyncEnumerable<T> Repeat<T>(this T t, [EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
while(!ct.IsCancellationRequested)
|
||||
|
|
|
@ -59,13 +59,14 @@ public static class GraphTraversal
|
|||
}
|
||||
|
||||
[SuppressMessage("ReSharper", "PossibleMultipleEnumeration")]
|
||||
private static IEnumerable<T> Traverse<T>(IEnumerable<T> sources,
|
||||
private static IEnumerable<T> Traverse<T>(
|
||||
IEnumerable<T> sources,
|
||||
Func<T , Func<T, IEnumerable<T>>,IEnumerable<T>> traversor,
|
||||
Func<T, IEnumerable<T>> getChildren,
|
||||
IEqualityComparer<T>? comparer = null)
|
||||
{
|
||||
var set = new HashSet<T>(sources, comparer ?? EqualityComparer<T>.Default);
|
||||
IEnumerable<T> GetUniqueChildren(T n) => getChildren(n).Where(set.Add);
|
||||
IEnumerable<T> GetUniqueChildren(T n) => getChildren(n).Where(set!.Add);
|
||||
|
||||
return from s in sources
|
||||
from e in traversor(s, GetUniqueChildren)
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.Runtime.CompilerServices;
|
||||
using static System.Runtime.CompilerServices.MethodImplOptions;
|
||||
|
@ -171,4 +171,12 @@ public static class Utils
|
|||
}
|
||||
|
||||
public static String ExecutingProcessName => Process.GetCurrentProcess().ProcessName;
|
||||
|
||||
|
||||
public static Func<A, R> Memoize<A, R>(Func<A, R> func) where A : notnull
|
||||
{
|
||||
var cache = new ConcurrentDictionary<A, R>();
|
||||
|
||||
return arg => cache.GetOrAdd(arg, func);
|
||||
}
|
||||
}
|
|
@ -109,8 +109,8 @@ public static class VePropertiesDbus
|
|||
VeDBusApi.GetValue => msg.Ok(p.Value),
|
||||
VeDBusApi.GetText => msg.Ok(p.Text),
|
||||
VeDBusApi.GetItems => msg.Ok(p.GetItem()),
|
||||
VeDBusApi.SetValue => (p.Writeable && msg.ObjectPath != null)
|
||||
? msg.Ok((p.Value==msg.Payload! || props.Set(path: msg.ObjectPath.Value, value: msg.Payload!, writable: true)) ? 0 : -1)
|
||||
VeDBusApi.SetValue => p.Writeable && msg.ObjectPath != null
|
||||
? msg.Ok(p.Value==msg.Payload! || props.Set(path: msg.ObjectPath.Value, value: msg.Payload!, writable: true) ? 0 : -1)
|
||||
: msg.ObjectNotWritable(msg.Member),
|
||||
_ => msg.UnknownMember(msg.Member)
|
||||
};
|
||||
|
@ -150,7 +150,6 @@ public static class VePropertiesDbus
|
|||
return CreateErrorReply(msg, $"Not Writable Property: {objectPath}");
|
||||
}
|
||||
|
||||
|
||||
private static Message CreateErrorReply(Message msg, String error)
|
||||
{
|
||||
Debug.WriteLine("ERROR: " + error);
|
||||
|
|
|
@ -2,6 +2,7 @@ using System.Net;
|
|||
using System.Text.Json.Nodes;
|
||||
using Flurl.Http;
|
||||
using Flurl.Http.Content;
|
||||
using InnovEnergy.Lib.Utils;
|
||||
using static System.Net.Http.HttpMethod;
|
||||
|
||||
namespace InnovEnergy.Lib.Victron.VictronVRM;
|
||||
|
@ -21,6 +22,12 @@ public static class FlurlExtensions
|
|||
return JsonNode.Parse(stream)!;
|
||||
}
|
||||
|
||||
public static async Task<JsonNode> TryPutJson(this IFlurlRequest request, JsonObject? content = null)
|
||||
{
|
||||
var stream = await TryRequest(request, Put, content).ReceiveStream();
|
||||
return JsonNode.Parse(stream)!;
|
||||
}
|
||||
|
||||
private static async Task<IFlurlResponse> TryRequest(this IFlurlRequest request, HttpMethod verb, JsonObject? data = null)
|
||||
{
|
||||
var jsonContent = data is null
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Flurl.Http;
|
||||
|
||||
namespace InnovEnergy.Lib.Victron.VictronVRM;
|
||||
|
||||
|
@ -10,21 +11,23 @@ public readonly partial record struct Installation
|
|||
public async Task<IReadOnlyList<String>> GetAlarms()
|
||||
{
|
||||
var reply = await VrmAccount.AlarmsRequest(IdSite).TryGetJson();
|
||||
// Console.WriteLine("Got");
|
||||
|
||||
var vrmReply = new Reply(reply);
|
||||
Console.WriteLine(vrmReply.ToString());
|
||||
if (!vrmReply.Success)
|
||||
throw new Exception(nameof(GetAlarms) + " failed");
|
||||
|
||||
Console.WriteLine(vrmReply.Alarms);
|
||||
return vrmReply.Alarms;
|
||||
}
|
||||
|
||||
public async Task<Boolean> SetAlarms(JsonObject tags)
|
||||
{
|
||||
var reply = await VrmAccount.AlarmsRequest(IdSite).TryPostJson(tags);
|
||||
var reply = await VrmAccount.AlarmsRequest(IdSite).TryPutJson(tags);
|
||||
var vrmReply = new Reply(reply);
|
||||
// Console.WriteLine(vrmReply.Success.ToString());
|
||||
if (!vrmReply.Success)
|
||||
throw new Exception(nameof(SetAlarms) + " failed");
|
||||
|
||||
return vrmReply.Success;
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue