Innovenergy
/
Innovenergy_trunk
2
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Rewrote Backend.csproj to use new S3Utils.csproj, Testing needed

This commit is contained in:
Kim 2023-10-09 14:23:31 +02:00
parent b92391efcd
commit 2a5f9a0cc6
25 changed files with 161 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
csharp/App/Backend/Backend.csproj
View File

@ -34,6 +34,7 @@
<ItemGroup>
<ProjectReference Include="../../Lib/Utils/Utils.csproj" />
<ProjectReference Include="../../Lib/Mailer/Mailer.csproj" />
<ProjectReference Include="..\..\Lib\S3Utils\S3Utils.csproj" />
</ItemGroup>
<ItemGroup>

2
csharp/App/Backend/DataTypes/Installation.cs
View File

@ -10,7 +10,7 @@ public class Installation : TreeNode
// TODO: make relation
//public IReadOnlyList<String> OrderNumbers { get; set; } = Array.Empty<String>();
public String OrderNumbers { get; set; } = "";
public String? OrderNumbers { get; set; } = "";
public Double Lat { get; set; }
public Double Long { get; set; }

94
csharp/App/Backend/DataTypes/Methods/ExoCmd.cs
View File

@ -1,71 +1,63 @@
using CliWrap;
using CliWrap.Buffered;
using InnovEnergy.Lib.Utils;
using System.Text.Json;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using System.Diagnostics.CodeAnalysis;
namespace InnovEnergy.App.Backend.DataTypes.Methods;
public static class ExoCmd
{
private static readonly Command Exo = Cli.Wrap("exo");
private const String ConfigFile = "./exoscale.toml";
[UnconditionalSuppressMessage("Trimming", "IL2026:Members annotated with 'RequiresUnreferencedCodeAttribute' require dynamic access otherwise can break functionality when trimming application code", Justification = "<Pending>")]
private static readonly S3Credentials? S3Creds = JsonSerializer.Deserialize<S3Credentials>("./exoscaleS3.json");
public static async Task<(String key, String secret)> CreateReadKey(this Installation installation)
{
//if (installation.Id != 1) return "help"; //Todo remove me I am for debugging
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
if (!await Iam.UserExists(iamService, $"READ{installation.BucketName()}"))
{
var readOnlyPolicy = $"{installation.BucketName()}"; // TODO make me
await Iam.CreateUserAsync(iamService, $"READ{installation.BucketName()}");
await Iam.PutUserPolicyAsync(iamService, $"READ{installation.BucketName()}", $"READ{installation.BucketName()}",readOnlyPolicy);
}
var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READ{installation.BucketName()}");
return (keySecret.AccessKeyId, keySecret.SecretAccessKey);
}
var preParse = await Exo
.WithArguments("iam access-key create " + installation.BucketName()
+ " --operation get-sos-object"
+ " --resource sos/bucket:" + installation.BucketName()
+ " -C " + ConfigFile
+ " -O text")
.ExecuteBufferedAsync();
public static async Task<Boolean> RevokeReadKey(this Installation installation)
{
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
if (!await Iam.UserExists(iamService, $"READ{installation.BucketName()}"))
{
return true;
}
var key = preParse.StandardOutput.Split("\t")[2];
var secret = preParse.StandardOutput.Split("\t")[3];
return (key, secret);
//return $"{key};{secret}";
return await Iam.RevokeAccessKey(iamService, $"READ{installation.BucketName()}");
}
public static async Task<(String key, String secret)> CreateWriteKey(this Installation installation)
{
//if (installation.Id != 1) return "help"; //Todo remove me I am for debugging
var preParse = await Exo
.WithArguments("iam access-key create " + installation.BucketName()
+ " --resource sos/bucket:" + installation.BucketName()
+ " -C " + ConfigFile
+ " -O text")
.ExecuteBufferedAsync();
var key = preParse.StandardOutput.Split("\t")[2];
var secret = preParse.StandardOutput.Split("\t")[3];
return (key, secret);
//return $"{key};{secret}";
}
public static async Task RevokeReadKey(this Installation installation)
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
if (!await Iam.UserExists(iamService, $"READWRITE{installation.BucketName()}"))
{
try
{
await Exo
.WithArguments("iam access-key revoke " + installation.S3Key + " -f " + " -C " + ConfigFile)
.WithValidation(CommandResultValidation.None)
.ExecuteAsync();
}
catch
{
// TODO
("Failed to revoke key for installation " + installation.Name).WriteLine();
}
var readWritePolicy = $"{installation.BucketName()}"; // TODO make me
await Iam.CreateUserAsync(iamService, $"READWRITE{installation.BucketName()}");
await Iam.PutUserPolicyAsync(iamService, $"READWRITE{installation.BucketName()}", $"READWRITE{installation.BucketName()}",readWritePolicy);
}
var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READWRITE{installation.BucketName()}");
return (keySecret.AccessKeyId, keySecret.SecretAccessKey);
}
public static async Task<Boolean> CreateBucket(this Installation installation)
{
var s3Region = new S3Region(installation.Region, S3Creds!);
return await s3Region.PutBucket(installation.BucketName()) != null;
}
}

26
csharp/App/Backend/DataTypes/Methods/Installation.cs
View File

@ -1,4 +1,5 @@
using InnovEnergy.App.Backend.Database;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.App.Backend.DataTypes.Methods;
@ -32,26 +33,11 @@ public static class InstallationMethods
return Db.Update(installation);
}
public static Task<Boolean> CreateBucket(this Installation installation)
public static async Task<Boolean> DeleteBucket(this Installation installation)
{
// TODO
throw new NotImplementedException();
// return S3Access
// .Admin
// .CreateBucket(installation.BucketName());
}
public static Task<Boolean> DeleteBucket(this Installation installation)
{
// TODO
throw new NotImplementedException();
// return S3Access
// .ReadWrite
// .DeleteBucket(installation.BucketName());
// TODO We dont do this here
return true;
}
public static IEnumerable<User> UsersWithAccess(this Installation installation)
@ -138,7 +124,7 @@ public static class InstallationMethods
public static Installation FillOrderNumbers(this Installation installation)
{
installation.OrderNumbers = installation.GetOrderNumbers();
installation.OrderNumbers = installation.GetOrderNumbers().ToString();
return installation;
}

4
csharp/App/Backend/DataTypes/Methods/Session.cs
View File

@ -87,7 +87,7 @@ public static class SessionMethods
&& user.HasWriteAccess
&& user.HasAccessToParentOf(installation)
&& Db.Create(installation) // TODO: these two in a transaction
&& installation.SetOrderNumbers()
// && installation.SetOrderNumbers()
&& Db.Create(new InstallationAccess { UserId = user.Id, InstallationId = installation.Id })
&& await installation.CreateBucket()
&& await installation.RenewS3Credentials(); // generation of access _after_ generation of
@ -101,7 +101,7 @@ public static class SessionMethods
var user = session?.User;
var original = Db.GetInstallationById(installation?.Id);
var originalOrderNumbers = original!.GetOrderNumbers();
var originalOrderNumbers = original.OrderNumbers;
if (!Equals(originalOrderNumbers, installation?.OrderNumbers))
{

10
csharp/App/Backend/DataTypes/Methods/exoscale.toml
View File

@ -1,10 +0,0 @@
defaultaccount = "Kim"
[[accounts]]
account = "innovenergy-ag"
defaultZone = "ch-dk-2"
endpoint = "https://api.exoscale.com/v1"
environment = ""
key = "EXOf67c5b528282988503ddab12"
name = "Kim"
secret = "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"

4
csharp/App/Backend/DataTypes/Methods/exoscaleS3.json Normal file
View File

@ -0,0 +1,4 @@
{
"Key": "EXOf67c5b528282988503ddab12",
"Secret": "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"
}

3
csharp/App/Backend/Properties/launchSettings.json
View File

@ -9,7 +9,8 @@
"launchUrl": "swagger",
"applicationUrl": "https://localhost:7087",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
"ASPNETCORE_ENVIRONMENT": "Development",
"HOME":"~/backend"
}
}
}

20
csharp/App/Backend/S3/S3Access.cs
View File

@ -1,20 +0,0 @@
// using System.Diagnostics.CodeAnalysis;
// using static System.IO.File;
// using static System.Text.Json.JsonSerializer;
//
// namespace InnovEnergy.App.Backend.S3;
//
// [SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")]
// public static class S3Access
// {
// public static S3Cmd ReadOnly => ParseJsonFile<S3Cmd>("./Resources/s3ReadOnlyKey.json")!;
//
// private static T? ParseJsonFile<T>(String file)
// {
// var fileStream = OpenRead(file);
// return Deserialize<T>(fileStream);
// }
//
// public static S3Cmd ReadWrite => Deserialize<S3Cmd>(OpenRead("./Resources/s3ReadWriteKey.json"))!;
// public static S3Cmd Admin => Deserialize<S3Cmd>(OpenRead("./Resources/s3AdminKey.json"))!;
// }

90
csharp/App/Backend/S3/S3Cmd.cs
View File

@ -1,90 +0,0 @@
// using System.Diagnostics.CodeAnalysis;
// using CliWrap;
// using CliWrap.Buffered;
// using InnovEnergy.Lib.Utils;
//
// #pragma warning disable CS8618
//
// namespace InnovEnergy.App.Backend.S3;
//
// [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")]
// public class S3Cmd
// {
// private static readonly Command Python = Cli.Wrap("python3");
//
// private const String S3CmdPath = "Resources/s3cmd.py";
// private const String S3Prefix = "s3://";
//
// public String Key { get; init; }
// public String Secret { get; init; }
// public String Region { get; init; } = "sos-ch-dk-2.exo.io";
//
// [Obsolete("Only to be used by Json-Deserializer")] public S3Cmd()
// {}
//
// public async Task<Boolean> CreateBucket(String bucketName)
// {
// const String cors = "./Resources/CORS";
//
// var makeBucket = await Run(bucketName, "mb");
//
// if (makeBucket.ExitCode != 0)
// return false;
//
// var setCors = await Run(bucketName, "setcors", cors);
//
// return setCors.ExitCode == 0;
// }
//
// public async Task<String> ListFilesInBucket(String bucketName)
// {
// var result = await Run(bucketName, "ls");
// return result.StandardOutput;
// }
//
// public async Task<IReadOnlyList<String>?> GetFileLines(String bucketName, String filename)
// {
// try
// {
// await Run(bucketName + "/" + filename, "get", "--force");
// }
// catch
// {
// return null;
// }
//
// var lines = File.ReadAllLines($"./{filename}");
// File.Delete(filename);
// return lines;
// }
//
// public async Task<Boolean> DeleteBucket(String bucketName)
// {
// var result = await Run(bucketName, "rb");
// return result.ExitCode == 0;
// }
//
// private Task<BufferedCommandResult> Run(String s3Path, String operation, params String[] optionalArgs)
// {
// var credentials = new[]
// {
// S3CmdPath,
// "--access_key", Key,
// "--secret_key", Secret,
// "--host" , Region
// };
//
// var args = credentials
// .Append(operation)
// .Concat(optionalArgs)
// .Append(s3Path.EnsureStartsWith(S3Prefix));
//
// var withArguments = Python
// .WithArguments(args);
//
// return withArguments
// .WithValidation(CommandResultValidation.None)
// .ExecuteBufferedAsync();
// }
//
// }

10
csharp/App/Backend/S3/exoscale.toml
View File

@ -1,10 +0,0 @@
defaultaccount = "Kim"
[[accounts]]
account = "innovenergy-ag"
defaultZone = "ch-dk-2"
endpoint = "https://api.exoscale.com/v1"
environment = ""
key = "EXOf67c5b528282988503ddab12"
name = "Kim"
secret = "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"

1
csharp/App/S3Explorer/Program.cs
View File

@ -4,6 +4,7 @@ using Amazon.S3.Model;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils;
using S3Cfg = InnovEnergy.Lib.S3Utils.S3Cfg;
namespace InnovEnergy.App.S3Explorer;

12
csharp/App/VrmGrabber/Controller.cs
View File

@ -191,17 +191,19 @@ th { /* header cell */
for (var batteryId = 3; batteryId < Int64.Parse(numberOfBatteries) + 2; batteryId++)
{
localCommand = localCommand.Append(
$" && sleep 3m && /opt/innovenergy/scripts/upload-bms-firmware {batteryTtyName} {batteryId} /opt/innovenergy/{FirmwareVersion}.bin");
$" && sleep 3m && /opt/innovenergy/scripts/upload-bms-firmware {batteryTtyName} {batteryId} /opt/innovenergy/bms-firmware/{FirmwareVersion}.bin");
}
#pragma warning disable CS4014
Db.ExecuteBufferedAsyncCommandOnIp(installationIp, localCommand)
.ContinueWith(t =>
.ContinueWith(async t =>
{
if (t.Status == TaskStatus.RanToCompletion)
{
installation.BatteryUpdateStatus = "Complete";
Db.Update(installation: installation);
UpdateVrmTagsToNewFirmware(installationIp);
var vrmInst = await FindVrmInstallationByIp(installation.Ip!);
await UpdateVrmTagsToNewFirmware(installationIp);
await Db.UpdateAlarms(vrmInst);
}
else
{
@ -236,8 +238,8 @@ th { /* header cell */
private static async Task SendNewBatteryFirmware(String installationIp)
{
await Cli.Wrap("rsync")
.WithArguments($@"-r {FirmwareVersion}.bin")
.AppendArgument($@"root@{installationIp}:/opt/innovenergy/bms-firmware/{FirmwareVersion}.bin")
.WithArguments($@"-r --relative bms-firmware/{FirmwareVersion}.bin")
.AppendArgument($@"root@{installationIp}:/opt/innovenergy")
.ExecuteAsync();
}
// [HttpGet(nameof(GetInstallation))]

26
csharp/App/VrmGrabber/Database/Db.cs
View File

@ -1,4 +1,5 @@
using System.Diagnostics.CodeAnalysis;
using System.IO.Enumeration;
using System.Runtime.InteropServices;
using System.Text.Json.Nodes;
using System.Web;
@ -82,7 +83,8 @@ public static partial class Db
Console.WriteLine(installation.Name);
sd_notify(0, "WATCHDOG=1");
var details = await GetInstallationDetails(installation);
await updateAlarms(installation);
// Thread.Sleep(1000);
await UpdateAlarms(installation);
var ip = Ip(details);
var updatedInstallation = new Installation(
installation.Name,
@ -113,23 +115,21 @@ public static partial class Db
}
}
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToElement<TValue>(TValue, JsonSerializerOptions)")]
private static async Task<Boolean> updateAlarms(VrmInstallation installation)
[RequiresUnreferencedCode(
"Calls System.Text.Json.JsonSerializer.SerializeToElement<TValue>(TValue, JsonSerializerOptions)")]
public static async Task<Boolean> UpdateAlarms(VrmInstallation installation)
{
// installation.GetDevices().batteryMonitor.setTemperatureAlarms(245,250,315,313);
var alarmJson = Deserialize<JsonObject>("""
{
"AlarmEnabled": 1,
"NotifyAfterSeconds": 60,
"highAlarm": 315,
"highAlarmHysteresis": 313,
"lowAlarm": 245,
"lowAlarmHysteresis": 250
}
""")!;
var alarmJson = JsonNode.Parse(File.ReadAllText("./alarm.json"))!.AsObject();
var tags = await installation.GetTags();
if (tags.Contains("FM-AF09"))
{
// Console.WriteLine(installation.IdSite.ToString());
// await installation.GetAlarms();
return await installation.SetAlarms(alarmJson);
}
return true;
}

10
csharp/App/VrmGrabber/alarm.json Normal file
View File

@ -0,0 +1,10 @@
{
"idDataAttribute":115,
"instance":1,
"lowAlarm":245,
"lowAlarmHysteresis":250,
"highAlarm":315,
"highAlarmHysteresis":313,
"AlarmEnabled":1,
"NotifyAfterSeconds":60,
"meta_info": {"icon":"device-battery-monitor","name":"Battery Monitor [1]","idDeviceType":2,"dataAttribute":"Battery temperature"}}

BIN
csharp/App/VrmGrabber/db.sqlite
View File

Binary file not shown.

2
csharp/Lib/S3Utils/DataTypes/S3Cfg.cs
View File

@ -57,7 +57,7 @@ public static class S3Cfg
Secret: cfg["secret_key"]
);
return new S3Region(Name: cfg["host_base"], Credentials: credentials);
return new S3Region(cfg["host_base"], credentials);
}
catch
{

14
csharp/Lib/S3Utils/DataTypes/S3Region.cs
View File

@ -1,7 +1,13 @@
namespace InnovEnergy.Lib.S3Utils.DataTypes;
public record S3Region
(
String Name,
S3Credentials Credentials
);
{
public S3Region(String name, S3Credentials creds)
{
Name = name;
Credentials = creds;
}
public String Name { get; init; }
public S3Credentials Credentials { get; init; }
}

44
csharp/Lib/S3Utils/Iam.cs
View File

@ -1,5 +1,7 @@
using System.Collections.Concurrent;
using System.Net;
using Amazon.IdentityManagement;
using Amazon.IdentityManagement.Model;
using Amazon.Runtime;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils;
@ -11,6 +13,8 @@ public static class Iam
// TODO
private static readonly ConcurrentDictionary<S3Region, AmazonIdentityManagementServiceClient> AimClientCache = new();
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Url url ) => url.Bucket.GetIamClient();
@ -23,10 +27,48 @@ public static class Iam
private static AmazonIdentityManagementServiceClient CreateIamClient(S3Region region) => new
(
credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret),
clientConfig: new() { ServiceURL = StringUtils.EnsureStartsWith(region.Name, "https://") }
clientConfig: new() { ServiceURL = region.Name.EnsureStartsWith("https://") }
);
public static async Task<User> CreateUserAsync(AmazonIdentityManagementServiceClient iamService,String userName)
{
var response = await iamService.CreateUserAsync(new CreateUserRequest { UserName = userName });
return response.User;
}
public static async Task<Boolean> PutUserPolicyAsync(AmazonIdentityManagementServiceClient iamService, String userName, String policyName, String policyDocument)
{
var request = new PutUserPolicyRequest()
{
UserName = userName,
PolicyName = policyName,
PolicyDocument = policyDocument
};
var response = await iamService.PutUserPolicyAsync(request);
return response.HttpStatusCode == System.Net.HttpStatusCode.OK;
}
public static async Task<AccessKey> CreateAccessKeyAsync(AmazonIdentityManagementServiceClient iamService, String userName)
{
var response = await iamService.CreateAccessKeyAsync(new CreateAccessKeyRequest
{
UserName = userName,
});
return response.AccessKey;
}
public static async Task<Boolean> UserExists(AmazonIdentityManagementServiceClient iamService, String userName)
{
var response = await iamService.GetUserAsync(new GetUserRequest { UserName = userName });
return response.HttpStatusCode == HttpStatusCode.OK;
}
public static async Task<Boolean> RevokeAccessKey(AmazonIdentityManagementServiceClient iamService, String userName)
{
var response = await iamService.DeleteAccessKeyAsync(new DeleteAccessKeyRequest{ AccessKeyId = userName });
return response.HttpStatusCode == HttpStatusCode.OK;
}
}

9
csharp/Lib/S3Utils/S3.cs
View File

@ -15,13 +15,6 @@ public static class S3
{
private static readonly ConcurrentDictionary<S3Region, AmazonS3Client> S3ClientCache = new();
// QOL method
public static S3Region Region(this S3Credentials credentials, String name) => new
(
Name: name,
Credentials: credentials
);
// QOL method
public static S3Bucket Bucket(this S3Region region, String name) => new
(
@ -179,7 +172,7 @@ public static class S3
credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret),
clientConfig: new()
{
ServiceURL = StringUtils.EnsureStartsWith(region.Name, "https://"),
ServiceURL = region.Name.EnsureStartsWith("https://"),
ForcePathStyle = true,
}
);

2
csharp/Lib/S3Utils/S3Cfg.cs
View File

@ -58,7 +58,7 @@ public static class S3Cfg
Secret: cfg["secret_key"]
);
return new S3Region(Name: cfg["host_base"], Credentials: credentials);
return new S3Region(cfg["host_base"], credentials);
}
catch
{

14
csharp/Lib/S3Utils/S3Utils.csproj
View File

@ -16,18 +16,4 @@
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
<Compile Remove="ExoScale\DefaultCredentials.cs" />
<Compile Remove="Data\S3Credentials.cs" />
<Compile Remove="Data\S3Path.cs" />
<Compile Remove="Data\S3Bucket.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="Data\" />
<Folder Include="ExoScale\" />
</ItemGroup>
</Project>

7
csharp/Lib/Victron/VictronVRM/FlurlExtensions.cs
View File

@ -2,6 +2,7 @@ using System.Net;
using System.Text.Json.Nodes;
using Flurl.Http;
using Flurl.Http.Content;
using InnovEnergy.Lib.Utils;
using static System.Net.Http.HttpMethod;
namespace InnovEnergy.Lib.Victron.VictronVRM;
@ -21,6 +22,12 @@ public static class FlurlExtensions
return JsonNode.Parse(stream)!;
}
public static async Task<JsonNode> TryPutJson(this IFlurlRequest request, JsonObject? content = null)
{
var stream = await TryRequest(request, Put, content).ReceiveStream();
return JsonNode.Parse(stream)!;
}
private static async Task<IFlurlResponse> TryRequest(this IFlurlRequest request, HttpMethod verb, JsonObject? data = null)
{
var jsonContent = data is null

9
csharp/Lib/Victron/VictronVRM/Installation.Alarms.cs
View File

@ -1,5 +1,6 @@
using System.Text.Json;
using System.Text.Json.Nodes;
using Flurl.Http;
namespace InnovEnergy.Lib.Victron.VictronVRM;
@ -10,21 +11,23 @@ public readonly partial record struct Installation
public async Task<IReadOnlyList<String>> GetAlarms()
{
var reply = await VrmAccount.AlarmsRequest(IdSite).TryGetJson();
// Console.WriteLine("Got");
var vrmReply = new Reply(reply);
Console.WriteLine(vrmReply.ToString());
if (!vrmReply.Success)
throw new Exception(nameof(GetAlarms) + " failed");
Console.WriteLine(vrmReply.Alarms);
return vrmReply.Alarms;
}
public async Task<Boolean> SetAlarms(JsonObject tags)
{
var reply = await VrmAccount.AlarmsRequest(IdSite).TryPostJson(tags);
var reply = await VrmAccount.AlarmsRequest(IdSite).TryPutJson(tags);
var vrmReply = new Reply(reply);
// Console.WriteLine(vrmReply.Success.ToString());
if (!vrmReply.Success)
throw new Exception(nameof(SetAlarms) + " failed");
return vrmReply.Success;
}
}