Innovenergy
/
Innovenergy_trunk
2
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Various S3 is WIP

This commit is contained in:
Kim 2023-10-16 11:27:19 +02:00
parent 44836b0bca
commit 6f4c1122f7
13 changed files with 176 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
csharp/App/Backend/Controller.cs
View File

@ -1,3 +1,5 @@
using System.Runtime.InteropServices.ComTypes;
using System.Text.Json.Nodes;
using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.DataTypes;
using InnovEnergy.App.Backend.DataTypes.Methods;
@ -399,6 +401,19 @@ public class Controller : ControllerBase
: Unauthorized();
}
[HttpPost(nameof(EditInstallationConfig))]
public async Task<ActionResult<IEnumerable<Object>>> EditInstallationConfig([FromBody] String config, Int64 installationId, Token authToken)
{
var session = Db.GetSession(authToken);
// var installationToUpdate = Db.GetInstallationById(installationId);
return await session.SendInstallationConfig(installationId, config)
? Ok()
: Unauthorized();
}
[HttpPut(nameof(MoveFolder))]
public ActionResult MoveFolder(Int64 folderId,Int64 parentId, Token authToken)
{
@ -458,8 +473,9 @@ public class Controller : ControllerBase
: Unauthorized();
}
[HttpGet(nameof(ResetPassword))]
public ActionResult<IEnumerable<Object>> ResetPassword(Token token)
public ActionResult<Object> ResetPassword(Token token)
{
var user = Db.GetSession(token)?.User;
@ -467,9 +483,13 @@ public class Controller : ControllerBase
return Unauthorized();
//todo dont hardcode url
return Db.DeleteUserPassword(user)
? RedirectToRoute("https://monitor.innov.energy")
: Unauthorized();
if (!Db.DeleteUserPassword(user))
{
return Unauthorized();
}
// Db.Sessions.Delete(s => s.Token == token);
return Redirect($"https://monitor.innov.energy/?username={user.Email}");
}
}

6
csharp/App/Backend/DataTypes/Installation.cs
View File

@ -7,10 +7,11 @@ public class Installation : TreeNode
public String Location { get; set; } = "";
public String Region { get; set; } = "";
public String Country { get; set; } = "";
public String InstallationName { get; set; } = "";
// TODO: make relation
//public IReadOnlyList<String> OrderNumbers { get; set; } = Array.Empty<String>();
public String? OrderNumbers { get; set; } = "";
// public String? OrderNumbers { get; set; } = "";
public Double Lat { get; set; }
public Double Long { get; set; }
@ -21,4 +22,7 @@ public class Installation : TreeNode
public String S3Key { get; set; } = "";
public String S3WriteSecret { get; set; } = "";
public String S3Secret { get; set; } = "";
[Ignore]
public IReadOnlyList<String>? OrderNumbers { get; set; }
}

57
csharp/App/Backend/DataTypes/Methods/ExoCmd.cs
View File

@ -8,17 +8,36 @@ namespace InnovEnergy.App.Backend.DataTypes.Methods;
public static class ExoCmd
{
[UnconditionalSuppressMessage("Trimming", "IL2026:Members annotated with 'RequiresUnreferencedCodeAttribute' require dynamic access otherwise can break functionality when trimming application code", Justification = "<Pending>")]
private static readonly S3Credentials? S3Creds = JsonSerializer.Deserialize<S3Credentials>("./exoscaleS3.json");
public static readonly S3Credentials? S3Creds = JsonSerializer.Deserialize<S3Credentials>(File.OpenRead("./Resources/exoscaleS3.json"));
public static async Task<(String key, String secret)> CreateReadKey(this Installation installation)
{
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
if (!await Iam.UserExists(iamService, $"READ{installation.BucketName()}"))
var iamService = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", S3Creds!).GetIamClient();
if (!await Iam.RoleExists(iamService, $"READ{installation.BucketName()}"))
{
var readOnlyPolicy = $"{installation.BucketName()}"; // TODO make me
await Iam.CreateUserAsync(iamService, $"READ{installation.BucketName()}");
await Iam.PutUserPolicyAsync(iamService, $"READ{installation.BucketName()}", $"READ{installation.BucketName()}",readOnlyPolicy);
var readOnlyPolicy =@"{
""default-service-strategy"": ""deny"",
""services"": {
""sos"": {
""type"": ""rules"",
""rules"": [
{
""expression"": ""operation == 'list-objects'"",
""action"": ""allow""
},
{
""expression"": ""operation == 'get-object'"",
""action"": ""allow""
}
],
""resource"": " + $@"{installation.BucketName()}
}}
}}
}}";
await Iam.CreateRoleAsync(iamService, $"READ{installation.BucketName()}");
await Iam.PutRolePolicyAsync(iamService, $"READ{installation.BucketName()}", $"READ{installation.BucketName()}",readOnlyPolicy);
}
var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READ{installation.BucketName()}");
@ -29,8 +48,8 @@ public static class ExoCmd
public static async Task<Boolean> RevokeReadKey(this Installation installation)
{
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
if (!await Iam.UserExists(iamService, $"READ{installation.BucketName()}"))
var iamService = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", S3Creds!).GetIamClient();
if (!await Iam.RoleExists(iamService, $"READ{installation.BucketName()}"))
{
return true;
}
@ -40,12 +59,20 @@ public static class ExoCmd
public static async Task<(String key, String secret)> CreateWriteKey(this Installation installation)
{
var iamService = new S3Region(installation.Region, S3Creds!).GetIamClient();
if (!await Iam.UserExists(iamService, $"READWRITE{installation.BucketName()}"))
var iamService = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", S3Creds!).GetIamClient();
if (!await Iam.RoleExists(iamService, $"READWRITE{installation.BucketName()}"))
{
var readWritePolicy = $"{installation.BucketName()}"; // TODO make me
await Iam.CreateUserAsync(iamService, $"READWRITE{installation.BucketName()}");
await Iam.PutUserPolicyAsync(iamService, $"READWRITE{installation.BucketName()}", $"READWRITE{installation.BucketName()}",readWritePolicy);
var readWritePolicy = @"{
""default-service-strategy"": ""deny"",
""services"": {
""sos"": {
""type"": ""allow"",
""resource"": " + $@"{installation.BucketName()}
}}
}}
}}";
await Iam.CreateRoleAsync(iamService, $"READWRITE{installation.BucketName()}");
await Iam.PutRolePolicyAsync(iamService, $"READWRITE{installation.BucketName()}", $"READWRITE{installation.BucketName()}",readWritePolicy);
}
var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READWRITE{installation.BucketName()}");
@ -55,9 +82,11 @@ public static class ExoCmd
}
public static async Task<Boolean> CreateBucket(this Installation installation)
{
var s3Region = new S3Region(installation.Region, S3Creds!);
var s3Region = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", S3Creds!);
return await s3Region.PutBucket(installation.BucketName()) != null;
}
}

31
csharp/App/Backend/DataTypes/Methods/Installation.cs
View File

@ -1,4 +1,8 @@
using System.Net;
using System.Text.Json.Nodes;
using CliWrap;
using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.Relations;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.Utils;
@ -40,6 +44,29 @@ public static class InstallationMethods
return true;
}
public static async Task<Boolean> SendConfig(this Installation installation, String config)
{
// This looks hacky but here we grab the vpn-Ip of the installation by its installation Name (e.g. Salimax0001)
// From the vpn server (here salidomo, but we use the vpn home ip for future-proofing)
using var client = new HttpClient();
var webRequest = client.GetAsync("10.2.0.1/vpnstatus.txt");
var text = webRequest.ToString();
var lines = text!.Split(new [] { Environment.NewLine }, StringSplitOptions.None);
var vpnIp = lines.First(l => l.Contains(installation.InstallationName)).Split(",")[1];
// Writing the config to a file and then sending that file with rsync sounds inefficient
// We should find a better solution...
// TODO The VPN server should do this not the backend!!!
await File.WriteAllTextAsync("./config.json", config);
var result = await Cli.Wrap("rsync")
.WithArguments("./config.json")
.AppendArgument($@"root@{vpnIp}:/salimax")
.ExecuteAsync();
return result.ExitCode == 200;
}
public static IEnumerable<User> UsersWithAccess(this Installation installation)
{
return installation
@ -114,7 +141,7 @@ public static class InstallationMethods
return Db.Installations.Any(i => i.Id == installation.Id);
}
public static IReadOnlyList<String> GetOrderNumbers(this Installation installation)
public static IReadOnlyList<String>? GetOrderNumbers(this Installation installation)
{
return Db.OrderNumber2Installation
.Where(i => i.InstallationId == installation.Id)
@ -124,7 +151,7 @@ public static class InstallationMethods
public static Installation FillOrderNumbers(this Installation installation)
{
installation.OrderNumbers = installation.GetOrderNumbers().ToString();
installation.OrderNumbers = installation.GetOrderNumbers();
return installation;
}

23
csharp/App/Backend/DataTypes/Methods/Session.cs
View File

@ -1,3 +1,4 @@
using System.Text.Json.Nodes;
using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.Relations;
using InnovEnergy.Lib.Utils;
@ -66,6 +67,18 @@ public static class SessionMethods
.Apply(Db.Update);
}
public static async Task<Boolean> SendInstallationConfig(this Session? session, Int64 installationId, String configuration)
{
var user = session?.User;
var installation = Db.GetInstallationById(installationId);
return user is not null
&& installation is not null
&& user.HasWriteAccess
&& user.HasAccessTo(installation)
&& await installation.SendConfig(configuration);
}
public static Boolean Delete(this Session? session, Folder? folder)
{
var user = session?.User;
@ -101,13 +114,13 @@ public static class SessionMethods
var user = session?.User;
var original = Db.GetInstallationById(installation?.Id);
var originalOrderNumbers = original.OrderNumbers;
var originalOrderNumbers = original!.GetOrderNumbers();
if (!Equals(originalOrderNumbers, installation?.OrderNumbers))
{
foreach (var orderNumber in installation!.OrderNumbers.Split(','))
foreach (var orderNumber in installation!.OrderNumbers!)
{
if (originalOrderNumbers.Contains(orderNumber)) continue;
if (originalOrderNumbers!.Contains(orderNumber)) continue;
var o2I = new OrderNumber2Installation
{
OrderNumber = orderNumber,
@ -116,9 +129,9 @@ public static class SessionMethods
Db.Create(o2I);
}
foreach (var orderNumberOld in originalOrderNumbers.Split(','))
foreach (var orderNumberOld in originalOrderNumbers!)
{
if (!installation!.OrderNumbers.Contains(orderNumberOld))
if (!installation.OrderNumbers.Contains(orderNumberOld))
{
Db.OrderNumber2Installation.Delete(i =>
i.InstallationId == installation.Id && i.OrderNumber == orderNumberOld);

4
csharp/App/Backend/DataTypes/Methods/exoscaleS3.json
View File

@ -1,4 +0,0 @@
{
"Key": "EXOf67c5b528282988503ddab12",
"Secret": "KBFh5HvoSQcTtGYcWSm4Qn4m-WFutKe89UqsOdOL-ts"
}

30
csharp/App/Backend/Database/Db.cs
View File

@ -5,6 +5,8 @@ using CliWrap.Buffered;
using InnovEnergy.App.Backend.DataTypes;
using InnovEnergy.App.Backend.DataTypes.Methods;
using InnovEnergy.App.Backend.Relations;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using SQLite;
using SQLiteConnection = SQLite.SQLiteConnection;
@ -149,18 +151,24 @@ public static partial class Db
private static async Task UpdateS3Urls()
{
var bucketList = await Cli.Wrap("exo")
.WithArguments("storage list -O json")
.ExecuteBufferedAsync();
var installationsToUpdate = Installations
.Select(i => i)
.Where(i => bucketList.StandardOutput.Contains("\"" + i.BucketName() + "\"")).ToList();
foreach (var installation in installationsToUpdate)
var regions = Installations
.Select(i => i.S3Region)
.Distinct().ToList();
const String provider = "exo.io";
foreach (var region in regions)
{
await installation.RenewS3Credentials();
var bucketList = await new S3Region($"https://{region}.{provider}", ExoCmd.S3Creds!).ListAllBuckets();
foreach (var bucket in bucketList.Buckets)
{
foreach (var installation in Installations)
{
if (installation.BucketName() == bucket.BucketName)
{
await installation.RenewS3Credentials();
}
}
}
}
}

2
csharp/App/Backend/Properties/launchSettings.json
View File

@ -7,7 +7,7 @@
"dotnetRunMessages": true,
"launchUrl": "swagger",
"applicationUrl": "https://localhost:7087",
"applicationUrl": "http://localhost:7087",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development",
"HOME":"~/backend"

4
csharp/App/Backend/Resources/exoscaleS3.json Normal file
View File

@ -0,0 +1,4 @@
{
"Key": "EXOea18f5a82bd358896154c783",
"Secret": "lYtzU7R5e0L6XKOgBaLVPFr41nEBDxDdXU47zBAEI6M"
}

7
csharp/App/VrmGrabber/Controller.cs
View File

@ -123,7 +123,7 @@ th { /* header cell */
<td>{{Serial}}</td>
<td>{{NumBatteries}}</td>
<td>{{BatteryVersion}}</td>
<td><a target='_blank' href=http://{{ServerIp}}/UpdateBatteryFirmware/{{Ip}}/{{NumBatteries}}>⬆️{{FirmwareVersion}}</a></td>
<td><a target='_blank' href=http://{{ServerIp}}/UpdateBatteryFirmware/{{Ip}}>⬆️{{FirmwareVersion}}</a></td>
<td>{{BatteryUpdateStatus}}</td>
</tr>";
@ -189,21 +189,22 @@ th { /* header cell */
installation.BatteryUpdateStatus = "Running";
Db.Update(installation: installation);
var batteryIdsResult = await Db.ExecuteBufferedAsyncCommandOnIp(installationIp, $"dbus-send --system --dest=com.victronenergy.battery.{batteryTtyName} --type=method_call --print-reply / com.victronenergy.BusItem.GetText | grep -E -o '_Battery/[0-9]+/' | grep -E -o '[0-9]+'| sort -u");
var batteryIds = batteryIdsResult.Split("\n").ToList();
batteryIds.Pop();
foreach (var batteryId in batteryIds)
{
localCommand = localCommand.Append(
$" && /opt/innovenergy/scripts/upload-bms-firmware {batteryTtyName} {batteryId} /opt/innovenergy/bms-firmware/{FirmwareVersion}.bin");
}
#pragma warning disable CS4014
Console.WriteLine(localCommand);
// Console.WriteLine(localCommand);
Db.ExecuteBufferedAsyncCommandOnIp(installationIp, localCommand)
.ContinueWith(async t =>
{
Console.WriteLine(t.Result);
installation.BatteryUpdateStatus = "Complete";
// installation.BatteryFirmwareVersion = FirmwareVersion;
Db.Update(installation: installation);
var vrmInst = await FindVrmInstallationByIp(installation.Ip!);
await UpdateVrmTagsToNewFirmware(installationIp);

25
csharp/Lib/S3Utils/Iam.cs
View File

@ -1,10 +1,12 @@
using System.Collections.Concurrent;
using System.Net;
using Amazon.CognitoIdentityProvider;
using Amazon.IdentityManagement;
using Amazon.IdentityManagement.Model;
using Amazon.Runtime;
using InnovEnergy.Lib.S3Utils.DataTypes;
using InnovEnergy.Lib.Utils;
using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
namespace InnovEnergy.Lib.S3Utils;
@ -13,8 +15,6 @@ public static class Iam
// TODO
private static readonly ConcurrentDictionary<S3Region, AmazonIdentityManagementServiceClient> AimClientCache = new();
public static AmazonIdentityManagementServiceClient GetIamClient(this S3Url url ) => url.Bucket.GetIamClient();
@ -30,39 +30,40 @@ public static class Iam
clientConfig: new() { ServiceURL = region.Name.EnsureStartsWith("https://") }
);
public static async Task<User> CreateUserAsync(AmazonIdentityManagementServiceClient iamService,String userName)
public static async Task<Role> CreateRoleAsync(AmazonIdentityManagementServiceClient iamService,String userName)
{
var response = await iamService.CreateUserAsync(new CreateUserRequest { UserName = userName });
return response.User;
var response = await iamService.CreateRoleAsync(new CreateRoleRequest() { RoleName = userName });
return response.Role;
}
public static async Task<Boolean> PutUserPolicyAsync(AmazonIdentityManagementServiceClient iamService, String userName, String policyName, String policyDocument)
public static async Task<Boolean> PutRolePolicyAsync(AmazonIdentityManagementServiceClient iamService, String userName, String policyName, String policyDocument)
{
var request = new PutUserPolicyRequest()
var request = new PutRolePolicyRequest()
{
UserName = userName,
RoleName = userName,
PolicyName = policyName,
PolicyDocument = policyDocument
};
var response = await iamService.PutUserPolicyAsync(request);
var response = await iamService.PutRolePolicyAsync(request);
return response.HttpStatusCode == System.Net.HttpStatusCode.OK;
}
public static async Task<AccessKey> CreateAccessKeyAsync(AmazonIdentityManagementServiceClient iamService, String userName)
{
// iamService.Role
var response = await iamService.CreateAccessKeyAsync(new CreateAccessKeyRequest
{
UserName = userName,
UserName= userName,
});
return response.AccessKey;
}
public static async Task<Boolean> UserExists(AmazonIdentityManagementServiceClient iamService, String userName)
public static async Task<Boolean> RoleExists(AmazonIdentityManagementServiceClient iamService, String roleName)
{
var response = await iamService.GetUserAsync(new GetUserRequest { UserName = userName });
var response = await iamService.GetRoleAsync(new GetRoleRequest{RoleName = roleName});
return response.HttpStatusCode == HttpStatusCode.OK;
}

7
csharp/Lib/S3Utils/S3.cs
View File

@ -42,6 +42,13 @@ public static class S3
.Select(o => new S3Url(o.Key, bucket));
}
public static async Task<ListBucketsResponse> ListAllBuckets(this S3Region region)
{
return await region
.GetS3Client()
.ListBucketsAsync();
}
public static Task<Boolean> PutObject(this S3Url path, String data, Encoding encoding) => path.PutObject(encoding.GetBytes(data));
public static Task<Boolean> PutObject(this S3Url path, String data) => path.PutObject(data, Encoding.UTF8);
public static Task<Boolean> PutObject(this S3Url path, Byte[] data) => path.PutObject(new MemoryStream(data));

1
csharp/Lib/S3Utils/S3Utils.csproj
View File

@ -11,6 +11,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.CognitoIdentityProvider" Version="3.7.203.6" />
<PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.200.39" />
<PackageReference Include="AWSSDK.S3" Version="3.7.203.12" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />