From a94116a58404967bc53aef2ca1c0d60f6ba76407 Mon Sep 17 00:00:00 2001 From: Kim Date: Mon, 23 Oct 2023 13:08:09 +0200 Subject: [PATCH] Finally implemented automatic IAM role and key generation and renewal --- csharp/App/Backend/DataTypes/Installation.cs | 10 +- .../App/Backend/DataTypes/Methods/ExoCmd.cs | 266 ++++++++++++++---- .../Backend/DataTypes/Methods/Installation.cs | 44 ++- .../App/Backend/DataTypes/Methods/Session.cs | 28 +- csharp/App/Backend/Database/Delete.cs | 5 + csharp/Lib/S3Utils/S3.cs | 10 + 6 files changed, 277 insertions(+), 86 deletions(-) diff --git a/csharp/App/Backend/DataTypes/Installation.cs b/csharp/App/Backend/DataTypes/Installation.cs index e33f47e84..dbcd527fb 100644 --- a/csharp/App/Backend/DataTypes/Installation.cs +++ b/csharp/App/Backend/DataTypes/Installation.cs @@ -16,13 +16,17 @@ public class Installation : TreeNode public Double Lat { get; set; } public Double Long { get; set; } - public String S3Region { get; set; } = ""; - public String S3Provider { get; set; } = ""; + public String S3Region { get; set; } = "sos-ch-dk-2"; + public String S3Provider { get; set; } = "exo.io"; public String S3WriteKey { get; set; } = ""; public String S3Key { get; set; } = ""; public String S3WriteSecret { get; set; } = ""; public String S3Secret { get; set; } = ""; + public String ReadRoleId { get; set; } = ""; + public String WriteRoleId { get; set; } = ""; [Ignore] - public IReadOnlyList? OrderNumbers { get; set; } + public String OrderNumbers { get; set; } + + } \ No newline at end of file diff --git a/csharp/App/Backend/DataTypes/Methods/ExoCmd.cs b/csharp/App/Backend/DataTypes/Methods/ExoCmd.cs index 2b26c2c76..8da04c6a9 100644 --- a/csharp/App/Backend/DataTypes/Methods/ExoCmd.cs +++ b/csharp/App/Backend/DataTypes/Methods/ExoCmd.cs @@ -2,76 +2,246 @@ using System.Text.Json; using InnovEnergy.Lib.S3Utils; using InnovEnergy.Lib.S3Utils.DataTypes; using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json.Nodes; +using InnovEnergy.App.Backend.Database; namespace InnovEnergy.App.Backend.DataTypes.Methods; public static class ExoCmd { + + private const String Key = "EXOea18f5a82bd358896154c783"; + private const String Secret = "lYtzU7R5e0L6XKOgBaLVPFr41nEBDxDdXU47zBAEI6M"; + [UnconditionalSuppressMessage("Trimming", "IL2026:Members annotated with 'RequiresUnreferencedCodeAttribute' require dynamic access otherwise can break functionality when trimming application code", Justification = "")] public static readonly S3Credentials? S3Creds = JsonSerializer.Deserialize(File.OpenRead("./Resources/exoscaleS3.json")); - - public static async Task<(String, String)> CreateReadKey(this Installation installation) + + private static Byte[] HmacSha256Digest(String message, String secret) { - var url = $"https://{installation.S3Region}-2.exoscale.com/v2/access-key"; + var encoding = new UTF8Encoding(); + var keyBytes = encoding.GetBytes(secret); + var messageBytes = encoding.GetBytes(message); + var cryptographer = new System.Security.Cryptography.HMACSHA256(keyBytes); - var content = new HttpMessageContent(new HttpRequestMessage(HttpMethod.Post, requestUri: $$""" - { - "name" : {{installation.Name}}, - "operations": [ - "list-objects", - "get-object" - ], - "resources": { - "resource-name": "{{installation.BucketName()}}" - } - } -""")); - - // await Iam.CreateRoleAsync(iamService, $"READ{installation.BucketName()}"); - // await Iam.PutRolePolicyAsync(iamService, $"READ{installation.BucketName()}", $"READ{installation.BucketName()}",readOnlyPolicy); - var client = new HttpClient(); - var postRequestResponse = await client.PostAsync(url, content); - // var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READ{installation.BucketName()}"); + var bytes = cryptographer.ComputeHash(messageBytes); + return bytes; + } - return (postRequestResponse.Content.ToString(), postRequestResponse.Content.ToString()); + + private static String BuildSignature(String method, String path, String? data, Int64 time) + { + var messageToSign = ""; + messageToSign += method + " /v2/" + path + "\n"; + messageToSign += data + "\n"; + + // query strings + messageToSign += "\n"; + // headers + messageToSign += "\n"; + + messageToSign += time; + + Console.WriteLine("Message to sign:\n" + messageToSign); + + + var hmac = HmacSha256Digest(messageToSign, Secret); + return Convert.ToBase64String(hmac); + } + + private static String BuildSignature(String method, String path, Int64 time) + { + return BuildSignature(method, path, null, time); + } + + public static async Task<(String,String)> CreateReadKey(this Installation installation) + { + var readRoleId = installation.ReadRoleId; + + if (String.IsNullOrEmpty(readRoleId) + ||! await CheckRoleExists(readRoleId)) + { + readRoleId = await installation.CreateReadRole(); + Thread.Sleep(4000); // Exoscale is to slow for us the role might not be there yet + } + return await CreateKey(installation, readRoleId); + } + + private static async Task CheckRoleExists(String roleId) + { + const String url = "https://api-ch-dk-2.exoscale.com/v2/iam-role"; + const String method = "iam-role"; + + var unixtime = DateTimeOffset.UtcNow.ToUnixTimeSeconds()+60; + + var authheader = "credential="+Key+",signed-query-args="+",expires="+unixtime+",signature="+BuildSignature("GET", method, unixtime); + + var client = new HttpClient(); + + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("EXO2-HMAC-SHA256", authheader); + + var response = await client.GetAsync(url); + var responseString = await response.Content.ReadAsStringAsync(); + return responseString.Contains(roleId); + } + + private static async Task<(String,String)> CreateKey(Installation installation, String roleName) + { + var url = "https://api-ch-dk-2.exoscale.com/v2/api-key"; + var method = "api-key"; + var contentString = $$"""{"role-id": "{{roleName}}", "name":"{{installation.BucketName()}}v2"}"""; + var unixtime = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + 60; + + var authheader = "credential=" + Key + ",expires=" + unixtime + ",signature=" + + BuildSignature("POST", method, contentString, unixtime); + + var client = new HttpClient(); + + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("EXO2-HMAC-SHA256", authheader); + var content = new StringContent(contentString, Encoding.UTF8, "application/json"); + + var response = await client.PostAsync(url, content); + if (response.StatusCode != HttpStatusCode.OK){ + Console.WriteLine("Fuck"); + } + Console.WriteLine($"Created Key for {installation.Name}"); + var responseString = await response.Content.ReadAsStringAsync(); + + var responseJson = JsonNode.Parse(responseString) ; + return (responseJson!["key"]!.GetValue(), responseJson!["secret"]!.GetValue()); + } + + + public static async Task CreateReadRole(this Installation installation) + { + const String url = "https://api-ch-dk-2.exoscale.com/v2/iam-role"; + const String method = "iam-role"; + + var contentString = $$""" + { + "name" : "{{installation.Id + installation.Name}}", + "policy" : { + "default-service-strategy": "deny", + "services": { + "sos": { + "type": "rules", + "rules": [ + { + "expression": "operation == 'get-object' && resources.bucket.startsWith('{{installation.BucketName()}}')", + "action": "allow" + } + ] + } + } + } + } + """; + + var unixtime = DateTimeOffset.UtcNow.ToUnixTimeSeconds()+60; + + var authheader = "credential="+Key+",signed-query-args="+",expires="+unixtime+",signature="+BuildSignature("POST", method, contentString, unixtime); + + var client = new HttpClient(); + + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("EXO2-HMAC-SHA256", authheader); + var content = new StringContent(contentString, Encoding.UTF8, "application/json"); + + + var response = await client.PostAsync(url, content); + + var responseString = await response.Content.ReadAsStringAsync(); + Console.WriteLine(responseString); + + //Put Role ID into database + var id = JsonNode.Parse(responseString)!["reference"]!["id"]!.GetValue(); + installation.ReadRoleId = id; + Db.Update(installation); + return id; } public static async Task RevokeReadKey(this Installation installation) { - var iamService = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", S3Creds!).GetIamClient(); - if (!await Iam.RoleExists(iamService, $"READ{installation.BucketName()}")) - { - return true; - } + var url = $"https://api-ch-dk-2.exoscale.com/v2/access-key/{installation.S3Key}"; + var method = $"access-key/{installation.S3Key}"; + + var unixtime = DateTimeOffset.UtcNow.ToUnixTimeSeconds()+60; - return await Iam.RevokeAccessKey(iamService, $"READ{installation.BucketName()}"); + var authheader = "credential="+Key+",expires="+unixtime+",signature="+BuildSignature("DELETE", method, unixtime); + + var client = new HttpClient(); + + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("EXO2-HMAC-SHA256", authheader); + + var response = await client.DeleteAsync(url); + return response.IsSuccessStatusCode; } - public static async Task<(String key, String secret)> CreateWriteKey(this Installation installation) + public static async Task<(String, String)> CreateWriteKey(this Installation installation) { - var iamService = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", S3Creds!).GetIamClient(); - if (!await Iam.RoleExists(iamService, $"READWRITE{installation.BucketName()}")) + var writeRoleId = installation.WriteRoleId; + + if (String.IsNullOrEmpty(writeRoleId) + || !await CheckRoleExists(writeRoleId)) { - var readWritePolicy = @"{ - ""default-service-strategy"": ""deny"", - ""services"": { - ""sos"": { - ""type"": ""allow"", - ""resource"": " + $@"{installation.BucketName()} - }} - }} - }}"; - await Iam.CreateRoleAsync(iamService, $"READWRITE{installation.BucketName()}"); - await Iam.PutRolePolicyAsync(iamService, $"READWRITE{installation.BucketName()}", $"READWRITE{installation.BucketName()}",readWritePolicy); + writeRoleId = await installation.CreateWriteRole(); + Thread.Sleep(4000); // Exoscale is to slow for us the role might not be there yet } - - var keySecret = await Iam.CreateAccessKeyAsync(iamService, $"READWRITE{installation.BucketName()}"); - - - return (keySecret.AccessKeyId, keySecret.SecretAccessKey); + return await CreateKey(installation, writeRoleId); } - + public static async Task CreateWriteRole(this Installation installation) + { + const String url = "https://api-ch-dk-2.exoscale.com/v2/iam-role"; + const String method = "iam-role"; + + var contentString = $$""" + { + "name" : "WRITE{{installation.Id + installation.Name}}", + "policy" : { + "default-service-strategy": "deny", + "services": { + "sos": { + "type": "rules", + "rules":[{ + "action" : "allow", + "expression": "resources.bucket.startsWith('{{installation.BucketName()}}')" + }] + } + } + } + } + """; + + var unixtime = DateTimeOffset.UtcNow.ToUnixTimeSeconds()+60; + + var authheader = "credential="+Key+",signed-query-args="+",expires="+unixtime+",signature="+BuildSignature("POST", method, contentString, unixtime); + + var client = new HttpClient(); + + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("EXO2-HMAC-SHA256", authheader); + var content = new StringContent(contentString, Encoding.UTF8, "application/json"); + + + var response = await client.PostAsync(url, content); + + var responseString = await response.Content.ReadAsStringAsync(); + Console.WriteLine(responseString); + + //Put Role ID into database + var id = JsonNode.Parse(responseString)!["reference"]!["id"]!.GetValue(); + installation.WriteRoleId = id; + ; + Db.Update(installation); + + return id; + } public static async Task CreateBucket(this Installation installation) { diff --git a/csharp/App/Backend/DataTypes/Methods/Installation.cs b/csharp/App/Backend/DataTypes/Methods/Installation.cs index d75f38230..f61a5446a 100644 --- a/csharp/App/Backend/DataTypes/Methods/Installation.cs +++ b/csharp/App/Backend/DataTypes/Methods/Installation.cs @@ -11,8 +11,11 @@ namespace InnovEnergy.App.Backend.DataTypes.Methods; public static class InstallationMethods { - private const String BucketNameSalt = "3e5b3069-214a-43ee-8d85-57d72000c19d"; - + private static readonly String BucketNameSalt = + Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "" + ? "stage" + :"3e5b3069-214a-43ee-8d85-57d72000c19d"; + public static String BucketName(this Installation installation) { return $"{installation.Id}-{BucketNameSalt}"; @@ -22,17 +25,17 @@ public static class InstallationMethods { if(installation.S3Key != "") await installation.RevokeReadKey(); - var (key, secret) = await installation.CreateReadKey(); + var (key,secret) = await installation.CreateReadKey(); + installation.S3Key = key; + installation.S3Secret = secret; + if (installation.S3WriteKey == "" || installation.S3WriteSecret == "") { - var (writeKey, writeSecret) = await installation.CreateWriteKey(); + var (writeKey,writeSecret) = await installation.CreateWriteKey(); installation.S3WriteSecret = writeSecret; installation.S3WriteKey = writeKey; } - - installation.S3Key = key; - installation.S3Secret = secret; return Db.Update(installation); } @@ -141,12 +144,12 @@ public static class InstallationMethods return Db.Installations.Any(i => i.Id == installation.Id); } - public static IReadOnlyList? GetOrderNumbers(this Installation installation) + public static String GetOrderNumbers(this Installation installation) { return Db.OrderNumber2Installation .Where(i => i.InstallationId == installation.Id) .Select(i => i.OrderNumber) - .ToReadOnlyList(); + .ToReadOnlyList().JoinWith(","); } public static Installation FillOrderNumbers(this Installation installation) @@ -155,4 +158,27 @@ public static class InstallationMethods return installation; } + public static Boolean SetOrderNumbers(this Installation installation) + { + var relations = Db.OrderNumber2Installation.Where(i => i.InstallationId == installation.Id).ToList(); + foreach (var orderNumber in installation.OrderNumbers.Split(",")) + { + var rel = relations.FirstOrDefault(i => i.OrderNumber == orderNumber); + if ( rel != null) relations.Remove(rel); + var o2I = new OrderNumber2Installation + { + OrderNumber = orderNumber, + InstallationId = installation.Id + }; + Db.Create(o2I); + } + + foreach (var rel in relations) + { + Db.Delete(rel); + } + + return true; + } + } \ No newline at end of file diff --git a/csharp/App/Backend/DataTypes/Methods/Session.cs b/csharp/App/Backend/DataTypes/Methods/Session.cs index bfe0308b3..e5ecf5ff9 100644 --- a/csharp/App/Backend/DataTypes/Methods/Session.cs +++ b/csharp/App/Backend/DataTypes/Methods/Session.cs @@ -100,12 +100,12 @@ public static class SessionMethods && user.HasWriteAccess && user.HasAccessToParentOf(installation) && Db.Create(installation) // TODO: these two in a transaction - // && installation.SetOrderNumbers() + && installation.SetOrderNumbers() && Db.Create(new InstallationAccess { UserId = user.Id, InstallationId = installation.Id }) && await installation.CreateBucket() && await installation.RenewS3Credentials(); // generation of access _after_ generation of // bucket to prevent "zombie" access-rights. - // This might fuck us over if the creation of access rights fails, + // This might ** us over if the creation of access rights fails, // as bucket-names are unique and bound to the installation id... -K } @@ -114,31 +114,7 @@ public static class SessionMethods var user = session?.User; var original = Db.GetInstallationById(installation?.Id); - var originalOrderNumbers = original!.GetOrderNumbers(); - - if (!Equals(originalOrderNumbers, installation?.OrderNumbers)) - { - foreach (var orderNumber in installation!.OrderNumbers!) - { - if (originalOrderNumbers!.Contains(orderNumber)) continue; - var o2I = new OrderNumber2Installation - { - OrderNumber = orderNumber, - InstallationId = installation.Id - }; - Db.Create(o2I); - } - foreach (var orderNumberOld in originalOrderNumbers!) - { - if (!installation.OrderNumbers.Contains(orderNumberOld)) - { - Db.OrderNumber2Installation.Delete(i => - i.InstallationId == installation.Id && i.OrderNumber == orderNumberOld); - } - } - } - return user is not null && installation is not null && original is not null diff --git a/csharp/App/Backend/Database/Delete.cs b/csharp/App/Backend/Database/Delete.cs index cb3895037..660d61539 100644 --- a/csharp/App/Backend/Database/Delete.cs +++ b/csharp/App/Backend/Database/Delete.cs @@ -78,4 +78,9 @@ public static partial class Db BackupDatabase(); return delete; } + + public static void Delete(OrderNumber2Installation relation) + { + OrderNumber2Installation.Delete(s => s.InstallationId == relation.InstallationId && s.OrderNumber == relation.OrderNumber); + } } \ No newline at end of file diff --git a/csharp/Lib/S3Utils/S3.cs b/csharp/Lib/S3Utils/S3.cs index 20aa9b492..71decbd73 100644 --- a/csharp/Lib/S3Utils/S3.cs +++ b/csharp/Lib/S3Utils/S3.cs @@ -96,6 +96,16 @@ public static class S3 return response.ResponseStream; } + + // public static async Task CheckRoleExists(this S3Region region, String roleId) + // { + // + // + // var response = await region + // .GetIamClient() + // .GetRoleAsync(new GetRoleRequest(){RoleName = roleId}); + // return response.HttpStatusCode != HttpStatusCode.NotFound; + // } public static async Task> GetObject(this S3Url url) {