Innovenergy
/
Innovenergy_trunk
2
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

introduce S3Utils, WIP

This commit is contained in:
ig 2023-09-14 14:33:45 +02:00
parent 3b3381247a
commit d93a066b36
15 changed files with 3658 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
csharp/Lib/S3Utils/Data/S3Bucket.cs Normal file
View File

@ -0,0 +1,6 @@
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3Bucket : S3RegionCredentials
{
public required String Bucket { get; init; }
}

47
csharp/Lib/S3Utils/Data/S3Credentials.cs Normal file
View File

@ -0,0 +1,47 @@
using static System.Environment;
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3Credentials
{
protected static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile).TrimEnd('\\', '/') + "/.s3cfg";
public required String Key { get; init; }
public required String Secret { get; init; }
public static S3Credentials? FromS3Cfg() => FromFile(S3CfgFile);
public static S3Credentials? FromFile(String file)
{
// [default]
// host_base = sos-ch-dk-2.exo.io
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
// access_key = xxxxxxxxxxxxxxx
// secret_key = xxxxxxxxxxxxxxx
// use_https = True
try
{
var cfg = ParseFile(file);
return new S3Credentials
{
Key = cfg["access_key"],
Secret = cfg["secret_key"]
};
}
catch
{
return null;
}
}
protected static Dictionary<String, String> ParseFile(String cfgFile)
{
return File
.ReadAllLines(cfgFile)
.Where(l => l.Contains("="))
.Select(l => l.Split("="))
.ToDictionary(l => l[0].Trim(), l => l[1].Trim());
}
}

6
csharp/Lib/S3Utils/Data/S3Path.cs Normal file
View File

@ -0,0 +1,6 @@
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3Path : S3Bucket
{
public required String Path { get; init; }
}

49
csharp/Lib/S3Utils/Data/S3RegionCredentials.cs Normal file
View File

@ -0,0 +1,49 @@
using Amazon.Runtime;
using Amazon.S3;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3Utils.Data;
public record S3RegionCredentials : S3Credentials
{
public required String Region { get; init; }
private AmazonS3Client? _Client;
internal AmazonS3Client Client => _Client ??= new AmazonS3Client
(
credentials: new BasicAWSCredentials(Key, Secret),
clientConfig: new()
{
ServiceURL = Region.EnsureStartsWith("https://"),
ForcePathStyle = true,
}
);
public new static S3RegionCredentials? FromS3Cfg() => FromFile(S3CfgFile);
public new static S3RegionCredentials? FromFile(String file)
{
// [default]
// host_base = sos-ch-dk-2.exo.io
// host_bucket = %(bucket)s.sos-ch-dk-2.exo.io
// access_key = xxxxxxxxxxxxxxx
// secret_key = xxxxxxxxxxxxxxx
// use_https = True
try
{
var cfg = ParseFile(file);
return new S3RegionCredentials
{
Key = cfg["access_key"],
Secret = cfg["secret_key"],
Region = cfg["host_base"],
};
}
catch
{
return null;
}
}
}

21
csharp/Lib/S3Utils/ExoScale/DefaultCredentials.cs Normal file
View File

@ -0,0 +1,21 @@
using InnovEnergy.Lib.S3Utils.Data;
namespace InnovEnergy.Lib.S3Utils.ExoScale;
public static class DefaultCredentials
{
public static S3Credentials ReadOnly => new S3Credentials
{
Key = "EXOb6d6dc1880cdd51f1ebc6692",
Secret = "kpIey4QJlQFuWG_WoTazcY7kBEjN2f_ll2cDBeg64m4",
};
public static S3Credentials ReadWrite => new S3Credentials
{
Key = "EXO87ca85e29dd412f1238f1cf0",
Secret = "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU",
};
}

6
csharp/Lib/S3Utils/ExoScale/Regions.cs Normal file
View File

@ -0,0 +1,6 @@
namespace InnovEnergy.Lib.S3Utils.ExoScale;
public static class Regions
{
public static String Default => "sos-ch-dk-2.exo.io";
}

9
csharp/Lib/S3Utils/Resources/CORS Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" ?>
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
</CORSRule>
</CORSConfiguration>

4
csharp/Lib/S3Utils/Resources/s3AdminKey.json Normal file
View File

@ -0,0 +1,4 @@
{
"Key": "EXO1abcb772bf43ab72951ba1dc",
"Secret": "_ym1KsGBSp90S5dwhZn18XD-u9Y4ghHvyIxg5gv5fHw"
}

5
csharp/Lib/S3Utils/Resources/s3ReadOnlyKey.json Normal file
View File

@ -0,0 +1,5 @@
{
"Key": "EXOb6d6dc1880cdd51f1ebc6692",
"Secret": "kpIey4QJlQFuWG_WoTazcY7kBEjN2f_ll2cDBeg64m4",
"Region": "sos-ch-dk-2.exo.io"
}

5
csharp/Lib/S3Utils/Resources/s3ReadWriteKey.json Normal file
View File

@ -0,0 +1,5 @@
{
"Key": "EXO87ca85e29dd412f1238f1cf0",
"Secret": "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU",
"Region": "sos-ch-dk-2.exo.io"
}

3380
csharp/Lib/S3Utils/Resources/s3cmd.py Executable file
View File

File diff suppressed because it is too large Load Diff

6
csharp/Lib/S3Utils/Resources/smtpConfig.json Normal file
View File

@ -0,0 +1,6 @@
{
"Url": "mail.agenturserver.de",
"Port": 587,
"Username": "p518526p69",
"Password": "i;b*xqm4iB5uhl"
}

6
csharp/Lib/S3Utils/Resources/urlAndKey.json Normal file
View File

@ -0,0 +1,6 @@
{
"ReadOnlyS3Key": "EXO44d2979c8e570eae81ead564",
"ReadOnlyS3Secret": "55MAqyO_FqUmh7O64VIO0egq50ERn_WIAWuc2QC44QU" ,
"ReadWriteS3Key": "EXO87ca85e29dd412f1238f1cf0",
"ReadWriteS3Secret": "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU"
}

90
csharp/Lib/S3Utils/S3.cs Normal file
View File

@ -0,0 +1,90 @@
using Amazon.S3.Model;
using InnovEnergy.Lib.S3Utils.Data;
using InnovEnergy.Lib.Utils;
using S3Bucket = InnovEnergy.Lib.S3Utils.Data.S3Bucket;
namespace InnovEnergy.Lib.S3Utils;
public static class S3
{
public static S3RegionCredentials Region(this S3Credentials credentials, String region) => new()
{
Secret = credentials.Secret,
Key = credentials.Key,
Region = region,
};
public static S3Bucket Bucket(this S3RegionCredentials region, String bucket) => new()
{
Region = region.Region,
Secret = region.Secret,
Key = region.Key,
Bucket = bucket,
};
public static S3Path Path(this S3Bucket bucket, String path) => new()
{
Bucket = bucket.Bucket,
Region = bucket.Region,
Secret = bucket.Secret,
Key = bucket.Key,
Path = path
};
public static IAsyncEnumerable<S3Path> ListObjects(this S3Bucket bucketOrPrefixPath)
{
var path = bucketOrPrefixPath as S3Path ?? bucketOrPrefixPath.Path(null!);
var request = new ListObjectsV2Request
{
BucketName = path.Bucket,
Prefix = path.Path
};
return bucketOrPrefixPath
.Client
.Paginators
.ListObjectsV2(request)
.Responses
.SelectMany(r => r.S3Objects)
.Select(o => path with { Path = o.Key });
}
public static async Task<String> GetObject(this S3Path path)
{
var request = new GetObjectRequest
{
BucketName = path.Bucket,
Key = path.Path
};
using var response = await path.Client.GetObjectAsync(request);
await using var responseStream = response.ResponseStream;
using var reader = new StreamReader(responseStream);
return await reader.ReadToEndAsync();
}
public static async IAsyncEnumerable<String> GetObjectLineByLine(this S3Path path)
{
var request = new GetObjectRequest
{
BucketName = path.Bucket,
Key = path.Path
};
using var response = await path.Client.GetObjectAsync(request);
await using var responseStream = response.ResponseStream;
using var reader = new StreamReader(responseStream);
while (true)
{
var line = await reader.ReadLineAsync();
if (line is not null)
yield return line;
else
yield break;
}
}
}

18
csharp/Lib/S3Utils/S3Utils.csproj Normal file
View File

@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../InnovEnergy.Lib.props" />
<PropertyGroup>
<RootNamespace>InnovEnergy.Lib.S3Utils</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../Utils/Utils.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.203.12" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>