Innovenergy
/
Innovenergy_trunk
2
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Remove S3 lib

This commit is contained in:
ig 2023-02-25 16:16:12 +01:00
parent aa46ad37b6
commit 1aec8e1fe9
49 changed files with 78 additions and 1813 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
csharp/InnovEnergy.props
View File

@ -9,7 +9,7 @@
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net6.0</TargetFramework>
<InvariantGlobalization>true</InvariantGlobalization> <InvariantGlobalization>true</InvariantGlobalization>
<SuppressTrimAnalysisWarnings>false</SuppressTrimAnalysisWarnings> <SuppressTrimAnalysisWarnings>false</SuppressTrimAnalysisWarnings>
<RootNamespace>$(Company).$(MSBuildProjectDirectory.Replace($(SolutionDir), "").Replace("lib/", "Lib/").Replace("app/", "App/").Replace("src/", "").Replace("/","."))</RootNamespace> <RootNamespace>$(Company).$(MSBuildProjectDirectory.Replace($(SolutionDir), "").Replace("src/", "").Replace("/","."))</RootNamespace>
<Authors>$(Company) Team</Authors> <Authors>$(Company) Team</Authors>
</PropertyGroup> </PropertyGroup>

12
csharp/InnovEnergy.sln
View File

@ -20,10 +20,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "App", "App", "{145597B4-3E3
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Lib", "Lib", "{AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Lib", "Lib", "{AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "S3", "Lib/S3/S3.csproj", "{C3639841-13F4-4F24-99C6-7D965593BF89}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "deprecated", "deprecated", "{46DE03C4-52D1-47AA-8E60-8BB15361D723}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SaliMax", "App/SaliMax/SaliMax.csproj", "{25073794-D859-4824-9984-194C7E928496}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SaliMax", "App/SaliMax/SaliMax.csproj", "{25073794-D859-4824-9984-194C7E928496}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusApi", "Lib/StatusApi/StatusApi.csproj", "{9D17E78C-8A70-43DB-A619-DC12D20D023D}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusApi", "Lib/StatusApi/StatusApi.csproj", "{9D17E78C-8A70-43DB-A619-DC12D20D023D}"
@ -106,10 +102,6 @@ Global
{40B45363-BE34-420B-8F87-775EE6EE3513}.Debug|Any CPU.Build.0 = Debug|Any CPU {40B45363-BE34-420B-8F87-775EE6EE3513}.Debug|Any CPU.Build.0 = Debug|Any CPU
{40B45363-BE34-420B-8F87-775EE6EE3513}.Release|Any CPU.ActiveCfg = Release|Any CPU {40B45363-BE34-420B-8F87-775EE6EE3513}.Release|Any CPU.ActiveCfg = Release|Any CPU
{40B45363-BE34-420B-8F87-775EE6EE3513}.Release|Any CPU.Build.0 = Release|Any CPU {40B45363-BE34-420B-8F87-775EE6EE3513}.Release|Any CPU.Build.0 = Release|Any CPU
{C3639841-13F4-4F24-99C6-7D965593BF89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C3639841-13F4-4F24-99C6-7D965593BF89}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C3639841-13F4-4F24-99C6-7D965593BF89}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C3639841-13F4-4F24-99C6-7D965593BF89}.Release|Any CPU.Build.0 = Release|Any CPU
{25073794-D859-4824-9984-194C7E928496}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {25073794-D859-4824-9984-194C7E928496}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{25073794-D859-4824-9984-194C7E928496}.Debug|Any CPU.Build.0 = Debug|Any CPU {25073794-D859-4824-9984-194C7E928496}.Debug|Any CPU.Build.0 = Debug|Any CPU
{25073794-D859-4824-9984-194C7E928496}.Release|Any CPU.ActiveCfg = Release|Any CPU {25073794-D859-4824-9984-194C7E928496}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -187,11 +179,8 @@ Global
{B2627B9F-41DF-44F7-A0D1-CA71FF4A007A} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {B2627B9F-41DF-44F7-A0D1-CA71FF4A007A} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{F65F33B0-3522-4008-8D1E-47EF8E4C7AC7} = {145597B4-3E30-45E6-9F72-4DD43194539A} {F65F33B0-3522-4008-8D1E-47EF8E4C7AC7} = {145597B4-3E30-45E6-9F72-4DD43194539A}
{E3A5F3A3-72A5-47CC-85C6-2D8E962A0EC1} = {145597B4-3E30-45E6-9F72-4DD43194539A} {E3A5F3A3-72A5-47CC-85C6-2D8E962A0EC1} = {145597B4-3E30-45E6-9F72-4DD43194539A}
{46DE03C4-52D1-47AA-8E60-8BB15361D723} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{4A67D79F-F0C9-4BBC-9601-D5948E6C05D3} = {46DE03C4-52D1-47AA-8E60-8BB15361D723}
{25073794-D859-4824-9984-194C7E928496} = {145597B4-3E30-45E6-9F72-4DD43194539A} {25073794-D859-4824-9984-194C7E928496} = {145597B4-3E30-45E6-9F72-4DD43194539A}
{9D17E78C-8A70-43DB-A619-DC12D20D023D} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {9D17E78C-8A70-43DB-A619-DC12D20D023D} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{C3639841-13F4-4F24-99C6-7D965593BF89} = {46DE03C4-52D1-47AA-8E60-8BB15361D723}
{4931A385-24DC-4E78-BFF4-356F8D6D5183} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {4931A385-24DC-4E78-BFF4-356F8D6D5183} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{794FD07C-93E9-4803-982E-1CA261504AB5} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {794FD07C-93E9-4803-982E-1CA261504AB5} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{BD8CBC5C-0B9E-48A3-BC4E-725E3FAB2348} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {BD8CBC5C-0B9E-48A3-BC4E-725E3FAB2348} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
@ -211,5 +200,6 @@ Global
{AF7E8DCA-8D48-498E-AB3D-208061B244DC} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {AF7E8DCA-8D48-498E-AB3D-208061B244DC} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{A56F58C2-B265-435B-A985-53B4D6F49B1A} = {145597B4-3E30-45E6-9F72-4DD43194539A} {A56F58C2-B265-435B-A985-53B4D6F49B1A} = {145597B4-3E30-45E6-9F72-4DD43194539A}
{C04FB6DA-23C6-46BB-9B21-8F4FBA32FFF7} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854} {C04FB6DA-23C6-46BB-9B21-8F4FBA32FFF7} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
{4A67D79F-F0C9-4BBC-9601-D5948E6C05D3} = {AD5B98A8-AB7F-4DA2-B66D-5B4E63E7D854}
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

11
csharp/Lib/S3/DataRec.cs
View File

@ -1,11 +0,0 @@
using InnovEnergy.Lib.S3.Records.Fields;
namespace InnovEnergy.Lib.S3;
public record DataRec
{
[Unit("V")] public Double Voltage { get; init; }
[Unit("A")] public Double Current { get; init; }
public Boolean Error { get; init; }
public String State { get; init; }
}

6
csharp/Lib/S3/Drivers/Internal/Delegates/ReadRecord.cs
View File

@ -1,6 +0,0 @@
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records.Specialized;
namespace InnovEnergy.Lib.S3.Drivers.Internal.Delegates;
public delegate Task<TimeStampedRecord> ReadRecord(AggregationLevel level, UInt32 index);

6
csharp/Lib/S3/Drivers/Internal/Delegates/WriteRecord.cs
View File

@ -1,6 +0,0 @@
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records.Specialized;
namespace InnovEnergy.Lib.S3.Drivers.Internal.Delegates;
public delegate Task WriteRecord(TimeStampedRecord record, AggregationLevel level, UInt32 index);

50
csharp/Lib/S3/Drivers/Internal/Reader.cs
View File

@ -1,50 +0,0 @@
using InnovEnergy.Lib.S3.Drivers.Internal.Delegates;
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Drivers.Internal;
public partial class Reader
{
public IReadOnlyList<AggregationLevel> AggregationLevels { get; }
public UnixTimeSpan SamplePeriod => AggregationLevels[0].SamplePeriod;
private readonly ReadRecord _ReadRecord;
internal Reader(IReadOnlyList<AggregationLevel> levels, ReadRecord readRecord)
{
_ReadRecord = readRecord;
AggregationLevels = InitLevels(levels);
}
public Task<TimeStampedRecord> ReadRecord(AggregationLevel level, UInt32 index)
{
return _ReadRecord(level, index);
}
private static IReadOnlyList<AggregationLevel> InitLevels(IReadOnlyList<AggregationLevel> levels)
{
if (!levels.Any())
throw new ArgumentOutOfRangeException(nameof(levels));
levels = levels
.OrderBy(l => l.SamplePeriod.Ticks)
.ToReadOnlyList(levels.Count);
var pairwise = levels
.Select(l => l.SamplePeriod)
.Pairwise()
.ToReadOnlyList(levels.Count - 1);
var greater = pairwise.Any(ls => ls.right <= ls.left);
var multiple = pairwise.Any(ls => ls.right % ls.left != 0 );
var retention = levels .Any(l => l.RetentionPeriod % l.SamplePeriod != 0);
if (greater || multiple || retention)
throw new ArgumentException(nameof(levels)); // TODO: error messages
return levels;
}
}

37
csharp/Lib/S3/Drivers/Internal/Reader.public.cs
View File

@ -1,37 +0,0 @@
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Drivers.Internal;
public partial class Reader
{
public Task<IReadOnlyList<TimeStampedRecord>> ReadRecords(UnixTime from, UnixTime to, Int32 minNumber)
{
var maxDt = (to - from) / minNumber;
return ReadRecords(from, to, maxDt);
}
public async Task<IReadOnlyList<TimeStampedRecord>> ReadRecords(UnixTime from, UnixTime to, UnixTimeSpan maxDt)
{
if (maxDt < SamplePeriod)
maxDt = SamplePeriod;
var level = AggregationLevels.LastOrDefault(l => l.SamplePeriod <= maxDt) ?? AggregationLevels[^1];
return await level
.RangeExclusive(from, to)
.Select(t => ReadRecord(level, t))
.WhenAll();
}
public async Task<TimeStampedRecord> ReadRecord(AggregationLevel level, UnixTime time)
{
var tsRecord = await ReadRecord(level, level.GetRetentionIndex(time));
return tsRecord.TimeStamp == time
? tsRecord
: TimeStampedRecord.Empty(time);
}
}

60
csharp/Lib/S3/Drivers/Internal/Util/Aggregator.cs
View File

@ -1,60 +0,0 @@
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records;
using InnovEnergy.Lib.S3.Records.Operations;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Drivers.Internal.Util;
public class Aggregator
{
public AggregationLevel AggregationLevel { get; }
private Record[] Buffer { get; }
private UInt32 Index { get; set; }
public Aggregator(AggregationLevel thisLevel,
AggregationLevel levelBelow,
IEnumerable<Record> initialRecords)
{
var ratio = thisLevel.SamplePeriod / levelBelow.SamplePeriod;
AggregationLevel = thisLevel;
Index = 0;
Buffer = new Record[ratio];
Clear();
foreach (var record in initialRecords)
Aggregate(record);
}
public Record? Aggregate(Record r)
{
Buffer[Index++] = r;
return IsFull
? ForceAggregation()
: null;
}
public Record? ForceAggregation()
{
if (IsEmpty)
return null; // nothing to aggregate
var aggregated = Buffer.Aggregate();
Clear();
return aggregated;
}
private void Clear()
{
Buffer.Fill(Record.Empty);
Index = 0;
}
private Boolean IsFull => Index == Buffer.Length;
private Boolean IsEmpty => Index == 0;
}

43
csharp/Lib/S3/Drivers/Internal/Util/Sampler.cs
View File

@ -1,43 +0,0 @@
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Drivers.Internal.Util;
internal class Sampler
{
public Record CurrentRecord { get; set; }
public UnixTime CurrentTimeStamp { get; set; }
public AggregationLevel AggregationLevel { get; }
private UnixTimeSpan SamplePeriod => AggregationLevel.SamplePeriod;
public Sampler(AggregationLevel aggregationLevel, Record record, UnixTime currentTime)
{
AggregationLevel = aggregationLevel;
CurrentRecord = record;
CurrentTimeStamp = aggregationLevel.GetPeriodStartTime(currentTime);
}
// TODO: repeat/max age
public IEnumerable<AggregatedRecord> Sample(Record record, UnixTime timeStamp)
{
timeStamp = AggregationLevel.GetPeriodStartTime(timeStamp);
if (timeStamp < CurrentTimeStamp)
yield break; //throw new IndexOutOfRangeException(nameof(index)); // TODO: log
if (timeStamp > CurrentTimeStamp)
yield return new AggregatedRecord(CurrentRecord, AggregationLevel, CurrentTimeStamp);
for (var t = CurrentTimeStamp + SamplePeriod; t < timeStamp; t += SamplePeriod)
yield return new AggregatedRecord(Record.Empty, AggregationLevel, t);
CurrentTimeStamp = timeStamp;
CurrentRecord = record;
}
}

129
csharp/Lib/S3/Drivers/Internal/Writer.cs
View File

@ -1,129 +0,0 @@
using System.Diagnostics;
using InnovEnergy.Lib.S3.Drivers.Internal.Delegates;
using InnovEnergy.Lib.S3.Drivers.Internal.Util;
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Drivers.Internal;
using AggregationLevels = IReadOnlyList<AggregationLevel>;
using Aggregators = IReadOnlyList<Aggregator>;
public partial class Writer : Reader, IDisposable
{
private Sampler Sampler { get; }
private Aggregators Aggregators { get; }
private readonly WriteRecord _WriteRecord;
internal Writer(UnixTime startTime,
AggregationLevels levels,
ReadRecord readRecord,
WriteRecord writeRecord) : base(levels, readRecord)
{
_WriteRecord = writeRecord;
startTime = AggregationLevels.First().GetPeriodStartTime(startTime);
Sampler = InitializeSampler(startTime);
Aggregators = InitializeAggregators(startTime);
}
private IReadOnlyList<Aggregator> InitializeAggregators(UnixTime startTime)
{
return AggregationLevels
.Pairwise()
.SelectTuple((lo, hi) => InitializeAggregator(lo, hi, startTime))
.ToReadOnlyList(AggregationLevels.Count - 1);
}
private Aggregator InitializeAggregator(AggregationLevel lo,
AggregationLevel hi,
UnixTime currentTime)
{
// This was a REAL brainfuck to get right
var loStartTime = lo.GetPeriodStartTime(currentTime);
var hiStartTime = hi.GetPeriodStartTime(currentTime);
Debug.Assert(hiStartTime <= loStartTime);
var initialRecords = lo
.RangeExclusive(hiStartTime, loStartTime)
.Select(t => ReadRecord(lo, t))
.WhenAll()
.Result
.Select(r => r.Record);
return new Aggregator(hi, lo, initialRecords);
}
private Sampler InitializeSampler(UnixTime startTime)
{
var samplerLevel = AggregationLevels.First();
var initialRecord = ReadRecord(samplerLevel, startTime).Result.Record;
return new Sampler(samplerLevel, initialRecord, startTime);
}
private IEnumerable<AggregatedRecord> Aggregate(Record record, UnixTime timeStamp)
{
return Sampler
.Sample(record, timeStamp)
.SelectMany(Aggregate);
}
private IEnumerable<AggregatedRecord> Aggregate(AggregatedRecord r)
{
yield return r;
var record = r.Record;
foreach (var a in Aggregators)
{
record = a.Aggregate(record);
if (record is null) break;
var timeStamp = a.AggregationLevel.GetPeriodStartTime(r.TimeStamp);
yield return new AggregatedRecord(record, a.AggregationLevel, timeStamp);
}
}
private Task WriteRecord(AggregatedRecord rec)
{
var tsr = rec.ToTimeStamped();
var level = rec.AggregationLevel;
var index = level.GetRetentionIndex(rec.TimeStamp);
return _WriteRecord(tsr, level, index);
}
void IDisposable.Dispose()
{
DisposeAsync().Wait();
}
private async Task DisposeAsync()
{
// feed the sampler an empty "next" record, so it writes and aggregates the current one.
await WriteRecord(Record.Empty, Sampler.CurrentTimeStamp + SamplePeriod);
foreach (var a in Aggregators)
{
// force and write incomplete aggregation for each level
var agg = a.ForceAggregation();
if (agg is null)
continue;
var lev = a.AggregationLevel;
var ts = lev.GetPeriodStartTime(Sampler.CurrentTimeStamp);
var tsr = agg!.TimeStamped(ts);
var idx = lev.GetRetentionIndex(ts);
await _WriteRecord(tsr, lev, idx);
}
}
}

26
csharp/Lib/S3/Drivers/Internal/Writer.public.cs
View File

@ -1,26 +0,0 @@
using InnovEnergy.Lib.S3.Records;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Drivers.Internal;
public partial class Writer
{
public Task WriteRecord(Record record)
{
return WriteRecord(record, UnixTime.Now);
}
public Task WriteRecord(TimeStampedRecord tsr)
{
return WriteRecord(tsr.Record, tsr.TimeStamp);
}
public Task WriteRecord(Record record, UnixTime time)
{
return Aggregate(record, time)
.Select(WriteRecord)
.WhenAll();
}
}

57
csharp/Lib/S3/Drivers/MemoryDriver.cs
View File

@ -1,57 +0,0 @@
using InnovEnergy.Lib.S3.Drivers.Internal;
using InnovEnergy.Lib.S3.Drivers.Internal.Delegates;
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Drivers;
using Levels = IReadOnlyList<AggregationLevel>;
using Memory = Dictionary<String, TimeStampedRecord>;
public class MemoryDriver : Writer
{
public MemoryDriver(Levels levels, UnixTime startTime, Memory memory) :
base(
startTime,
levels,
ReadRecord(memory),
WriteRecord(memory)
)
{
}
private static String GetKey(AggregationLevel level, UInt32 index)
{
return $"{level}/{index}";
}
private static ReadRecord ReadRecord(Memory memory)
{
Task<TimeStampedRecord> Read(AggregationLevel level, UInt32 index)
{
var key = GetKey(level, index);
var result = memory.TryGetValue(key, out var tsRecord)
? tsRecord
: TimeStampedRecord.Empty();
return Task.FromResult(result);
}
return Read;
}
private static WriteRecord WriteRecord(Memory memory)
{
Task Write(TimeStampedRecord record, AggregationLevel level, UInt32 index)
{
var key = GetKey(level, index);
memory[key] = record;
return Task.CompletedTask;
}
return Write;
}
}

83
csharp/Lib/S3/Drivers/S3Config.cs
View File

@ -1,83 +0,0 @@
using System.Security.Cryptography;
using Flurl;
using Flurl.Http;
using InnovEnergy.Lib.Utils;
using static System.Text.Encoding;
using Convert = System.Convert;
namespace InnovEnergy.Lib.S3.Drivers;
public record S3Config
{
public String Bucket { get; init; } = "";
public String Region { get; init; } = "";
public String Provider { get; init; } = "";
public String Key { get; init; } = "";
public String Secret { get; init; } = "";
public String ContentType { get; init; } = "";
public String Host => $"{Bucket}.{Region}.{Provider}";
public String Url => $"https://{Host}";
public IFlurlRequest CreatePutRequest(String s3Path) => CreateRequest("PUT", s3Path);
public IFlurlRequest CreateGetRequest(String s3Path) => CreateRequest("GET", s3Path);
private IFlurlRequest CreateRequest(String method, String s3Path)
{
var date = DateTime.UtcNow.ToString("r");
var auth = CreateAuthorization(method, s3Path, date);
return Url
.AppendPathSegment(s3Path)
.WithHeader("Host", Host)
.WithHeader("Date", date)
.WithHeader("Authorization", auth)
.AllowAnyHttpStatus();
}
private String CreateAuthorization(String method,
String s3Path,
String date)
{
return CreateAuthorization
(
method : method,
bucket : Bucket,
s3Path : s3Path,
date : date,
s3Key : Key,
s3Secret : Secret,
contentType: ContentType
);
}
private static String CreateAuthorization(String method,
String bucket,
String s3Path,
String date,
String s3Key,
String s3Secret,
String contentType = "",
String md5Hash = "")
{
// StringToSign = HTTP-Verb + "\n" +
// Content-MD5 + "\n" +
// Content-Type + "\n" +
// Date + "\n" +
// CanonicalizedAmzHeaders +
// CanonicalizedResource;
var payload = $"{method}\n{md5Hash}\n{contentType}\n{date}\n/{bucket.Trim('/')}/{s3Path.Trim('/')}";
using var hmacSha1 = new HMACSHA1(UTF8.GetBytes(s3Secret));
var signature = UTF8
.GetBytes(payload)
.Apply(hmacSha1.ComputeHash)
.Apply(Convert.ToBase64String);
return $"AWS {s3Key}:{signature}";
}
}

82
csharp/Lib/S3/Drivers/S3Driver.cs
View File

@ -1,82 +0,0 @@
using Flurl.Http;
using InnovEnergy.Lib.S3.Drivers.Internal;
using InnovEnergy.Lib.S3.Drivers.Internal.Delegates;
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.S3.Records.Serialization;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Drivers;
using Levels = IReadOnlyList<AggregationLevel>;
public class S3Driver : Writer
{
public S3Driver(Levels levels, UnixTime startTime, S3Config config) :
base(
startTime,
levels,
ReadRecord(config),
WriteRecord(config)
)
{
}
private static String GetS3Path(AggregationLevel level, UInt32 index)
{
return $"/{level}/{index}";
}
private static ReadRecord ReadRecord(S3Config config)
{
async Task<TimeStampedRecord> Read(AggregationLevel level, UInt32 index)
{
var s3Path = GetS3Path(level, index);
var request = config.CreateGetRequest(s3Path);
var response = await request.GetAsync();
if (response.StatusCode != 200)
{
Console.WriteLine("ERROR: Get " + s3Path);
var error = await response.GetStringAsync();
Console.WriteLine(error);
return TimeStampedRecord.Empty();
}
var payload = await response.GetBytesAsync();
Console.WriteLine("GET " + s3Path);
return Parser.ParseTimeStampedRecord(payload);
}
return Read;
}
private static WriteRecord WriteRecord(S3Config config)
{
async Task Write(TimeStampedRecord record, AggregationLevel level, UInt32 index)
{
var payload = record.Serialize().ToArray();
var s3Path = GetS3Path(level, index);
var request = config.CreatePutRequest(s3Path);
var response = await request.PutAsync(new ByteArrayContent(payload));
if (response.StatusCode == 200)
{
//Console.WriteLine("PUT " + s3Path);
}
else
{
Console.WriteLine("ERROR: PUT");
var error = await response.GetStringAsync();
Console.WriteLine(error);
}
}
return Write;
}
}

52
csharp/Lib/S3/Metadata/AggregationLevel.cs
View File

@ -1,52 +0,0 @@
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Metadata;
public record AggregationLevel
{
public UnixTimeSpan RetentionPeriod { get; }
public UnixTimeSpan SamplePeriod { get; }
public UInt32 RetentionBufferSize { get; }
public AggregationLevel(UnixTimeSpan samplePeriod, UnixTimeSpan retentionPeriod)
{
SamplePeriod = samplePeriod;
RetentionPeriod = retentionPeriod;
RetentionBufferSize = retentionPeriod / samplePeriod;
}
public IEnumerable<UnixTime> RangeExclusive(UnixTime from, UnixTime to)
{
if (from > to)
throw new ArgumentOutOfRangeException(nameof(to));
for (var t = GetPeriodStartTime(from); t < to; t += SamplePeriod)
yield return t;
}
// TODO:
public IEnumerable<UnixTime> RangeInclusive(UnixTime from, UnixTime to)
{
if (from >= to)
throw new ArgumentOutOfRangeException(nameof(to));
from = GetPeriodStartTime(from);
to = GetPeriodStartTime(to) + SamplePeriod;
for (var t = GetPeriodStartTime(from); t < to; t += SamplePeriod)
yield return t;
}
public UInt32 GetRetentionIndex(UnixTime t)
{
return t / SamplePeriod % RetentionBufferSize;
}
public UnixTime GetPeriodStartTime(UnixTime t)
{
return UnixTime.Epoch + t / SamplePeriod * SamplePeriod; // integer division!
}
public override String ToString() => SamplePeriod.ToString();
}

8
csharp/Lib/S3/Metadata/FieldType.cs
View File

@ -1,8 +0,0 @@
namespace InnovEnergy.Lib.S3.Metadata;
public enum FieldType
{
Number,
Text,
Boolean,
}

211
csharp/Lib/S3/Program.cs
View File

@ -1,211 +0,0 @@
using System.Reactive.Concurrency;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
using static InnovEnergy.Lib.Time.Unix.UnixTimeSpan;
namespace InnovEnergy.Lib.S3;
public enum MyEnum
{
Foo = 0x01,
Bar = 0x02,
Baz = 0x04
}
public record Test(Double X, Double Y, Double Z);
public static class Program
{
private static readonly Random Rng = new Random(0);
public static void Main(String[] args)
{
// var x = Observable
// .Range(0, 22)
// .Select(i=> new Dictionary<String, int>
// {
// {"i", i},
// {"i*2", i * 2},
// {"i/2", i / 2},
// })
// .DictObserve(o => Observable.Where(o, i=> i < 10));
//
// x.Subscribe(ints => Console.WriteLine(String.Join(Environment.NewLine,ints) + "\n") );
var src = Observable
.Interval(TimeSpan.FromSeconds(1))
.BufferBy(i => i / 5)
.Subscribe(b =>
{
var values = b.Select(e => e.ToString()).Aggregate("", (x, y) => $"{x}\n{y}");
var average = b.Average();
var msg = $"{values}\nAverage: {average}";
Console.WriteLine(msg);
});
Console.ReadLine();
// var t = new Test(12, 2, 1);
//
// foreach (var m in typeof(Test).GetMembers().OfType<MethodInfo>())
// {
// var parameters = m
// .GetParameters()
// .Select(p => $"{p.ParameterType.Name} {p.Name}")
// .Aggregate("", (a, b) => a + ", " + b)
// .TrimStart(", ".ToCharArray());
//
// Console.WriteLine($"{m.ReturnType.Name} {m.Name}({parameters})");
// }
return;
var s = new Subject<String>();
Observable.Interval(TimeSpan.FromSeconds(2), TaskPoolScheduler.Default)
.WithLatestFrom(s, (_, x) => x)
.Subscribe(Console.WriteLine);
while (true)
{
var k = Console.ReadKey(true);
s.OnNext(k.KeyChar.ToString());
}
var levels = new []
{
new AggregationLevel( 2.Seconds(), 54.Weeks()),
new AggregationLevel(10.Seconds(), 54.Weeks()),
new AggregationLevel( 1.Minutes(), 54.Weeks()),
new AggregationLevel( 5.Minutes(), 540.Weeks()),
new AggregationLevel(15.Minutes(), 540.Weeks()),
new AggregationLevel( 1.Hours() , 540.Weeks()),
new AggregationLevel( 6.Hours() , 540.Weeks()),
new AggregationLevel( 1.Days() , 540.Weeks()),
new AggregationLevel( 1.Weeks() , Forever)
};
var resolution = 2.Seconds();
for (int i = 0; i < 10; i++)
{
var startTime = UnixTime.Epoch + Rng.Next(0, 2.Weeks().Ticks.ConvertTo<Int32>()).Seconds();
var split = Rng.Next(2, 1000);
//Test(startTime, resolution, levels, split);
}
Console.WriteLine("Done");
}
// private static void Test(UnixTime startTime, UnixTimeSpan resolution, AggregationLevel[] aggregationLevels, Int32 split)
// {
// var times = Enumerable
// .Range(0, 2.Weeks().Ticks.ConvertTo<Int32>())
// .Select(t => startTime + t * resolution)
// .ToReadOnlyList();
//
// var records = times
// //.Where(_ => Rng.NextDouble() > .25) // "loose" 1 in 4
// .Select(t => (
// record: new Record
// (
// // new NumberField("Current", t.Ticks / 2, "A"),
// new SubRecord("Sub1", new Record(new NumberField("Current1", t.Ticks / 4, "A"))),
// new SubRecord("Sub2", new Record(new NumberField("Current2", t.Ticks / 8, "A")))
// ),
// time: t + Rng.NextDouble().Apply(Math.Round).ConvertTo<Int32>().Seconds() // add noise
// ))
// .ToList();
//
//
// var allMemory = new Dictionary<String, TimeStampedRecord>();
// using (var driverAll = new MemoryDriver(aggregationLevels, startTime, allMemory))
// {
// foreach (var record in records)
// driverAll.WriteRecord(record.record, record.time);
// }
//
// var list = allMemory.Where(kv => kv.Key.StartsWith("1w")).ToList();
//
// var splitMemory = new Dictionary<String, TimeStampedRecord>();
// using (var driver1 = new MemoryDriver(aggregationLevels, startTime, splitMemory))
// {
// foreach (var record in records.Take(split))
// driver1.WriteRecord(record.record, record.time);
// }
//
// using (var driver2 = new MemoryDriver(aggregationLevels, records.ElementAt(split-1).time, splitMemory))
// {
// foreach (var record in records.Skip(split-1))
// driver2.WriteRecord(record.record, record.time);
// }
//
//
//
// var zip = Enumerable.Zip(
// allMemory.OrderBy(kv => kv.Key),
// splitMemory.OrderBy(kv => kv.Key)
// );
//
// foreach (var (l, r) in zip)
// {
// var nl = l.Value.Record.Fields.OfType<NumberField>().FirstOrDefault()?.Value;
// var nr = r.Value.Record.Fields.OfType<NumberField>().FirstOrDefault()?.Value;
//
// if (l.Key != r.Key)
// {
// Console.WriteLine($"{l.Key} <=> {r.Key}");
// break;
// }
//
// if (nl != nr)
// {
// Console.WriteLine($"{l.Value.TimeStamp.ToUtcDateTime()}: {nl} <=> {nr}");
// }
// }
// }
public static String CreateRandomState()
{
var r = Rng.NextDouble() * 100;
return r switch
{
>= 90 => "Heating",
>= 10 => "SelfConsumption",
_ => "CalibrationCharge"
};
}
}

3
csharp/Lib/S3/Records/Fields/Field.cs
View File

@ -1,3 +0,0 @@
namespace InnovEnergy.Lib.S3.Records.Fields;
public abstract record Field;

25
csharp/Lib/S3/Records/Fields/NumberField.cs
View File

@ -1,25 +0,0 @@
namespace InnovEnergy.Lib.S3.Records.Fields;
public record NumberField : Field
{
public Double Value { get; }
public Double Min { get; }
public Double Max { get; }
public String Unit { get; }
public NumberField(Double value, Double min, Double max, String unit = "")
{
Value = value;
Min = min;
Max = max;
Unit = unit;
}
public NumberField(Double value, String unit = "") : this(value, value, value, unit)
{
}
public NumberField(Boolean value) : this(value?100:0, "%")
{
}
}

38
csharp/Lib/S3/Records/Fields/TextField.cs
View File

@ -1,38 +0,0 @@
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Records.Fields;
public record TextField : Field
{
public IReadOnlyList<TextFrequency> Frequencies { get; }
public TextField(IReadOnlyList<TextFrequency> frequencies)
{
Frequencies = frequencies;
}
public TextField(params TextFrequency[] frequencies) : this((IReadOnlyList<TextFrequency>)frequencies)
{
}
public TextField(String text) : this(new TextFrequency(text))
{
}
public TextField(Enum enumValue) : this(GetEnumFlagsAsFrequencies(enumValue))
{
}
private static IReadOnlyList<TextFrequency> GetEnumFlagsAsFrequencies(Enum enumValue)
{
return enumValue
.ToString()
.Split(',') // debug view is '|' separated, toString() uses ',' !!
.Select(v => v.Trim())
.Where(v => !v.IsInteger()) // filter "unknown" enum flags
.Select(f => new TextFrequency(f))
.ToReadOnlyList();
}
}

13
csharp/Lib/S3/Records/Fields/TextFrequency.cs
View File

@ -1,13 +0,0 @@
namespace InnovEnergy.Lib.S3.Records.Fields;
public record TextFrequency
{
public TextFrequency(String text, Double percent = 100)
{
Text = text;
Percent = percent;
}
public String Text { get; }
public Double Percent { get; }
}

16
csharp/Lib/S3/Records/Fields/UnitAttribute.cs
View File

@ -1,16 +0,0 @@
namespace InnovEnergy.Lib.S3.Records.Fields;
[AttributeUsage(AttributeTargets.Property)]
public sealed class UnitAttribute : Attribute
{
private readonly String _Unit;
public UnitAttribute(String unit)
{
_Unit = unit;
}
public override String ToString() => _Unit;
public static implicit operator String(UnitAttribute a) => a.ToString();
}

30
csharp/Lib/S3/Records/Fields/WithUnit.cs
View File

@ -1,30 +0,0 @@
namespace InnovEnergy.Lib.S3.Records.Fields;
public readonly struct WithUnit
{
public String Unit { get; }
public Double Value { get; }
public WithUnit(Double value, String unit)
{
Unit = unit;
Value = value;
}
public override String ToString() => Unit;
}
public static class UnitExtensions
{
public static WithUnit Unit(this Double number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this Single number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this Half number, String unit) => new WithUnit((Double) number, unit);
public static WithUnit Unit(this SByte number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this Byte number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this Int16 number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this UInt16 number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this Int32 number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this UInt32 number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this Int64 number, String unit) => new WithUnit(number, unit);
public static WithUnit Unit(this UInt64 number, String unit) => new WithUnit(number, unit);
}

83
csharp/Lib/S3/Records/Operations/Aggregation.cs
View File

@ -1,83 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using InnovEnergy.Lib.S3.Records.Fields;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Records.Operations;
// ReSharper disable ArgumentsStyleOther
public static class Aggregation
{
[SuppressMessage("ReSharper", "PossibleMultipleEnumeration")]
public static Record Aggregate(this IReadOnlyList<Record> records)
{
var availability = records.Sum(r => r.Availability) / records.Count;
if (availability == 0)
return Record.Empty;
var numbers = records.GetColumns<NumberField>().Select(AggregateNumbers);
var texts = records.GetColumns<TextField>() .Select(AggregateTexts);
var subRecords = records.GetColumns<Record>() .Select(AggregateRecords);
var aggregated = numbers.Concat(texts).Concat(subRecords);
var fields = new Dictionary<String, Field>(aggregated);
return new Record
(
fields,
availability
);
}
private static IEnumerable<(String name, F field)> GetFields<F>(this IEnumerable<Record> records) where F : Field
{
foreach (var record in records)
foreach (var (name, field) in record.Fields)
if (field is F f)
yield return (name, f);
}
private static ILookup<String, F> GetColumns<F>(this IEnumerable<Record> records) where F : Field
{
return records
.GetFields<F>()
.ToLookup(nf => nf.name, nf => nf.field);
}
private static KeyValuePair<String, Field> AggregateTexts(IGrouping<String, TextField> column)
{
var textFrequencies = column
.SelectMany(s => s.Frequencies)
.GroupBy(f => f.Text, f => f.Percent)
.Select(g => new TextFrequency(text: g.Key, percent: g.Sum() / g.Count()))
.OrderBy(f => f.Text)
.ToReadOnlyList();
return new (column.Key, new TextField(textFrequencies));
}
private static KeyValuePair<String, Field> AggregateNumbers(IGrouping<String, NumberField> column)
{
var aggregated = new NumberField
(
min: column.Min(f => f.Min),
max: column.Max(f => f.Max),
value: column.Average(f => f.Value),
unit: column.First().Unit
);
return new (column.Key, aggregated);
}
private static KeyValuePair<String, Field> AggregateRecords(IGrouping<String, Record> column)
{
var aggregate = column.ToReadOnlyList().Apply(Aggregate);
return new(column.Key, aggregate);
}
}

46
csharp/Lib/S3/Records/Operations/Conversion.cs
View File

@ -1,46 +0,0 @@
using System.Reflection;
using InnovEnergy.Lib.S3.Records.Fields;
using InnovEnergy.Lib.Utils;
using static System.Reflection.BindingFlags;
namespace InnovEnergy.Lib.S3.Records.Operations;
public static class Conversion
{
public static Record ToRecord(this Object t)
{
var fields = t.GetType()
.Apply(GetProperties)
.ToDictionary(p => p.Name, PropertyToField);
return new Record(fields);
Field PropertyToField(PropertyInfo p) => p.GetValue(t) switch
{
Double d => new NumberField(d, Unit(p)),
Boolean b => new NumberField(b),
Enum e => new TextField(e),
String s => new TextField(s), // ReSharper disable once PatternAlwaysOfType
Object o => ToRecord(o),
_ => throw new ArgumentException()
};
}
private static String Unit(MemberInfo member)
{
return member
.GetCustomAttributes()
.OfType<UnitAttribute>()
.FirstOrDefault()?
.ToString()
?? "";
}
private static IEnumerable<PropertyInfo> GetProperties<T>() => typeof(T).GetProperties(Instance | Public);
private static IEnumerable<PropertyInfo> GetProperties(Type t) => t.GetProperties(Instance | Public);
}

59
csharp/Lib/S3/Records/Record.cs
View File

@ -1,59 +0,0 @@
using InnovEnergy.Lib.S3.Records.Fields;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Records;
public record Record : Field
{
public static Record Empty { get; } = new Record(new Dictionary<String, Field>(), 0);
public IReadOnlyDictionary<String, Field> Fields { get; }
public Double Availability { get; }
public Boolean IsEmpty => Fields.Count == 0;
public Record(IReadOnlyDictionary<String, Field> fields, Double availability = 1.0)
{
Fields = fields;
Availability = availability;
}
public F GetField<F>(String name) where F : Field
{
return (F) Fields[name];
}
public TimeStampedRecord TimeStamped(UnixTime unixTime) => new TimeStampedRecord(this, unixTime);
public static Record ParseDict(Dictionary<String, Object> dict, Double availability = 1.0)
{
var fields = dict.ToDictionary(kv => kv.Key, kv => ParseField(kv.Value));
return new Record(fields, availability);
}
private static Field ParseField(Object value) =>
value switch
{
WithUnit v => new NumberField(v.Value, v.Unit),
Enum v => new TextField(v),
String v => new TextField(v),
Double v => new NumberField(v),
Field v => v,
Single v => new NumberField(v),
SByte v => new NumberField(v),
Byte v => new NumberField(v),
Int16 v => new NumberField(v),
UInt16 v => new NumberField(v),
Int32 v => new NumberField(v),
UInt32 v => new NumberField(v),
Int64 v => new NumberField(v),
UInt64 v => new NumberField(v),
Half v => new NumberField((Double)v),
_ => throw new ArgumentOutOfRangeException(nameof(value))
};
}

8
csharp/Lib/S3/Records/Serialization/FieldTag.cs
View File

@ -1,8 +0,0 @@
namespace InnovEnergy.Lib.S3.Records.Serialization;
public enum FieldTag : Byte
{
NumberField = 0,
TextField = 1,
RecordField = 2,
}

134
csharp/Lib/S3/Records/Serialization/Parser.cs
View File

@ -1,134 +0,0 @@
using System.Text;
using InnovEnergy.Lib.S3.Records.Fields;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.S3.Records.Serialization;
public class Parser
{
private Int32 Position { get; set; }
private Byte[] Data { get; }
private Parser(Byte[] data, Int32 position = 0)
{
Data = data;
Position = position;
}
public Boolean IsPastEnd => Position >= Data.Length;
public static TimeStampedRecord ParseTimeStampedRecord(Byte[] data)
{
var parser = new Parser(data);
var timeStamp = parser.ParseUnixTime();
var record = parser.ParseRecord();
return new TimeStampedRecord(record, timeStamp);
}
private UnixTime ParseUnixTime()
{
var span = new ReadOnlySpan<Byte>(Data, Position, sizeof(UInt32));
var ticks = BitConverter.ToUInt32(span);
Position += span.Length;
return UnixTime.FromTicks(ticks);
}
private Single ParseFloat()
{
var span = new ReadOnlySpan<Byte>(Data, Position, sizeof(Single));
Position += span.Length;
return BitConverter.ToSingle(span);
}
private String ParseString()
{
var len = GetByte();
var span = new ReadOnlySpan<Byte>(Data, Position, len);
Position += span.Length;
return Encoding.UTF8.GetString(span);
}
private NumberField ParseNumberField()
{
var value = ParseFloat();
var min = ParseFloat();
var max = ParseFloat();
var unit = ParseString();
return new NumberField(value, min, max, unit);
}
private TextFrequency ParseTextFrequency()
{
var txt = ParseString();
var frq = ParseFloat();
return new TextFrequency(txt, frq);
}
private Record ParseRecord()
{
var availability = ParseFloat();
var fields = ParseFields();
return new Record(fields, availability);
}
private IReadOnlyDictionary<String, Field> ParseFields()
{
var len = GetByte();
var fields = Enumerable
.Range(0, len)
.Select(_ => ParseNamedField());
return new Dictionary<String, Field>(fields);
}
private Field ParseField()
{
var tag = ParseFieldTag();
return tag switch
{
FieldTag.NumberField => ParseNumberField(),
FieldTag.TextField => ParseTextField(),
FieldTag.RecordField => ParseRecord(),
_ => throw new ArgumentOutOfRangeException(nameof(FieldTag))
};
}
private KeyValuePair<String, Field> ParseNamedField()
{
var name = ParseString();
var field = ParseField();
return new KeyValuePair<String, Field>(name, field);
}
private TextField ParseTextField()
{
var len = GetByte();
var fqs = Enumerable
.Range(0, len)
.Select(_ => ParseTextFrequency())
.ToReadOnlyList(len);
return new TextField(fqs);
}
private Byte GetByte() => Data[Position++];
private FieldTag ParseFieldTag() => (FieldTag) GetByte();
}

121
csharp/Lib/S3/Records/Serialization/Serializer.cs
View File

@ -1,121 +0,0 @@
using InnovEnergy.Lib.S3.Records.Fields;
using InnovEnergy.Lib.S3.Records.Specialized;
using InnovEnergy.Lib.Time.Unix;
using InnovEnergy.Lib.Utils;
using static System.Text.Encoding;
namespace InnovEnergy.Lib.S3.Records.Serialization;
public static class Serializer
{
public static IEnumerable<Byte> Serialize(this TimeStampedRecord tsr)
{
if (!BitConverter.IsLittleEndian)
throw new ArgumentException(nameof(BitConverter));
var timeStamp = tsr.TimeStamp.Serialize();
var record = tsr.Record.Serialize();
return timeStamp.Concat(record);
}
private static IEnumerable<Byte> Serialize(this UnixTime t)
{
return BitConverter.GetBytes(t.Ticks);
}
private static IEnumerable<Byte> Serialize(this Record r)
{
var availability = r.Availability.Serialize();
var fields = r.Fields.Serialize();
return availability.Concat(fields);
}
private static IEnumerable<Byte> Serialize(this IReadOnlyDictionary<String, Field> fields)
{
var len = fields.GetLength();
return fields
.SelectMany(Serialize)
.Prepend(len);
}
private static IEnumerable<Byte> Serialize(this KeyValuePair<String, Field> kv)
{
var name = kv.Key.Serialize();
var field = kv.Value.Serialize();
return name.Concat(field);
}
public static IEnumerable<Byte> Serialize(this Field f)
{
return f switch
{
NumberField nf => Serialize(nf).PrependTag(FieldTag.NumberField),
TextField tf => Serialize(tf).PrependTag(FieldTag.TextField),
Record r => Serialize(r).PrependTag(FieldTag.RecordField),
_ => throw new ArgumentException(nameof(f))
};
}
public static IEnumerable<Byte> Serialize(NumberField nf)
{
var value = nf.Value.Serialize();
var min = nf.Min .Serialize();
var max = nf.Max .Serialize();
var unit = nf.Unit .Serialize();
return value
.Concat(min)
.Concat(max)
.Concat(unit);
}
public static IEnumerable<Byte> Serialize(this TextField tf)
{
var len = tf.Frequencies.GetLength();
return tf
.Frequencies
.SelectMany(Serialize)
.Prepend(len);
}
public static IEnumerable<Byte> Serialize(this TextFrequency tf)
{
var txt = tf.Text.Serialize();
var frq = tf.Percent.Serialize();
return txt.Concat(frq);
}
public static IEnumerable<Byte> Serialize(this Double d)
{
return BitConverter.GetBytes((Single) d); // use float to save space
}
public static IEnumerable<Byte> Serialize(this String s)
{
return UTF8
.GetBytes(s)
.Apply(PrependLength);
}
private static IEnumerable<Byte> PrependTag(this IEnumerable<Byte> data, FieldTag fieldTag)
{
return data.Prepend((Byte) fieldTag);
}
private static IEnumerable<Byte> PrependLength(IReadOnlyCollection<Byte> data)
{
var len = (Byte) data.Count;
return data.Prepend(len);
}
private static Byte GetLength<T>(this IReadOnlyCollection<T> data)
{
return (Byte) data.Count;
}
}

23
csharp/Lib/S3/Records/Specialized/AggregatedRecord.cs
View File

@ -1,23 +0,0 @@
using InnovEnergy.Lib.S3.Metadata;
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Records.Specialized;
public readonly struct AggregatedRecord
{
public UnixTime TimeStamp { get; }
public Record Record { get; }
public AggregationLevel AggregationLevel { get; }
public AggregatedRecord(Record record, AggregationLevel level, UnixTime timeStamp)
{
AggregationLevel = level;
TimeStamp = timeStamp;
Record = record;
}
public TimeStampedRecord ToTimeStamped()
{
return new TimeStampedRecord(Record, TimeStamp);
}
}

27
csharp/Lib/S3/Records/Specialized/TimeStampedRecord.cs
View File

@ -1,27 +0,0 @@
using InnovEnergy.Lib.Time.Unix;
namespace InnovEnergy.Lib.S3.Records.Specialized;
public readonly struct TimeStampedRecord
{
public UnixTime TimeStamp { get; }
public Record Record { get; }
public TimeStampedRecord(Record record, UnixTime timeStamp)
{
Record = record;
TimeStamp = timeStamp;
}
public Boolean IsEmpty => Record.IsEmpty;
public static TimeStampedRecord Empty()
{
return Empty(UnixTime.Epoch);
}
public static TimeStampedRecord Empty(UnixTime timeStamp)
{
return new TimeStampedRecord(Record.Empty, timeStamp);
}
}

17
csharp/Lib/S3/S3.csproj
View File

@ -1,17 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="../InnovEnergy.Lib.props" />
<ItemGroup>
<ProjectReference Include="../Time/Time.csproj" />
<ProjectReference Include="../Utils/Utils.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Flurl.Http" Version="3.2.0" />
<PackageReference Include="System.Reactive.Linq" Version="5.0.0" />
</ItemGroup>
</Project>

167
csharp/Lib/S3/Serialization/JsonConverters.cs
View File

@ -1,167 +0,0 @@
// using System;
// using System.Text.Json;
// using System.Text.Json.Serialization;
// using InnovEnergy.S3.Records;
// using InnovEnergy.S3.Records.Fields;
//
// namespace InnovEnergy.S3.Serialization
// {
// public class SubClassConverterFactory : JsonConverterFactory
// {
// public override Boolean CanConvert(Type t)
// {
// return t.IsAbstract && t.IsClass;
// }
//
// public override JsonConverter CreateConverter(Type type, JsonSerializerOptions options)
// {
// var converterType = typeof(SubClassConverter<>).MakeGenericType(type);
// return (JsonConverter) Activator.CreateInstance(converterType)!;
// }
//
// public class SubClassConverter<T> : JsonConverter<T>
// {
// public override Boolean CanConvert(Type type)
// {
// return type == typeof(T);
// }
//
// public override T Read(ref Utf8JsonReader r, Type t, JsonSerializerOptions o)
// {
// throw new NotImplementedException();
// }
//
// public override void Write(Utf8JsonWriter writer, T obj, JsonSerializerOptions options)
// {
// JsonSerializer.Serialize(writer, obj, obj!.GetType());
// }
// }
// }
//
// public class DataRecordConverter : JsonConverter<Record>
// {
// public override Boolean CanConvert(Type type)
// {
// return typeof(Record).IsAssignableFrom(type);
// }
//
// public override Record Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
// {
// throw new NotImplementedException();
// }
//
// public override void Write(Utf8JsonWriter writer, Record @record, JsonSerializerOptions options)
// {
// writer.WriteStartObject();
//
// foreach (var field in record.Fields)
// {
// writer.WritePropertyName(field.Name);
//
// if (field is NumberField an)
// {
// writer.WriteStartObject();
// writer.WriteNumber("Min", an.Min);
// writer.WriteNumber("Max", an.Max);
// writer.WriteNumber("Mean", an.Value);
// writer.WriteEndObject();
// }
// else if (field is TextField at)
// {
// writer.WriteStartObject();
// foreach (var f in at.Frequencies)
// {
// writer.WriteNumber(f.Text, f.Percent);
// }
// writer.WriteEndObject();
// }
// else if (field is BooleanField ab)
// {
// writer.WriteStartObject();
// writer.WriteNumber("PercentTrue", ab.PercentTrue);
// writer.WriteEndObject();
// }
// else
// throw new NotSupportedException();
// }
//
// writer.WriteNumber("Time", record.TimeStamp.Ticks);
// writer.WriteNumber("Availability", record.Availability);
//
// writer.WriteEndObject();
// }
// }
//
// // public class AggregatedRecordConverter : JsonConverter<AggregatedRecord>
// // {
// // public override Boolean CanConvert(Type type)
// // {
// // return typeof(AggregatedRecord).IsAssignableFrom(type);
// // }
// //
// // public override AggregatedRecord Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => throw new NotImplementedException();
// //
// // public override void Write(Utf8JsonWriter writer, AggregatedRecord dataRecord, JsonSerializerOptions options)
// // {
// // writer.WriteStartObject();
// //
// // foreach (var field in dataRecord.Fields)
// // {
// // if (field is AggregatedNumber n)
// // {
// // writer.WritePropertyName(n.Name);
// // writer.WriteStartObject();
// // writer.WritePropertyName("Mean");
// // writer.WriteNumberValue(n.Mean);
// //
// // writer.WritePropertyName("Min");
// // writer.WriteNumberValue(n.Min);
// //
// // writer.WritePropertyName("Max");
// // writer.WriteNumberValue(n.Max);
// //
// // writer.WriteEndObject();
// // }
// // else if (field is AggregatedText t)
// // {
// // writer.WritePropertyName(t.Name);
// // JsonSerializer.Serialize(writer, t);
// // }
// // else if (field is AggregatedBoolean b)
// // {
// // writer.WritePropertyName(b.Name);
// // JsonSerializer.Serialize(writer, b);
// // }
// //
// // else throw new NotSupportedException();
// // }
// //
// // writer.WriteEndObject();
// // }
// //
// //
// //
// //
// // }
//
// // public class TextFrequencyConverter : JsonConverter<TextFrequency>
// // {
// // public override Boolean CanConvert(Type type)
// // {
// // return typeof(TextFrequency).IsAssignableFrom(type);
// // }
// //
// // public override TextFrequency Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => throw new NotImplementedException();
// //
// // public override void Write(Utf8JsonWriter writer, TextFrequency textFrequency, JsonSerializerOptions options)
// // {
// // writer.WriteStartObject();
// // writer.WritePropertyName(textFrequency.Text);
// // writer.WriteNumberValue(textFrequency.Percent);
// // writer.WriteEndObject();
// // }
// // }
//
//
//
// }

23
csharp/Lib/SysTools/FileIo.cs
View File

@ -1,8 +1,9 @@
using System.Text; using System.Text;
using InnovEnergy.Lib.SysTools.Utils; using InnovEnergy.Lib.Utils;
namespace InnovEnergy.Lib.SysTools; namespace InnovEnergy.Lib.SysTools;
[Obsolete("Needs rework before use")]
public static class FileIo public static class FileIo
{ {
public static Boolean Exists (this SysPath path) => path.FileExists() || path.DirectoryExists(); public static Boolean Exists (this SysPath path) => path.FileExists() || path.DirectoryExists();
@ -91,7 +92,7 @@ public static class FileIo
SysPath Target(SysPath path) => targetDir.Append(path.RelativeTo(sourceDir)); SysPath Target(SysPath path) => targetDir.Append(path.RelativeTo(sourceDir));
Utils.Utils.Traverse(sourceDir, Directories) sourceDir.Traverse(Directories)
.Do(d => Target(d).CreateDirectory()) .Do(d => Target(d).CreateDirectory())
.SelectMany(Files) .SelectMany(Files)
.ForEach(f => f.CopyFileTo(Target(f))); .ForEach(f => f.CopyFileTo(Target(f)));
@ -115,7 +116,7 @@ public static class FileIo
public static IEnumerable<SysPath> DescendantDirectories(this SysPath path) public static IEnumerable<SysPath> DescendantDirectories(this SysPath path)
{ {
return Utils.Utils.Traverse(path, Directories); return path.Traverse(Directories);
} }
public static IEnumerable<SysPath> DescendantFiles(this SysPath path) public static IEnumerable<SysPath> DescendantFiles(this SysPath path)
@ -140,21 +141,19 @@ public static class FileIo
{ {
var buf = new Byte[4096]; var buf = new Byte[4096];
using (var fs = File.OpenRead(path)) using var fs = File.OpenRead(path);
{
var n = fs.Read(buf, 0, buf.Length); var n = fs.Read(buf, 0, buf.Length);
foreach (var b in buf.Take(Math.Max(0, n))) foreach (var b in buf.Take(Math.Max(0, n)))
yield return b; yield return b;
} }
}
public static IEnumerable<String> ReadLines(this SysPath path) => path.ReadLines(Encoding.UTF8); public static IEnumerable<String> ReadLines(this SysPath path) => path.ReadLines(Encoding.UTF8);
public static IEnumerable<String> ReadLines(this SysPath path, Encoding encoding) public static IEnumerable<String> ReadLines(this SysPath path, Encoding encoding)
{ {
using (var sr = new StreamReader(path, encoding)) using var sr = new StreamReader(path, encoding);
while (true) while (true)
{ {
var str = sr.ReadLine(); var str = sr.ReadLine();
@ -169,13 +168,13 @@ public static class FileIo
public static String ReadText(this SysPath path, Encoding encoding) public static String ReadText(this SysPath path, Encoding encoding)
{ {
using (var sr = new StreamReader(path, encoding)) using var sr = new StreamReader(path, encoding);
return sr.ReadToEnd(); return sr.ReadToEnd();
} }
public static SysPath WriteText(this SysPath filePath, String text) public static SysPath WriteText(this SysPath filePath, String text)
{ {
using (var sw = new StreamWriter(filePath, append: false)) using var sw = new StreamWriter(filePath, append: false);
sw.Write(text); sw.Write(text);
return filePath; return filePath;
@ -189,7 +188,7 @@ public static class FileIo
public static SysPath WriteLines(this SysPath filePath, IEnumerable<String> lines) public static SysPath WriteLines(this SysPath filePath, IEnumerable<String> lines)
{ {
using (var sw = new StreamWriter(filePath, append: false)) using var sw = new StreamWriter(filePath, append: false);
foreach (var line in lines) foreach (var line in lines)
sw.WriteLine(line); sw.WriteLine(line);
@ -198,7 +197,7 @@ public static class FileIo
public static SysPath AppendText(this SysPath filePath, String text) public static SysPath AppendText(this SysPath filePath, String text)
{ {
using (var sw = new StreamWriter(filePath, append: true)) using var sw = new StreamWriter(filePath, append: true);
sw.Write(text); sw.Write(text);
return filePath; return filePath;
@ -212,7 +211,7 @@ public static class FileIo
public static SysPath AppendLines(this SysPath filePath, IEnumerable<String> lines) public static SysPath AppendLines(this SysPath filePath, IEnumerable<String> lines)
{ {
using (var sw = new StreamWriter(filePath, append: true)) using var sw = new StreamWriter(filePath, append: true);
foreach (var line in lines) foreach (var line in lines)
sw.WriteLine(line); sw.WriteLine(line);

1
csharp/Lib/SysTools/Process/AsyncProcess.cs
View File

@ -9,6 +9,7 @@ namespace InnovEnergy.Lib.SysTools.Process;
using Env = Dictionary<String, String>; using Env = Dictionary<String, String>;
[Obsolete("Use CliWrap instead")]
public class AsyncProcess public class AsyncProcess
{ {
private readonly Subject<String> _StandardIn; private readonly Subject<String> _StandardIn;

1
csharp/Lib/SysTools/Process/ProcessResult.cs
View File

@ -2,6 +2,7 @@ using InnovEnergy.Lib.SysTools.Utils;
namespace InnovEnergy.Lib.SysTools.Process; namespace InnovEnergy.Lib.SysTools.Process;
[Obsolete("Use CliWrap instead")]
public readonly struct ProcessResult public readonly struct ProcessResult
{ {
public ProcessResult(Int32 exitCode, public ProcessResult(Int32 exitCode,

2
csharp/Lib/SysTools/Process/SyncProcess.cs
View File

@ -7,7 +7,7 @@ namespace InnovEnergy.Lib.SysTools.Process;
using Env = Dictionary<String, String>; using Env = Dictionary<String, String>;
[Obsolete("Use CliWrap instead")]
public class SyncProcess public class SyncProcess
{ {
public SysCommand Command { get; } public SysCommand Command { get; }

1
csharp/Lib/SysTools/Remote/RemoteCommand.cs
View File

@ -2,6 +2,7 @@ using InnovEnergy.Lib.SysTools.Utils;
namespace InnovEnergy.Lib.SysTools.Remote; namespace InnovEnergy.Lib.SysTools.Remote;
[Obsolete("Use CliWrap instead")]
public readonly struct RemoteCommand public readonly struct RemoteCommand
{ {
public SshHost Host { get; } public SshHost Host { get; }

1
csharp/Lib/SysTools/Remote/RemoteFileIo.cs
View File

@ -3,6 +3,7 @@ using InnovEnergy.Lib.SysTools.Utils;
namespace InnovEnergy.Lib.SysTools.Remote; namespace InnovEnergy.Lib.SysTools.Remote;
[Obsolete("Needs rework before use")]
public static class RemoteFileIo public static class RemoteFileIo
{ {

1
csharp/Lib/SysTools/Remote/RemotePath.cs
View File

@ -1,5 +1,6 @@
namespace InnovEnergy.Lib.SysTools.Remote; namespace InnovEnergy.Lib.SysTools.Remote;
[Obsolete]
public readonly struct RemotePath public readonly struct RemotePath
{ {
public SysPath Path { get; } public SysPath Path { get; }

1
csharp/Lib/SysTools/Remote/SshHost.cs
View File

@ -3,6 +3,7 @@ using InnovEnergy.Lib.SysTools.Utils;
namespace InnovEnergy.Lib.SysTools.Remote; namespace InnovEnergy.Lib.SysTools.Remote;
[Obsolete("Needs rework before use")]
public readonly struct SshHost public readonly struct SshHost
{ {
public const Int32 DefaultPort = 22; public const Int32 DefaultPort = 22;

1
csharp/Lib/SysTools/SysCommand.cs
View File

@ -2,6 +2,7 @@ using InnovEnergy.Lib.SysTools.Utils;
namespace InnovEnergy.Lib.SysTools; namespace InnovEnergy.Lib.SysTools;
[Obsolete("Use CliWrap instead")]
public readonly struct SysCommand public readonly struct SysCommand
{ {
public SysPath Path { get; } public SysPath Path { get; }

1
csharp/Lib/SysTools/SysDirs.cs
View File

@ -3,6 +3,7 @@ using static System.Environment.SpecialFolder;
namespace InnovEnergy.Lib.SysTools; namespace InnovEnergy.Lib.SysTools;
[Obsolete]
public static class SysDirs public static class SysDirs
{ {

1
csharp/Lib/SysTools/SysPath.cs
View File

@ -5,6 +5,7 @@ using static System.IO.Path;
namespace InnovEnergy.Lib.SysTools; namespace InnovEnergy.Lib.SysTools;
[Obsolete("Needs rework before use")]
public readonly struct SysPath public readonly struct SysPath
{ {
private readonly String _Path; private readonly String _Path;

4
csharp/Lib/SysTools/SysTools.csproj
View File

@ -5,4 +5,8 @@
<PackageReference Include="System.Reactive.Linq" Version="5.0.0" /> <PackageReference Include="System.Reactive.Linq" Version="5.0.0" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Utils\Utils.csproj" />
</ItemGroup>
</Project> </Project>

22
csharp/Lib/SysTools/Utils/EnumerableUtils.cs
View File

@ -5,15 +5,13 @@ internal static class EnumerableUtils
public static IEnumerable<T> Pad<T>(this IEnumerable<T> src, Int32 length, T padding) public static IEnumerable<T> Pad<T>(this IEnumerable<T> src, Int32 length, T padding)
{ {
using (var enumerator = src.GetEnumerator()) using var enumerator = src.GetEnumerator();
{
while (enumerator.MoveNext() && length-- > 0) while (enumerator.MoveNext() && length-- > 0)
yield return enumerator.Current; yield return enumerator.Current;
while (length-- > 0) while (length-- > 0)
yield return padding; yield return padding;
} }
}
public static Dictionary<T, IReadOnlyList<T>> IndexColumn<T>(this IEnumerable<IEnumerable<T>> src, UInt16 index) public static Dictionary<T, IReadOnlyList<T>> IndexColumn<T>(this IEnumerable<IEnumerable<T>> src, UInt16 index)
{ {
@ -33,8 +31,8 @@ internal static class EnumerableUtils
public static IEnumerable<(TLeft left, TRight right)> Zip<TLeft, TRight>(IEnumerable<TLeft> left, public static IEnumerable<(TLeft left, TRight right)> Zip<TLeft, TRight>(IEnumerable<TLeft> left,
IEnumerable<TRight> right) IEnumerable<TRight> right)
{ {
using (var l = left.GetEnumerator()) using var l = left.GetEnumerator();
using (var r = right.GetEnumerator()) using var r = right.GetEnumerator();
while (l.MoveNext() && r.MoveNext()) while (l.MoveNext() && r.MoveNext())
yield return (l.Current, r.Current); yield return (l.Current, r.Current);
} }
@ -57,20 +55,6 @@ internal static class EnumerableUtils
action(e); action(e);
} }
public static IEnumerable<T> Do<T>(this IEnumerable<T> enumerable, Action<T> action)
{
return enumerable.Select(e =>
{
action(e);
return e;
});
}
public static void ForEach<T,R>(this IEnumerable<T> enumerable, Func<T,R> func)
{
foreach (var e in enumerable)
func(e);
}
public static IEnumerable<T> WhereNot<T>(this IEnumerable<T> enumerable, Func<T,Boolean> predicate) public static IEnumerable<T> WhereNot<T>(this IEnumerable<T> enumerable, Func<T,Boolean> predicate)

50
csharp/Lib/SysTools/Utils/Utils.cs
View File

@ -2,55 +2,6 @@ namespace InnovEnergy.Lib.SysTools.Utils;
public static class Utils public static class Utils
{ {
public static IEnumerable<T> Traverse<T>(T root, Func<T, IEnumerable<T>> getChildren)
{
var stack = new Stack<IEnumerator<T>>();
var it = root.Enumerator();
it.MoveNext();
while (true)
{
//////// going down ////////
while (true)
{
var cit = getChildren(it.Current).GetEnumerator();
if (cit.MoveNext()) // node has children, must be a branch
{
yield return it.Current;
stack.Push(it);
it = cit;
}
else // no children, hence a leaf
{
var node = it.Current;
yield return node;
if (!it.MoveNext())
break; // no more siblings: goto parent
}
}
//////// going up ////////
while (true)
{
it.Dispose();
if (stack.Count == 0) yield break; // we got to the bottom of the stack, were done
it = stack.Pop();
if (it.MoveNext())
break;
}
}
}
private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
@ -59,7 +10,6 @@ public static class Utils
return Epoch.AddSeconds(unixTime); return Epoch.AddSeconds(unixTime);
} }
public static R ValueOrDefault<T, R>(this Dictionary<T, R> dict, T key) public static R ValueOrDefault<T, R>(this Dictionary<T, R> dict, T key)
{ {
return ValueOrDefault(dict, key, default); return ValueOrDefault(dict, key, default);