Update lOGGER, we only save one file now by set of status data.

Add the Concatenate function for sending to S3
This commit is contained in:
atef 2024-06-20 12:02:06 +02:00
parent fadea0a483
commit 416be15c65
6 changed files with 207 additions and 156 deletions

View File

@ -46,7 +46,7 @@ public class AggregatedData
}
catch (Exception e)
{
$"Failed to write config file {dataFilePath}\n{e}".LogInfo();
$"Failed to write config file {dataFilePath}\n{e}".WriteLine();
throw;
}
}

View File

@ -0,0 +1,34 @@
using System.Text;
namespace InnovEnergy.App.SaliMax;
public class LogFileConcatenator
{
private readonly string _logDirectory;
public LogFileConcatenator(String logDirectory = "LogDirectory/")
{
_logDirectory = logDirectory;
}
public String ConcatenateFiles(int numberOfFiles)
{
var logFiles = Directory
.GetFiles(_logDirectory, "log_*.csv")
.OrderByDescending(file => file)
.Take(numberOfFiles)
.OrderBy(file => file)
.ToList();
var concatenatedContent = new StringBuilder();
foreach (var fileContent in logFiles.Select(File.ReadAllText))
{
concatenatedContent.AppendLine(fileContent);
concatenatedContent.AppendLine(); // Append an empty line to separate the files // maybe we don't need this
}
return concatenatedContent.ToString();
}
}

View File

@ -5,17 +5,16 @@ namespace InnovEnergy.App.SaliMax;
public class CustomLogger : ILogger
{
private readonly String _LogFilePath;
private readonly Int64 _MaxFileSizeBytes;
private readonly Int32 _MaxLogFileCount;
private Int64 _CurrentFileSizeBytes;
private readonly String _logFilePath;
//private readonly Int64 _maxFileSizeBytes;
private readonly Int32 _maxLogFileCount;
private Int64 _currentFileSizeBytes;
public CustomLogger(String logFilePath, Int64 maxFileSizeBytes, Int32 maxLogFileCount)
public CustomLogger(String logFilePath, Int32 maxLogFileCount)
{
_LogFilePath = logFilePath;
_MaxFileSizeBytes = maxFileSizeBytes;
_MaxLogFileCount = maxLogFileCount;
_CurrentFileSizeBytes = File.Exists(logFilePath) ? new FileInfo(logFilePath).Length : 0;
_logFilePath = logFilePath;
_maxLogFileCount = maxLogFileCount;
_currentFileSizeBytes = File.Exists(logFilePath) ? new FileInfo(logFilePath).Length : 0;
}
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => throw new NotImplementedException();
@ -25,38 +24,25 @@ public class CustomLogger : ILogger
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception, String> formatter)
{
var logMessage = formatter(state, exception!);
// Check the log file count and delete the oldest file if necessary
var logFileDir = Path.GetDirectoryName(_logFilePath)!;
var logFileExt = Path.GetExtension(_logFilePath);
var logFileBaseName = Path.GetFileNameWithoutExtension(_logFilePath);
var logFiles = Directory
.GetFiles(logFileDir, $"{logFileBaseName}_*{logFileExt}")
.OrderBy(file => file)
.ToList();
// Check the file size and rotate the log file if necessary
if (_CurrentFileSizeBytes + logMessage.Length >= _MaxFileSizeBytes)
if (logFiles.Count >= _maxLogFileCount)
{
RotateLogFile();
_CurrentFileSizeBytes = 0;
File.Delete(logFiles.First());
}
// Write the log message to the file
File.AppendAllText(_LogFilePath, logMessage + Environment.NewLine);
_CurrentFileSizeBytes += logMessage.Length;
var timestamp = DateTime.Now.ToUnixTime() + Environment.NewLine;
//Console.WriteLine(logMessage);
}
private void RotateLogFile()
{
// Check the log file count and delete the oldest file if necessary
var logFileDir = Path.GetDirectoryName(_LogFilePath)!;
var logFileExt = Path.GetExtension(_LogFilePath);
var logFileBaseName = Path.GetFileNameWithoutExtension(_LogFilePath);
var logFiles = Directory
.GetFiles(logFileDir, $"{logFileBaseName}_*{logFileExt}")
.OrderBy(file => file)
.ToList();
if (logFiles.Count >= _MaxLogFileCount)
File.Delete(logFiles.First());
// Rename the current log file with a timestamp
var logFileBackupPath = Path.Combine(logFileDir, $"{logFileBaseName}_{DateTime.Now.ToUnixTime()}{logFileExt}");
File.Move(_LogFilePath, logFileBackupPath);
File.AppendAllText(logFileBackupPath, timestamp + logMessage + Environment.NewLine);
}
}

View File

@ -6,12 +6,12 @@ public static class Logger
{
// Specify the maximum log file size in bytes (e.g., 1 MB)
private const Int32 MaxFileSizeBytes = 2024 * 30; // TODO: move to settings
//private const Int32 MaxFileSizeBytes = 2024 * 30; // TODO: move to settings
private const Int32 MaxLogFileCount = 5000; // TODO: move to settings
private const String LogFilePath = "LogDirectory/log.csv"; // TODO: move to settings
// ReSharper disable once InconsistentNaming
private static readonly ILogger _logger = new CustomLogger(LogFilePath, MaxFileSizeBytes, MaxLogFileCount);
private static readonly ILogger _logger = new CustomLogger(LogFilePath, MaxLogFileCount);
public static T LogInfo<T>(this T t) where T : notnull
{

View File

@ -1,9 +1,12 @@
#undef Amax
#undef GridLimit
using System.Diagnostics;
using System.IO.Compression;
using System.Reactive.Linq;
using System.Reactive.Threading.Tasks;
using System.Reflection.Metadata;
using System.Security;
using System.Text;
using Flurl.Http;
using InnovEnergy.App.SaliMax.Devices;
@ -56,6 +59,8 @@ internal static class Program
private static Boolean _subscribeToQueueForTheFirstTime = false;
private static SalimaxAlarmState _prevSalimaxState = SalimaxAlarmState.Green;
private static Int32 _heartBitInterval = 0;
private const UInt16 NbrOfFileToConcatenate = 15;
private static UInt16 _counterOfFile = 0;
static Program()
{
@ -112,7 +117,7 @@ internal static class Program
private static async Task Run()
{
"Starting SaliMax".LogInfo();
"Starting SaliMax".WriteLine();
Watchdog.NotifyReady();
@ -219,54 +224,18 @@ internal static class Program
var record = ReadStatus();
/******************************************** For Battery Debug *************************************/
//foreach (var batteryNodeRecord in record.Battery.Devices)
//{
// var ioStates = batteryNodeRecord.IoStates;
//
// batteryNodeRecord.FwVersion.WriteLine(" FwVersion ");
// batteryNodeRecord.IoStatus.ConnectedToDcBus.WriteLine(" ConnectedToDcBus");
// batteryNodeRecord.IoStatus.AlarmOutActive.WriteLine(" AlarmOutActive");
// batteryNodeRecord.IoStatus.InternalFanActive.WriteLine(" InternalFanActive");
// batteryNodeRecord.IoStatus.VoltMeasurementAllowed.WriteLine(" VoltMeasurementAllowed");
// batteryNodeRecord.IoStatus.AuxRelayBus.WriteLine(" AuxRelayBus");
// batteryNodeRecord.IoStatus.RemoteStateActive.WriteLine(" RemoteStateActive");
// batteryNodeRecord.IoStatus.RiscActive.WriteLine(" RiscActive");
// Convert.ToString(ioStates, 2).PadLeft(16, '0').WriteLine($" IoStates Battery node ");
// batteryNodeRecord.TimeSinceTOC.WriteLine(" TOC");
// batteryNodeRecord.Eoc.WriteLine(" EOC");
//strings.String1Active.WriteLine(" BatteryString1");
//strings.String2Active.WriteLine(" BatteryString2");
//strings.String3Active.WriteLine(" BatteryString3");
//strings.String4Active.WriteLine(" BatteryString4");
//strings.String5Active.WriteLine(" BatteryString5");
// ("********************************************************").WriteLine();
//}
//record.GridMeter?.ActivePowerImportT1.WriteLine("kWh Export 6004");
//record.GridMeter?.ActivePowerExportT1.WriteLine("kWh Import 6024");
//record.GridMeter?.ActivePowerImportT2.WriteLine("Wh Import 6008");
//record.GridMeter?.ActivePowerExportT2.WriteLine("Wh Export 6028");
//record.GridMeter?.ActivePowerExportT3.WriteLine("KWh Export 8012");
//record.GridMeter?.ActivePowerImportT3.WriteLine("kWh Import 8002");
//record.GridMeter?.ActivePowerImportT4.WriteLine("Wh Import 8000");
//record.GridMeter?.ActivePowerExportT4.WriteLine("Wh Export 8010");
var currentSalimaxState = GetSalimaxStateAlarm(record);
SendSalimaxStateAlarm(currentSalimaxState, record);
//record.ControlPvPower(record.Config.CurtailP);
record.ControlConstants();
record.ControlSystemState();
var essControl = record.ControlEss().WriteLine().LogInfo();
var essControl = record.ControlEss().WriteLine();
record.EssControl = essControl;
@ -277,8 +246,7 @@ internal static class Program
WriteControl(record);
$"{DateTime.Now.Round(UpdateInterval).ToUnixTime()} : {record.StateMachine.State}: {record.StateMachine.Message}".WriteLine()
.LogInfo();
$"{DateTime.Now.Round(UpdateInterval).ToUnixTime()} : {record.StateMachine.State}: {record.StateMachine.Message}".WriteLine();
record.CreateTopologyTextBlock().WriteLine();
@ -287,7 +255,7 @@ internal static class Program
record.Config.Save();
"===========================================".LogInfo();
"===========================================".WriteLine();
return record;
}
@ -340,7 +308,7 @@ internal static class Program
private static StatusMessage GetSalimaxStateAlarm(StatusRecord record)
{
var alarmCondition = record.DetectAlarmStates();
var alarmCondition = record.DetectAlarmStates(); // this need to be emailed to support or customer
var s3Bucket = Config.Load().S3?.Bucket;
var alarmList = new List<AlarmOrWarning>();
@ -418,7 +386,7 @@ internal static class Program
if (alarmCondition is not null)
{
alarmCondition.LogInfo();
alarmCondition.WriteLine();
alarmList.Add(new AlarmOrWarning
{
@ -510,15 +478,6 @@ internal static class Program
var dcDevices = r.DcDc.Devices;
var configFile = r.Config;
var maxBatteryDischargingCurrentLive = 0.0;
// This adapting the max discharging current to the current connected batteries // Connected to Dc Bus is not reliable yet
// if (r.Battery != null)
// {
// var numberOfBatteries = r.Battery.Devices.Count;;
// var dischargingCurrentByBattery = configFile.MaxBatteryDischargingCurrent / numberOfBatteries;
// var numberOfConnectedBatteries = r.Battery.Devices.Where(d => d.IoStatus.ConnectedToDcBus).ToList().Count;
// maxBatteryDischargingCurrentLive = dischargingCurrentByBattery * numberOfConnectedBatteries;
// }
// This adapting the max discharging current to the current Active Strings
if (r.Battery != null)
@ -545,11 +504,6 @@ internal static class Program
{
maxBatteryDischargingCurrentLive = dischargingCurrentByString * numberOfBatteriesStringActive;
}
//dischargingCurrentByString.WriteLine(" dischargingCurrentByString");
//numberOfBatteriesStringActive.WriteLine(" numberOfBatteriesStringActive");
//numberOfTotalStrings.WriteLine(" numberOfTotalStrings");
//maxBatteryDischargingCurrentLive.WriteLine(" maxBatteryDischargingCurrentLive");
}
// TODO The discharging current is well calculated but not communicated to live. But Written in S3
@ -578,31 +532,92 @@ internal static class Program
r.AcDc.ResetAlarms();
}
// This is will be used for provider throttling, this example is only for either 100% or 0 %
// This will be used for provider throttling, this example is only for either 100% or 0 %
private static void ControlPvPower(this StatusRecord r, Int16 exportLimit = 100)
{
var inverters = r.AcDc.Devices;
var dcDevices = r.DcDc.Devices;
var configFile = r.Config;
var devicesConfig = r.AcDc.Devices.All(d => d.Control.Ac.GridType == GridType.GridTied400V50Hz) ? configFile.GridTie : configFile.IslandMode; // TODO if any of the grid tie mode
// 950 V = 0%
// 750 V = 100%
const Int32 maxDcLinkVoltage = 950;
const Int32 minDcLinkVoltage = 850; // we may dont need this;
const Int32 referenceDcLinkVoltage = 900; // we may dont need this;
UInt16 stableFactor = 500;
var inverters = r.AcDc.Devices;
var dcDevices = r.DcDc.Devices;
var configFile = r.Config;
var systemProduction = inverters.Count * 25000;
var limitSystemProduction = systemProduction * exportLimit / 100;
var devicesConfig = r.AcDc.Devices.All(d => d.Control.Ac.GridType == GridType.GridTied400V50Hz) ? configFile.GridTie : configFile.IslandMode; // TODO if any of the grid tie mode
var targetReferenceVoltage = devicesConfig.AcDc.ReferenceDcLinkVoltage;
inverters.ForEach(d => d.Control.Dc.MaxVoltage = exportLimit == 100 ? devicesConfig.AcDc.MaxDcLinkVoltage : maxDcLinkVoltage);
inverters.ForEach(d => d.Control.Dc.MinVoltage = exportLimit == 100 ? devicesConfig.AcDc.MinDcLinkVoltage : minDcLinkVoltage);
inverters.ForEach(d => d.Control.Dc.ReferenceVoltage = exportLimit == 100 ? devicesConfig.AcDc.ReferenceDcLinkVoltage : referenceDcLinkVoltage);
dcDevices.ForEach(d => d.Control.DroopControl.UpperVoltage = exportLimit == 100 ? devicesConfig.DcDc.UpperDcLinkVoltage : maxDcLinkVoltage);
dcDevices.ForEach(d => d.Control.DroopControl.LowerVoltage = exportLimit == 100 ? devicesConfig.DcDc.LowerDcLinkVoltage : minDcLinkVoltage);
dcDevices.ForEach(d => d.Control.DroopControl.ReferenceVoltage = exportLimit == 100 ? devicesConfig.DcDc.ReferenceDcLinkVoltage : referenceDcLinkVoltage);
if (r.PvOnDc.Dc.Power != null && r.PvOnDc.Dc.Power > limitSystemProduction)
{
exportLimit.WriteLine(" exportLimit");
systemProduction.WriteLine(" systemProduction");
limitSystemProduction.WriteLine(" limitSystemexport");
targetReferenceVoltage.WriteLine("targetReferenceVoltage");
if (r.GridMeter?.Ac.Power.Active != null)
{
if (r.GridMeter.Ac.Power.Active < -limitSystemProduction)
{
"We are openning the window".WriteLine();
r.Config.GridSetPoint = -limitSystemProduction + stableFactor;
r.Config.GridSetPoint.WriteLine(" Grid set point");
var maxDcLinkVoltage = (UInt16)(r.Config.GridTie.AcDc.MaxDcLinkVoltage + 10);
var minDcLinkVoltage = (UInt16)(r.Config.GridTie.AcDc.MinDcLinkVoltage - 10);
var maxDcDcLinkVoltage = (UInt16)(r.Config.GridTie.DcDc.UpperDcLinkVoltage + 10);
var minDcDcLinkVoltage = (UInt16)(r.Config.GridTie.DcDc.LowerDcLinkVoltage + 10);
r.Config.GridTie.AcDc.MaxDcLinkVoltage = maxDcLinkVoltage;
r.Config.GridTie.AcDc.MinDcLinkVoltage = minDcLinkVoltage;
r.Config.GridTie.DcDc.UpperDcLinkVoltage = maxDcDcLinkVoltage;
r.Config.GridTie.DcDc.LowerDcLinkVoltage = minDcDcLinkVoltage;
maxDcLinkVoltage.WriteLine("maxDcLinkVoltage");
minDcLinkVoltage.WriteLine("minxDcLinkVoltage");
maxDcDcLinkVoltage.WriteLine("maxDcDcLinkVoltage");
minDcDcLinkVoltage.WriteLine("minDcDcLinkVoltage");
}
else if (r.GridMeter.Ac.Power.Active > -limitSystemProduction + stableFactor * 2)
{
"We are closing the window".WriteLine();
r.Config.GridSetPoint = -limitSystemProduction + stableFactor;
r.Config.GridSetPoint.WriteLine(" Grid set point");
if ((r.Config.GridTie.AcDc.MaxDcLinkVoltage - r.Config.GridTie.AcDc.MinDcLinkVoltage) > 60)
{
var maxDcLinkVoltage = (UInt16)(r.Config.GridTie.AcDc.MaxDcLinkVoltage - 10);
var minDcLinkVoltage = (UInt16)(r.Config.GridTie.AcDc.MinDcLinkVoltage + 10);
var maxDcDcLinkVoltage = (UInt16)(r.Config.GridTie.DcDc.UpperDcLinkVoltage - 10);
var minDcDcLinkVoltage = (UInt16)(r.Config.GridTie.DcDc.LowerDcLinkVoltage - 10);
r.Config.GridTie.AcDc.MaxDcLinkVoltage = maxDcLinkVoltage;
r.Config.GridTie.AcDc.MinDcLinkVoltage = minDcLinkVoltage;
r.Config.GridTie.DcDc.UpperDcLinkVoltage = maxDcDcLinkVoltage;
r.Config.GridTie.DcDc.LowerDcLinkVoltage = minDcDcLinkVoltage;
maxDcLinkVoltage.WriteLine("maxDcLinkVoltage");
minDcLinkVoltage.WriteLine("minxDcLinkVoltage");
maxDcDcLinkVoltage.WriteLine("maxDcDcLinkVoltage");
minDcDcLinkVoltage.WriteLine("minDcDcLinkVoltage");
}
else
{
"do nothing".WriteLine();
}
}
else
{
r.Config.GridTie.AcDc.MaxDcLinkVoltage.WriteLine("maxDcLinkVoltage");
r.Config.GridTie.AcDc.MinDcLinkVoltage.WriteLine("minxDcLinkVoltage");
r.Config.GridTie.DcDc.UpperDcLinkVoltage.WriteLine("maxDcDcLinkVoltage");
r.Config.GridTie.DcDc.LowerDcLinkVoltage.WriteLine("minDcDcLinkVoltage");
}
}
if (exportLimit == 100)
{
r.Config.GridSetPoint = 0;
}
}
}
// why this is not in Controller?
@ -670,53 +685,69 @@ internal static class Program
{
var s3Config = status.Config.S3;
var csv = status.ToCsv().LogInfo();
if (s3Config is null)
return false;
var s3Path = timeStamp.ToUnixTime() + ".csv";
var request = s3Config.CreatePutRequest(s3Path);
// This is temporary for Wittman, but now it's for all Installation
//await File.WriteAllTextAsync("/var/www/html/status.csv", csv.SplitLines().Where(l => !l.Contains("Secret")).JoinLines());
//Use this for no compression
//var response = await request.PutAsync(new StringContent(csv));
//Compress CSV data to a byte array
byte[] compressedBytes;
using (var memoryStream = new MemoryStream())
if (s3Config is null)
return false;
//this for concatenating in one file
if (_counterOfFile == NbrOfFileToConcatenate)
{
//Create a zip directory and put the compressed file inside
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
_counterOfFile = 0;
var logFileConcatenator = new LogFileConcatenator();
var csvToSend = logFileConcatenator.ConcatenateFiles(NbrOfFileToConcatenate);
File.WriteAllText("test.csv",csvToSend);
var s3Path = timeStamp.ToUnixTime() + ".csv";
var request = s3Config.CreatePutRequest(s3Path);
"Sending to S3".WriteLine();
//Use this for no compression
//var response = await request.PutAsync(new StringContent(csv));
//Compress CSV data to a byte array
byte[] compressedBytes;
using (var memoryStream = new MemoryStream())
{
var entry = archive.CreateEntry("data.csv", CompressionLevel.SmallestSize); // Add CSV data to the ZIP archive
using (var entryStream = entry.Open())
using (var writer = new StreamWriter(entryStream))
//Create a zip directory and put the compressed file inside
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
writer.Write(csv);
var entry = archive.CreateEntry("data.csv", CompressionLevel.SmallestSize); // Add CSV data to the ZIP archive
using (var entryStream = entry.Open())
using (var writer = new StreamWriter(entryStream))
{
writer.Write(csvToSend);
}
}
compressedBytes = memoryStream.ToArray();
}
compressedBytes = memoryStream.ToArray();
// Encode the compressed byte array as a Base64 string
string base64String = Convert.ToBase64String(compressedBytes);
// Create StringContent from Base64 string
var stringContent = new StringContent(base64String, Encoding.UTF8, "application/base64");
// Upload the compressed data (ZIP archive) to S3
var response = await request.PutAsync(stringContent);
if (response.StatusCode != 200)
{
Console.WriteLine("ERROR: PUT");
var error = await response.GetStringAsync();
Console.WriteLine(error);
return false;
}
}
// Encode the compressed byte array as a Base64 string
string base64String = Convert.ToBase64String(compressedBytes);
// Create StringContent from Base64 string
var stringContent = new StringContent(base64String, Encoding.UTF8, "application/base64");
// Upload the compressed data (ZIP archive) to S3
var response = await request.PutAsync(stringContent);
if (response.StatusCode != 200)
{
Console.WriteLine("ERROR: PUT");
var error = await response.GetStringAsync();
Console.WriteLine(error);
return false;
}
_counterOfFile++;
return true;
}

View File

@ -219,7 +219,7 @@ public class Config //TODO: let IE choose from config files (Json) and connect t
}
catch (Exception e)
{
$"Failed to write config file {configFilePath}\n{e}".LogInfo();
$"Failed to write config file {configFilePath}\n{e}".WriteLine();
throw;
}
}