update daily data aggregator

2023-12-18 16:09:43 +01:00 · 2023-12-18 16:09:43 +01:00 · 85b248dc6f
parent c36ebeeb6a
commit 85b248dc6f
5 changed files with 606 additions and 498 deletions
--- a/csharp/App/SaliMax/src/AggregationService/Aggregator.cs
+++ b/csharp/App/SaliMax/src/AggregationService/Aggregator.cs
@ -55,7 +55,6 @@ public static class Aggregator
        Console.WriteLine("Waiting for " + timeUntilNextDay.TotalHours + " hours...");
        Console.WriteLine("-----------------------------------------------------------------------------------------------------------------");
        
-        
        // Wait until the next rounded hour
        await Task.Delay(timeUntilNextDay);
        
@ -63,8 +62,15 @@ public static class Aggregator
        {
            try
            {
-                AggregatedData dailyAggregatedData = CreateAverage("HourlyData",DateTime.Now.AddDays(-1).ToUnixTime(),DateTime.Now.ToUnixTime());
+                var currentTime = DateTime.Now;
+                AggregatedData dailyAggregatedData = CreateAverage("HourlyData",currentTime.AddDays(-1).ToUnixTime(),currentTime.ToUnixTime());
                dailyAggregatedData.Save("DailyData");
+                if (await dailyAggregatedData.PushToS3())
+                {
+                    DeleteHourlyData("HourlyData",currentTime.ToUnixTime());
+                    dailyAggregatedData.DeleteDailyData("DailyData");
+                }
+                
            }
            catch (Exception e)
            {
@ -74,6 +80,21 @@ public static class Aggregator
        }
    }

+    private static void DeleteHourlyData(String myDirectory, Int64 beforeTimestamp)
+    {
+        var csvFiles            = Directory.GetFiles(myDirectory, "*.csv");
+        Console.WriteLine("Delete data before"+beforeTimestamp);
+        foreach (var csvFile in csvFiles)
+        {
+            if (IsFileWithinTimeRange(csvFile, 0, beforeTimestamp))
+            {
+                File.Delete(csvFile);
+                Console.WriteLine($"Deleted hourly data file: {csvFile}");
+            }
+        }
+    }
+
+    
    private static AggregatedData CreateAverage(String myDirectory, Int64 afterTimestamp, Int64 beforeTimestamp)
    {
        // Get all CSV files in the specified directory
--- a/csharp/App/SaliMax/src/AggregationService/HourlyData.cs
+++ b/csharp/App/SaliMax/src/AggregationService/HourlyData.cs
@ -1,5 +1,7 @@
 using System.Text.Json;
+using Flurl.Http;
 using InnovEnergy.App.SaliMax.Devices;
+using InnovEnergy.App.SaliMax.SystemConfig;
 using InnovEnergy.Lib.Units;
 using InnovEnergy.Lib.Utils;
 using static System.Text.Json.JsonSerializer;
@ -14,6 +16,8 @@ public class AggregatedData
    public required Double  AvgPvPower                  { get; set; }
    public required Double  BatteryPowerAverage         { get; set; }
    
+    private readonly S3Config? _S3Config = Config.Load().S3;
+
    public void Save(String directory)
    {
        var date             = DateTime.Now.ToUnixTime();
@ -39,6 +43,38 @@ public class AggregatedData
        }
    }
    
+    public void DeleteDailyData(String directory)
+    {
+        
+        var csvFiles            = Directory.GetFiles(directory, "*.csv");
+        foreach (var csvFile in csvFiles)
+        {
+            File.Delete(csvFile);
+            Console.WriteLine($"Deleted daily data file: {csvFile}");
+        }
+    }
+    
+    public async Task<Boolean> PushToS3()
+    {
+        var csv = this.ToCsv();
+        if (_S3Config is null)
+            return false;
+
+        var s3Path   = DateTime.Now.ToString("yyyy-MM-dd") + ".csv";
+        var request  = _S3Config.CreatePutRequest(s3Path);
+        var response = await request.PutAsync(new StringContent(csv));
+
+        if (response.StatusCode != 200)
+        {
+            Console.WriteLine("ERROR: PUT");
+            var error = await response.GetStringAsync();
+            Console.WriteLine(error);
+            return false;
+        }
+
+        return true;
+    }
+    
    // public static HourlyData? Load(String dataFilePath)
    // {
    //     try
--- a/csharp/App/SaliMax/src/Program.cs
+++ b/csharp/App/SaliMax/src/Program.cs
@ -19,7 +19,6 @@ using InnovEnergy.Lib.Devices.Trumpf.TruConvertDc.Control;
 using InnovEnergy.Lib.Protocols.Modbus.Channels;
 using InnovEnergy.Lib.Units;
 using InnovEnergy.Lib.Utils;
-using System.Text.Json;
 using InnovEnergy.App.SaliMax.AggregationService;
 using InnovEnergy.App.SaliMax.DataTypes;
 using static InnovEnergy.Lib.Devices.Trumpf.SystemControl.DataTypes.SystemConfig;
@ -98,7 +97,6 @@ internal static class Program
    }


-
    private static async Task Run()
    {
        "Starting SaliMax".LogInfo();
@ -144,8 +142,8 @@ internal static class Program
                ? new DcPowerDevice { Power = 0 }
                : Topology.CalculateDcLoad(acDc, pvOnDc, dcDc);

-            var acDcToDcLink = devices.LoadOnDc.DeviceState == DeviceState.Disabled ?
-                                                              Topology.CalculateAcDcToDcLink(pvOnDc, dcDc, acDc) 
+            var acDcToDcLink = devices.LoadOnDc.DeviceState == DeviceState.Disabled
+                ? Topology.CalculateAcDcToDcLink(pvOnDc, dcDc, acDc)
                : new DcPowerDevice { Power = acDc.Dc.Power };

            return new StatusRecord
@ -250,6 +248,7 @@ internal static class Program
            _subscribeToQueueForTheFirstTime = true;
            _subscribedToQueue = RabbitMqManager.SubscribeToQueue(currentSalimaxState, s3Bucket, VpnServerIp);
        }
+
        //If already subscribed to the queue and the status has been changed, update the queue
        if (_subscribedToQueue && currentSalimaxState.Status != _prevSalimaxState)
        {
@ -274,7 +273,6 @@ internal static class Program
        {
            record.ApplyConfigFile(config);
        }
-       
    }

    private static StatusMessage GetSalimaxStateAlarm(StatusRecord record)
@ -338,7 +336,6 @@ internal static class Program
                    });
                }
            }
-           
        }

        foreach (var warning in record.AcDc.Warnings)
@ -487,7 +484,6 @@ internal static class Program

    private static void ApplyDcDcDefaultSettings(this SystemControlRegisters? sc)
    {
-
        if (sc is null)
            return;

@ -502,7 +498,6 @@ internal static class Program
        sc.UseSlaveIdForAddressing = true;
        sc.SlaveErrorHandling = SlaveErrorHandling.Relaxed;
        sc.SubSlaveErrorHandling = SubSlaveErrorHandling.Off;
-                
        sc.ResetAlarmsAndWarnings = true;
    }

@ -536,5 +531,4 @@ internal static class Program
        status.Config.GridSetPoint = config.GridSetPoint * 1000; // converted from kW to W
        status.Config.ForceCalibrationCharge = config.ForceCalibrationCharge;
    }
-    
 }
--- a/typescript/frontend-marios2/src/content/dashboards/Overview/chartOptions.tsx
+++ b/typescript/frontend-marios2/src/content/dashboards/Overview/chartOptions.tsx
@ -3,26 +3,14 @@
 import { ApexOptions } from 'apexcharts';
 import { chartInfoInterface } from 'src/interfaces/Chart';
 import { findPower, formatPowerForGraph } from 'src/Resources/formatPower';
-import { addHours, format } from 'date-fns';

-export const getChartOptions = (chartInfo: chartInfoInterface): ApexOptions => {
-  // Custom datetime formatter for GMT+2
-  const customDatetimeFormatter = (timestamp, options) => {
-    const gmtDate = new Date(timestamp); // Convert Unix timestamp to milliseconds
-    const gmtPlus2Date = addHours(gmtDate, 4); // Add 2 hours to convert to GMT+2
-
-    // Use the specified options to format the date and time
-    const year = format(gmtDate, 'yyyy');
-    const month = format(gmtDate, "MMM 'yy");
-    const day = format(gmtDate, 'dd MMM');
-    const hour = format(gmtDate, 'HH:mm');
-    const minute = format(gmtDate, 'mm');
-
-    // Return the formatted date and time based on the provided options
-    return `  ${hour}:${minute}`;
-  };
-
-  const chartOptions: ApexOptions = {
+export const getChartOptions = (
+  chartInfo: chartInfoInterface,
+  type: string
+): ApexOptions => {
+  const chartOptions: ApexOptions =
+    type == 'daily'
+      ? {
          chart: {
            id: 'area-datetime',
            toolbar: {
@ -34,10 +22,7 @@ export const getChartOptions = (chartInfo: chartInfoInterface): ApexOptions => {
              autoScaleYaxis: false
            }
          },
-    // markers: {
-    //   size: 1,
-    //   strokeColors: 'black'
-    // },
+
          dataLabels: {
            enabled: false
          },
@ -121,6 +106,126 @@ export const getChartOptions = (chartInfo: chartInfoInterface): ApexOptions => {
              }
            }
          }
+        }
+      : type == 'monthly'
+      ? {
+          chart: {
+            height: 350,
+            type: 'bar'
+          },
+          plotOptions: {
+            bar: {
+              borderRadius: 10,
+              dataLabels: {
+                position: 'top' // top, center, bottom
+              }
+            }
+          },
+          dataLabels: {
+            enabled: true,
+            formatter: function (val) {
+              return val + '%';
+            },
+            offsetY: -20,
+            style: {
+              fontSize: '12px',
+              colors: ['#304758']
+            }
+          },
+
+          xaxis: {
+            categories: [
+              'Jan',
+              'Feb',
+              'Mar',
+              'Apr',
+              'May',
+              'Jun',
+              'Jul',
+              'Aug',
+              'Sep',
+              'Oct',
+              'Nov',
+              'Dec'
+            ],
+            position: 'bottom',
+            axisBorder: {
+              show: false
+            },
+            axisTicks: {
+              show: false
+            },
+            tooltip: {
+              enabled: true
+            }
+          },
+          yaxis: {
+            axisBorder: {
+              show: false
+            },
+            axisTicks: {
+              show: false
+            },
+            labels: {
+              show: false,
+              formatter: function (val) {
+                return val + '%';
+              }
+            }
+          }
+        }
+      : {
+          chart: {
+            height: 350,
+            type: 'bar'
+          },
+          plotOptions: {
+            bar: {
+              borderRadius: 10,
+              dataLabels: {
+                position: 'top' // top, center, bottom
+              }
+            }
+          },
+          dataLabels: {
+            enabled: true,
+            formatter: function (val) {
+              return val + '%';
+            },
+            offsetY: -20,
+            style: {
+              fontSize: '12px',
+              colors: ['#304758']
+            }
+          },
+
+          xaxis: {
+            categories: ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'],
+            position: 'bottom',
+            axisBorder: {
+              show: false
+            },
+            axisTicks: {
+              show: false
+            },
+            tooltip: {
+              enabled: true
+            }
+          },
+          yaxis: {
+            axisBorder: {
+              show: false
+            },
+            axisTicks: {
+              show: false
+            },
+            labels: {
+              show: false,
+              formatter: function (val) {
+                return val + '%';
+              }
+            }
+          }
        };

  return chartOptions;
--- a/typescript/frontend-marios2/src/content/dashboards/Overview/overview.tsx
+++ b/typescript/frontend-marios2/src/content/dashboards/Overview/overview.tsx
@ -1,11 +1,4 @@
-import {
-  Box,
-  Card,
-  Container,
-  Grid,
-  Typography,
-  useTheme
-} from '@mui/material';
+import { Box, Card, Container, Grid, Typography } from '@mui/material';
 import ReactApexChart from 'react-apexcharts';
 import React, { useEffect, useMemo, useState } from 'react';
 import DataCache from 'src/dataCache/dataCache';
@ -25,7 +18,6 @@ import { chartDataInterface, overviewInterface } from 'src/interfaces/Chart';
 import { fetchData } from 'src/content/dashboards/Installations/fetchData';
 import Button from '@mui/material/Button';
 import { FormattedMessage } from 'react-intl';
-import { ApexOptions } from 'apexcharts';

 const prefixes = ['', 'k', 'M', 'G', 'T'];
 const MAX_NUMBER = 9999999;
@ -35,7 +27,6 @@ interface OverviewProps {
 }

 function Overview(props: OverviewProps) {
-  const theme = useTheme();
  const numOfPointsToFetch = 100;
  const [timeRange, setTimeRange] = useState(
    createTimes(
@ -80,6 +71,9 @@ function Overview(props: OverviewProps) {
  }, []);

  const times$ = useMemo(() => new BehaviorSubject(timeRange), []);
+  const [dailyData, setDailyData] = useState(true);
+  const [weeklyData, setWeeklyData] = useState(false);
+  const [monthlyData, setMonthlyData] = useState(false);

  const transformToGraphData = (
    input: RecordSeries
@ -157,12 +151,10 @@ function Overview(props: OverviewProps) {
        Math.abs(overviewData[path].max),
        Math.abs(overviewData[path].min)
      );
-      let negative = false;
      let magnitude = 0;

      if (value < 0) {
        value = -value;
-        negative = true;
      }
      while (value >= 1000) {
        value /= 1000;
@ -289,6 +281,9 @@ function Overview(props: OverviewProps) {
  };

  const handle24HourData = () => {
+    setDailyData(true);
+    setWeeklyData(false);
+    setMonthlyData(false);
    const times = createTimes(
      UnixTime.now().rangeBefore(TimeSpan.fromDays(1)),
      numOfPointsToFetch
@ -298,21 +293,28 @@ function Overview(props: OverviewProps) {
  };

  const handleWeekData = () => {
-    const times = createTimes(
-      UnixTime.now().rangeBefore(TimeSpan.fromWeeks(1)),
-      numOfPointsToFetch
-    );
-    cache.getSeries(times);
-    times$.next(times);
+    setDailyData(false);
+    setWeeklyData(true);
+    setMonthlyData(false);
+    //fetchData(12312,props.s3Credentials);
+    // const times = createTimes(
+    //   UnixTime.now().rangeBefore(TimeSpan.fromWeeks(1)),
+    //   numOfPointsToFetch
+    // );
+    // cache.getSeries(times);
+    // times$.next(times);
  };

  const handleMonthData = () => {
-    const times = createTimes(
-      UnixTime.now().rangeBefore(TimeSpan.fromWeeks(4)),
-      numOfPointsToFetch
-    );
-    cache.getSeries(times);
-    times$.next(times);
+    setDailyData(false);
+    setWeeklyData(false);
+    setMonthlyData(true);
+    // const times = createTimes(
+    //   UnixTime.now().rangeBefore(TimeSpan.fromWeeks(4)),
+    //   numOfPointsToFetch
+    // );
+    // cache.getSeries(times);
+    // times$.next(times);
  };

  const series = [
@ -322,73 +324,6 @@ function Overview(props: OverviewProps) {
    }
  ];

-  const state: ApexOptions = {
-    chart: {
-      height: 350,
-      type: 'bar'
-    },
-    plotOptions: {
-      bar: {
-        borderRadius: 10,
-        dataLabels: {
-          position: 'top' // top, center, bottom
-        }
-      }
-    },
-    dataLabels: {
-      enabled: true,
-      formatter: function (val) {
-        return val + '%';
-      },
-      offsetY: -20,
-      style: {
-        fontSize: '12px',
-        colors: ['#304758']
-      }
-    },
-
-    xaxis: {
-      categories: [
-        'Jan',
-        'Feb',
-        'Mar',
-        'Apr',
-        'May',
-        'Jun',
-        'Jul',
-        'Aug',
-        'Sep',
-        'Oct',
-        'Nov',
-        'Dec'
-      ],
-      position: 'bottom',
-      axisBorder: {
-        show: false
-      },
-      axisTicks: {
-        show: false
-      },
-      tooltip: {
-        enabled: true
-      }
-    },
-    yaxis: {
-      axisBorder: {
-        show: false
-      },
-      axisTicks: {
-        show: false
-      },
-      labels: {
-        show: false,
-        formatter: function (val) {
-          return val + '%';
-        }
-      }
-    }
-  };
-
  const renderGraphs = () => {
    return (
      <Container maxWidth="xl">
@ -475,26 +410,43 @@ function Overview(props: OverviewProps) {
                    ></Box>
                  </Box>

+                  {dailyData && (
                    <ReactApexChart
-                    options={state}
+                      options={{
+                        ...getChartOptions(chartOverview.soc, 'daily'),
+                        chart: {
+                          events: {
+                            beforeZoom: handleBeforeZoom
+                          }
+                        }
+                      }}
+                      series={chartData.soc}
+                      type="area"
+                      height={350}
+                    />
+                  )}
+
+                  {weeklyData && (
+                    <ReactApexChart
+                      options={{
+                        ...getChartOptions(chartOverview.soc, 'weekly')
+                      }}
                      series={series}
                      type="bar"
                      height={350}
                    />
+                  )}

-                  {/*<ReactApexChart*/}
-                  {/*  options={{*/}
-                  {/*    ...getChartOptions(chartOverview.soc),*/}
-                  {/*    chart: {*/}
-                  {/*      events: {*/}
-                  {/*        beforeZoom: handleBeforeZoom*/}
-                  {/*      }*/}
-                  {/*    }*/}
-                  {/*  }}*/}
-                  {/*  series={chartData.soc}*/}
-                  {/*  type="area"*/}
-                  {/*  height={350}*/}
-                  {/*/>*/}
+                  {monthlyData && (
+                    <ReactApexChart
+                      options={{
+                        ...getChartOptions(chartOverview.soc, 'monthly')
+                      }}
+                      series={series}
+                      type="bar"
+                      height={350}
+                    />
+                  )}
                </Card>
              </Grid>
              <Grid item md={6} xs={12}>
@ -532,7 +484,7 @@ function Overview(props: OverviewProps) {
                  <div onDoubleClick={handleDoubleClick}>
                    <ReactApexChart
                      options={{
-                        ...getChartOptions(chartOverview.temperature),
+                        ...getChartOptions(chartOverview.temperature, 'daily'),
                        chart: {
                          events: {
                            beforeZoom: handleBeforeZoom
@ -590,7 +542,7 @@ function Overview(props: OverviewProps) {
                  <div onDoubleClick={handleDoubleClick}>
                    <ReactApexChart
                      options={{
-                        ...getChartOptions(chartOverview.pvProduction),
+                        ...getChartOptions(chartOverview.pvProduction, 'daily'),
                        chart: {
                          events: {
                            beforeZoom: handleBeforeZoom
@ -639,7 +591,7 @@ function Overview(props: OverviewProps) {
                  <div onDoubleClick={handleDoubleClick}>
                    <ReactApexChart
                      options={{
-                        ...getChartOptions(chartOverview.gridPower),
+                        ...getChartOptions(chartOverview.gridPower, 'daily'),
                        chart: {
                          events: {
                            beforeZoom: handleBeforeZoom
@ -696,7 +648,7 @@ function Overview(props: OverviewProps) {
                  <div onDoubleClick={handleDoubleClick}>
                    <ReactApexChart
                      options={{
-                        ...getChartOptions(chartOverview.dcPower),
+                        ...getChartOptions(chartOverview.dcPower, 'daily'),
                        chart: {
                          events: {
                            beforeZoom: handleBeforeZoom
@ -745,7 +697,7 @@ function Overview(props: OverviewProps) {
                  <div onDoubleClick={handleDoubleClick}>
                    <ReactApexChart
                      options={{
-                        ...getChartOptions(chartOverview.dcBusVoltage),
+                        ...getChartOptions(chartOverview.dcBusVoltage, 'daily'),
                        chart: {
                          events: {
                            beforeZoom: handleBeforeZoom