Innovenergy
/
Innovenergy_trunk
2
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

add datacache (ivos code), first working version with diagrams, only sample data

This commit is contained in:
Sina Blattmann 2023-04-24 09:35:01 +02:00
parent ee289d27b2
commit 39b414d0f0
40 changed files with 4738 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
csharp/App/Backend/db.sqlite
View File

Binary file not shown.

2450
typescript/Frontend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
typescript/Frontend/package.json
View File

@ -3,6 +3,7 @@
"version": "0.1.0", "version": "0.1.0",
"private": true, "private": true,
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.316.0",
"@emotion/react": "^11.10.5", "@emotion/react": "^11.10.5",
"@emotion/styled": "^11.10.5", "@emotion/styled": "^11.10.5",
"@minoru/react-dnd-treeview": "^3.4.1", "@minoru/react-dnd-treeview": "^3.4.1",
@ -21,6 +22,7 @@
"chart.js": "^4.2.1", "chart.js": "^4.2.1",
"css-loader": "^6.7.3", "css-loader": "^6.7.3",
"formik": "^2.2.9", "formik": "^2.2.9",
"linq-to-typescript": "^11.0.0",
"package.json": "^2.0.1", "package.json": "^2.0.1",
"plotly.js": "^2.20.0", "plotly.js": "^2.20.0",
"react": "^18.2.0", "react": "^18.2.0",
@ -34,8 +36,10 @@
"react-router-dom": "^6.8.0", "react-router-dom": "^6.8.0",
"react-scripts": "5.0.1", "react-scripts": "5.0.1",
"reactflow": "^11.5.6", "reactflow": "^11.5.6",
"rxjs": "^7.8.0",
"sass": "^1.58.3", "sass": "^1.58.3",
"sass-loader": "^13.2.0", "sass-loader": "^13.2.0",
"simplytyped": "^3.3.0",
"style-loader": "^3.3.1", "style-loader": "^3.3.1",
"testcafe": "^2.4.0", "testcafe": "^2.4.0",
"typescript": "^4.9.5", "typescript": "^4.9.5",

41
typescript/Frontend/src/components/Installations/InstallationForm.tsx
View File

@ -9,6 +9,7 @@ import MoveDialog from "../Groups/Tree/MoveDialog";
import InnovenergyButton from "../Layout/InnovenergyButton"; import InnovenergyButton from "../Layout/InnovenergyButton";
import InnovenergyTextfield from "../Layout/InnovenergyTextfield"; import InnovenergyTextfield from "../Layout/InnovenergyTextfield";
import { UserContext } from "../Context/UserContextProvider"; import { UserContext } from "../Context/UserContextProvider";
import * as Yup from "yup";
interface I_InstallationFormProps { interface I_InstallationFormProps {
values: I_Installation; values: I_Installation;
@ -19,11 +20,32 @@ const InstallationForm = (props: I_InstallationFormProps) => {
const { values, id, hasMoveButton } = props; const { values, id, hasMoveButton } = props;
const [open, setOpen] = useState(false); const [open, setOpen] = useState(false);
const intl = useIntl();
const { fetchData } = useContext(InstallationContext); const { fetchData } = useContext(InstallationContext);
const { getCurrentUser } = useContext(UserContext); const { getCurrentUser } = useContext(UserContext);
const readOnly = !getCurrentUser().hasWriteAccess; const readOnly = !getCurrentUser().hasWriteAccess;
const intl = useIntl();
const validationSchema = Yup.object().shape({
name: Yup.string().required(
intl.formatMessage({
id: "requiredName",
defaultMessage: "Name is required",
})
),
region: Yup.string().required(
intl.formatMessage({
id: "requiredRegion",
defaultMessage: "Region is required",
})
),
location: Yup.string().required(
intl.formatMessage({
id: "requiredLocation",
defaultMessage: "Location is required",
})
),
});
const formik = useFormik({ const formik = useFormik({
initialValues: { initialValues: {
@ -44,6 +66,7 @@ const InstallationForm = (props: I_InstallationFormProps) => {
fetchData(); fetchData();
}); });
}, },
validationSchema,
}); });
const handleClose = () => { const handleClose = () => {
@ -62,6 +85,10 @@ const InstallationForm = (props: I_InstallationFormProps) => {
name="name" name="name"
value={formik.values.name} value={formik.values.name}
handleChange={formik.handleChange} handleChange={formik.handleChange}
helperText={
formik.errors.name && formik.touched.name ? formik.errors.name : ""
}
error={!!formik.errors.name && formik.touched.name}
/> />
<InnovenergyTextfield <InnovenergyTextfield
disabled={readOnly} disabled={readOnly}
@ -73,6 +100,12 @@ const InstallationForm = (props: I_InstallationFormProps) => {
name="region" name="region"
value={formik.values.region} value={formik.values.region}
handleChange={formik.handleChange} handleChange={formik.handleChange}
helperText={
formik.errors.region && formik.touched.region
? formik.errors.region
: ""
}
error={!!formik.errors.region && formik.touched.region}
/> />
<InnovenergyTextfield <InnovenergyTextfield
disabled={readOnly} disabled={readOnly}
@ -84,6 +117,12 @@ const InstallationForm = (props: I_InstallationFormProps) => {
name="location" name="location"
value={formik.values.location} value={formik.values.location}
handleChange={formik.handleChange} handleChange={formik.handleChange}
helperText={
formik.errors.location && formik.touched.location
? formik.errors.location
: ""
}
error={!!formik.errors.location && formik.touched.location}
/> />
<InnovenergyTextfield <InnovenergyTextfield
disabled={readOnly} disabled={readOnly}

63
typescript/Frontend/src/components/Installations/Log/Log.tsx
View File

@ -1,8 +1,67 @@
import React from "react"; import React, { useState } from "react";
import ScalarGraph from "./ScalarGraph"; import ScalarGraph from "./ScalarGraph";
import DataCache, { FetchResult } from "../../../dataCache/dataCache";
import { TimeSpan, UnixTime } from "../../../dataCache/time";
import { S3Access } from "../../../dataCache/S3/S3Access";
import { map, debounceTime } from "rxjs/operators";
import { RecordSeries } from "../../../dataCache/data";
import { parseCsv } from "../../../util/graph.util";
const Log = () => { const Log = () => {
return <ScalarGraph />; const [timeSeries, setTimeSeries] = useState<RecordSeries>([]);
const s3Access = new S3Access(
"saliomameiringen",
"sos-ch-dk-2",
"exo.io",
"EXO18e7ae9e53fae71ee55cf35b",
"3Cyonq8gMQ0a3elTH2vP7Yv-czcCj8iE2lBcPB9XhSc",
""
);
const fetchData = (
timestamp: UnixTime
): Promise<FetchResult<Record<string, number>>> => {
const s3Path = `${timestamp.ticks}.csv`;
return s3Access
.get(s3Path)
.then(async (r) => {
if (r.status === 404) {
return Promise.resolve(FetchResult.notAvailable);
} else if (r.status === 200) {
const text = await r.text();
return parseCsv(text);
} else {
console.error("unexpected status code");
return Promise.resolve(FetchResult.notAvailable);
}
})
.catch((e) => {
console.log(e);
return Promise.resolve(FetchResult.tryLater);
});
};
const cache = new DataCache(fetchData, TimeSpan.fromSeconds(2));
const sampleTimes = UnixTime.fromTicks(1682085650)
.earlier(TimeSpan.fromMinutes(1))
.rangeBefore(TimeSpan.fromMinutes(1))
.sample(TimeSpan.fromSeconds(2));
cache.getSeries(sampleTimes);
const update = cache.gotData.pipe(
debounceTime(2000),
map((_) => setTimeSeries(cache.getSeries(sampleTimes)))
);
update.subscribe();
return (
<>
<ScalarGraph data={timeSeries} />
</>
);
}; };
export default Log; export default Log;

58
typescript/Frontend/src/components/Installations/Log/ScalarGraph.tsx
View File

@ -1,59 +1,15 @@
import Plot from "react-plotly.js"; import Plot from "react-plotly.js";
import { TimeSeries, timeSeries } from "../ExampleLogData"; import { RecordSeries } from "../../../dataCache/data";
import { transformToGraphData } from "../../../util/graph.util";
const ScalarGraph = () => { interface I_ScalarGraphProps {
const transformToGraphData = (timeStampData: TimeSeries) => { data: RecordSeries;
return Object.keys(timeSeries).reduce(
(timeSeriesAcc, timeSeriesKey) => {
const logData = timeStampData[parseInt(timeSeriesKey)];
const transformedTimeSeries = Object.keys(logData).reduce(
(logAcc, logDataKey) => {
return {
...logAcc,
[logDataKey]: {
[timeSeriesKey]: logData[logDataKey],
},
};
},
{} as any
);
Object.keys(transformedTimeSeries).forEach(
(transformedTimeSeriesKey) => {
const date = new Date(parseInt(timeSeriesKey));
if (timeSeriesAcc[transformedTimeSeriesKey]) {
timeSeriesAcc[transformedTimeSeriesKey] = {
x: [...timeSeriesAcc[transformedTimeSeriesKey].x, date],
y: [
...timeSeriesAcc[transformedTimeSeriesKey].y,
transformedTimeSeries[transformedTimeSeriesKey][
timeSeriesKey
],
],
};
} else {
timeSeriesAcc[transformedTimeSeriesKey] = {
x: [date],
y: [
transformedTimeSeries[transformedTimeSeriesKey][
timeSeriesKey
],
],
};
} }
}
);
return timeSeriesAcc;
},
{} as {
[path: string]: { x: Date[]; y: number[] };
}
);
};
const ScalarGraph = (props: I_ScalarGraphProps) => {
const renderGraphs = () => { const renderGraphs = () => {
const coordinateTimeSeries = transformToGraphData(timeSeries); const coordinateTimeSeries = transformToGraphData(props.data);
return Object.keys(coordinateTimeSeries).map((path) => { return Object.keys(coordinateTimeSeries).map((path) => {
console.log(timeSeries[parseInt(path)]);
return ( return (
<Plot <Plot
data={[ data={[
@ -74,7 +30,7 @@ const ScalarGraph = () => {
], ],
}} }}
onUpdate={(figure) => { onUpdate={(figure) => {
console.log(figure); //console.log(figure);
}} }}
/> />
); );

7
typescript/Frontend/src/components/Layout/InnovenergyTextfield.tsx
View File

@ -9,6 +9,8 @@ interface I_InnovenergyTextfieldProps {
type?: string; type?: string;
readOnly?: boolean; readOnly?: boolean;
disabled?: boolean; disabled?: boolean;
helperText?: string;
error?: boolean;
} }
const InnovenergyTextfield = (props: I_InnovenergyTextfieldProps) => { const InnovenergyTextfield = (props: I_InnovenergyTextfieldProps) => {
@ -30,6 +32,9 @@ const InnovenergyTextfield = (props: I_InnovenergyTextfieldProps) => {
"-webkit-text-fill-color": "black", "-webkit-text-fill-color": "black",
color: "black", color: "black",
}, },
".MuiFormHelperText-root": {
marginLeft: 0,
},
}} }}
value={props.value || ""} value={props.value || ""}
onChange={props.handleChange} onChange={props.handleChange}
@ -37,6 +42,8 @@ const InnovenergyTextfield = (props: I_InnovenergyTextfieldProps) => {
readOnly: props.readOnly, readOnly: props.readOnly,
}} }}
disabled={props.disabled} disabled={props.disabled}
helperText={props.helperText}
error={props.error}
/> />
</Grid> </Grid>
</Grid> </Grid>

79
typescript/Frontend/src/dataCache/S3/S3Access.ts Normal file
View File

@ -0,0 +1,79 @@
import {sha1Hmac} from "./Sha1";
import {Utf8} from "./Utf8";
import {toBase64} from "./UInt8Utils";
export class S3Access
{
constructor
(
readonly bucket: string,
readonly region: string,
readonly provider: string,
readonly key: string,
readonly secret: string,
readonly contentType: string
)
{}
get host() : string { return `${this.bucket}.${this.region}.${this.provider}` }
get url() : string { return `https://${this.host}` }
public get(s3Path : string): Promise<Response>
{
const method = "GET";
const auth = this.createAuthorizationHeader(method, s3Path, "");
const url = this.url + "/" + s3Path
const headers = {"Host": this.host, "Authorization": auth};
try
{
return fetch(url, {method: method, mode: "cors", headers: headers})
}
catch
{
return Promise.reject()
}
}
private createAuthorizationHeader(method: string,
s3Path: string,
date: string)
{
return createAuthorizationHeader
(
method,
this.bucket,
s3Path,
date,
this.key,
this.secret,
this.contentType
);
}
}
function createAuthorizationHeader(method: string,
bucket: string,
s3Path: string,
date: string,
s3Key: string,
s3Secret: string,
contentType: string,
md5Hash: string = "")
{
// StringToSign = HTTP-Verb + "\n" +
// Content-MD5 + "\n" +
// Content-Type + "\n" +
// Date + "\n" +
// CanonicalizedAmzHeaders +
// CanonicalizedResource;
const payload = Utf8.encode(`${method}\n${md5Hash}\n${contentType}\n${date}\n/${bucket}/${s3Path}`)
//console.log(`${method}\n${md5Hash}\n${contentType}\n${date}\n/${bucket}/${s3Path}`)
const secret = Utf8.encode(s3Secret)
const signature = toBase64(sha1Hmac(payload, secret));
return `AWS ${s3Key}:${signature}`
}

125
typescript/Frontend/src/dataCache/S3/Sha1.ts Normal file
View File

@ -0,0 +1,125 @@
import {concat, pad} from "./UInt8Utils";
const BigEndian = false
export function sha1Hmac(msg: Uint8Array, key: Uint8Array): Uint8Array
{
if (key.byteLength > 64)
key = sha1(key)
if (key.byteLength < 64)
key = pad(key, 64)
const oKey = key.map(b => b ^ 0x5C);
const iKey = key.map(b => b ^ 0x36);
const iData = concat(iKey, msg);
const iHash = sha1(iData);
const oData = concat(oKey, iHash);
return sha1(oData);
}
export function sha1(data: Uint8Array): Uint8Array
{
const paddedData: DataView = initData(data)
const H = new Uint32Array([0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0])
const S = new Uint32Array(5) // State
const round = new Uint32Array(80);
function initRound(startOffset: number)
{
for (let i = 0; i < 16; i++)
round[i] = paddedData.getUint32((startOffset + i) * 4, BigEndian);
for (let i = 16; i < 80; i++)
{
const int32 = round[i - 3] ^ round[i - 8] ^ round[i - 14] ^ round[i - 16];
round[i] = rotate1(int32); // SHA0 has no rotate
}
}
const functions =
[
() => (S[1] & S[2] | ~S[1] & S[3]) + 0x5A827999,
() => (S[1] ^ S[2] ^ S[3]) + 0x6ED9EBA1,
() => (S[1] & S[2] | S[1] & S[3] | S[2] & S[3]) + 0x8F1BBCDC,
() => (S[1] ^ S[2] ^ S[3]) + 0xCA62C1D6
]
for (let startOffset = 0; startOffset < paddedData.byteLength / 4; startOffset += 16)
{
initRound(startOffset);
S.set(H)
for (let r = 0, i = 0; r < 4; r++)
{
const f = functions[r]
const end = i + 20;
do
{
const S0 = rotate5(S[0]) + f() + S[4] + round[i];
S[4] = S[3];
S[3] = S[2];
S[2] = rotate30(S[1]);
S[1] = S[0];
S[0] = S0;
}
while (++i < end)
}
for (let i = 0; i < 5; i++)
H[i] += S[i]
}
swapEndianness(H);
return new Uint8Array(H.buffer)
}
function rotate5(int32: number)
{
return (int32 << 5) | (int32 >>> 27); // >>> for unsigned shift
}
function rotate30(int32: number)
{
return (int32 << 30) | (int32 >>> 2);
}
function rotate1(int32: number)
{
return (int32 << 1) | (int32 >>> 31);
}
function initData(data: Uint8Array): DataView
{
const dataLength = data.length
const extendedLength = dataLength + 9; // add 8 bytes for UInt64 length + 1 byte for "stop-bit" (0x80)
const paddedLength = Math.ceil(extendedLength / 64) * 64; // pad to 512 bits block
const paddedData = new Uint8Array(paddedLength)
paddedData.set(data)
paddedData[dataLength] = 0x80 // append single 1 bit at end of data
const dataView = new DataView(paddedData.buffer)
// append UInt64 length
dataView.setUint32(paddedData.length - 4, dataLength << 3 , BigEndian) // dataLength in *bits* LO, (<< 3: x8 bits per byte)
dataView.setUint32(paddedData.length - 8, dataLength >>> 29, BigEndian) // dataLength in *bits* HI
return dataView
}
function swapEndianness(uint32Array: Uint32Array)
{
const dv = new DataView(uint32Array.buffer)
for (let i = 0; i < uint32Array.byteLength; i += 4)
{
const uint32 = dv.getUint32(i, false)
dv.setUint32(i, uint32, true)
}
}

56
typescript/Frontend/src/dataCache/S3/UInt8Utils.ts Normal file
View File

@ -0,0 +1,56 @@
export function pad(data: Uint8Array, length: number): Uint8Array
{
if (length < data.byteLength)
throw new RangeError("length")
const padded = new Uint8Array(length)
padded.set(data)
return padded;
}
export function concat(left: Uint8Array, right: Uint8Array): Uint8Array
{
const c = new Uint8Array(left.length + right.length);
c.set(left);
c.set(right, left.length);
return c
}
export function toHexString(data: Uint8Array)
{
return [...data].map(byteToHex).join('');
}
function byteToHex(b: number)
{
return b.toString(16).padStart(2, "0");
}
const b64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
export function toBase64(data : Uint8Array) : string
{
const byteLength = data.byteLength
const base64LengthPadded = 4 * Math.ceil(byteLength / 3)
const base64Length = Math.ceil(byteLength / 3 * 4);
const base64 = new Array<String>(base64LengthPadded)
for (let i = 0, o = 0; i < byteLength;)
{
const x = data[i++]
const y = data[i++] ?? 0
const z = data[i++] ?? 0
base64[o++] = b64Chars[x >>> 2]
base64[o++] = b64Chars[(x << 4 | y >>> 4) & 63]
base64[o++] = b64Chars[(y << 2 | z >>> 6) & 63]
base64[o++] = b64Chars[z & 63]
}
for (let i = base64LengthPadded; i > base64Length ;)
base64[--i] = "="
return base64.join('')
}

10
typescript/Frontend/src/dataCache/S3/Utf8.ts Normal file
View File

@ -0,0 +1,10 @@
export namespace Utf8
{
const encoder = new TextEncoder()
const decoder = new TextDecoder()
export const encode = (text: string): Uint8Array => encoder.encode(text);
export const decode = (data: Uint8Array): string => decoder.decode(data);
}

21
typescript/Frontend/src/dataCache/data.ts Normal file
View File

@ -0,0 +1,21 @@
import { Maybe } from "yup";
import {Timestamped} from "./types";
import { isDefined } from "./utils/maybe";
export type DataRecord = Record<string, number>
export type DataPoint = Timestamped<Maybe<DataRecord>>
export type RecordSeries = Array<DataPoint>
export type PointSeries = Array<Timestamped<Maybe<number>>>
export type DataSeries = Array<Maybe<number>>
export function getPoints(recordSeries: RecordSeries, series: keyof DataRecord): PointSeries
{
return recordSeries.map(p => ({time: p.time, value: isDefined(p.value) ? p.value[series] : undefined}))
}
export function getData(recordSeries: RecordSeries, series: keyof DataRecord): DataSeries
{
return recordSeries.map(p => (isDefined(p.value) ? p.value[series] : undefined))
}

169
typescript/Frontend/src/dataCache/dataCache.ts Normal file
View File

@ -0,0 +1,169 @@
/* eslint-disable no-mixed-operators */
import {TimeSpan, UnixTime} from "./time";
import {Observable, Subject} from "rxjs";
import {SkipList} from "./skipList/skipList";
import {createDispatchQueue} from "./promiseQueue";
import {SkipListNode} from "./skipList/skipListNode";
import {RecordSeries} from "./data";
import { Maybe, isUndefined } from "./utils/maybe";
export const FetchResult =
{
notAvailable : "N/A",
tryLater : "Try Later"
} as const
export type FetchResult<T> =
| T
| typeof FetchResult.notAvailable
| typeof FetchResult.tryLater
function reverseBits(x : number): number
{
// https://stackoverflow.com/a/60227327/141397
x = (x & 0x55555555) << 1 | (x & 0xAAAAAAAA) >> 1;
x = (x & 0x33333333) << 2 | (x & 0xCCCCCCCC) >> 2;
x = (x & 0x0F0F0F0F) << 4 | (x & 0xF0F0F0F0) >> 4;
x = (x & 0x00FF00FF) << 8 | (x & 0xFF00FF00) >> 8;
x = (x & 0x0000FFFF) << 16 | (x & 0xFFFF0000) >> 16;
return x >>> 0;
}
export default class DataCache<T extends Record<string, number>>
{
private readonly cache: SkipList<Maybe<T>> = new SkipList<Maybe<T>>()
private readonly resolution: TimeSpan;
readonly _fetch: (t: UnixTime) => Promise<FetchResult<T>>;
private readonly fetchQueue = createDispatchQueue(6)
private readonly fetching: Set<number> = new Set<number>()
public readonly gotData: Observable<UnixTime>;
constructor(fetch: (t: UnixTime) => Promise<FetchResult<T>>, resolution: TimeSpan)
{
this._fetch = fetch;
this.resolution = resolution;
this.gotData = new Subject<UnixTime>()
}
public prefetch(times: Array<UnixTime>, clear = true)
{
if (clear)
{
this.fetching.clear()
this.fetchQueue.clear()
}
const timesWithPriority = times.map((time, index) => ({time, priority: reverseBits(index)}))
timesWithPriority.sort((x, y) => x.priority - y.priority)
for (let i = 0; i < timesWithPriority.length; i++)
{
const time = timesWithPriority[i].time.round(this.resolution)
const t = time.ticks;
const node = this.cache.find(t);
if (node.index !== t)
this.fetchData(time);
}
}
public get(timeStamp: UnixTime, fetch = true): Maybe<T>
{
const time = timeStamp.round(this.resolution)
const t = time.ticks;
const node = this.cache.find(t);
if (node.index === t)
return node.value
if (fetch)
this.fetchData(time);
return this.interpolate(node, t)
}
public getSeries(sampleTimes: UnixTime[]): RecordSeries
{
this.prefetch(sampleTimes)
return sampleTimes.map(time => ({time, value: this.get(time, false)}))
}
private interpolate(before: SkipListNode<Maybe<T>>, t: number): Maybe<T>
{
const dataBefore = before.value
const after = before.next[0];
const dataAfter = after.value
if (isUndefined(dataBefore) && isUndefined(dataAfter))
return undefined
if (isUndefined(dataBefore))
return dataAfter
if (isUndefined(dataAfter))
return dataBefore
const p = t - before.index
const n = after.index - t
const pn = p + n
let interpolated: Partial<Record<string, number>> = {}
//What about string nodes? like Alarms
for (const k of Object.keys(dataBefore))
{
interpolated[k] = (dataBefore[k] * n + dataAfter[k] * p) / pn
}
return interpolated as T
}
private fetchData(time: UnixTime)
{
const t = time.ticks;
if (this.fetching.has(t)) // we are already fetching t
return
const fetchTask = () =>
{
const onSuccess = (data: FetchResult<T>) =>
{
if (data === FetchResult.tryLater)
{
console.warn(FetchResult.tryLater)
return
}
const value = data === FetchResult.notAvailable ? undefined : data;
this.cache.insert(value, t)
}
const onFailure = (_: unknown) =>
{
console.error(time.ticks + " FAILED!") // should not happen
}
const dispatch = () =>
{
this.fetching.delete(time.ticks);
(this.gotData as Subject<UnixTime>).next(time);
}
return this._fetch(time)
.then(d => onSuccess(d), f => onFailure(f))
.finally(() => dispatch())
};
this.fetching.add(t)
this.fetchQueue.dispatch(() => fetchTask());
}
}

20
typescript/Frontend/src/dataCache/linq.ts Normal file
View File

@ -0,0 +1,20 @@
// 0. Import Module
import { initializeLinq, IEnumerable } from "linq-to-typescript"
// 1. Declare that the JS types implement the IEnumerable interface
declare global {
interface Array<T> extends IEnumerable<T> { }
interface Uint8Array extends IEnumerable<number> { }
interface Uint8ClampedArray extends IEnumerable<number> { }
interface Uint16Array extends IEnumerable<number> { }
interface Uint32Array extends IEnumerable<number> { }
interface Int8Array extends IEnumerable<number> { }
interface Int16Array extends IEnumerable<number> { }
interface Int32Array extends IEnumerable<number> { }
interface Float32Array extends IEnumerable<number> { }
interface Float64Array extends IEnumerable<number> { }
interface Map<K, V> extends IEnumerable<[K, V]> { }
interface Set<T> extends IEnumerable<T> { }
interface String extends IEnumerable<string> { }
}
// 2. Bind Linq Functions to Array, Map, etc
initializeLinq()

35
typescript/Frontend/src/dataCache/observableUtils.ts Normal file
View File

@ -0,0 +1,35 @@
import {map, MonoTypeOperatorFunction, Observable, tap} from "rxjs";
import {fastHash} from "./utils";
type ConcatX<T extends readonly (readonly any[])[]> = [
...T[0], ...T[1], ...T[2], ...T[3], ...T[4],
...T[5], ...T[6], ...T[7], ...T[8], ...T[9],
...T[10], ...T[11], ...T[12], ...T[13], ...T[14],
...T[15], ...T[16], ...T[17], ...T[18], ...T[19]
];
type Flatten<T extends readonly any[]> =
ConcatX<[...{ [K in keyof T]: T[K] extends any[] ? T[K] : [T[K]] }, ...[][]]>
export function flatten()
{
return function<T extends Array<unknown>>(source: Observable<T>)
{
return source.pipe
(
map(a => a.flat() as Flatten<T>)
)
}
}
type RecursiveObject<T> = T extends object ? T : never;
type Terminals<TModel, T> =
{
[Key in keyof TModel]: TModel[Key] extends RecursiveObject<TModel[Key]>
? Terminals<TModel[Key], T>
: T;
};

51
typescript/Frontend/src/dataCache/promiseQueue.ts Normal file
View File

@ -0,0 +1,51 @@
export function createDispatchQueue(maxInflight: number, debug = false): { dispatch: (task: () => Promise<void>) => number; clear: () => void }
{
const queue: Array<() => Promise<void>> = []
let inflight = 0;
function done()
{
inflight--
if (debug && inflight + queue.length === 0)
console.log("queue empty")
if (inflight < maxInflight && queue.length > 0)
{
const task = queue.pop()!
inflight++
task().finally(() => done())
}
}
function dispatch(task: () => Promise<void>) : number
{
if (inflight < maxInflight)
{
inflight++;
task().finally(() => done())
}
else
{
if (debug && queue.length === 0)
console.log("queue in use")
queue.push(task)
}
return queue.length
}
function clear()
{
// https://stackoverflow.com/questions/1232040/how-do-i-empty-an-array-in-javascript
queue.length = 0
if (debug)
console.log("queue cleared")
}
return {dispatch, clear}
}

80
typescript/Frontend/src/dataCache/skipList/skipList.ts Normal file
View File

@ -0,0 +1,80 @@
import {find, findPath, insert, Path, SkipListNode} from "./skipListNode";
export class SkipList<T>
{
public readonly head: SkipListNode<T>;
public readonly tail: SkipListNode<T>;
private readonly nLevels: number;
private _length = 0
constructor(nLevels: number = 20)
{
// TODO: auto-levels
this.tail =
{
index: Number.MAX_VALUE,
next: [],
value: undefined!
};
this.head =
{
index: Number.MIN_VALUE,
next: Array(nLevels).fill(this.tail),
value: undefined!
};
this.nLevels = nLevels
}
public find(index: number, startNode = this.head, endNode = this.tail): SkipListNode<T>
{
return find(index, startNode, endNode)
}
private findPath(index: number, startNode = this.head, endNode = this.tail): Path<T>
{
return findPath(index, startNode, endNode)
}
public insert(value: T, index: number): SkipListNode<T>
{
const path = this.findPath(index)
const node = path[0];
if (node.index === index) // overwrite
{
node.value = value
return node
}
const nodeToInsert = {value, index, next: []} as SkipListNode<T>
const rnd = (Math.random() * (1 << this.nLevels)) << 0;
for (let level = 0; level < this.nLevels; level++)
{
insert(nodeToInsert, path[level], level)
if ((rnd & (1 << level)) === 0)
break
}
this._length += 1
return nodeToInsert;
}
get length(): number
{
return this._length;
}
// public remove(index: number): void
// {
// // TODO
// }
}

52
typescript/Frontend/src/dataCache/skipList/skipListNode.ts Normal file
View File

@ -0,0 +1,52 @@
import {asMutableArray} from "../types";
export type Next<T> = { readonly next: ReadonlyArray<SkipListNode<T>> }
export type Index = { readonly index: number }
export type Indexed<T> = Index & { value: T }
export type SkipListNode<T> = Next<T> & Indexed<T>
export type Path<T> = SkipListNode<T>[];
export function find<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>): SkipListNode<T>
{
let node = startNode
for (let level = startNode.next.length - 1; level >= 0; level--)
node = findOnLevel(index, node, endNode, level)
return node
}
export function findOnLevel<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>, level: number): SkipListNode<T>
{
let node: SkipListNode<T> = startNode
while (true)
{
const next = node.next[level]
if (index < next.index || endNode.index < next.index)
return node
node = next
}
}
export function findPath<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>): Path<T>
{
const path = Array(startNode.next.length - 1)
let node = startNode
for (let level = startNode.next.length - 1; level >= 0; level--)
{
node = findOnLevel(index, node, endNode, level)
path[level] = node
}
return path
}
export function insert<T>(nodeToInsert: SkipListNode<T>, after: SkipListNode<T>, onLevel: number): void
{
asMutableArray(nodeToInsert.next)[onLevel] = after.next[onLevel]
asMutableArray(after.next)[onLevel] = nodeToInsert
}

5
typescript/Frontend/src/dataCache/stringUtils.ts Normal file
View File

@ -0,0 +1,5 @@
export function trim(str: string, string: string = " "): string
{
const pattern = '^[' + string + ']*(.*?)[' + string + ']*$';
return str.replace(new RegExp(pattern), '$1')
}

302
typescript/Frontend/src/dataCache/time.ts Normal file
View File

@ -0,0 +1,302 @@
import {trim} from "./stringUtils";
export class UnixTime
{
private constructor(readonly ticks: number)
{
}
public static readonly Epoch = new UnixTime(0)
public static now(): UnixTime
{
return UnixTime.fromTicks(Date.now() / 1000)
}
public static fromDate(date: Date): UnixTime
{
return UnixTime.fromTicks(date.getTime() / 1000)
}
public toDate(): Date
{
return new Date(this.ticks * 1000)
}
public static fromTicks(ticks: number): UnixTime
{
return new UnixTime(ticks)
}
public later(timeSpan: TimeSpan): UnixTime
{
return new UnixTime(this.ticks + timeSpan.ticks)
}
public move(ticks: number): UnixTime
{
return new UnixTime(this.ticks + ticks)
}
public earlier(timeSpan: TimeSpan): UnixTime
{
return new UnixTime(this.ticks - timeSpan.ticks)
}
public isEarlierThan(time: UnixTime): boolean
{
return this.ticks < time.ticks
}
public isEarlierThanOrEqual(time: UnixTime): boolean
{
return this.ticks <= time.ticks
}
public isLaterThan(time: UnixTime): boolean
{
return this.ticks > time.ticks
}
public isLaterThanOrEqual(time: UnixTime): boolean
{
return this.ticks >= time.ticks
}
public isEqual(time: UnixTime): boolean
{
return this.ticks === time.ticks
}
public isInTheFuture(): boolean
{
return this.isLaterThan(UnixTime.now())
}
public isInThePast(): boolean
{
return this.ticks < UnixTime.now().ticks
}
public round(ticks:number) : UnixTime
public round(duration: TimeSpan) : UnixTime
public round(durationOrTicks: TimeSpan | number) : UnixTime
{
const ticks = (typeof durationOrTicks === "number") ? durationOrTicks : durationOrTicks.ticks
return new UnixTime(Math.round(this.ticks / ticks) * ticks)
}
public rangeTo(time: UnixTime): TimeRange
{
return TimeRange.fromTimes(this, time);
}
public rangeBefore(timeSpan: TimeSpan): TimeRange
{
return TimeRange.fromTimes(this.earlier(timeSpan), this);
}
public rangeAfter(timeSpan: TimeSpan): TimeRange
{
return TimeRange.fromTimes(this, this.later(timeSpan));
}
public toString() : string
{
return this.ticks.toString()
}
}
export class TimeSpan
{
private constructor(readonly ticks: number) {}
get milliSeconds(): number { return this.ticks * 1000 }
get seconds() : number { return this.ticks }
get minutes() : number { return this.ticks / 60 }
get hours() : number { return this.minutes / 60 }
get days() : number { return this.hours / 24 }
get weeks() : number { return this.days / 7 }
public static fromTicks (t: number): TimeSpan { return new TimeSpan(t) }
public static fromSeconds(t: number): TimeSpan { return TimeSpan.fromTicks(t) }
public static fromMinutes(t: number): TimeSpan { return TimeSpan.fromSeconds(t*60) }
public static fromHours (t: number): TimeSpan { return TimeSpan.fromMinutes(t*60) }
public static fromDays (t: number): TimeSpan { return TimeSpan.fromHours(t*24) }
public static fromWeeks (t: number): TimeSpan { return TimeSpan.fromDays(t*7) }
public static span(from: UnixTime, to: UnixTime) : TimeSpan
{
return TimeSpan.fromTicks(Math.abs(to.ticks - from.ticks))
}
public add(timeSpan: TimeSpan) : TimeSpan
{
return TimeSpan.fromTicks(this.ticks + timeSpan.ticks)
}
public subtract(timeSpan: TimeSpan) : TimeSpan
{
return TimeSpan.fromTicks(this.ticks - timeSpan.ticks)
}
public divide(n: number) : TimeSpan
{
if (n <= 0)
throw 'n must be positive';
return TimeSpan.fromTicks(this.ticks/n)
}
public multiply(n: number) : TimeSpan
{
if (n < 0)
throw 'n cannot be negative';
return TimeSpan.fromTicks(this.ticks * n)
}
public round(ticks:number) : TimeSpan
public round(duration: TimeSpan) : TimeSpan
public round(durationOrTicks: TimeSpan | number) : TimeSpan
{
const ticks = (typeof durationOrTicks === "number")
? durationOrTicks
: durationOrTicks.ticks
return TimeSpan.fromTicks(Math.round(this.ticks / ticks) * ticks)
}
public toString() : string
{
let dt = 60*60*24*7
let ticks = this.ticks;
if (ticks === 0)
return "0s"
ticks = Math.abs(ticks)
const nWeeks = Math.floor(ticks / dt)
ticks -= nWeeks * dt
dt /= 7
const nDays = Math.floor(ticks / dt)
ticks -= nDays * dt
dt /= 24
const nHours = Math.floor(ticks / dt)
ticks -= nHours * dt
dt /= 60
const nMinutes = Math.floor(ticks / dt)
ticks -= nMinutes * dt
dt /= 60
const nSeconds = Math.floor(ticks / dt)
let s = ""
if (nWeeks > 0) s += nWeeks .toString() + "w "
if (nDays > 0) s += nDays .toString() + "d "
if (nHours > 0) s += nHours .toString() + "h "
if (nMinutes > 0) s += nMinutes.toString() + "m "
if (nSeconds > 0) s += nSeconds.toString() + "s"
return trim(s);
}
}
export class TimeRange
{
private constructor(private readonly from: number, private readonly to: number)
{
}
public get start(): UnixTime
{
return UnixTime.fromTicks(this.from)
}
public get mid(): UnixTime
{
return UnixTime.fromTicks((this.from + this.to) / 2)
}
public get end(): UnixTime
{
return UnixTime.fromTicks(this.to)
}
public get duration(): TimeSpan
{
return TimeSpan.fromTicks(this.to - this.from)
}
public static fromTimes(from: UnixTime, to: UnixTime): TimeRange
{
return from.isLaterThan(to)
? new TimeRange(to.ticks, from.ticks)
: new TimeRange(from.ticks, to.ticks)
}
public isInside(time: number) : boolean;
public isInside(time: UnixTime) : boolean;
public isInside(time: UnixTime | number)
{
const t = time instanceof UnixTime ? time.ticks : time
return t >= this.from && t < this.to
}
public sample(period: TimeSpan): UnixTime[]
{
const samples = []
for (let t = this.from; t < this.to; t += period.ticks)
samples.push(UnixTime.fromTicks(t));
return samples
}
public subdivide(n: number) : TimeRange[]
{
if (n <= 0)
throw 'n must be positive';
const period = TimeSpan.fromTicks(this.duration.ticks / n);
if (period === this.duration)
return [this];
const samples = this.sample(period);
const ranges : TimeRange[] = []
for (let i = 0; i < samples.length;)
ranges.push(TimeRange.fromTimes(samples[i], samples[++i]))
return ranges
}
public earlier(dt: TimeSpan) : TimeRange
{
return new TimeRange(this.from - dt.ticks, this.to - dt.ticks)
}
public later(dt: TimeSpan) : TimeRange
{
return new TimeRange(this.from + dt.ticks, this.to + dt.ticks)
}
public move(ticks: number) : TimeRange
{
return new TimeRange(this.from + ticks, this.to + ticks)
}
}

21
typescript/Frontend/src/dataCache/types.ts Normal file
View File

@ -0,0 +1,21 @@
import {UnixTime} from "./time";
export type Timestamped<T> = { time: UnixTime, value: T }
export type Pair<T1, T2 = T1> = [T1, T2]
export type Position = { readonly x: number, readonly y: number }
export type Direction = { readonly dx: number, readonly dy: number }
export type Size = { readonly width: number, readonly height: number }
export type Rect = Position & Size
export type Mutable<T> = { -readonly [P in keyof T]: T[P] };
export type FieldKey<T> = { [P in keyof T]: T[P] extends (...args: any) => any ? never : P }[keyof T];
export type AllFields<T> = Pick<T, FieldKey<T>>;
export type SomeFields<T> = Partial<AllFields<T>>
export const asMutable = <T>(t: T) => (t as Mutable<T>);
export const asMutableArray = <T>(t: ReadonlyArray<T>) => (t as Array<T>);
export const cast = <T>(t: unknown) => (t as T);
export type Rename<T, K extends keyof T, N extends string> = Pick<T, Exclude<keyof T, K>> & { [P in N]: T[K] }

119
typescript/Frontend/src/dataCache/utils.ts Normal file
View File

@ -0,0 +1,119 @@
import {IEnumerable} from "linq-to-typescript";
import { isDefined } from "./utils/maybe";
//export type UnionToIntersection<U> = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I : never
export type Nothing = Record<string, never>
// eslint-disable-next-line @typescript-eslint/no-empty-function
export function fastHash(str: string): number
{
const signed = str
.split('')
.reduce((p, c) => ((p << 5) - p) + c.charCodeAt(0) | 0, 0);
return Math.abs(signed);
}
// export function flattenObject(obj: object) : object
// {
// const flattened = {}
//
// for (const key of Object.keys(obj))
// {
// // @ts-ignore
// const value = obj[key]
//
// if (typeof value === 'object' && value !== null && !Array.isArray(value))
// {
// Object.assign(flattened, flattenObject(value))
// }
// else
// {
// // @ts-ignore
// flattened[key] = value
// }
// }
//
// return flattened
// }
//return function<TTerminal,TTree extends Terminals<TTree, TTerminal>>(source: Observable<TTerminal>)
export function* pairwise<T>(iterable: Iterable<T>, init?: T): Generator<[T, T]>
{
const it = iterable[Symbol.iterator]()
let first : T;
if (isDefined(init))
{
first = init
}
else
{
const f = it.next()
if (f.done)
return
first = f.value
}
let second = it.next()
while(!second.done)
{
yield [first, second.value]
first = second.value
second = it.next()
}
}
export function arraysEqual<T>(a: Array<T>, b: Array<T>)
{
if (a === b) return true;
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; ++i)
{
if (a[i] !== b[i]) return false;
}
return true;
}
export function mod(a:number, b:number)
{
return ((a % b) + b) % b;
}
export function clamp(a: number, min: number, max: number)
{
return a > max ? max
: a < min ? min
: a
}
export function isDST(d : Date)
{
const jan = new Date(d.getFullYear(), 0, 1).getTimezoneOffset();
const jul = new Date(d.getFullYear(), 6, 1).getTimezoneOffset();
return Math.max(jan, jul) != d.getTimezoneOffset();
}
export function Transpose<T>(src: IEnumerable<IEnumerable<T>>): IEnumerable<IEnumerable<T>>
{
return src
.selectMany(line => line.select((element, column) => ({element, column})))
.groupBy(i => i.column)
.select(g => g.select(e => e.element));
}

46
typescript/Frontend/src/dataCache/utils/fileSystem.ts Normal file
View File

@ -0,0 +1,46 @@
import fs from "fs";
export function doesFileExist(path: string): boolean
{
try
{
fs.accessSync(path, fs.constants.F_OK);
return true;
}
catch (e)
{
return false;
}
}
export function doesDirExist(path: string): boolean
{
try
{
fs.readdirSync(path)
return true;
}
catch (e)
{
return false;
}
}
export function readJsonFile<T>(path: string)
{
const data = fs.readFileSync(path, "utf-8")
return JSON.parse(data) as T
}
export function writeJsonFile<T>(path: string, contents: T)
{
const data = JSON.stringify(contents)
return fs.writeFileSync(path, data, "utf-8")
}
export function writeJsonFilePretty<T>(path: string, contents: T)
{
const data = JSON.stringify(contents,undefined,2)
return fs.writeFileSync(path, data, "utf-8")
}

127
typescript/Frontend/src/dataCache/utils/httpResponse.ts Normal file
View File

@ -0,0 +1,127 @@
import MimeType from "./mime";
import fs from "fs";
import http, {IncomingMessage, ServerResponse} from "http";
import { Maybe } from "yup";
import { getLogger } from "./logging";
import { isDefined, isUndefined } from "./maybe";
import { Dictionary } from "./utilityTypes";
import { promisify } from "util";
import { entries } from "./utils";
const log = getLogger("HTTP")
const readFile = promisify(fs.readFile)
export type HttpResponse = {
body: Maybe<Buffer | string>
headers : Dictionary<string>
statusCode: number
}
function contentTypeHeader(mimeType: string)
{
return {'Content-type': mimeType};
}
function forbidden(message = "403 : forbidden", headers: Dictionary<string> = {}): HttpResponse
{
return text(message, 403, headers)
}
function notFound(message ="404 : not found", headers: Dictionary<string> = {}): HttpResponse
{
return text(message, 404, headers)
}
function text(text: string, statusCode = 200, headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: statusCode,
headers : {...headers, ...contentTypeHeader('text/plain')},
body : text
};
}
function json(json: Dictionary,
replacer?: (k: string, v: unknown) => unknown,
headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: 200,
headers : {...headers, 'Content-type': MimeType.json},
body: JSON.stringify(json, replacer)
}
}
function empty(headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: 200,
headers,
body: undefined
}
}
function ok(body: Maybe<Buffer | string>, headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: 200,
headers,
body
}
}
async function file(localRootPath: string, urlPath: string, headers: Dictionary<string> = {}, defaultPath = "/"): Promise<HttpResponse>
{
if (urlPath.includes('..'))
return HTTP.forbidden();
const localPath = localRootPath + (urlPath === "/" ? defaultPath : urlPath);
const body = await readFile(localPath).catch(_ => undefined)
if (isUndefined(body))
return HTTP.notFound();
if (!('Content-type' in headers))
{
headers = {...headers, ...contentTypeHeader(MimeType.guessFromPath(localPath))}
}
return HTTP.ok(body, headers)
}
function createServer(serve: (request: IncomingMessage) => Promise<HttpResponse>)
{
async function wrapServe(request: IncomingMessage, response: ServerResponse): Promise<void>
{
const r = await serve(request)
entries(r.headers).forEach(([k, v]) => response.setHeader(k, v as any))
response.statusCode = r.statusCode
if (isDefined(r.body))
response.end(r.body)
else
response.end()
}
return http.createServer(wrapServe);
}
const HTTP =
{
contentTypeHeader,
forbidden,
notFound,
ok,
json,
empty,
file,
createServer
}
export default HTTP;

21
typescript/Frontend/src/dataCache/utils/linq.ts Normal file
View File

@ -0,0 +1,21 @@
// 0. Import Module
import {IEnumerable, initializeLinq} from "linq-to-typescript"
// 1. Declare that the JS types implement the IEnumerable interface
declare global {
interface Array<T> extends IEnumerable<T> { }
interface Uint8Array extends IEnumerable<number> { }
interface Uint8ClampedArray extends IEnumerable<number> { }
interface Uint16Array extends IEnumerable<number> { }
interface Uint32Array extends IEnumerable<number> { }
interface Int8Array extends IEnumerable<number> { }
interface Int16Array extends IEnumerable<number> { }
interface Int32Array extends IEnumerable<number> { }
interface Float32Array extends IEnumerable<number> { }
interface Float64Array extends IEnumerable<number> { }
interface Map<K, V> extends IEnumerable<[K, V]> { }
interface Set<T> extends IEnumerable<T> { }
interface String extends IEnumerable<string> { }
}
// 2. Bind Linq Functions to Array, Map, etc
initializeLinq()

11
typescript/Frontend/src/dataCache/utils/logging.ts Normal file
View File

@ -0,0 +1,11 @@
let subsystemPadding = 0
export function getLogger(subsystem: string): (msg: string) => void
{
subsystemPadding = Math.max(subsystem.length, subsystemPadding)
// eslint-disable-next-line no-console
return (msg: string) => console.log(`${new Date().toLocaleString()} | ${(subsystem.padEnd(subsystemPadding))} | ${msg}`);
}

176
typescript/Frontend/src/dataCache/utils/match.ts Normal file
View File

@ -0,0 +1,176 @@
import {Dictionary, Func, Normalize1} from "./utilityTypes";
import {isUndefined} from "./maybe";
import {UnionToIntersection} from "simplytyped";
import {current} from "immer";
import { keys, valueToFunction } from "./utils";
// Type Compatibility
// https://www.typescriptlang.org/docs/handbook/type-compatibility.html
//TODO: review
export type IsUnionCase<T> =
T extends Dictionary
? [UnionToIntersection<keyof T>] extends [keyof T]
? [keyof T] extends [UnionToIntersection<keyof T>]
? true
: false
: false
: false
//TODO: review
export type IsTaggedUnion<T> = true extends UnionToIntersection<IsUnionCase<T>> ? Dictionary : never
export type Unwrap<U extends IsTaggedUnion<U>> = UnionToIntersection<U>[keyof UnionToIntersection<U>] ;
export function update<U extends IsTaggedUnion<U>>(u: U, e: Partial<Unwrap<U>>)
{
const v = u as UnionToIntersection<U>
const o = current(v)
const ks = keys(v)
if (ks.length != 1)
throw new Error("not a valid union case")
const tag = ks[0]
const before = v[tag];
const before2 = current(before);
const newVar = {...before, ...e};
v[tag] = newVar
}
export function unwrap<U extends IsTaggedUnion<U>>(u: U) : Normalize1<Unwrap<U>>
{
const v = u as UnionToIntersection<U>
const ks = keys(v)
if (ks.length != 1)
throw new Error("not a valid union case")
const key = ks[0]
return v[key] as any;
}
export function base<U extends IsTaggedUnion<U>>(u: U): Normalize1<Unwrap<U> & Partial<UnionToIntersection<Unwrap<U>>>>
{
return unwrap(u) as Normalize1<Unwrap<U> & Partial<UnionToIntersection<Unwrap<U>>>>
}
export function tag<U extends IsTaggedUnion<U>>(u: U) : keyof UnionToIntersection<U>
{
const v = u as UnionToIntersection<U>
const ks = keys(v)
if (ks.length != 1)
throw new Error("not a valid union case")
return ks[0]
}
export function tagsEqual<U extends IsTaggedUnion<U>>(u: U, v: U) : v is U
{
return tag(u) === tag(v)
}
type MapFuncs<U> = { [k in keyof UnionToIntersection<U>]: Func<UnionToIntersection<U>[k]> }
type OtherwiseKeys<U,M> = Exclude<keyof UnionToIntersection<U>, keyof M>;
type OtherwiseArg<U,M> = {
[k in keyof UnionToIntersection<U>]: Record<k, UnionToIntersection<U>[k]>
}[OtherwiseKeys<U,M>]
type OtherwiseFunc<U, M extends Partial<MapFuncs<U>>, R> = Func<OtherwiseArg<U,M> extends never ? unknown : OtherwiseArg<U,M>, R>;
export function match<U extends IsTaggedUnion<U>, M extends Partial<MapFuncs<U>>, R>(uCase: U, matchFuncs: M, otherwise: OtherwiseFunc<U, M, R> | R):{ [k in keyof M]: M[k] extends Func<any, infer O> ? O : never }[keyof M] | R
{
const otw = valueToFunction(otherwise)
const c = uCase as UnionToIntersection<U>
const ks = keys(c)
if (ks.length != 1)
return otw(c)
const key = ks[0]
const arg = c[key]
const matchFunc = matchFuncs[key]
if (isUndefined(matchFunc))
return otw(c);
return matchFunc(arg as any) as any;
}
export function dispatch<U extends IsTaggedUnion<U>>()
{
// type Intersection = UnionToIntersection<U>;
//
// type MapFuncs = { [k in keyof Intersection]: Func<Intersection[k]> }
// type OtherwiseKeys<M> = Exclude<keyof Intersection, keyof M>;
//
// type OtherwiseArg<M> = {
// [k in keyof Intersection]: Record<k, Intersection[k]>
// }[OtherwiseKeys<M>]
//
// type OtherwiseFunc<M extends Partial<MapFuncs>, R> = Func<OtherwiseArg<M> extends never ? unknown : OtherwiseArg<M>, R>;
return <M extends Partial<MapFuncs<U>>, R>(matchFuncs: M, otherwise: OtherwiseFunc<U,M, R> | R) =>
{
const otw = valueToFunction(otherwise)
return (uCase: U): { [k in keyof M]: M[k] extends Func<any, infer O> ? O : never }[keyof M] | R =>
{
const c = uCase as UnionToIntersection<U>
const ks = keys(c)
if (ks.length != 1)
return otw(c)
const key = ks[0]
const arg = c[key]
const matchFunc = matchFuncs[key]
if (isUndefined(matchFunc))
return otw(c);
return matchFunc(arg as any) as any;
}
};
}
export function concat<R extends Record<keyof any, any>, T extends Dictionary>(rec: R, t:T)
{
const result = {} as {
[k in keyof UnionToIntersection<R>]: Record<k, UnionToIntersection<R>[k] & T>
}[keyof UnionToIntersection<R>]
for (const k in rec)
{
// @ts-ignore
result[k] = { ...rec[k], ...t}
}
return result
}

16
typescript/Frontend/src/dataCache/utils/maybe.ts Normal file
View File

@ -0,0 +1,16 @@
export type Maybe<T> = T | undefined | null;
export function isDefined<T>(e: Maybe<T>): e is T
{
return e != undefined // != by design to include null
}
export function isUndefined<T>(e: Maybe<T>): e is undefined | null
{
return e == undefined // == by design to include null
}
export function toArray<T>(e: Maybe<T>): T[]
{
return isDefined(e) ? [e] : []
}

18
typescript/Frontend/src/dataCache/utils/milliseconds.ts Normal file
View File

@ -0,0 +1,18 @@
export type Milliseconds = number
export const Milliseconds =
{
fromSeconds: (count: number): Milliseconds => count * 1000,
fromMinutes: (count: number): Milliseconds => count * 1000 * 60,
fromHours : (count: number): Milliseconds => count * 1000 * 60 * 60,
fromDays : (count: number): Milliseconds => count * 1000 * 60 * 60 * 24,
fromWeeks : (count: number): Milliseconds => count * 1000 * 60 * 60 * 24 * 7,
toSeconds: (count: Milliseconds): number => count / 1000,
toMinutes: (count: Milliseconds): number => count / 1000 / 60,
toHours : (count: Milliseconds): number => count / 1000 / 60 / 60,
toDays : (count: Milliseconds): number => count / 1000 / 60 / 60 / 24,
toWeeks : (count: Milliseconds): number => count / 1000 / 60 / 60 / 24 / 7,
} as const

32
typescript/Frontend/src/dataCache/utils/mime.ts Normal file
View File

@ -0,0 +1,32 @@
import PlatformPath from "path";
import { isDefined } from "./maybe";
function guessFromPath(path: string) : string
{
const ext = PlatformPath.parse(path).ext?.substring(1) as keyof typeof MimeType;
const mimeType = MimeType[ext]
return isDefined(mimeType) && typeof mimeType === "string"
? mimeType
: 'application/octet-stream'
}
const MimeType =
{
ico : 'image/x-icon',
html: 'text/html; charset=UTF-8',
js : 'text/javascript',
json: 'application/json; charset=UTF-8',
css : 'text/css; charset=UTF-8',
png : 'image/png',
jpg : 'image/jpeg',
wav : 'audio/wav',
mp3 : 'audio/mpeg',
svg : 'image/svg+xml; charset=UTF-8',
pdf : 'application/pdf',
guessFromPath
};
export default MimeType

81
typescript/Frontend/src/dataCache/utils/path.ts Normal file
View File

@ -0,0 +1,81 @@
import {isUndefined} from "./maybe";
import {from, IEnumerable} from "linq-to-typescript";
import {isBoolean, isNumber, isPlainObject, isString} from "./runtimeTypeChecking";
function getAt(root: any, path: (keyof any)[])
{
return path.reduce((v, p) => v[p], root)
}
function iterate(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
{
if (isUndefined(root))
return []
return from(iterate(root))
function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
{
if (isString(node) || isNumber(node) || isBoolean(node))
yield {path, node}
else if (isPlainObject(node))
for (const key in node)
{
path.push(key)
yield {path, node}
yield* iterate(node[key], path)
path.pop()
}
}
}
function iterateLeafs(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
{
if (isUndefined(root))
return []
return from(iterate(root))
function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
{
if (isString(node) || isNumber(node) || isBoolean(node))
yield {path, node}
else if (isPlainObject(node))
for (const key in node)
{
path.push(key)
yield* iterate(node[key], path)
path.pop()
}
}
}
function iterateBranches(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
{
if (isUndefined(root))
return []
return from(iterate(root))
function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
{
if (isPlainObject(node))
for (const key in node)
{
path.push(key)
yield {path, node}
yield* iterate(node[key], path)
path.pop()
}
}
}
export const Path =
{
iterate,
iterateLeafs,
iterateBranches,
getAt
} as const

45
typescript/Frontend/src/dataCache/utils/requestUtils.ts Normal file
View File

@ -0,0 +1,45 @@
import {IncomingMessage} from "http";
import {firstValueFrom, map, Observable, startWith, toArray} from "rxjs";
export function observeData(request: IncomingMessage, maxLength: number = Number.POSITIVE_INFINITY): Observable<Uint8Array>
{
let nBytes = 0;
return new Observable<Uint8Array>(subscriber =>
{
request.on('end', () => subscriber.complete());
request.on('data', (data: Uint8Array) =>
{
nBytes += data.byteLength
if (nBytes <= maxLength)
subscriber.next(data);
else
{
const error = `too much data: expected ${maxLength} bytes or less, got ${nBytes} bytes.`;
subscriber.error(error);
request.destroy(new Error(error))
}
});
});
}
export async function getRequestJson<T = unknown>(request: IncomingMessage, maxLength = 500000): Promise<T>
{
const data = await getData(request, maxLength)
return JSON.parse(data.toString())
}
const noData = new Uint8Array(0);
export function getData(request: IncomingMessage, maxLength: number = Number.POSITIVE_INFINITY): Promise<Buffer>
{
const data = observeData(request, maxLength).pipe
(
startWith(noData),
toArray(),
map(b => Buffer.concat(b)), // cannot inline!
)
return firstValueFrom(data);
}

66
typescript/Frontend/src/dataCache/utils/runtimeTypeChecking.ts Normal file
View File

@ -0,0 +1,66 @@
export type TypeCode =
| "undefined"
| "object"
| "boolean"
| "number"
| "string"
| "function"
| "symbol"
| "bigint";
export type PlainObject<K extends keyof any = keyof any, V = unknown> = Record<K, V>
export function isObject(thing: unknown) : thing is object
{
return typeof thing === "object"
}
export function isDate(thing: unknown) : thing is Date
{
return thing instanceof Date
}
export function isPlainObject(thing: unknown) : thing is PlainObject
{
return isObject(thing) && !isDate(thing)
}
export function isArray(thing: unknown) : thing is Array<unknown>
{
return Array.isArray(thing)
}
export function isNumber(thing: unknown) : thing is number
{
return typeof thing === "number"
}
export function isBoolean(thing: unknown) : thing is boolean
{
return typeof thing === "boolean"
}
export function isString(thing: unknown) : thing is string
{
return typeof thing === "string"
}
// export function isFunction(thing: unknown): thing is (...args: unknown[]) => unknown
// {
// return typeof thing === "function"
// }
export function isFunction(obj: unknown): obj is (...args: any[]) => any
{
return obj instanceof Function;
}
export function isSymbol(thing: unknown) : thing is symbol
{
return typeof thing === "symbol"
}
export function isBigint(thing: unknown) : thing is bigint
{
return typeof thing === "bigint"
}

22
typescript/Frontend/src/dataCache/utils/stringUtils.ts Normal file
View File

@ -0,0 +1,22 @@
export function toLowercaseAscii(string: string)
{
return string
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase();
}
export function containsIgnoringAccents(string: string, substring: string)
{
if (substring === "") return true;
if (string === "") return false;
substring = "" + substring;
if (substring.length > string.length)
return false;
return toLowercaseAscii(string).includes(toLowercaseAscii(substring));
}

71
typescript/Frontend/src/dataCache/utils/tree.ts Normal file
View File

@ -0,0 +1,71 @@
import {from, IEnumerable} from "linq-to-typescript";
import {isDefined, isUndefined, Maybe} from "./maybe";
export function Tree<T>(getChildren: (t: T) => IEnumerable<T>)
{
function iterate(root: Maybe<T>): IEnumerable<T>
{
if (isUndefined(root))
return []
return from(iterateTree())
function* iterateTree()
{
const queue: T[] = [root!]
do
{
const element = queue.shift()!
yield element
for (const child of getChildren(element))
queue.push(child)
}
while (queue.length > 0)
}
}
function iterateWithPath(root: Maybe<T>): IEnumerable<T[]>
{
return isDefined(root)
? from(iterateTreeWithPath())
: [];
function* iterateTreeWithPath()
{
const stack: Array<Array<T>> = [[root!]]
while (true)
{
const head = stack[0];
if (head.length > 0)
{
yield stack
.select(l => l[0])
.toArray()
const children = getChildren(head[0]).toArray()
stack.unshift(children)
}
else
{
stack.shift() // remove empty array in front
if(stack.length > 0)
stack[0].shift()
else
break;
}
}
}
}
return {
iterate,
iterateWithPath
} as const
}

115
typescript/Frontend/src/dataCache/utils/type.ts Normal file
View File

@ -0,0 +1,115 @@
export {}
// export type Type =
// | "number"
// | "object"
// | "string"
// | "never"
// | "any"
// | "unknown"
// | "undefined"
// | "boolean"
// | "bigint"
// | "symbol"
// | Property[]
// | Func
//
// export type Key = "string" | "number" | "symbol"
//
// export type Property = Func |
// {
// key: Key,
// type: Type,
// readonly? : boolean,
// nullable? : boolean
// }
//
// export type Arg =
// {
// name: string,
// type: Type,
// nullable? : boolean
// }
//
// export type Func =
// {
// args: Arg[],
// returnType: Type,
// }
//
//
// type X = Partial<any>
//
// export function render(t: Type, indent = 0)
// {
// if (typeof t === "string")
// return t
//
//
// return "ERROR"
// }
//
type DeviceType =
| "Pv"
| "Load"
| "Battery"
| "Grid"
| "Inverter"
| "AcInToAcOut"
| "DcDc"
| "AcInBus"
| "AcOutBus"
| "DcBus"
| "Dc48Bus" // low voltage DC Bus, to be eliminated in later versions
type Phase =
{
voltage : number // U, non-negative
current : number // I, sign depends on device type, see sign convention below
}
type AcPhase = Phase &
{
phi : number // [0,2pi)
}
type Device =
{
Type: DeviceType,
Name?: string,
}
type Stack =
{
Top? : Device[], // 0 to N
Right? : Device // 0 or 1
Bottom? : Device[] // 0 to N
Disconnected?: boolean // not present = false
}
/// A DC device must have a field denoting its DC connection
type DcDevice = Device &
{
Dc : Phase
}
/// An AC device can have 1 to 3 AC phases
/// An AC device also needs a Frequency measurement
/// Total power can be obtained by summing the power of the phases
type AcDevice = Device &
{
Ac: AcPhase[]
Frequency: number
}
/// A low voltage 48V DC device
/// Needed to distinguish the two sides of the DCDC
/// Will be dropped once we get HV batteries
type Dc48Device = Device &
{
dc48 : Phase
}

52
typescript/Frontend/src/dataCache/utils/utilityTypes.ts Normal file
View File

@ -0,0 +1,52 @@
import {UnionToIntersection} from "simplytyped";
export type Dictionary<T = unknown> = Record<string, T>
export type Nothing = Dictionary<never>
export type IsUnion<T> = [T] extends [UnionToIntersection<T>] ? false : true;
export type UnionToDeepPartialIntersection<U> = DeepPartial<UnionToIntersection<U>>
export type UnionToPartialIntersection<U> = Partial<UnionToIntersection<U>>
export type Func<T = unknown, R = unknown> = (arg: T) => R
export type AsyncFunc<T = unknown, R = unknown> = (arg: T) => Promise<R>
export type SyncAction<T> = (arg: T) => void
export type AsyncAction<T> = (arg: T) => Promise<void>
export type Action<T> = SyncAction<T> | AsyncAction<T>
export type Lazy<T> = () => T
export type Base64 = string
export type ValueOf<T> = T[keyof T];
export type DeepPartial<T> = T extends object ? { [P in keyof T]?: DeepPartial<T[P]>; } : T;
export type DeepMutable<T> = { -readonly [P in keyof T]: DeepMutable<T[P]> };
export type Mutable<T> = { -readonly [P in keyof T]: T[P] };
export type NumberLiteralToStringLiteral<T> = T extends number ? `${T}` : T
export type KeyedChildren<T> = { children?: Dictionary<T> }
export type Union<K extends string, V extends string = string> = { [S in K] : V}
export type IntersectionToUnion<T extends Dictionary> = { [Prop in keyof T]: Record<Prop, T[Prop]> }[keyof T] // not sure if this is aptly named
// helper to flatten (instantiate) types in editor popups
// eslint-disable-next-line @typescript-eslint/ban-types
export type Normalize<T> = T extends (...args: infer A) => infer R ? (...args: Normalize<A>) => Normalize<R>
: [T] extends [any] ? { [K in keyof T]: Normalize<T[K]> }
: never
export type Normalize1<T> = T extends (...args: infer A) => infer R ? (...args: A) => R
: [T] extends [any] ? { [K in keyof T]: T[K] }
: T
export type Normalize2<T> = T extends (...args: infer A) => infer R ? (...args: Normalize1<A>) => Normalize1<R>
: [T] extends [any] ? { [K in keyof T]: Normalize1<T[K]> }
: never
export function mutable<T>(t: T)
{
return t as Mutable<T>
}

55
typescript/Frontend/src/dataCache/utils/utils.ts Normal file
View File

@ -0,0 +1,55 @@
import {IncomingMessage} from "http";
import {from} from "linq-to-typescript";
import { Maybe, isUndefined } from "./maybe";
import { Dictionary, Func } from "./utilityTypes";
type StringValued<T> =
{
[Key in keyof T]: T[Key] extends number ? Maybe<string>
: T[Key] extends string ? Maybe<string>
: T[Key] extends boolean ? Maybe<string>
: never
}
export function getQueryParams<T>(request: IncomingMessage): Maybe<StringValued<T>>
{
if (isUndefined(request.url))
return undefined
const url = new URL(request.url, `https://${request.headers.host}/`);
const query: Dictionary = {}
const urlSearchParams = new URLSearchParams(url.search);
if (!from(urlSearchParams.entries()).any())
return undefined
for (const [key, value] of urlSearchParams.entries())
query[key] = value;
return query as StringValued<T>;
}
export function getPath(req: IncomingMessage)
{
return new URL(req.url!, `https://${req.headers.host}/`).pathname;
}
export function entries<T>(t: T)
{
return Object.entries(t as Dictionary)
}
export function keys<T>(t: T): (keyof T)[]
{
return Object.keys(t as object) as (keyof T)[]
}
export function valueToFunction<T,R>(tr: Func<T,R> | R) : Func<T,R>
{
if (typeof tr === "function")
return tr as Func<T,R>
return (_: T) => tr
}

68
typescript/Frontend/src/util/graph.util.tsx Normal file
View File

@ -0,0 +1,68 @@
import { Datum, TypedArray } from "plotly.js";
import { RecordSeries } from "../dataCache/data";
import { isDefined } from "../dataCache/utils/maybe";
export const mergeDeep = (...objects: any[]) => {
const isObject = (obj: GraphCoordinates) => obj && typeof obj === "object";
return objects.reduce((prev, obj) => {
Object.keys(obj).forEach((key) => {
const pVal = prev[key];
const oVal = obj[key];
if (Array.isArray(pVal) && Array.isArray(oVal)) {
prev[key] = pVal.concat(...oVal);
} else if (isObject(pVal) && isObject(oVal)) {
prev[key] = mergeDeep(pVal, oVal);
} else {
prev[key] = oVal;
}
});
return prev;
}, {} as GraphData);
};
export const transformToGraphData = (timeStampData: RecordSeries) => {
return timeStampData.reduce((acc, curr) => {
if (isDefined(curr.value)) {
const timeStampObj = Object.keys(curr.value).reduce(
(pathAcc, currPath) => {
if (currPath) {
return {
...pathAcc,
[currPath]: {
x: [curr.time.ticks],
y: [curr.value ? curr.value[currPath] : 0],
},
};
}
return pathAcc;
},
{} as GraphData
);
return mergeDeep(acc, timeStampObj);
}
return acc;
}, {} as GraphData);
};
export interface GraphCoordinates {
x: Datum[] | Datum[][] | TypedArray;
y: Datum[] | Datum[][] | TypedArray;
}
export interface GraphData {
[path: string]: GraphCoordinates;
}
export const parseCsv = (text: string) => {
const y = text
.split(/\r?\n/)
.map((l) => l.split(";"))
.filter((fields) => !isNaN(parseFloat(fields[1])));
const x = y
.map((fields) => ({ [fields[0]]: parseFloat(fields[1]) }))
.reduce((acc, current) => ({ ...acc, ...current }), {});
return x;
};

1
typescript/Frontend/tsconfig.json
View File

@ -7,6 +7,7 @@
"esnext" "esnext"
], ],
"allowJs": true, "allowJs": true,
"downlevelIteration": true,
"skipLibCheck": true, "skipLibCheck": true,
"esModuleInterop": true, "esModuleInterop": true,
"allowSyntheticDefaultImports": true, "allowSyntheticDefaultImports": true,