add WIP DataCache

typescript/DataCache/dist/www/icons

@@ -0,0 +1 @@
+../../src/client/icons

typescript/DataCache/dist/www/index.html

@@ -0,0 +1 @@
+../../src/client/index.html

typescript/DataCache/package.json

@@ -0,0 +1,30 @@
+{
+  "main": "src/server.ts",
+  "type": "commonjs",
+  "devDependencies": {
+    "@types/node": "^17.0.45",
+    "@typescript-eslint/eslint-plugin": "^5.12.1",
+    "@typescript-eslint/parser": "^5.12.1",
+    "eslint": "^8.10.0",
+    "node-fetch": "^3.2.0",
+    "simplytyped": "^3.3.0",
+    "ts-auto-guard": "^2.4.1",
+    "typescript": "^4.8.3"
+  },
+  "scripts": {
+    "buildServer": "esbuild src/server.ts --bundle --sourcemap --outfile=dist/server.js --platform=node",
+    "buildAndRunServer": "npm run buildServer && node dist/server.js",
+    "buildClient": "esbuild src/client.ts --bundle --sourcemap --outfile=dist/www/client.js",
+    "buildClientMinified": "esbuild src/client.ts --bundle --minify --sourcemap --outfile=dist/www/client.js"
+  },
+  "dependencies": {
+    "@types/react": "^18.0.14",
+    "@types/react-dom": "^18.0.5",
+    "esbuild": "^0.14.23",
+    "immer": "^9.0.16",
+    "linq-to-typescript": "^10.0.0",
+    "preact": "^10.8.2",
+    "rxjs": "^7.5.5",
+    "ts-proto": "^1.104.0"
+  }
+}

typescript/DataCache/src/server.ts

@@ -0,0 +1,30 @@
+import './utils/linq'
+import DataCache            from "./server/dataCache/dataCache";
+import {TimeSpan, UnixTime} from "./server/dataCache/time";
+import {filter}             from "rxjs";
+
+
+const resolution = TimeSpan.fromSeconds(2);
+
+const cache = new DataCache(x => (Promise.resolve({foo: 0})), resolution)
+
+const sampleTimes = [1, 2, 3, 4, 5, 6].select(e => UnixTime.fromTicks(e)).toArray();
+
+const series = cache.getSeries(sampleTimes)
+
+console.log(series)
+
+console.log("done")
+
+
+
+
+
+
+
+
+
+
+
+
+

typescript/DataCache/src/server/dataCache/S3/S3Access.ts

@@ -0,0 +1,79 @@
+import {sha1Hmac} from "./Sha1";
+import {Utf8} from "./Utf8";
+import {toBase64} from "./UInt8Utils";
+
+export class S3Access
+{
+    constructor
+    (
+        readonly bucket:      string,
+        readonly region:      string,
+        readonly provider:    string,
+        readonly key:         string,
+        readonly secret:      string,
+        readonly contentType: string
+    )
+    {}
+
+    get host() : string { return `${this.bucket}.${this.region}.${this.provider}` }
+    get url()  : string { return `https://${this.host}` }
+
+    public get(s3Path : string): Promise<Response>
+    {
+        const method = "GET";
+        const auth    = this.createAuthorizationHeader(method, s3Path, "");
+        const url     = this.url + "/" + s3Path
+        const headers = {"Host": this.host, "Authorization": auth};
+
+        try
+        {
+            return fetch(url, {method: method, mode: "cors", headers: headers})
+        }
+        catch
+        {
+            return Promise.reject()
+        }
+    }
+
+    private createAuthorizationHeader(method: string,
+                                      s3Path: string,
+                                        date: string)
+    {
+        return createAuthorizationHeader
+                       (
+                           method,
+                           this.bucket,
+                           s3Path,
+                           date,
+                           this.key,
+                           this.secret,
+                           this.contentType
+                       );
+    }
+}
+
+function createAuthorizationHeader(method: string,
+                                   bucket: string,
+                                   s3Path: string,
+                                     date: string,
+                                    s3Key: string,
+                                 s3Secret: string,
+                              contentType: string,
+                                  md5Hash: string = "")
+{
+    // StringToSign = HTTP-Verb + "\n" +
+    //                Content-MD5 + "\n" +
+    //                Content-Type + "\n" +
+    //                Date + "\n" +
+    //                CanonicalizedAmzHeaders +
+    //                CanonicalizedResource;
+
+    const payload   = Utf8.encode(`${method}\n${md5Hash}\n${contentType}\n${date}\n/${bucket}/${s3Path}`)
+
+    //console.log(`${method}\n${md5Hash}\n${contentType}\n${date}\n/${bucket}/${s3Path}`)
+
+    const secret    = Utf8.encode(s3Secret)
+    const signature = toBase64(sha1Hmac(payload, secret));
+
+    return `AWS ${s3Key}:${signature}`
+}

typescript/DataCache/src/server/dataCache/S3/Sha1.ts

@@ -0,0 +1,125 @@
+import {concat, pad} from "./UInt8Utils";
+
+const BigEndian = false
+
+export function sha1Hmac(msg: Uint8Array, key: Uint8Array): Uint8Array
+{
+    if (key.byteLength > 64)
+        key = sha1(key)
+    if (key.byteLength < 64)
+        key = pad(key, 64)
+
+    const oKey = key.map(b => b ^ 0x5C);
+    const iKey = key.map(b => b ^ 0x36);
+
+    const iData = concat(iKey, msg);
+    const iHash = sha1(iData);
+    const oData = concat(oKey, iHash);
+
+    return sha1(oData);
+}
+
+export function sha1(data: Uint8Array): Uint8Array
+{
+    const paddedData: DataView = initData(data)
+
+    const H = new Uint32Array([0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0])
+    const S = new Uint32Array(5)  // State
+    const round = new Uint32Array(80);
+
+    function initRound(startOffset: number)
+    {
+        for (let i = 0; i < 16; i++)
+            round[i] = paddedData.getUint32((startOffset + i) * 4, BigEndian);
+
+        for (let i = 16; i < 80; i++)
+        {
+            const int32 = round[i - 3] ^ round[i - 8] ^ round[i - 14] ^ round[i - 16];
+            round[i] = rotate1(int32);  // SHA0 has no rotate
+        }
+    }
+
+    const functions =
+        [
+            () => (S[1] & S[2] | ~S[1] & S[3])              + 0x5A827999,
+            () => (S[1] ^ S[2] ^ S[3])                      + 0x6ED9EBA1,
+            () => (S[1] & S[2] | S[1] & S[3] | S[2] & S[3]) + 0x8F1BBCDC,
+            () => (S[1] ^ S[2] ^ S[3])                      + 0xCA62C1D6
+        ]
+
+    for (let startOffset = 0; startOffset < paddedData.byteLength / 4; startOffset += 16)
+    {
+        initRound(startOffset);
+
+        S.set(H)
+
+        for (let r = 0, i = 0; r < 4; r++)
+        {
+            const f = functions[r]
+            const end = i + 20;
+
+            do
+            {
+                const S0 = rotate5(S[0]) + f() + S[4] + round[i];
+                S[4] = S[3];
+                S[3] = S[2];
+                S[2] = rotate30(S[1]);
+                S[1] = S[0];
+                S[0] = S0;
+            }
+            while (++i < end)
+        }
+
+        for (let i = 0; i < 5; i++)
+            H[i] += S[i]
+    }
+
+    swapEndianness(H);
+
+    return new Uint8Array(H.buffer)
+}
+
+function rotate5(int32: number)
+{
+    return (int32 << 5) | (int32 >>> 27);  // >>> for unsigned shift
+}
+
+function rotate30(int32: number)
+{
+    return (int32 << 30) | (int32 >>> 2);
+}
+
+function rotate1(int32: number)
+{
+    return (int32 << 1) | (int32 >>> 31);
+}
+
+function initData(data: Uint8Array): DataView
+{
+    const dataLength     = data.length
+    const extendedLength = dataLength + 9;                      // add 8 bytes for UInt64 length + 1 byte for "stop-bit" (0x80)
+    const paddedLength   = Math.ceil(extendedLength / 64) * 64; // pad to 512 bits block
+    const paddedData     = new Uint8Array(paddedLength)
+
+    paddedData.set(data)
+    paddedData[dataLength] = 0x80          // append single 1 bit at end of data
+
+    const dataView = new DataView(paddedData.buffer)
+
+    // append UInt64 length
+    dataView.setUint32(paddedData.length - 4, dataLength << 3  , BigEndian)   // dataLength in *bits* LO, (<< 3: x8 bits per byte)
+    dataView.setUint32(paddedData.length - 8, dataLength >>> 29, BigEndian)   // dataLength in *bits* HI
+
+    return dataView
+}
+
+function swapEndianness(uint32Array: Uint32Array)
+{
+    const dv = new DataView(uint32Array.buffer)
+    for (let i = 0; i < uint32Array.byteLength; i += 4)
+    {
+        const uint32 = dv.getUint32(i, false)
+        dv.setUint32(i, uint32, true)
+    }
+}
+

typescript/DataCache/src/server/dataCache/S3/UInt8Utils.ts

@@ -0,0 +1,56 @@
+export function pad(data: Uint8Array, length: number): Uint8Array
+{
+    if (length < data.byteLength)
+        throw new RangeError("length")
+
+    const padded = new Uint8Array(length)
+    padded.set(data)
+
+    return padded;
+}
+
+export function concat(left: Uint8Array, right: Uint8Array): Uint8Array
+{
+    const c = new Uint8Array(left.length + right.length);
+    c.set(left);
+    c.set(right, left.length);
+    return c
+}
+
+export function toHexString(data: Uint8Array)
+{
+    return [...data].map(byteToHex).join('');
+}
+
+function byteToHex(b: number)
+{
+    return b.toString(16).padStart(2, "0");
+}
+
+const b64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+
+export function toBase64(data : Uint8Array) : string
+{
+    const byteLength = data.byteLength
+    const base64LengthPadded = 4 * Math.ceil(byteLength / 3)
+    const base64Length       = Math.ceil(byteLength / 3 * 4);
+
+    const base64 = new Array<String>(base64LengthPadded)
+
+    for (let i = 0, o = 0; i < byteLength;)
+    {
+        const x = data[i++]
+        const y = data[i++] ?? 0
+        const z = data[i++] ?? 0
+
+        base64[o++] = b64Chars[x >>> 2]
+        base64[o++] = b64Chars[(x << 4 | y >>> 4) & 63]
+        base64[o++] = b64Chars[(y << 2 | z >>> 6) & 63]
+        base64[o++] = b64Chars[z & 63]
+    }
+
+    for (let i = base64LengthPadded; i > base64Length ;)
+        base64[--i] = "="
+
+    return base64.join('')
+}

typescript/DataCache/src/server/dataCache/S3/Utf8.ts

@@ -0,0 +1,10 @@
+export namespace Utf8
+{
+    const encoder = new TextEncoder()
+    const decoder = new TextDecoder()
+
+    export const encode = (text: string): Uint8Array => encoder.encode(text);
+    export const decode = (data: Uint8Array): string => decoder.decode(data);
+}
+
+

typescript/DataCache/src/server/dataCache/data.ts

@@ -0,0 +1,20 @@
+import {Timestamped} from "./types";
+import {isDefined, Maybe} from "../../utils/maybe";
+
+export type DataRecord = Record<string, number>
+
+export type DataPoint    = Timestamped<Maybe<DataRecord>>
+export type RecordSeries = Array<DataPoint>
+export type PointSeries  = Array<Timestamped<Maybe<number>>>
+export type DataSeries   = Array<Maybe<number>>
+
+export function getPoints(recordSeries: RecordSeries, series: keyof DataRecord): PointSeries
+{
+    return recordSeries.map(p => ({time: p.time, value: isDefined(p.value) ? p.value[series] : undefined}))
+}
+
+export function getData(recordSeries: RecordSeries, series: keyof DataRecord): DataSeries
+{
+    return recordSeries.map(p => (isDefined(p.value) ? p.value[series] : undefined))
+}
+

typescript/DataCache/src/server/dataCache/dataCache.ts

@@ -0,0 +1,167 @@
+import {TimeSpan, UnixTime} from "./time";
+import {Observable, Subject} from "rxjs";
+import {SkipList} from "./skipList/skipList";
+import {createDispatchQueue} from "./promiseQueue";
+import {SkipListNode} from "./skipList/skipListNode";
+import {RecordSeries} from "./data";
+import {isUndefined, Maybe} from "../../utils/maybe";
+
+
+export const FetchResult =
+{
+    notAvailable : "N/A",
+    tryLater : "Try Later"
+} as const
+
+export type FetchResult<T> =
+    | T
+    | typeof FetchResult.notAvailable
+    | typeof FetchResult.tryLater
+
+function reverseBits(x : number): number
+{
+    // https://stackoverflow.com/a/60227327/141397
+
+    x = (x & 0x55555555)  <<   1 | (x & 0xAAAAAAAA) >>  1;
+    x = (x & 0x33333333)  <<   2 | (x & 0xCCCCCCCC) >>  2;
+    x = (x & 0x0F0F0F0F)  <<   4 | (x & 0xF0F0F0F0) >>  4;
+    x = (x & 0x00FF00FF)  <<   8 | (x & 0xFF00FF00) >>  8;
+    x = (x & 0x0000FFFF)  <<  16 | (x & 0xFFFF0000) >> 16;
+
+    return x >>> 0;
+}
+
+
+export default class DataCache<T extends Record<string, number>>
+{
+    private readonly cache: SkipList<Maybe<T>> =  new SkipList<Maybe<T>>()
+    private readonly resolution: TimeSpan;
+
+    readonly #fetch: (t: UnixTime) => Promise<FetchResult<T>>;
+
+    private readonly fetchQueue = createDispatchQueue(6)
+    private readonly fetching: Set<number> = new Set<number>()
+
+    public readonly gotData: Observable<UnixTime>;
+
+    constructor(fetch: (t: UnixTime) => Promise<FetchResult<T>>, resolution: TimeSpan)
+    {
+        this.#fetch = fetch;
+        this.resolution = resolution;
+        this.gotData = new Subject<UnixTime>()
+    }
+
+    public prefetch(times: Array<UnixTime>, clear = true)
+    {
+        if (clear)
+        {
+            this.fetching.clear()
+            this.fetchQueue.clear()
+        }
+
+        const timesWithPriority = times.map((time, index) => ({time, priority: reverseBits(index)}))
+        timesWithPriority.sort((x, y) => x.priority - y.priority)
+
+        for (let i = 0; i < timesWithPriority.length; i++)
+        {
+            const time = timesWithPriority[i].time.round(this.resolution)
+            const t = time.ticks;
+
+            const node = this.cache.find(t);
+            if (node.index !== t)
+                this.fetchData(time);
+        }
+    }
+
+    public get(timeStamp: UnixTime, fetch = true): Maybe<T>
+    {
+        const time = timeStamp.round(this.resolution)
+        const t = time.ticks;
+
+        const node = this.cache.find(t);
+        if (node.index === t)
+            return node.value
+
+        if (fetch)
+            this.fetchData(time);
+
+        return this.interpolate(node, t)
+    }
+
+    public getSeries(sampleTimes: UnixTime[]): RecordSeries
+    {
+        this.prefetch(sampleTimes)
+        return sampleTimes.map(time => ({time, value: this.get(time, false)}))
+    }
+
+    private interpolate(before: SkipListNode<Maybe<T>>, t: number): Maybe<T>
+    {
+        const dataBefore = before.value
+        const after = before.next[0];
+        const dataAfter = after.value
+
+        if (isUndefined(dataBefore) && isUndefined(dataAfter))
+            return undefined
+
+        if (isUndefined(dataBefore))
+            return dataAfter
+
+        if (isUndefined(dataAfter))
+            return dataBefore
+
+        const p = t - before.index
+        const n = after.index - t
+        const pn = p + n
+
+        let interpolated: Partial<Record<string, number>> = {}
+
+        for (const k of Object.keys(dataBefore))
+        {
+            interpolated[k] = (dataBefore[k] * n + dataAfter[k] * p) / pn
+        }
+
+        return interpolated as T
+    }
+
+    private fetchData(time: UnixTime)
+    {
+        const t = time.ticks;
+
+        if (this.fetching.has(t))  // we are already fetching t
+            return
+
+        const fetchTask = () =>
+        {
+            const onSuccess = (data: FetchResult<T>) =>
+            {
+                if (data === FetchResult.tryLater)
+                {
+                    console.warn(FetchResult.tryLater)
+                    return
+                }
+
+                const value = data === FetchResult.notAvailable ? undefined : data;
+                this.cache.insert(value, t)
+            }
+
+            const onFailure = (_: unknown) =>
+            {
+                console.error(time.ticks + " FAILED!")   // should not happen
+            }
+
+            const dispatch = () =>
+            {
+                this.fetching.delete(time.ticks);
+                (this.gotData as Subject<UnixTime>).next(time);
+            }
+
+            return this.#fetch(time)
+                       .then(d => onSuccess(d), f => onFailure(f))
+                       .finally(() => dispatch())
+        };
+
+        this.fetching.add(t)
+        this.fetchQueue.dispatch(() => fetchTask());
+    }
+
+}

typescript/DataCache/src/server/dataCache/graphCanvas.ts

@@ -0,0 +1,167 @@
+import {isDefined} from "../../utils/maybe";
+
+export class GraphCanvas
+{
+    private constructor(private readonly canvasElement: HTMLCanvasElement,
+                        private readonly ctx: CanvasRenderingContext2D,
+                        private readonly scaleX: number = 1,
+                        private readonly scaleY: number = 1,
+                        private readonly translateX: number = 0,
+                        private readonly translateY: number = 0,
+    )
+    {
+
+    }
+
+    public static fromCanvasElement(canvasElement: HTMLCanvasElement): GraphCanvas
+    {
+        const ctx = canvasElement.getContext("2d") as CanvasRenderingContext2D
+        return new GraphCanvas(canvasElement, ctx)
+    }
+
+    public transform(scaleX: number, scaleY: number, translateX: number, translateY: number): GraphCanvas
+    {
+        return new GraphCanvas(this.canvasElement,
+                               this.ctx,
+                               this.scaleX * scaleX,
+                               this.scaleY * scaleY,
+                               this.translateX + translateX * this.scaleX,
+                               this.translateY + translateY * this.scaleY)
+    }
+
+
+    public clear(bgStyle?: string | CanvasGradient | CanvasPattern)
+    {
+        if (isDefined(bgStyle))
+        {
+            this.ctx.fillStyle = bgStyle
+            this.ctx.fillRect(0, 0, this.canvasElement.width, this.canvasElement.height)
+        }
+        else
+        {
+            this.ctx.clearRect(0, 0, this.canvasElement.width, this.canvasElement.height)
+        }
+    }
+
+
+    public beginPath()
+    {
+        this.ctx.beginPath()
+    }
+
+    public closePath()
+    {
+        this.ctx.closePath()
+    }
+
+    public moveTo(x: number, y: number)
+    {
+        this.ctx.moveTo(this.transformX(x), this.transformY(y))
+    }
+
+    public lineTo(x: number, y: number)
+    {
+        this.ctx.lineTo(this.transformX(x), this.transformY(y))
+    }
+
+    public fill()
+    {
+        this.ctx.fill();
+    }
+
+    public stroke()
+    {
+        this.ctx.stroke();
+    }
+
+    public save()
+    {
+        this.ctx.save();
+    }
+
+    public restore()
+    {
+        this.ctx.restore();
+    }
+
+    public set fillStyle(style: string | CanvasGradient | CanvasPattern)
+    {
+        this.ctx.fillStyle = style
+    }
+
+    public get fillStyle(): string | CanvasGradient | CanvasPattern
+    {
+        return this.ctx.fillStyle
+    }
+
+    public set strokeStyle(style: string | CanvasGradient | CanvasPattern)
+    {
+        this.ctx.strokeStyle = style
+    }
+
+    public get strokeStyle(): string | CanvasGradient | CanvasPattern
+    {
+        return this.ctx.strokeStyle
+    }
+
+    public set lineWidth(width: number)
+    {
+        this.ctx.lineWidth = width
+    }
+
+    public get lineWidth(): number
+    {
+        return this.ctx.lineWidth
+    }
+
+    public strokeCircle(x: number, y: number, radius: number, stroke?: string | CanvasGradient | CanvasPattern)
+    {
+        if (isDefined(stroke))
+            this.ctx.strokeStyle = stroke
+
+        this.ctx.beginPath();
+        this.ctx.ellipse(this.transformX(x), this.transformY(y), radius, radius, 0, 0, 2 * Math.PI)
+        this.ctx.stroke();
+    }
+
+    public fillCircle(x: number, y: number, radius: number, fill?: string | CanvasGradient | CanvasPattern)
+    {
+        if (isDefined(fill))
+            this.ctx.fillStyle = fill
+
+        this.ctx.beginPath();
+        this.ctx.ellipse(this.transformX(x), this.transformY(y), radius, radius, 0, 0, 2 * Math.PI)
+        this.ctx.fill();
+    }
+
+
+    public fillRect(x: number, y: number, w: number, h: number)
+    {
+        this.ctx.fillRect(this.transformX(x), this.transformY(y), w, h)
+    }
+
+    public strokeRect(x: number, y: number, w: number, h: number)
+    {
+        this.ctx.strokeRect(this.transformX(x), this.transformY(y), w, h)
+    }
+
+    public transformX(x: number)
+    {
+        return x * this.scaleX + this.translateX
+    }
+
+    public transformY(y: number)
+    {
+        return y * this.scaleY + this.translateY
+    }
+
+    public transformBackX(x: number)
+    {
+        return (x - this.translateX) / this.scaleX
+    }
+
+    public transformBackY(y: number)
+    {
+        return (y - this.translateY) / this.scaleY
+    }
+}

typescript/DataCache/src/server/dataCache/linq.ts

@@ -0,0 +1,20 @@
+// 0. Import Module
+import { initializeLinq, IEnumerable } from "linq-to-typescript"
+// 1. Declare that the JS types implement the IEnumerable interface
+declare global {
+    interface Array<T> extends IEnumerable<T> { }
+    interface Uint8Array extends IEnumerable<number> { }
+    interface Uint8ClampedArray extends IEnumerable<number> { }
+    interface Uint16Array extends IEnumerable<number> { }
+    interface Uint32Array extends IEnumerable<number> { }
+    interface Int8Array extends IEnumerable<number> { }
+    interface Int16Array extends IEnumerable<number> { }
+    interface Int32Array extends IEnumerable<number> { }
+    interface Float32Array extends IEnumerable<number> { }
+    interface Float64Array extends IEnumerable<number> { }
+    interface Map<K, V> extends IEnumerable<[K, V]> { }
+    interface Set<T> extends IEnumerable<T> { }
+    interface String extends IEnumerable<string> { }
+}
+// 2. Bind Linq Functions to Array, Map, etc
+initializeLinq()

typescript/DataCache/src/server/dataCache/observableUtils.ts

@@ -0,0 +1,63 @@
+import {map, MonoTypeOperatorFunction, Observable, tap} from "rxjs";
+import {fastHash} from "./utils";
+import {HslColor, toCssColor} from "./color";
+
+type ConcatX<T extends readonly (readonly any[])[]> = [
+    ...T[0], ...T[1], ...T[2], ...T[3], ...T[4],
+    ...T[5], ...T[6], ...T[7], ...T[8], ...T[9],
+    ...T[10], ...T[11], ...T[12], ...T[13], ...T[14],
+    ...T[15], ...T[16], ...T[17], ...T[18], ...T[19]
+];
+type Flatten<T extends readonly any[]> =
+    ConcatX<[...{ [K in keyof T]: T[K] extends any[] ? T[K] : [T[K]] }, ...[][]]>
+
+
+export function flatten()
+{
+    return function<T extends Array<unknown>>(source: Observable<T>)
+    {
+        return source.pipe
+        (
+            map(a => a.flat() as Flatten<T>)
+        )
+    }
+}
+
+type RecursiveObject<T> = T extends object ? T : never;
+
+type Terminals<TModel, T> =
+{
+    [Key in keyof TModel]: TModel[Key] extends RecursiveObject<TModel[Key]>
+                         ? Terminals<TModel[Key], T>
+                         : T;
+};
+
+
+
+export function debug<T>(tag: string = "debug"): MonoTypeOperatorFunction<T>
+{
+    const hslColor: HslColor =
+    {
+        h: fastHash(tag),
+        s: (fastHash('s' + tag) % 30 + 30) / 100,
+        l: (fastHash('l' + tag) % 30 + 30) / 100
+    }
+
+    const color = toCssColor(hslColor)
+    const css = `background: ${color}; color: #fff; padding: 3px; font-size: 9px;`
+
+    return tap({
+        next(value)
+        {
+            console.log(`%c[${tag}]`, css, value)
+        },
+        error(error)
+        {
+            console.log(`%c[${tag}]`, css, error)
+        },
+        complete()
+        {
+            console.log(`%c[${tag}]`, css)
+        }
+    })
+}

typescript/DataCache/src/server/dataCache/promiseQueue.ts

@@ -0,0 +1,51 @@
+
+
+export function createDispatchQueue(maxInflight: number, debug = false): { dispatch: (task: () => Promise<void>) => number; clear: () => void }
+{
+    const queue: Array<() => Promise<void>> = []
+
+    let inflight = 0;
+
+    function done()
+    {
+        inflight--
+
+        if (debug && inflight + queue.length === 0)
+            console.log("queue empty")
+
+        if (inflight < maxInflight && queue.length > 0)
+        {
+            const task = queue.pop()!
+            inflight++
+            task().finally(() => done())
+        }
+    }
+
+    function dispatch(task: () => Promise<void>) : number
+    {
+        if (inflight < maxInflight)
+        {
+            inflight++;
+            task().finally(() => done())
+        }
+        else
+        {
+            if (debug && queue.length === 0)
+                console.log("queue in use")
+
+            queue.push(task)
+        }
+
+        return queue.length
+    }
+
+    function clear()
+    {
+        // https://stackoverflow.com/questions/1232040/how-do-i-empty-an-array-in-javascript
+        queue.length = 0
+        if (debug)
+            console.log("queue cleared")
+    }
+
+    return {dispatch, clear}
+}

typescript/DataCache/src/server/dataCache/skipList/skipList.ts

@@ -0,0 +1,80 @@
+import {find, findPath, insert, Path, SkipListNode} from "./skipListNode";
+
+export class SkipList<T>
+{
+    public readonly head: SkipListNode<T>;
+    public readonly tail: SkipListNode<T>;
+
+    private readonly nLevels: number;
+
+    private _length = 0
+
+    constructor(nLevels: number = 20)
+    {
+        // TODO: auto-levels
+
+        this.tail =
+        {
+            index: Number.MAX_VALUE,
+            next: [],
+            value: undefined!
+        };
+
+        this.head =
+        {
+            index: Number.MIN_VALUE,
+            next: Array(nLevels).fill(this.tail),
+            value: undefined!
+        };
+
+        this.nLevels = nLevels
+    }
+
+    public find(index: number, startNode = this.head, endNode = this.tail): SkipListNode<T>
+    {
+        return find(index, startNode, endNode)
+    }
+
+    private findPath(index: number, startNode = this.head, endNode = this.tail): Path<T>
+    {
+        return findPath(index, startNode, endNode)
+    }
+
+    public insert(value: T, index: number): SkipListNode<T>
+    {
+        const path = this.findPath(index)
+        const node = path[0];
+
+        if (node.index === index)  // overwrite
+        {
+            node.value = value
+            return node
+        }
+
+        const nodeToInsert = {value, index, next: []} as SkipListNode<T>
+
+        const rnd = (Math.random() * (1 << this.nLevels)) << 0;
+
+        for (let level = 0; level < this.nLevels; level++)
+        {
+            insert(nodeToInsert, path[level], level)
+
+            if ((rnd & (1 << level)) === 0)
+                break
+        }
+
+        this._length += 1
+
+        return nodeToInsert;
+    }
+
+    get length(): number
+    {
+        return this._length;
+    }
+
+    // public remove(index: number): void
+    // {
+    //     // TODO
+    // }
+}

typescript/DataCache/src/server/dataCache/skipList/skipListNode.ts

@@ -0,0 +1,52 @@
+import {asMutableArray} from "../types";
+
+export type Next<T> = { readonly next: ReadonlyArray<SkipListNode<T>> }
+export type Index = { readonly index: number }
+export type Indexed<T> = Index & { value: T }
+export type SkipListNode<T> = Next<T> & Indexed<T>
+export type Path<T> = SkipListNode<T>[];
+
+export function find<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>): SkipListNode<T>
+{
+    let node = startNode
+
+    for (let level = startNode.next.length - 1; level >= 0; level--)
+        node = findOnLevel(index, node, endNode, level)
+
+    return node
+}
+
+export function findOnLevel<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>, level: number): SkipListNode<T>
+{
+    let node: SkipListNode<T> = startNode
+
+    while (true)
+    {
+        const next = node.next[level]
+
+        if (index < next.index || endNode.index < next.index)
+            return node
+
+        node = next
+    }
+}
+
+export function findPath<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>): Path<T>
+{
+    const path = Array(startNode.next.length - 1)
+    let node = startNode
+
+    for (let level = startNode.next.length - 1; level >= 0; level--)
+    {
+        node = findOnLevel(index, node, endNode, level)
+        path[level] = node
+    }
+
+    return path
+}
+
+export function insert<T>(nodeToInsert: SkipListNode<T>, after: SkipListNode<T>, onLevel: number): void
+{
+    asMutableArray(nodeToInsert.next)[onLevel] = after.next[onLevel]
+    asMutableArray(after.next)[onLevel] = nodeToInsert
+}

typescript/DataCache/src/server/dataCache/stringUtils.ts

@@ -0,0 +1,5 @@
+export function trim(str: string, string: string = " "): string
+{
+    const pattern = '^[' + string + ']*(.*?)[' + string + ']*$';
+    return str.replace(new RegExp(pattern), '$1')
+}

typescript/DataCache/src/server/dataCache/time.ts

@@ -0,0 +1,302 @@
+import {trim} from "./stringUtils";
+
+export class UnixTime
+{
+    private constructor(readonly ticks: number)
+    {
+    }
+
+    public static readonly Epoch = new UnixTime(0)
+
+    public static now(): UnixTime
+    {
+        return UnixTime.fromTicks(Date.now() / 1000)
+    }
+
+    public static fromDate(date: Date): UnixTime
+    {
+        return UnixTime.fromTicks(date.getTime() / 1000)
+    }
+
+    public toDate(): Date
+    {
+        return new Date(this.ticks * 1000)
+    }
+
+    public static fromTicks(ticks: number): UnixTime
+    {
+        return new UnixTime(ticks)
+    }
+
+    public later(timeSpan: TimeSpan): UnixTime
+    {
+        return new UnixTime(this.ticks + timeSpan.ticks)
+    }
+
+    public move(ticks: number): UnixTime
+    {
+        return new UnixTime(this.ticks + ticks)
+    }
+
+    public earlier(timeSpan: TimeSpan): UnixTime
+    {
+        return new UnixTime(this.ticks - timeSpan.ticks)
+    }
+
+    public isEarlierThan(time: UnixTime): boolean
+    {
+        return this.ticks < time.ticks
+    }
+
+    public isEarlierThanOrEqual(time: UnixTime): boolean
+    {
+        return this.ticks <= time.ticks
+    }
+
+    public isLaterThan(time: UnixTime): boolean
+    {
+        return this.ticks > time.ticks
+    }
+
+    public isLaterThanOrEqual(time: UnixTime): boolean
+    {
+        return this.ticks >= time.ticks
+    }
+
+    public isEqual(time: UnixTime): boolean
+    {
+        return this.ticks === time.ticks
+    }
+
+    public isInTheFuture(): boolean
+    {
+        return this.isLaterThan(UnixTime.now())
+    }
+
+    public isInThePast(): boolean
+    {
+        return this.ticks < UnixTime.now().ticks
+    }
+
+    public round(ticks:number) : UnixTime
+    public round(duration: TimeSpan) : UnixTime
+    public round(durationOrTicks: TimeSpan | number) : UnixTime
+    {
+        const ticks = (typeof durationOrTicks === "number") ? durationOrTicks : durationOrTicks.ticks
+
+        return new UnixTime(Math.round(this.ticks / ticks) * ticks)
+    }
+
+    public rangeTo(time: UnixTime): TimeRange
+    {
+        return TimeRange.fromTimes(this, time);
+    }
+
+    public rangeBefore(timeSpan: TimeSpan): TimeRange
+    {
+        return TimeRange.fromTimes(this.earlier(timeSpan), this);
+    }
+
+    public rangeAfter(timeSpan: TimeSpan): TimeRange
+    {
+        return TimeRange.fromTimes(this, this.later(timeSpan));
+    }
+
+    public toString() : string
+    {
+        return this.ticks.toString()
+    }
+}
+
+
+export class TimeSpan
+{
+    private constructor(readonly ticks: number) {}
+
+    get milliSeconds(): number { return this.ticks * 1000 }
+    get seconds()     : number { return this.ticks }
+    get minutes()     : number { return this.ticks / 60 }
+    get hours()       : number { return this.minutes / 60 }
+    get days()        : number { return this.hours / 24 }
+    get weeks()       : number { return this.days / 7 }
+
+    public static fromTicks  (t: number): TimeSpan { return new TimeSpan(t) }
+    public static fromSeconds(t: number): TimeSpan { return TimeSpan.fromTicks(t) }
+    public static fromMinutes(t: number): TimeSpan { return TimeSpan.fromSeconds(t*60) }
+    public static fromHours  (t: number): TimeSpan { return TimeSpan.fromMinutes(t*60) }
+    public static fromDays   (t: number): TimeSpan { return TimeSpan.fromHours(t*24) }
+    public static fromWeeks  (t: number): TimeSpan { return TimeSpan.fromDays(t*7) }
+
+    public static span(from: UnixTime, to: UnixTime) : TimeSpan
+    {
+        return TimeSpan.fromTicks(Math.abs(to.ticks - from.ticks))
+    }
+
+    public add(timeSpan: TimeSpan) : TimeSpan
+    {
+        return TimeSpan.fromTicks(this.ticks + timeSpan.ticks)
+    }
+
+    public subtract(timeSpan: TimeSpan) : TimeSpan
+    {
+        return TimeSpan.fromTicks(this.ticks - timeSpan.ticks)
+    }
+
+    public divide(n: number) : TimeSpan
+    {
+        if (n <= 0)
+            throw 'n must be positive';
+
+        return TimeSpan.fromTicks(this.ticks/n)
+    }
+
+    public multiply(n: number) : TimeSpan
+    {
+        if (n < 0)
+            throw 'n cannot be negative';
+
+        return TimeSpan.fromTicks(this.ticks * n)
+    }
+
+    public round(ticks:number) : TimeSpan
+    public round(duration: TimeSpan) : TimeSpan
+    public round(durationOrTicks: TimeSpan | number) : TimeSpan
+    {
+        const ticks = (typeof durationOrTicks === "number")
+                    ? durationOrTicks
+                    : durationOrTicks.ticks
+
+        return TimeSpan.fromTicks(Math.round(this.ticks / ticks) * ticks)
+    }
+
+
+    public toString() : string
+    {
+        let dt = 60*60*24*7
+
+        let ticks = this.ticks;
+
+        if (ticks === 0)
+            return "0s"
+
+        ticks = Math.abs(ticks)
+
+        const nWeeks = Math.floor(ticks / dt)
+        ticks -= nWeeks * dt
+
+        dt /= 7
+        const nDays = Math.floor(ticks / dt)
+        ticks -= nDays * dt
+
+        dt /= 24
+        const nHours = Math.floor(ticks / dt)
+        ticks -= nHours * dt
+
+        dt /= 60
+        const nMinutes = Math.floor(ticks / dt)
+        ticks -= nMinutes * dt
+
+        dt /= 60
+        const nSeconds = Math.floor(ticks / dt)
+
+        let s = ""
+
+        if (nWeeks > 0)   s += nWeeks  .toString() + "w "
+        if (nDays > 0)    s += nDays   .toString() + "d "
+        if (nHours > 0)   s += nHours  .toString() + "h "
+        if (nMinutes > 0) s += nMinutes.toString() + "m "
+        if (nSeconds > 0) s += nSeconds.toString() + "s"
+
+        return trim(s);
+    }
+}
+
+export class TimeRange
+{
+    private constructor(private readonly from: number, private readonly to: number)
+    {
+    }
+
+    public get start(): UnixTime
+    {
+        return UnixTime.fromTicks(this.from)
+    }
+
+    public get mid(): UnixTime
+    {
+        return UnixTime.fromTicks((this.from + this.to) / 2)
+    }
+
+
+    public get end(): UnixTime
+    {
+        return UnixTime.fromTicks(this.to)
+    }
+
+    public get duration(): TimeSpan
+    {
+        return TimeSpan.fromTicks(this.to - this.from)
+    }
+
+    public static fromTimes(from: UnixTime, to: UnixTime): TimeRange
+    {
+        return from.isLaterThan(to)
+             ? new TimeRange(to.ticks, from.ticks)
+             : new TimeRange(from.ticks, to.ticks)
+    }
+
+    public isInside(time: number) : boolean;
+    public isInside(time: UnixTime) : boolean;
+    public isInside(time: UnixTime | number)
+    {
+        const t = time instanceof UnixTime ? time.ticks : time
+
+        return t >= this.from && t < this.to
+    }
+
+    public sample(period: TimeSpan): UnixTime[]
+    {
+        const samples = []
+
+        for (let t = this.from; t < this.to; t += period.ticks)
+            samples.push(UnixTime.fromTicks(t));
+
+        return samples
+    }
+
+    public subdivide(n: number) : TimeRange[]
+    {
+        if (n <= 0)
+            throw 'n must be positive';
+
+        const period = TimeSpan.fromTicks(this.duration.ticks / n);
+        if (period === this.duration)
+            return [this];
+
+        const samples = this.sample(period);
+
+        const ranges : TimeRange[] = []
+
+        for (let i = 0; i < samples.length;)
+            ranges.push(TimeRange.fromTimes(samples[i], samples[++i]))
+
+        return ranges
+    }
+
+
+    public earlier(dt: TimeSpan) : TimeRange
+    {
+        return new TimeRange(this.from - dt.ticks, this.to - dt.ticks)
+    }
+
+    public later(dt: TimeSpan) : TimeRange
+    {
+        return new TimeRange(this.from + dt.ticks, this.to + dt.ticks)
+    }
+
+    public move(ticks: number) : TimeRange
+    {
+        return new TimeRange(this.from + ticks, this.to + ticks)
+    }
+}
+

typescript/DataCache/src/server/dataCache/types.ts

@@ -0,0 +1,21 @@
+import {UnixTime} from "./time";
+
+export type Timestamped<T> = { time: UnixTime, value: T }
+
+export type Pair<T1, T2 = T1> = [T1, T2]
+
+export type Position  = { readonly x: number, readonly y: number }
+export type Direction = { readonly dx: number, readonly dy: number }
+export type Size      = { readonly width: number, readonly height: number }
+export type Rect = Position & Size
+
+export type Mutable<T> = { -readonly [P in keyof T]: T[P] };
+
+export type FieldKey<T> = { [P in keyof T]: T[P] extends (...args: any) => any ? never : P }[keyof T];
+export type AllFields<T> = Pick<T, FieldKey<T>>;
+export type SomeFields<T> = Partial<AllFields<T>>
+export const asMutable = <T>(t: T) => (t as Mutable<T>);
+export const asMutableArray = <T>(t: ReadonlyArray<T>) => (t as Array<T>);
+export const cast = <T>(t: unknown) => (t as T);
+
+export type Rename<T, K extends keyof T, N extends string> = Pick<T, Exclude<keyof T, K>> & { [P in N]: T[K] }

typescript/DataCache/src/server/dataCache/utils.ts

@@ -0,0 +1,119 @@
+import {IEnumerable} from "linq-to-typescript";
+import {isDefined}   from "../../utils/maybe";
+
+//export type UnionToIntersection<U> = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I : never
+
+export type Nothing = Record<string, never>
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+
+
+
+export function fastHash(str: string): number
+{
+    const signed = str
+                  .split('')
+                  .reduce((p, c) => ((p << 5) - p) + c.charCodeAt(0) | 0, 0);
+
+    return Math.abs(signed);
+}
+
+
+
+
+
+// export function flattenObject(obj: object) : object
+// {
+//     const flattened = {}
+//
+//     for (const key of Object.keys(obj))
+//     {
+//         // @ts-ignore
+//         const value = obj[key]
+//
+//         if (typeof value === 'object' && value !== null && !Array.isArray(value))
+//         {
+//             Object.assign(flattened, flattenObject(value))
+//         }
+//         else
+//         {
+//             // @ts-ignore
+//             flattened[key] = value
+//         }
+//     }
+//
+//     return flattened
+// }
+//return function<TTerminal,TTree extends Terminals<TTree, TTerminal>>(source: Observable<TTerminal>)
+
+
+
+export function* pairwise<T>(iterable: Iterable<T>, init?: T): Generator<[T, T]>
+{
+    const it = iterable[Symbol.iterator]()
+
+    let first : T;
+
+    if (isDefined(init))
+    {
+        first = init
+    }
+    else
+    {
+        const f = it.next()
+        if (f.done)
+            return
+
+        first = f.value
+    }
+
+    let second = it.next()
+
+    while(!second.done)
+    {
+        yield [first, second.value]
+        first = second.value
+        second = it.next()
+    }
+}
+
+export function arraysEqual<T>(a: Array<T>, b: Array<T>)
+{
+    if (a === b) return true;
+    if (a.length !== b.length) return false;
+
+    for (let i = 0; i < a.length; ++i)
+    {
+        if (a[i] !== b[i]) return false;
+    }
+
+    return true;
+}
+
+export function mod(a:number, b:number)
+{
+    return ((a % b) + b) % b;
+}
+
+export function clamp(a: number, min: number, max: number)
+{
+    return a > max ? max
+         : a < min ? min
+         : a
+}
+
+
+export function isDST(d : Date)
+{
+    const jan = new Date(d.getFullYear(), 0, 1).getTimezoneOffset();
+    const jul = new Date(d.getFullYear(), 6, 1).getTimezoneOffset();
+
+    return Math.max(jan, jul) != d.getTimezoneOffset();
+}
+
+export function Transpose<T>(src: IEnumerable<IEnumerable<T>>): IEnumerable<IEnumerable<T>>
+{
+    return src
+          .selectMany(line => line.select((element, column) => ({element, column})))
+          .groupBy(i => i.column)
+          .select(g => g.select(e => e.element));
+}

typescript/DataCache/src/server/utils/fileSystem.ts

@@ -0,0 +1,46 @@
+import fs from "fs";
+
+export function doesFileExist(path: string): boolean
+{
+    try
+    {
+        fs.accessSync(path, fs.constants.F_OK);
+        return true;
+    }
+    catch (e)
+    {
+        return false;
+    }
+}
+
+export function doesDirExist(path: string): boolean
+{
+    try
+    {
+        fs.readdirSync(path)
+        return true;
+    }
+    catch (e)
+    {
+        return false;
+    }
+}
+
+export function readJsonFile<T>(path: string)
+{
+    const data = fs.readFileSync(path, "utf-8")
+    return JSON.parse(data) as T
+}
+
+export function writeJsonFile<T>(path: string, contents: T)
+{
+    const data = JSON.stringify(contents)
+    return fs.writeFileSync(path, data, "utf-8")
+}
+
+export function writeJsonFilePretty<T>(path: string, contents: T)
+{
+    const data = JSON.stringify(contents,undefined,2)
+    return fs.writeFileSync(path, data, "utf-8")
+}
+

typescript/DataCache/src/server/utils/httpResponse.ts

@@ -0,0 +1,124 @@
+import {isDefined, isUndefined, Maybe} from "../../utils/maybe";
+import {Dictionary} from "../../utils/utilityTypes";
+import MimeType from "./mime";
+import {promisify} from "util";
+import fs from "fs";
+import http, {IncomingMessage, ServerResponse} from "http";
+import {entries} from "../../utils/utils";
+import {getLogger} from "../../utils/logging";
+
+const log = getLogger("HTTP")
+const readFile = promisify(fs.readFile)
+
+export type HttpResponse = {
+    body: Maybe<Buffer | string>
+    headers : Dictionary<string>
+    statusCode: number
+}
+
+
+function contentTypeHeader(mimeType: string)
+{
+    return {['Content-type']: mimeType};
+}
+
+function forbidden(message = "403 : forbidden", headers: Dictionary<string> = {}): HttpResponse
+{
+    return text(message, 403, headers)
+}
+
+function notFound(message ="404 : not found", headers: Dictionary<string> = {}): HttpResponse
+{
+    return text(message, 404, headers)
+}
+
+function text(text: string, statusCode = 200, headers: Dictionary<string> = {}): HttpResponse
+{
+    return {
+        statusCode: statusCode,
+        headers   : {...headers, ...contentTypeHeader('text/plain')},
+        body      : text
+    };
+}
+
+function json(json: Dictionary,
+              replacer?: (k: string, v: unknown) => unknown,
+              headers: Dictionary<string> = {}): HttpResponse
+{
+    return {
+        statusCode: 200,
+        headers : {...headers, ['Content-type']: MimeType.json},
+        body: JSON.stringify(json, replacer)
+    }
+}
+
+function empty(headers: Dictionary<string> = {}): HttpResponse
+{
+    return {
+        statusCode: 200,
+        headers,
+        body: undefined
+    }
+}
+
+function ok(body: Maybe<Buffer | string>, headers: Dictionary<string> = {}): HttpResponse
+{
+    return {
+        statusCode: 200,
+        headers,
+        body
+    }
+}
+
+async function file(localRootPath: string, urlPath: string, headers: Dictionary<string> = {}, defaultPath = "/"): Promise<HttpResponse>
+{
+    if (urlPath.contains('..'))
+        return HTTP.forbidden();
+
+    const localPath = localRootPath + (urlPath === "/" ? defaultPath : urlPath);
+
+    const body = await readFile(localPath).catch(_ => undefined)
+
+    if (isUndefined(body))
+        return HTTP.notFound();
+
+    if (!('Content-type' in headers))
+    {
+        headers = {...headers, ...contentTypeHeader(MimeType.guessFromPath(localPath))}
+    }
+
+    return HTTP.ok(body, headers)
+}
+
+function createServer(serve: (request: IncomingMessage) => Promise<HttpResponse>)
+{
+    async function wrapServe(request: IncomingMessage, response: ServerResponse): Promise<void>
+    {
+        const r = await serve(request)
+
+        entries(r.headers).forEach(([k, v]) => response.setHeader(k, v))
+
+        response.statusCode = r.statusCode
+
+        if (isDefined(r.body))
+            response.end(r.body)
+        else
+            response.end()
+    }
+
+    return http.createServer(wrapServe);
+}
+
+const HTTP =
+{
+    contentTypeHeader,
+    forbidden,
+    notFound,
+    ok,
+    json,
+    empty,
+    file,
+    createServer
+}
+
+export default HTTP;

typescript/DataCache/src/server/utils/mime.ts

@@ -0,0 +1,32 @@
+import PlatformPath from "path";
+import {isDefined, isUndefined} from "../../utils/maybe";
+
+
+function guessFromPath(path: string) : string
+{
+    const ext = PlatformPath.parse(path).ext?.substring(1) as keyof typeof MimeType;
+
+    const mimeType = MimeType[ext]
+
+    return isDefined(mimeType) && typeof mimeType === "string"
+         ? mimeType
+         : 'application/octet-stream'
+}
+
+const MimeType =
+{
+    ico : 'image/x-icon',
+    html: 'text/html; charset=UTF-8',
+    js  : 'text/javascript',
+    json: 'application/json; charset=UTF-8',
+    css : 'text/css; charset=UTF-8',
+    png : 'image/png',
+    jpg : 'image/jpeg',
+    wav : 'audio/wav',
+    mp3 : 'audio/mpeg',
+    svg : 'image/svg+xml; charset=UTF-8',
+    pdf : 'application/pdf',
+    guessFromPath
+};
+
+export default MimeType

typescript/DataCache/src/server/utils/requestUtils.ts

@@ -0,0 +1,45 @@
+import {IncomingMessage}                                     from "http";
+import {firstValueFrom, map, Observable, startWith, toArray} from "rxjs";
+
+export function observeData(request: IncomingMessage, maxLength: number = Number.POSITIVE_INFINITY): Observable<Uint8Array>
+{
+    let nBytes = 0;
+
+    return new Observable<Uint8Array>(subscriber =>
+    {
+        request.on('end', () => subscriber.complete());
+        request.on('data', (data: Uint8Array) =>
+        {
+            nBytes += data.byteLength
+
+            if (nBytes <= maxLength)
+                subscriber.next(data);
+            else
+            {
+                const error = `too much data: expected ${maxLength} bytes or less, got ${nBytes} bytes.`;
+                subscriber.error(error);
+                request.destroy(new Error(error))
+            }
+        });
+    });
+}
+
+export async function getRequestJson<T = unknown>(request: IncomingMessage, maxLength = 500000): Promise<T>
+{
+    const data = await getData(request, maxLength)
+    return JSON.parse(data.toString())
+}
+
+const noData = new Uint8Array(0);
+
+export function getData(request: IncomingMessage, maxLength: number = Number.POSITIVE_INFINITY): Promise<Buffer>
+{
+    const data = observeData(request, maxLength).pipe
+    (
+        startWith(noData),
+        toArray(),
+        map(b => Buffer.concat(b)),  // cannot inline!
+    )
+
+    return firstValueFrom(data);
+}

typescript/DataCache/src/server/utils/stringUtils.ts

@@ -0,0 +1,22 @@
+export function toLowercaseAscii(string: string)
+{
+    return string
+        .normalize("NFD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .toLowerCase();
+}
+
+export function containsIgnoringAccents(string: string, substring: string)
+{
+    if (substring === "") return true;
+    if (string === "") return false;
+
+    substring = "" + substring;
+
+    if (substring.length > string.length)
+        return false;
+
+    return toLowercaseAscii(string).includes(toLowercaseAscii(substring));
+}
+
+

typescript/DataCache/src/server/utils/utils.ts

@@ -0,0 +1,37 @@
+
+import {IncomingMessage} from "http";
+import {Dictionary} from "../../utils/utilityTypes";
+import {from} from "linq-to-typescript";
+import {isUndefined, Maybe} from "../../utils/maybe";
+
+type StringValued<T> =
+{
+    [Key in keyof T]: T[Key] extends number ? Maybe<string>
+                    : T[Key] extends string ? Maybe<string>
+                    : T[Key] extends boolean ? Maybe<string>
+                    : never
+}
+
+export function getQueryParams<T>(request: IncomingMessage): Maybe<StringValued<T>>
+{
+    if (isUndefined(request.url))
+        return undefined
+
+    const url = new URL(request.url, `https://${request.headers.host}/`);
+
+    const query: Dictionary = {}
+    const urlSearchParams = new URLSearchParams(url.search);
+
+    if (!from(urlSearchParams.entries()).any())
+        return undefined
+
+    for (const [key, value] of urlSearchParams.entries())
+        query[key] = value;
+
+    return query as StringValued<T>;
+}
+
+export function getPath(req: IncomingMessage)
+{
+    return new URL(req.url!, `https://${req.headers.host}/`).pathname;
+}

typescript/DataCache/src/server/vpn/vpn.ts

@@ -0,0 +1,114 @@
+import fs from "fs";
+import fetch from "node-fetch";
+import {getLogger} from "../../utils/logging";
+import {doesDirExist, doesFileExist} from "../utils/fileSystem";
+import {Dictionary} from "../../utils/utilityTypes";
+
+const log = getLogger('VPM')
+
+const vpnIp = "http://10.2.0.1"
+const rxIp = /ifconfig-push +10\.2\.\d+\.\d+/g;
+const ccdDir = "/etc/openvpn/server/Salino/ccd";
+const vpnStatusFile = "/var/log/openvpn/status";
+
+const useFs = doesFileExist(vpnStatusFile) && doesDirExist(ccdDir)
+
+export type VpnIps = Dictionary<string>;
+
+function vpn(vpnName: string): Promise<string>
+{
+    return fetch(`${vpnIp}/vpn/${vpnName}`)
+          .then(r => r.text())
+          .then(t => t.match(rxIp)?.firstOrDefault()?.replace("ifconfig-push", "").trim() ?? "")
+}
+
+function getVpnIpFromFs(vpnName: string): string
+{
+    return fs
+          .readFileSync(`${ccdDir}/${vpnName}`, 'utf-8')
+          .match(rxIp)
+         ?.firstOrDefault()
+         ?.replace("ifconfig-push", "")
+          .trim() ?? ""
+}
+
+// function getVpnOnlineStatusFromHttp(): Promise<string[]>
+// {
+//     return fetch(`${vpnIp}/vpnstatus.txt`)
+//           .then(r => r.text())
+//           .then(s => s.split('\n'));
+// }
+//
+// function getVpnOnlineStatusFromFs(): string[]
+// {
+//     return fs
+//           .readFileSync(vpnStatusFile, 'utf8')
+//           .split('\n');
+// }
+
+function getVpnNamesFromHttp(): Promise<string[]>
+{
+    return fetch(`${vpnIp}/vpn/`)
+          .then(r => r.json() as Promise<{ name: string, type: string }[]>)
+          .then(fs => fs
+                     .where(n => n.type === "file")
+                     .select(n => n.name)
+                     .toArray()
+          );
+}
+
+function getCcdFilesFromFs()
+{
+    return fs
+        .readdirSync(ccdDir)
+        .where(p => !p.endsWith(".bak"))
+        .where(f => fs.lstatSync(`${ccdDir}/${f}`).isFile())
+        .toArray();
+}
+
+function getVpnIpsFromFs(): VpnIps
+{
+    const ccdFiles = getCcdFilesFromFs();
+    const vpnIps : VpnIps = {}
+
+    for (const vpnName of ccdFiles)
+        vpnIps[vpnName] = getVpnIpFromFs(vpnName)
+
+    return vpnIps
+}
+
+export async function getVpnIpsFromHttp(): Promise<VpnIps>
+{
+    log(`getting VPN status (http)`)
+
+    const vpnNames = await getVpnNamesFromHttp();
+
+    const vpnIps : VpnIps = {}
+
+    for (const vpnName of vpnNames)
+        vpnIps[vpnName] = await vpn(vpnName)
+
+    return vpnIps
+}
+
+// export function updateVpnStats(installation: VrmInstallation, vpnStats: VpnStatus[])
+// {
+//     if (!isDefined(installation.machineSerialNumber))
+//         return;
+//
+//     const stat = vpnStats.firstOrDefault(s => s.vpnName === installation.machineSerialNumber)
+//     if (stat === null)
+//         return;
+//
+//     installation.vpnIp = stat.vpnIp;
+//     installation.vpnOnline = stat.vpnOnline;
+//     installation.vpnName = stat.vpnName;
+//
+// }
+
+export async function getVpnIps(): Promise<VpnIps>
+{
+    return useFs
+         ? getVpnIpsFromFs()
+         : await getVpnIpsFromHttp()
+}

typescript/DataCache/src/server/vpn/vpnSync.ts

@@ -0,0 +1,25 @@
+import {Milliseconds} from "../../utils/milliseconds";
+
+import {getLogger} from "../../utils/logging";
+import {exhaustMap, Observable, of, repeat, share, tap} from "rxjs";
+import {getVpnIps, VpnIps} from "./vpn";
+
+const updateDelay = Milliseconds.fromMinutes(5)
+
+const log = getLogger('VPN_SYNC')
+
+const vpnIps: Observable<VpnIps> = of(0).pipe
+(
+    tap(_ => log('fetching VPN info')),
+    exhaustMap(_ => getVpnIps()),
+    repeat({delay: updateDelay}),
+    tap(_ => log("VPN info updated")),
+    share()
+)
+
+export default vpnIps
+
+
+
+
+

typescript/DataCache/src/server/vrm/api/api.ts

@@ -0,0 +1,222 @@
+import fetch, {Request}                                                                                                                                                       from 'node-fetch';
+import {delay}                                                                                                                                                                from "../../../utils/utils";
+import {parseCreateTokenResponse, parseDevicesResponse, parseDiagnosticsResponse, parseGenericResponse, parseInstallationDataResponse, parseLoginResponse, parseTagsResponse} from "./parser";
+import {createTokenAuth}                                                                                                                                                      from "./auth";
+import {getLogger}                                                                                                                                                            from "../../../utils/logging";
+import {isDefined, isUndefined, Maybe}                                                                                                                                        from "../../../utils/maybe";
+import {VrmApiAuth, VrmApiDevice, VrmApiDiagnostic, VrmApiError, VrmApiInstallation, VrmApiInstallationData}                                                                  from "./apiTypes";
+
+const vrmApiPrefix = 'https://vrmapi.victronenergy.com/v2/'             // https://docs.victronenergy.com/vrmapi/overview.html
+const nAttempts = 7;
+const initialRetryDelayMs = 250;
+const timeoutMs = 5000;
+const log = getLogger('VRMAPI')
+
+
+const tokenAuth = createTokenAuth(55450, "ccf470115ca6b729475b9f6a8722a7eb4036df043e7cbaf5002c2c18ccd2e4ee")
+
+
+export function login(username: string, password: string): Promise<VrmApiAuth>
+{
+    log(`logging into VRM, user: ${username}`)
+
+    const auth = undefined;
+    return vrmPost("auth/login", auth, {username, password}, parseLoginResponse)
+}
+
+export async function createToken(auth: VrmApiAuth, tokenName: string): Promise<VrmApiAuth>
+{
+    log(`creating token "${tokenName}"`)
+
+    const path  = `users/${auth.idUser}/accesstokens/create`;
+    const token = await vrmPost(path, auth, {name: tokenName}, parseCreateTokenResponse);
+
+    return createTokenAuth(auth.idUser, token)
+}
+
+function createVrmRequest(path: string, method: "GET" | "POST", auth: Maybe<VrmApiAuth>, payload?: unknown)
+{
+    const url = path.startsWith(vrmApiPrefix) ? path
+              : path.startsWith('/') ? vrmApiPrefix + path.substring(1)
+              : vrmApiPrefix + path
+
+    const body = isDefined(payload)
+               ? JSON.stringify(payload)
+               : undefined;
+
+    const headers = isDefined(auth)
+                  ? auth.header
+                  : undefined
+
+    return new Request(url, {method, body, headers});
+}
+
+function vrmGet<T>(path: string, auth: Maybe<VrmApiAuth>, parseResponse: (response: unknown) => T)
+{
+    const request = createVrmRequest(path, "GET", auth)
+    return callVrmApi(request, parseResponse)
+}
+
+function vrmPost<T>(path: string, auth: Maybe<VrmApiAuth>, payload: unknown, parseResponse: (response: unknown) => T): Promise<T>
+{
+    const request = createVrmRequest(path, "POST", auth, payload)
+    return callVrmApi(request, parseResponse)
+}
+
+//https://vrmapi.victronenergy.com/v2/installations/34807/settings
+//{"description":"Graber, Zürich/ZH | InnovEnergy","phonenumber":""}
+
+export function editInstallation(idSite: number,
+                                 name?: string,
+                                 tags?: string[],
+                                 auth: VrmApiAuth = tokenAuth): Promise<boolean>
+{
+    if (isUndefined(name) && isUndefined(tags))
+        return Promise.resolve(true)  // nothing to do
+
+    return vrmPost(`installations/${idSite}/settings`,
+                   auth,
+                   {
+                       description: name,
+                       tags: isDefined(tags) ? tags.map(t => t.toUpperCase()) : undefined
+                   },
+                   parseGenericResponse)
+}
+
+export function deleteInstallation(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<boolean>
+{
+    return vrmPost(`installations/${idSite}/remove`, auth, undefined, parseGenericResponse)
+}
+
+export function addInstallation(installationName: string, vrmPortalId: string, auth: VrmApiAuth = tokenAuth): Promise<boolean>
+{
+    const payload =
+    {
+        installation_identifier: vrmPortalId,
+        description: installationName
+    }
+
+    return vrmPost(`users/${auth.idUser}/addsite`, auth, payload, parseGenericResponse)
+}
+
+
+export function listTokens(auth: VrmApiAuth): Promise<unknown>
+{
+    log(`listing tokens of user: ${auth.idUser}`)
+
+    return vrmGet(`users/${auth.idUser}/accesstokens/list`, auth, (r: unknown) => r)
+}
+
+export function getInstallationSummaries(auth: VrmApiAuth = tokenAuth): Promise<VrmApiInstallation[]>
+{
+    return vrmGet(`users/${auth.idUser}/installations`, auth, parseInstallationDataResponse)
+}
+
+
+function getDiagnostics(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<VrmApiDiagnostic[]>
+{
+    return vrmGet(`installations/${idSite}/diagnostics?count=1000`, auth, parseDiagnosticsResponse)
+}
+
+export async function getDevices(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<VrmApiDevice[]>
+{
+    const allDevices = await vrmGet(`installations/${idSite}/system-overview`, auth, parseDevicesResponse);
+
+    allDevices.devices.forEach(d => d.configured = true)
+    allDevices.unconfigured_devices.forEach(d => d.configured = false)
+
+    return [...allDevices.devices, ...allDevices.unconfigured_devices];
+}
+
+export function getTags(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<string[]>
+{
+    return vrmGet(`installations/${idSite}/tags`, auth, parseTagsResponse);
+}
+
+
+const isVrmError = (e: unknown): e is VrmApiError => e instanceof VrmApiError;
+
+function error(msg: string, error?: unknown): Promise<never>
+{
+    if (isVrmError(error))
+        return Promise.reject(error);  // do not repackage existing errors (multiple catch clauses)
+
+    if (isDefined(error))
+    {
+        if (error instanceof Error)
+        return Promise.reject(new VrmApiError(`${msg}: ${error}`));
+    }
+
+    return Promise.reject(new VrmApiError(msg));
+}
+
+export async function getAllInstallationData(auth: VrmApiAuth = tokenAuth): Promise<VrmApiInstallationData[]>
+{
+    const installationSummaries = await getInstallationSummaries(auth)
+
+    const installationData: VrmApiInstallationData[] = []
+
+    for (const installationSummary of installationSummaries/*.take(70)*/) // TODO: remove take
+    {
+        log(`fetching installation "${installationSummary.name}"`)
+        //await delay(1000)
+
+        const tags    = await getTags(installationSummary.idSite, auth);
+        const devices = await getDevices(installationSummary.idSite, auth);
+
+        installationData.push({installation: installationSummary, devices, tags})
+    }
+
+    return installationData;
+}
+
+export async function getInstallationData(vrmPortalId: string, auth: VrmApiAuth = tokenAuth): Promise<Maybe<VrmApiInstallationData>>
+{
+    const installationSummaries = await getInstallationSummaries(auth)
+
+    const installation = installationSummaries.firstOrDefault(s => s.identifier === vrmPortalId)
+    if (!isDefined(installation))
+        return undefined
+
+    const tags    = await getTags(installation.idSite, auth);
+    const devices = await getDevices(installation.idSite, auth);
+
+    return {installation, devices, tags}
+}
+
+
+async function callVrmApi<T>(request: Request, parseVrmResponse: (r: unknown) => T): Promise<T>
+{
+    let attempt = 1;
+    let retryDelayMs = initialRetryDelayMs;
+
+    // eslint-disable-next-line no-constant-condition
+    while (true)
+    {
+        const response = fetch(request)
+                        .catch(e => error("fetch failed", e))
+                        .then(r => r.ok ? r : error(`request failed: [${r.status}] ${r.statusText}`))
+                        .then(r => r.json())
+                        .catch(e => error("json parsing failed", e))
+                        .then(parseVrmResponse)
+                        .catch(e => error("vrm parser failed", e))
+                        .catch(e => isVrmError(e) ? e : new VrmApiError(`unexpected error: ${e}`)) // gotta catch 'em all
+
+        const timeout = delay(timeoutMs)
+                       .then(_ => new VrmApiError(`the request timed out after ${timeoutMs}ms`));
+
+        const result: T | VrmApiError = await Promise.race([response, timeout])
+
+        if (!isVrmError(result))
+            return result
+
+        if (attempt >= nAttempts)
+            throw `Giving up after ${attempt} attempts`
+
+        //log(`attempt ${attempt} failed: "${result.message}". retrying in ${retryDelayMs}ms`);
+
+        attempt++
+        await delay(retryDelayMs)   // wait before retrying
+        retryDelayMs *= 2;          // exponential backoff
+    }
+}

typescript/DataCache/src/server/vrm/api/apiTypes.ts

@@ -0,0 +1,130 @@
+import {Maybe} from "../../../utils/maybe";
+
+
+
+export type VrmApiInstallationData =
+{
+    installation : VrmApiInstallation,
+    tags : string[],
+    devices : VrmApiDevice[]
+}
+
+    // "idSite":171797,
+    // "accessLevel":1,
+    // "owner":true,
+    // "is_admin":true,
+    // "name":"_ IBN tbd Weisshaubt, Neunkirch\/SH | Lutz Bodenm\u00fcller AG (2022-00070)",
+    // "identifier":"48e7da8755b5",
+    // "idUser":55450,
+    // "pvMax":0,
+    // "timezone":"Europe\/Berlin",
+    // "phonenumber":null,
+    // "notes":null,
+    // "geofence":null,
+    // "geofenceEnabled":false,
+    // "realtimeUpdates":true,
+    // "hasMains":1,
+    // "hasGenerator":0,
+    // "noDataAlarmTimeout":null,
+    // "alarmMonitoring":1,
+    // "invalidVRMAuthTokenUsedInLogRequest":0,
+    // "syscreated":1650888746,
+    // "grafanaEnabled":0,
+    // "shared":false,
+    // "device_icon":"battery",
+    // "alarm":false,
+    // "last_timestamp":1650964098,
+    //  "current_time":"15:40",
+    // "timezone_offset":7200,
+    // "images":false,
+    // "view_permissions"
+    // "demo_mode":false,
+    // "mqtt_webhost":"webmqtt61.victronenergy.com",
+    // "high_workload":false,
+
+export type VrmApiInstallation =
+{
+    accessLevel: number,
+    name: string,
+    identifier: string,
+    hasMains: number,
+    hasGenerator: number,
+    noDataAlarmTimeout: Maybe<number>,
+    syscreated: number,
+    tags: string[],
+    vrmLink: Maybe<string>,
+    inverter: string,
+    inverterFw: string,
+    nbMppts: number,
+    nbPvInverters: number,
+    idSite: number,
+}
+
+// "name":"PV Inverter",
+//     "customName":"",
+//     "productCode":"",
+//     "idSite":98487,
+//     "productName":"Fronius PV Inverter",
+//     "lastConnection":1652968325,
+//     "class":"device-pv-inverter device-icon-fronius-pv-inverter",
+//     "froniusDeviceType":"Fronius Symo 10.0-3-M",
+//     "pL":"AC input 1",
+//     "connection":"AC current sensor",
+//     "instance":21,
+//     "idDeviceType":7,
+//     "settings"
+
+export type VrmApiDevice =
+{
+    name: string,
+    firmwareVersion: Maybe<string>,
+    instance: number,
+    lastConnection: number,
+    productName: string,
+    froniusDeviceType: Maybe<string>,
+    pL: Maybe<string>,
+    configured : boolean,
+}
+
+export type VrmApiGateway =
+{
+    autoUpdate: string,
+    updateTo: string,
+    lastConnection: number,
+    identifier: string,
+    lastPowerUpOrRestart: number,
+    twoWayCommunication: boolean,
+    machineSerialNumber: string,
+};
+
+
+export type VrmApiGatewayDevice = VrmApiDevice & VrmApiGateway
+export type VrmApiDevices = { devices: VrmApiDevice[], unconfigured_devices: VrmApiDevice[] };
+
+export type VrmApiDiagnostic =
+{
+    idSite: number,
+    timestamp: number,
+    Device: string,
+    instance: number,
+    idDataAttribute: number,
+    description: string,
+    formatWithUnit: string,
+    dbusServiceType: string,
+    dbusPath: string,
+    code: string,
+    bitmask: number,
+    formattedValue: string,
+    rawValue: string | number | boolean,
+    id: number
+}
+
+export type VrmApiSuccess = { success: boolean }
+export type VrmApiLoginResponse = VrmApiSuccess & { idUser: number, token: string }
+export type VrmApiCreateTokenResponse = VrmApiSuccess & { idAccessToken: number, token: string }
+export type VrmApiTagsResponse = VrmApiSuccess & { tags: string[] }
+export type VrmApiRecordsResponse<T> = VrmApiSuccess & { records: T }
+export type VrmApiAuthHeader = { 'X-Authorization': string }
+export type VrmApiAuth = { idUser: number, header: VrmApiAuthHeader }
+
+export class VrmApiError extends Error {constructor(message: string) { super(message); }}

typescript/DataCache/src/server/vrm/api/auth.ts

@@ -0,0 +1,13 @@
+import {VrmApiAuth} from "./apiTypes";
+
+export function createUserAuth(idUser: number, token: string): VrmApiAuth
+{
+    const header = {"X-Authorization": `Bearer ${token}`};
+    return {idUser, header}
+}
+
+export function createTokenAuth(idUser: number, token: string): VrmApiAuth
+{
+    const header = {"X-Authorization": `Token ${token}`};
+    return {idUser, header}
+}

typescript/DataCache/src/server/vrm/api/parser.ts

@@ -0,0 +1,68 @@
+import {createUserAuth}                                                                                                                                                            from "./auth";
+import {isUndefined}                                                                                                                                                               from "../../../utils/maybe";
+import {VrmApiAuth, VrmApiCreateTokenResponse, VrmApiDevices, VrmApiDiagnostic, VrmApiInstallation, VrmApiLoginResponse, VrmApiRecordsResponse, VrmApiSuccess, VrmApiTagsResponse} from "./apiTypes";
+
+
+export function parseLoginResponse(response: unknown): VrmApiAuth
+{
+    const r = response as VrmApiLoginResponse
+
+    if (isUndefined(r.idUser) || isUndefined(r.token))
+        throw "failed to parse VrmTokenResponse"
+
+    return createUserAuth(r.idUser, r.token);
+}
+
+
+export function parseCreateTokenResponse(response: unknown) : string
+{
+    const r = response as VrmApiCreateTokenResponse
+
+    if (isUndefined(r.token) || isUndefined(r.idAccessToken) || isUndefined(r.success))
+        throw "failed to parse VrmCreateTokenResponse"
+
+    if (!r.success)
+        throw "VrmCreateToken failed"
+
+    return r.token
+}
+
+export function parseTagsResponse(response: unknown): string[]
+{
+    const tagsResponse = response as VrmApiTagsResponse
+
+    if (isUndefined(tagsResponse.tags) || isUndefined(tagsResponse.success))
+        throw "failed to parse VrmTagsResponse"
+
+    if (!tagsResponse.success)
+        throw "VrmTagsResponse failed"
+
+    return tagsResponse.tags
+}
+
+function parseRecordsResponse<T>(response: unknown): T
+{
+    const recordsResponse = response as VrmApiRecordsResponse<T>
+
+    if (isUndefined(recordsResponse.records) || isUndefined(recordsResponse.success))
+        throw "failed to parse VrmRecordsResponse"
+
+    if (!recordsResponse.success)
+        throw "VrmRecordsResponse failed"
+
+    return recordsResponse.records
+}
+
+export function parseGenericResponse(response: unknown): boolean
+{
+    const recordsResponse = response as VrmApiSuccess
+
+    if (isUndefined(recordsResponse.success))
+        throw "failed to parse VrmRecordsResponse"
+
+    return recordsResponse.success
+}
+
+export const parseInstallationDataResponse = (r: unknown) => parseRecordsResponse<VrmApiInstallation[]>(r);
+export const parseDiagnosticsResponse      = (r: unknown) => parseRecordsResponse<VrmApiDiagnostic[]>(r);
+export const parseDevicesResponse          = (r: unknown) => parseRecordsResponse<VrmApiDevices>(r);

typescript/DataCache/src/server/vrm/convert.ts

@@ -0,0 +1,174 @@
+import {VrmApiGatewayDevice, VrmApiInstallationData} from "./api/apiTypes";
+import {VpnIps}                                      from "../vpn/vpn";
+import {unwrap, Unwrap}                              from "../../utils/match";
+import {Database, Installation, VrmInstallation}     from "../../api/data/types";
+import {getDescendantsWithPath}                      from "../../api/data/query";
+import {isDefined, isUndefined, Maybe}               from "../../utils/maybe";
+
+
+export function getGatewayDevice(iVrm: VrmApiInstallationData)
+{
+    return iVrm
+          .devices
+          .firstOrDefault(d => d.name === "Gateway") as Maybe<VrmApiGatewayDevice>;
+}
+
+export function getInverter(iVrm: VrmApiInstallationData)
+{
+    return iVrm
+          .devices
+          .where(d => d.name === "VE.Bus System")
+          .select(d => `${d.productName ?? "unknown"} [${d.firmwareVersion ?? "unknown"}]`)
+          .firstOrDefault() ?? "unknown";
+}
+
+export function getPvOnAcIn(iVrm: VrmApiInstallationData)
+{
+    return iVrm
+          .devices
+          .where(d => d.name === "PV Inverter" && isDefined(d.pL))
+          .where(d => d.pL!.startsWith("AC input"))
+          .where(d => isDefined(d.productName))
+          .select(d => `${d.productName} [${d.firmwareVersion ?? "unknown"}]`)
+          .orderBy(d => d)
+          .toArray();
+}
+
+export function getPvOnAcOut(iVrm: VrmApiInstallationData)
+{
+    return iVrm
+          .devices
+          .where(d => d.name === "PV Inverter" && isDefined(d.pL))
+          .where(d => d.pL!.startsWith("AC output"))
+          .where(d => isDefined(d.productName))
+          .select(d => `${d.productName} [${d.firmwareVersion ?? "unknown"}]`)
+          .orderBy(d => d)
+          .toArray();
+}
+
+export function getPvOnDc(iVrm: VrmApiInstallationData)
+{
+    return iVrm
+          .devices
+          .where(d => d.name === "Solar Charger")
+          .where(d => isDefined(d.productName))
+          .select(d => `${d.productName} [${d.firmwareVersion ?? "unknown"}]`)
+          .orderBy(d => d)
+          .toArray();
+}
+
+
+
+
+function unmangleVrmInstallationName(installationName: string)
+{
+    //   "name": "_IBN Frischknecht, Waldstatt/AR | St.Gallisch-Appenzellisch Kraftwerke AG (2022-00128)",
+
+    //const rx = / *(_IBN)? *([^,]+?),([^/]+?) *\/ *(.+?)( *\/ *.+?)? *| * (\(.+\)) */
+
+    const [ibnName, rest1] = installationName.split(/,(.*)/s)
+
+    if (isUndefined(rest1) || isUndefined(ibnName))
+        return undefined
+
+    const [locality, rest2] = rest1.split(/\/(.*)/s)
+
+    if (isUndefined(rest2))
+        return undefined
+
+    const [regionCountry, rest3] = rest2.split(/\|(.*)/s)
+
+    if (isUndefined(rest3))
+        return undefined
+
+    const [region, country] = regionCountry.split(/\/(.*)/s)
+
+    const [folder, orderNumber] = rest3.split(/\((.*)/s)
+
+    if (isUndefined(orderNumber))
+        return undefined
+
+    const orderNumbers = orderNumber.replace(")", "").split(',')
+
+    const rxIbn = /^\s*_\s*IBN\s*/;
+
+    return {
+        ibn     : rxIbn.test(ibnName),
+        name    : ibnName.replace(rxIbn, "").trim(),
+        locality: locality.trim(),
+        region  : region.trim(),
+        country : country?.trim(),
+        folder  : folder.trim(),
+        orderNumbers
+    }
+}
+
+export function toVrmInstallation(vrmData: VrmApiInstallationData, vpnIps: VpnIps = {}): VrmInstallation
+{
+    // TODO: VRM users
+    const tags = vrmData
+                .tags
+                .select(t => t.toUpperCase())
+                .where(t => t !== "ALARM" && t !== "NO-ALARM" && t !== "NODATA")
+                .orderBy(t => t)
+                .toArray();
+
+    const gateway = getGatewayDevice(vrmData)
+
+    const parsed = unmangleVrmInstallationName(vrmData.installation.name);
+
+    if (parsed?.ibn === true && !tags.includes("IBN"))
+        tags.push("IBN")
+
+    const vrmInstallation : Unwrap<VrmInstallation> = {
+
+        id          : `vrm${vrmData.installation.idSite}`,
+        name        : parsed?.name ?? vrmData.installation.name,
+        tags        : tags,
+        orderNumbers: parsed?.orderNumbers ?? [],
+        vpnName     : gateway?.machineSerialNumber ?? "unknown",
+        vpnIp       : vpnIps?.[gateway!.machineSerialNumber] ?? "unknown",
+        country     : parsed?.country,
+        region      : parsed?.region,
+        locality    : parsed?.locality,
+        children    : [],
+
+        controllerType: gateway?.productName ?? "unknown",
+        firmware      : gateway?.firmwareVersion ?? "unknown",
+        accessLevel   : vrmData.installation.accessLevel, // TODO
+        autoUpdate    : gateway?.autoUpdate ?? "unknown",
+        updateTo      : gateway?.updateTo ?? "unknown",
+        vrmId         : vrmData.installation.idSite,
+        vrmPortalId   : vrmData.installation.identifier,
+        pvOnAcIn      : getPvOnAcIn(vrmData),
+        pvOnAcOut     : getPvOnAcOut(vrmData),
+        pvOnDc        : getPvOnDc(vrmData),
+        inverter      : getInverter(vrmData),
+        vrmName       : vrmData.installation.name
+    };
+
+    return {vrmInstallation};
+}
+
+
+function getMangledVrmInstallationName(db: Database, installation: Unwrap<Installation>)
+{
+    const path = getDescendantsWithPath(db.rootFolder)
+                .where(e => unwrap(e[0]).id === installation.id)
+                .take(1)
+                .selectMany(e => e)
+                .skip(1)  // skip installation name
+                .select(e => unwrap(e).name)
+                .toArray()
+
+    path.pop() // remove root "InnovEnergy" folder
+
+    const {country, locality, region} = installation;
+    const lrc = [locality, region, country].filter(isDefined).join("/")
+    path.unshift(`${installation.name}, ${lrc}`)
+
+    const ibn          = installation.tags.includes("IBN") ? "_IBN " : ""
+    const orderNumbers = installation.orderNumbers.any() ? ` (${installation.orderNumbers.join(",")})` : "";
+
+    return `${ibn + path.join(" | ") + orderNumbers}`
+}

typescript/DataCache/src/server/vrm/vrmSync.ts

@@ -0,0 +1,42 @@
+import {Milliseconds}                                                        from "../../utils/milliseconds";
+import {getAllInstallationData}                                              from "./api/api";
+import {getLogger}                                                           from "../../utils/logging";
+import {exhaustMap, map, Observable, of, repeat, share, tap, withLatestFrom} from "rxjs";
+import {VrmApiInstallationData}                                              from "./api/apiTypes";
+import vpnIps                                                                from "../vpn/vpnSync";
+import {VpnIps}                                                              from "../vpn/vpn";
+import {toVrmInstallation}                                                   from "./convert";
+import {VrmInstallation}                                                     from "../../api/data/types";
+
+const updateDelay = Milliseconds.fromMinutes(5)
+
+const log = getLogger('VRM_SYNC')
+
+export const vrmInstallations: Observable<VrmInstallation[]> = of(0).pipe
+(
+    tap(_ => log('iteration started')),
+    exhaustMap(_ => getAllInstallationData()),
+    withLatestFrom(vpnIps),
+    map(([installationData, vpnIps]) => toVrmInstallations(installationData, vpnIps)),
+    repeat({delay: updateDelay}),
+    tap(_ => log(`iteration finished. Waiting for ${Milliseconds.toMinutes(updateDelay)} minutes`)),
+    share()
+)
+
+export function toVrmInstallations(vrmData: VrmApiInstallationData[], vpnIps: VpnIps) : VrmInstallation[]
+{
+    return vrmData.map(d => toVrmInstallation(d, vpnIps))
+}
+
+
+
+
+
+// const vrmUser = "victron@innov.energy";
+// const vrmPassword = "NnoVctr201002";
+
+
+
+
+
+

typescript/DataCache/src/utils/diffing.ts

@@ -0,0 +1,161 @@
+import {from, IEnumerable}                  from "linq-to-typescript";
+import {compactArray, hasOwnProperty, keys} from "./utils";
+
+export type Change<T = any> = {path: string[], value: T}
+
+const primitiveTypes = ["number", "string", "boolean", "null", "function", "bigint"]
+
+const _keep = {} as const
+
+export function keep<T>()
+{
+    return _keep as T
+}
+
+export function getAt(target: any, path: IEnumerable<string>): any
+{
+    let t = target
+
+    for (const p of path)
+    {
+        if (!hasOwnProperty(t, p))
+            throw "invalid path"  // TODO: improve error msg
+
+        t = t[p]
+    }
+
+    return t
+}
+
+export function getParent(target: any, path: IEnumerable<string>): any
+{
+    let t = target
+
+    const parentPath = path.toArray()
+    parentPath.pop()
+
+    for (const p of parentPath)
+    {
+        if (!hasOwnProperty(t, p))
+            throw "invalid path"  // TODO: improve error msg
+
+        t = t[p]
+    }
+
+    return t
+}
+
+export function applyChanges(target: any, changes : IEnumerable<Change>)
+{
+    for (const change of changes)
+    {
+        const path   = [...change.path]  // copy
+        const head   = path.pop()!
+        const parent = getAt(target, path)
+
+        if (change.value === undefined)
+            delete parent[head]
+        else
+        {
+            console.log(`${change.path.join("/")}: ${parent[head]} => ${change.value}`)
+            parent[head] = change.value
+        }
+
+        if (Array.isArray(parent))
+            compactArray(parent)   // TODO: review
+    }
+}
+
+export function getChanges(old_: any, new_: any): IEnumerable<Change>
+{
+    return from(diff(old_, new_))
+}
+
+export function isArrayOp(p: Change)
+{
+    return !isNaN(parseInt(p.path.lastOrDefault()!, 10));
+}
+
+// function observe<T extends object>(src: T, subject: Subject<Patch>, path: Path = []): T
+// {
+//     function get(target: T, p: string ): any
+//     {
+//         return observe(target, subject, [...path, p])
+//     }
+//
+//     function set(target: T, p: string, newValue: any, receiver: any): boolean
+//     {
+//
+//         return true
+//     }
+//
+//     return new Proxy(src,
+//                      {
+//         get(target, prop, receiver)
+//         {
+//             alert(`GET ${prop}`);
+//             return Reflect.get(target, prop, receiver); // (1)
+//         },
+//         set(target, prop:string, val, receiver)
+//         {
+//             alert(`SET ${prop}=${val}`);
+//             const success = Reflect.set(target, prop, val, receiver);
+//
+//             if (success)
+//             {
+//                 const patch : Patch = { path: [...path, prop], }
+//                 subject.next()
+//             }
+//
+//             return success; // (2)
+//         }
+//     });
+//
+//     //return new Proxy<T>(src, {get, set})
+// }
+
+
+
+
+function* diff(old_: any, new_: any, path: string[] = []): Generator<Change>
+{
+    const oldKeys = keys(old_);
+    const newKeys = keys(new_);
+    const allKeys = oldKeys
+                   .concatenate(newKeys)
+                   .distinct()
+
+    for (const k of allKeys)
+    {
+        const p = [...path, k] as string[];
+
+        const oldValue = old_[k]
+        const newValue = new_[k]
+
+        if (oldValue === newValue /*|| newValue === _keep*/)
+            continue // "reference equal": easy out
+
+        const oldType = typeof oldValue
+        const newType = typeof newValue
+
+        if (oldType !== newType || primitiveTypes.contains(newType))
+        {
+            // oldValue !== newValue because of test already made above
+
+            yield {path: p, value: newValue}
+            continue
+        }
+
+        if (newType !== "object")
+            throw "unsupported data type!"
+
+        if (keys(oldValue).any() && !keys(oldValue).intersect(keys(newValue)).any())
+        {
+            // no common keys: replace whole object
+            yield {path: p, value: newValue}
+            continue
+        }
+
+        yield* diff(oldValue, newValue, p)
+    }
+}

typescript/DataCache/src/utils/linq.ts

@@ -0,0 +1,21 @@
+// 0. Import Module
+
+import {IEnumerable, initializeLinq} from "linq-to-typescript"
+// 1. Declare that the JS types implement the IEnumerable interface
+declare global {
+    interface Array<T> extends IEnumerable<T> { }
+    interface Uint8Array extends IEnumerable<number> { }
+    interface Uint8ClampedArray extends IEnumerable<number> { }
+    interface Uint16Array extends IEnumerable<number> { }
+    interface Uint32Array extends IEnumerable<number> { }
+    interface Int8Array extends IEnumerable<number> { }
+    interface Int16Array extends IEnumerable<number> { }
+    interface Int32Array extends IEnumerable<number> { }
+    interface Float32Array extends IEnumerable<number> { }
+    interface Float64Array extends IEnumerable<number> { }
+    interface Map<K, V> extends IEnumerable<[K, V]> { }
+    interface Set<T> extends IEnumerable<T> { }
+    interface String extends IEnumerable<string> { }
+}
+// 2. Bind Linq Functions to Array, Map, etc
+initializeLinq()

typescript/DataCache/src/utils/logging.ts

@@ -0,0 +1,11 @@
+let subsystemPadding = 0
+
+export function getLogger(subsystem: string): (msg: string) => void
+{
+    subsystemPadding = Math.max(subsystem.length, subsystemPadding)
+
+    // eslint-disable-next-line no-console
+    return (msg: string) => console.log(`${new Date().toLocaleString()} | ${(subsystem.padEnd(subsystemPadding))} | ${msg}`);
+}
+
+

typescript/DataCache/src/utils/match.ts

@@ -0,0 +1,176 @@
+import {Dictionary, Func, Normalize1} from "./utilityTypes";
+import {keys, valueToFunction}        from "./utils";
+import {isUndefined}                  from "./maybe";
+import {UnionToIntersection}          from "simplytyped";
+import {current}                      from "immer";
+
+// Type Compatibility
+// https://www.typescriptlang.org/docs/handbook/type-compatibility.html
+
+
+//TODO: review
+export type IsUnionCase<T> =
+    T extends Dictionary
+    ? [UnionToIntersection<keyof T>] extends [keyof T]
+      ? [keyof T] extends [UnionToIntersection<keyof T>]
+        ? true
+        : false
+      : false
+    : false
+
+//TODO: review
+export type IsTaggedUnion<T> = true extends UnionToIntersection<IsUnionCase<T>> ? Dictionary : never
+
+export type Unwrap<U extends IsTaggedUnion<U>> = UnionToIntersection<U>[keyof UnionToIntersection<U>] ;
+
+export function update<U extends IsTaggedUnion<U>>(u: U, e: Partial<Unwrap<U>>)
+{
+    const v = u as UnionToIntersection<U>
+    const o = current(v)
+
+    const ks = keys(v)
+
+    if (ks.length != 1)
+        throw new Error("not a valid union case")
+
+    const tag = ks[0]
+
+    const before = v[tag];
+    const before2 = current(before);
+
+
+    const newVar = {...before, ...e};
+    v[tag] = newVar
+}
+
+export function unwrap<U extends IsTaggedUnion<U>>(u: U) : Normalize1<Unwrap<U>>
+{
+    const v = u as UnionToIntersection<U>
+
+    const ks = keys(v)
+
+    if (ks.length != 1)
+        throw new Error("not a valid union case")
+
+    const key = ks[0]
+    return v[key] as any;
+}
+
+export function base<U extends IsTaggedUnion<U>>(u: U): Normalize1<Unwrap<U> & Partial<UnionToIntersection<Unwrap<U>>>>
+{
+    return unwrap(u) as Normalize1<Unwrap<U> & Partial<UnionToIntersection<Unwrap<U>>>>
+}
+
+export function tag<U extends IsTaggedUnion<U>>(u: U) : keyof UnionToIntersection<U>
+{
+    const v = u as UnionToIntersection<U>
+
+    const ks = keys(v)
+
+    if (ks.length != 1)
+        throw new Error("not a valid union case")
+
+    return ks[0]
+}
+
+export function tagsEqual<U extends IsTaggedUnion<U>>(u: U, v: U) : v is U
+{
+    return tag(u) === tag(v)
+}
+
+
+type MapFuncs<U> = { [k in keyof UnionToIntersection<U>]: Func<UnionToIntersection<U>[k]> }
+type OtherwiseKeys<U,M> = Exclude<keyof UnionToIntersection<U>, keyof M>;
+
+type OtherwiseArg<U,M> = {
+    [k in keyof UnionToIntersection<U>]: Record<k, UnionToIntersection<U>[k]>
+}[OtherwiseKeys<U,M>]
+
+type OtherwiseFunc<U, M extends Partial<MapFuncs<U>>, R> = Func<OtherwiseArg<U,M> extends never ? unknown : OtherwiseArg<U,M>, R>;
+
+
+export function match<U extends IsTaggedUnion<U>, M extends Partial<MapFuncs<U>>, R>(uCase: U, matchFuncs: M, otherwise: OtherwiseFunc<U, M, R> | R):{ [k in keyof M]: M[k] extends Func<any, infer O> ? O : never }[keyof M] | R
+{
+    const otw = valueToFunction(otherwise)
+
+    const c = uCase as UnionToIntersection<U>
+
+    const ks = keys(c)
+
+    if (ks.length != 1)
+        return otw(c)
+
+    const key = ks[0]
+    const arg = c[key]
+
+    const matchFunc = matchFuncs[key]
+
+    if (isUndefined(matchFunc))
+        return otw(c);
+
+    return matchFunc(arg as any) as any;
+}
+
+
+export function dispatch<U extends IsTaggedUnion<U>>()
+{
+    // type Intersection = UnionToIntersection<U>;
+    //
+    // type MapFuncs = { [k in keyof Intersection]: Func<Intersection[k]> }
+    // type OtherwiseKeys<M> = Exclude<keyof Intersection, keyof M>;
+    //
+    // type OtherwiseArg<M> = {
+    //     [k in keyof Intersection]: Record<k, Intersection[k]>
+    // }[OtherwiseKeys<M>]
+    //
+    // type OtherwiseFunc<M extends Partial<MapFuncs>, R> = Func<OtherwiseArg<M> extends never ? unknown : OtherwiseArg<M>, R>;
+
+    return <M extends Partial<MapFuncs<U>>, R>(matchFuncs: M, otherwise: OtherwiseFunc<U,M, R> | R) =>
+    {
+        const otw = valueToFunction(otherwise)
+
+        return (uCase: U): { [k in keyof M]: M[k] extends Func<any, infer O> ? O : never }[keyof M] | R =>
+        {
+            const c = uCase as UnionToIntersection<U>
+
+            const ks = keys(c)
+
+            if (ks.length != 1)
+                return otw(c)
+
+            const key = ks[0]
+            const arg = c[key]
+
+            const matchFunc = matchFuncs[key]
+
+            if (isUndefined(matchFunc))
+                return otw(c);
+
+            return matchFunc(arg as any) as any;
+        }
+    };
+}
+
+
+export function concat<R extends Record<keyof any, any>, T extends Dictionary>(rec: R, t:T)
+{
+
+    const result = {} as {
+        [k in keyof UnionToIntersection<R>]: Record<k, UnionToIntersection<R>[k] & T>
+    }[keyof UnionToIntersection<R>]
+
+    for (const k in rec)
+    {
+
+        // @ts-ignore
+        result[k] = { ...rec[k], ...t}
+    }
+
+    return result
+}
+
+
+
+
+
+

typescript/DataCache/src/utils/maybe.ts

@@ -0,0 +1,16 @@
+export type Maybe<T> = T | undefined | null;
+
+export function isDefined<T>(e: Maybe<T>): e is T
+{
+    return e != undefined // != by design to include null
+}
+
+export function isUndefined<T>(e: Maybe<T>): e is undefined | null
+{
+    return e == undefined   // == by design to include null
+}
+
+export function toArray<T>(e: Maybe<T>): T[]
+{
+    return isDefined(e) ? [e] : []
+}

typescript/DataCache/src/utils/milliseconds.ts

@@ -0,0 +1,18 @@
+
+export type Milliseconds = number
+
+export const Milliseconds =
+{
+    fromSeconds: (count: number): Milliseconds => count * 1000,
+    fromMinutes: (count: number): Milliseconds => count * 1000 * 60,
+    fromHours  : (count: number): Milliseconds => count * 1000 * 60 * 60,
+    fromDays   : (count: number): Milliseconds => count * 1000 * 60 * 60 * 24,
+    fromWeeks  : (count: number): Milliseconds => count * 1000 * 60 * 60 * 24 * 7,
+
+    toSeconds: (count: Milliseconds): number => count / 1000,
+    toMinutes: (count: Milliseconds): number => count / 1000 / 60,
+    toHours  : (count: Milliseconds): number => count / 1000 / 60 / 60,
+    toDays   : (count: Milliseconds): number => count / 1000 / 60 / 60 / 24,
+    toWeeks  : (count: Milliseconds): number => count / 1000 / 60 / 60 / 24 / 7,
+} as const
+

typescript/DataCache/src/utils/path.ts

@@ -0,0 +1,81 @@
+import {isUndefined}                                  from "./maybe";
+import {from, IEnumerable}                            from "linq-to-typescript";
+import {isBoolean, isNumber, isPlainObject, isString} from "./runtimeTypeChecking";
+
+function getAt(root: any, path: (keyof any)[])
+{
+    return path.reduce((v, p) => v[p], root)
+}
+
+
+function iterate(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
+{
+    if (isUndefined(root))
+        return []
+
+    return from(iterate(root))
+
+    function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
+    {
+        if (isString(node) || isNumber(node) || isBoolean(node))
+            yield {path, node}
+        else if (isPlainObject(node))
+            for (const key in node)
+            {
+                path.push(key)
+                yield {path, node}
+                yield* iterate(node[key], path)
+                path.pop()
+            }
+    }
+}
+
+function iterateLeafs(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
+{
+    if (isUndefined(root))
+        return []
+
+    return from(iterate(root))
+
+    function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
+    {
+        if (isString(node) || isNumber(node) || isBoolean(node))
+            yield {path, node}
+        else if (isPlainObject(node))
+            for (const key in node)
+            {
+                path.push(key)
+                yield* iterate(node[key], path)
+                path.pop()
+            }
+    }
+}
+
+
+function iterateBranches(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
+{
+    if (isUndefined(root))
+        return []
+
+    return from(iterate(root))
+
+    function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
+    {
+        if (isPlainObject(node))
+            for (const key in node)
+            {
+                path.push(key)
+                yield {path, node}
+                yield* iterate(node[key], path)
+                path.pop()
+            }
+    }
+}
+
+export const Path =
+{
+    iterate,
+    iterateLeafs,
+    iterateBranches,
+    getAt
+} as const

typescript/DataCache/src/utils/runtimeTypeChecking.ts

@@ -0,0 +1,66 @@
+export type TypeCode =
+    | "undefined"
+    | "object"
+    | "boolean"
+    | "number"
+    | "string"
+    | "function"
+    | "symbol"
+    | "bigint";
+
+export type PlainObject<K extends keyof any = keyof any, V = unknown> = Record<K, V>
+
+export function isObject(thing: unknown) : thing is object
+{
+    return typeof thing === "object"
+}
+
+export function isDate(thing: unknown) : thing is Date
+{
+    return thing instanceof Date
+}
+
+export function isPlainObject(thing: unknown) : thing is PlainObject
+{
+    return isObject(thing) && !isDate(thing)
+}
+
+export function isArray(thing: unknown) : thing is Array<unknown>
+{
+    return Array.isArray(thing)
+}
+
+export function isNumber(thing: unknown) : thing is number
+{
+    return typeof thing === "number"
+}
+
+export function isBoolean(thing: unknown) : thing is boolean
+{
+    return typeof thing === "boolean"
+}
+
+export function isString(thing: unknown) : thing is string
+{
+    return typeof thing === "string"
+}
+
+// export function isFunction(thing: unknown): thing is (...args: unknown[]) => unknown
+// {
+//     return typeof thing === "function"
+// }
+
+export function isFunction(obj: unknown): obj is (...args: any[]) => any
+{
+    return obj instanceof Function;
+}
+
+export function isSymbol(thing: unknown) : thing is symbol
+{
+    return typeof thing === "symbol"
+}
+
+export function isBigint(thing: unknown) : thing is bigint
+{
+    return typeof thing === "bigint"
+}

typescript/DataCache/src/utils/tree.ts

@@ -0,0 +1,71 @@
+import {from, IEnumerable}             from "linq-to-typescript";
+import {isDefined, isUndefined, Maybe} from "./maybe";
+
+
+export function Tree<T>(getChildren: (t: T) => IEnumerable<T>)
+{
+    function iterate(root: Maybe<T>): IEnumerable<T>
+    {
+        if (isUndefined(root))
+            return []
+
+        return from(iterateTree())
+
+        function* iterateTree()
+        {
+            const queue: T[] = [root!]
+
+            do
+            {
+                const element = queue.shift()!
+                yield element
+                for (const child of getChildren(element))
+                    queue.push(child)
+            }
+            while (queue.length > 0)
+        }
+    }
+
+
+    function iterateWithPath(root: Maybe<T>): IEnumerable<T[]>
+    {
+        return isDefined(root)
+             ? from(iterateTreeWithPath())
+             : [];
+
+        function* iterateTreeWithPath()
+        {
+            const stack: Array<Array<T>> = [[root!]]
+
+            while (true)
+            {
+                const head = stack[0];
+
+                if (head.length > 0)
+                {
+                    yield stack
+                         .select(l => l[0])
+                         .toArray()
+
+                    const children = getChildren(head[0]).toArray()
+                    stack.unshift(children)
+                }
+                else
+                {
+                    stack.shift()  // remove empty array in front
+                    if(stack.length > 0)
+                        stack[0].shift()
+                    else
+                        break;
+                }
+            }
+
+        }
+    }
+
+    return {
+        iterate,
+        iterateWithPath
+    } as const
+}
+

typescript/DataCache/src/utils/type.ts

@@ -0,0 +1,115 @@
+export {}
+// export type Type =
+//     | "number"
+//     | "object"
+//     | "string"
+//     | "never"
+//     | "any"
+//     | "unknown"
+//     | "undefined"
+//     | "boolean"
+//     | "bigint"
+//     | "symbol"
+//     | Property[]
+//     | Func
+//
+// export type Key = "string" | "number" | "symbol"
+//
+// export type Property = Func |
+// {
+//     key: Key,
+//     type: Type,
+//     readonly? : boolean,
+//     nullable? : boolean
+// }
+//
+// export type Arg =
+// {
+//     name: string,
+//     type: Type,
+//     nullable? : boolean
+// }
+//
+// export type Func =
+// {
+//     args: Arg[],
+//     returnType: Type,
+// }
+//
+//
+// type X = Partial<any>
+//
+// export function render(t: Type, indent = 0)
+// {
+//     if (typeof t === "string")
+//         return t
+//
+//
+//     return "ERROR"
+// }
+//
+
+
+type DeviceType =
+    | "Pv"
+    | "Load"
+    | "Battery"
+    | "Grid"
+    | "Inverter"
+    | "AcInToAcOut"
+    | "DcDc"
+    | "AcInBus"
+    | "AcOutBus"
+    | "DcBus"
+    | "Dc48Bus" // low voltage DC Bus, to be eliminated in later versions
+
+
+type Phase =
+{
+    voltage : number  // U, non-negative
+    current : number  // I, sign depends on device type, see sign convention below
+}
+
+type AcPhase = Phase &
+{
+     phi : number  // [0,2pi)
+}
+
+type Device =
+{
+     Type: DeviceType,
+     Name?: string,
+}
+
+type Stack =
+{
+    Top?         : Device[], // 0 to N
+    Right?       : Device    // 0 or 1
+    Bottom?      : Device[]  // 0 to N
+    Disconnected?: boolean   // not present = false
+}
+
+/// A DC device must have a field denoting its DC connection
+type DcDevice = Device &
+{
+    Dc : Phase
+}
+
+
+/// An AC device can have 1 to 3 AC phases
+/// An AC device also needs a Frequency measurement
+/// Total power can be obtained by summing the power of the phases
+type AcDevice = Device &
+{
+    Ac: AcPhase[]
+    Frequency: number
+}
+
+
+/// A low voltage 48V DC device
+/// Needed to distinguish the two sides of the DCDC
+/// Will be dropped once we get HV batteries
+type Dc48Device = Device &
+{
+     dc48 : Phase
+}

typescript/DataCache/src/utils/utilityTypes.ts

@@ -0,0 +1,52 @@
+import {UnionToIntersection} from "simplytyped";
+
+export type Dictionary<T = unknown> = Record<string, T>
+export type Nothing = Dictionary<never>
+
+export type IsUnion<T> = [T] extends [UnionToIntersection<T>] ? false : true;
+
+export type UnionToDeepPartialIntersection<U> = DeepPartial<UnionToIntersection<U>>
+export type UnionToPartialIntersection<U> = Partial<UnionToIntersection<U>>
+
+export type Func<T = unknown, R = unknown>      = (arg: T) => R
+export type AsyncFunc<T = unknown, R = unknown> = (arg: T) => Promise<R>
+
+export type SyncAction<T> = (arg: T) => void
+export type AsyncAction<T> = (arg: T) => Promise<void>
+export type Action<T> = SyncAction<T> | AsyncAction<T>
+
+export type Lazy<T> = () => T
+export type Base64 = string
+export type ValueOf<T> = T[keyof T];
+
+export type DeepPartial<T> = T extends object ? { [P in keyof T]?: DeepPartial<T[P]>; } : T;
+export type DeepMutable<T> = { -readonly [P in keyof T]: DeepMutable<T[P]> };
+export type Mutable<T>     = { -readonly [P in keyof T]: T[P] };
+export type NumberLiteralToStringLiteral<T> = T extends number ? `${T}` : T
+
+export type KeyedChildren<T> = { children?: Dictionary<T> }
+
+export type Union<K extends string, V extends string = string> = { [S in K] : V}
+
+export type IntersectionToUnion<T extends Dictionary> = { [Prop in keyof T]: Record<Prop, T[Prop]> }[keyof T] // not sure if this is aptly named
+
+// helper to flatten (instantiate) types in editor popups
+// eslint-disable-next-line @typescript-eslint/ban-types
+
+export type Normalize<T> = T extends (...args: infer A) => infer R ? (...args: Normalize<A>) => Normalize<R>
+                         : [T] extends [any]                       ? { [K in keyof T]: Normalize<T[K]> }
+                         : never
+
+export type Normalize1<T> = T extends (...args: infer A) => infer R ? (...args: A) => R
+                          : [T] extends [any]                       ? { [K in keyof T]: T[K] }
+                          : T
+
+export type Normalize2<T> = T extends (...args: infer A) => infer R ? (...args: Normalize1<A>) => Normalize1<R>
+                          : [T] extends [any]                       ? { [K in keyof T]: Normalize1<T[K]> }
+                          : never
+
+export function mutable<T>(t: T)
+{
+    return t as Mutable<T>
+}
+

typescript/DataCache/src/utils/utils.ts

@@ -0,0 +1,281 @@
+import {Dictionary, Func, Lazy} from "./utilityTypes";
+import {isUndefined, Maybe}     from "./maybe";
+import {IEnumerable}            from "linq-to-typescript";
+import {Builder}                from "../client/njsx/njsx";
+
+export const id  = <T>(e:T) => e;
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+export const nop = (_?:unknown) => {};
+
+export function sleep(ms: number): Promise<never>
+{
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+
+export function fail<T>(error?: string) : T
+{
+    throw new Error(error)
+}
+
+export function first<T>(a: Array<T>) : Maybe<T>
+{
+    return a[0]
+}
+
+export function last<T>(array: T[]) : Maybe<T>
+{
+    const last = array.length - 1
+
+    return last < 0
+         ? undefined
+         : array[last]
+}
+
+
+export function compactArray<T>(array: T[])
+{
+    let j = 0;
+
+    array.forEach((e, i) =>
+              {
+                  if (i !== j)
+                      array[j] = e;
+                  j++;
+              });
+
+    array.length = j;
+
+    return array;
+}
+
+
+export const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(() => resolve(), ms));
+
+
+export function hasOwnProperty(thing: unknown, key : keyof any)
+{
+    return Object.prototype.hasOwnProperty.call(thing, key)
+}
+
+export function toDictionary<T>(ts: IEnumerable<T>, keySelector: (element: T) => string): Dictionary<T>
+{
+    function addEntry(d: Dictionary<T>, t: T)
+    {
+        const key = keySelector(t);
+        d[key] = t;
+        return d
+    }
+
+    return ts.aggregate({}, addEntry)
+}
+
+export function keys<T>(t: T): (keyof T)[]
+{
+    return Object.keys(t as object) as (keyof T)[]
+}
+
+
+export function values<V>(t: Record<keyof any, V>): V[]
+{
+    return Object.values(t)
+}
+
+type Entry<T> =
+{
+    [P in keyof T]: P extends number ? [`${P}`, T[P]]
+                  : P extends string ? [P, T[P]]
+                  : never
+};
+
+export function entries<T>(t: T)
+{
+    return Object.entries(t as Dictionary)
+}
+
+export function shallowEqual(left: any, right: any)
+{
+    const keysLeft  = keys(left) ;
+    const keysRight = keys(right);
+
+    return keysLeft.length === keysRight.length &&
+           keysLeft.every(key => Object.prototype.hasOwnProperty.call(right, key) && left[key] === right[key]);
+}
+
+
+export function objectEquals(x: any, y: any): boolean
+{
+    if (isUndefined(x) || isUndefined(y))
+        return x === y;
+
+    // after this just checking type of one would be enough
+    if (x.constructor !== y.constructor)
+        return false;
+
+    // if they are functions, they should exactly refer to same one (because of closures)
+    // if they are regexps, they should exactly refer to same one (it is hard to better equality check on current ES)
+    if (x instanceof Function || x instanceof RegExp)
+        return x === y;
+
+    if (x === y || x.valueOf() === y.valueOf())
+        return true;
+
+    // optimization
+    if (Array.isArray(x) && x.length !== y.length)
+        return false;
+
+    // optimization: if they are dates, they must have had equal valueOf (above)
+    if (x instanceof Date)
+        return false;
+
+    // if they are strictly equal, they both need to be an object at least
+    if (!(x instanceof Object && y instanceof Object)) return false;
+
+
+    const xKeys = Object.keys(x);
+    const yKeys = Object.keys(y);
+
+    // recursive object equality check
+    return xKeys.length === yKeys.length
+        && yKeys.every(yKey => yKey in x)
+        && xKeys.every(xKey => objectEquals(x[xKey], y[xKey]));
+}
+
+export function structurallyEqual(left: any, right: any)
+{
+    if (left === right)
+        return true;
+
+    if (typeof left !== typeof right)
+        return false;
+
+    if (typeof left !== 'object')
+        return left === right;
+
+    if (keys(left).length !== keys(right).length)
+        return false;
+
+    for (const key in left)
+    {
+        if (!(key in right))
+            return false;
+
+        if (!structurallyEqual(left[key], right[key]))
+            return false;
+    }
+
+    return true;
+}
+
+
+
+type FuncDict<T> ={ [P in keyof T]: (a: T[P]) => any };
+type Out<T extends FuncDict<any>> = { [P in keyof T]: ReturnType<T[P]> };
+
+// TODO: use Ramda ?
+export function mapObject<S, M extends FuncDict<S>>(src: S, map: M): Out<FuncDict<S>>
+{
+    const result : Partial<Out<FuncDict<S>>> = {}
+
+    for (const k in src)
+        result[k] = map[k](src[k])
+
+    return result as Out<FuncDict<S>>
+}
+
+export function mapDict<T, R>(src: Dictionary<T>, map: (arg: T) => R): Dictionary<R>
+{
+    const result : Partial<Dictionary<R>> = {}
+
+    for (const k in src)
+        result[k] = map(src[k])
+
+    return result as Dictionary<R>
+}
+
+
+export function valueToFunction<T,R>(tr: Func<T,R> | R) : Func<T,R>
+{
+    if (typeof tr === "function")
+        return tr as Func<T,R>
+
+    return (_: T) => tr
+}
+
+
+// export function mapEntries<V, R>(src: Dictionary<V>, map: (k: string, v: V) => [string, R]): Dictionary<R>
+// {
+//     const result: Dictionary<R> = {}
+//
+//     entries(src).forEach(([key, value]) =>
+//                          {
+//                              const [k, v] = map(key, value)
+//                              result[k] = v;
+//                          })
+//
+//     return result
+// }
+
+export function once<T>(action: Builder<T>) : Builder<T>
+export function once<T>(action: Lazy<T>) : Lazy<Maybe<T>>
+{
+    let done = false;
+    return () =>
+    {
+        if (done)
+            return undefined
+
+        done = true
+        return action()
+    }
+}
+
+
+// export function deepMerge<K extends keyof any, V>(...objects: Record<K, V>[])
+// {
+//     function extracted(prev: Record<K, V>, cur: Record<K, V>)
+//     {
+//         for (const key of keys(cur))
+//         {
+//            const k = key as K
+//
+//             const pVal = prev[k];
+//             const oVal = cur[k];
+//
+//             if (isArray(pVal) && isArray(oVal))
+//             {
+//                 prev[k] = pVal.concat(...oVal);
+//             }
+//             else if (isObject(pVal) && isObject(oVal))
+//             {
+//                 prev[k] = deepMerge(pVal, oVal);
+//             }
+//             else
+//             {
+//                 prev[k] = oVal;
+//             }
+//         }
+//
+//         return prev;
+//     }
+//
+//     return objects.reduce((prev, cur) => extracted(prev, cur), {} as Record<K, V>);
+// }
+
+// // Test objects
+// const obj1 = {
+//     a: 1,
+//     b: 1,
+//     c: { x: 1, y: 1 },
+//     d: [ 1, 1 ]
+// }
+// const obj2 = {
+//     b: 2,
+//     c: { y: 2, z: 2 },
+//     d: [ 2, 2 ],
+//     e: 2
+// }
+// const obj3 = deepMerge(obj1, obj2);
+
+

typescript/DataCache/tsconfig.json

@@ -0,0 +1,82 @@
+{
+    "compilerOptions": 
+    {
+        "lib": ["es2021"],
+        "module": "ESNext",
+        "target": "es2021",
+        "strict": true,
+        "esModuleInterop": true,
+        //"skipLibCheck": true,
+        "forceConsistentCasingInFileNames": true,
+        "useDefineForClassFields": true,
+        "outDir": "dist",
+        "allowSyntheticDefaultImports": true,
+
+
+        // d.ts files
+        // "declaration": true,
+        // "emitDeclarationOnly": true,
+
+        /* Basic Options */
+        "incremental": true,                         /* Enable incremental compilation */
+        //"module": "ES6",                           /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
+
+        "allowJs": false,                             /* Allow javascript files to be compiled. */
+        // "checkJs": true,                             /* Report errors in .js files. */
+        "jsx": "preserve",                           /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
+        // "declaration": true,                         /* Generates corresponding '.d.ts' file. */
+        // "declarationMap": true,                      /* Generates a sourcemap for each corresponding '.d.ts' file. */
+        "sourceMap": true,                           /* Generates corresponding '.map' file. */
+        // "outFile": "./",                             /* Concatenate and emit output to single file. */
+        // "outDir": "./",                              /* Redirect output structure to the directory. */
+        // "rootDir": "./",                             /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+        // "composite": true,                           /* Enable project compilation */
+        "tsBuildInfoFile": "./node_modules/tsconfig.tsbuildinfo",                     /* Specify file to store incremental compilation information */
+        "removeComments": true,                      /* Do not emit comments to output. */
+        "noEmit": true,                              /* Do not emit outputs. */
+        // "importHelpers": true,                       /* Import emit helpers from 'tslib'. */
+        "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+        "isolatedModules": true,                     /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+        /* Strict Type-Checking Options */
+        "noImplicitAny": true,                       /* Raise error on expressions and declarations with an implied 'any' type. */
+        "strictNullChecks": true,                    /* Enable strict null checks. */
+        "strictFunctionTypes": true,                 /* Enable strict checking of function types. */
+        "strictBindCallApply": true,                 /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+        "strictPropertyInitialization": true,        /* Enable strict checking of property initialization in classes. */
+        "noImplicitThis": true,                      /* Raise error on 'this' expressions with an implied 'any' type. */
+        "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
+
+        /* Additional Checks */
+        //"noUnusedLocals": true,                      /* Report errors on unused locals. */
+        // "noUnusedParameters": true,                  /* Report errors on unused parameters. */
+        "noImplicitReturns": true,                   /* Report error when not all code paths in function return a value. */
+        "noFallthroughCasesInSwitch": true,          /* Report errors for fallthrough cases in switch statement. */
+        //"noUncheckedIndexedAccess": true,            /* Include 'undefined' in index signature results */
+        // "noPropertyAccessFromIndexSignature": true,  /* Require undeclared properties from index signatures to use element accesses. */
+
+        /* Module Resolution Options */
+        "moduleResolution": "node",                  /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+        // "baseUrl": "./",                             /* Base directory to resolve non-absolute module names. */
+        // "paths": {},                                 /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+        // "rootDirs": [],                              /* List of root folders whose combined content represents the structure of the project at runtime. */
+        // "typeRoots": [],                             /* List of folders to include type definitions from. */
+        // "types": [],                                 /* Type declaration files to be included in compilation. */
+        // "allowSyntheticDefaultImports": true,        /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+        // "preserveSymlinks": true,                    /* Do not resolve the real path of symlinks. */
+        // "allowUmdGlobalAccess": true,                /* Allow accessing UMD globals from modules. */
+
+        /* Source Map Options */
+        // "sourceRoot": "",                            /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+        // "mapRoot": "",                               /* Specify the location where debugger should locate map files instead of generated locations. */
+        // "inlineSourceMap": true,                     /* Emit a single file with source maps instead of having a separate file. */
+        // "inlineSources": true,                       /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+        /* Experimental Options */
+        // "experimentalDecorators": true,              /* Enables experimental support for ES7 decorators. */
+        // "emitDecoratorMetadata": true,               /* Enables experimental support for emitting type metadata for decorators. */
+    },
+    
+    "include": ["src/**/*.ts","src/**/*.tsx"],
+    "exclude": ["node_modules", "dist"]
+}