add WIP DataCache

This commit is contained in:
ig 2023-04-13 08:08:36 +02:00
parent ba54f9e325
commit 8de076d8de
52 changed files with 50187 additions and 0 deletions

19394
typescript/DataCache/dist/server.js vendored Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

12577
typescript/DataCache/dist/www/client.js vendored Normal file

File diff suppressed because it is too large Load Diff

1
typescript/DataCache/dist/www/icons vendored Symbolic link
View File

@ -0,0 +1 @@
../../src/client/icons

1
typescript/DataCache/dist/www/index.html vendored Symbolic link
View File

@ -0,0 +1 @@
../../src/client/index.html

4456
typescript/DataCache/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,30 @@
{
"main": "src/server.ts",
"type": "commonjs",
"devDependencies": {
"@types/node": "^17.0.45",
"@typescript-eslint/eslint-plugin": "^5.12.1",
"@typescript-eslint/parser": "^5.12.1",
"eslint": "^8.10.0",
"node-fetch": "^3.2.0",
"simplytyped": "^3.3.0",
"ts-auto-guard": "^2.4.1",
"typescript": "^4.8.3"
},
"scripts": {
"buildServer": "esbuild src/server.ts --bundle --sourcemap --outfile=dist/server.js --platform=node",
"buildAndRunServer": "npm run buildServer && node dist/server.js",
"buildClient": "esbuild src/client.ts --bundle --sourcemap --outfile=dist/www/client.js",
"buildClientMinified": "esbuild src/client.ts --bundle --minify --sourcemap --outfile=dist/www/client.js"
},
"dependencies": {
"@types/react": "^18.0.14",
"@types/react-dom": "^18.0.5",
"esbuild": "^0.14.23",
"immer": "^9.0.16",
"linq-to-typescript": "^10.0.0",
"preact": "^10.8.2",
"rxjs": "^7.5.5",
"ts-proto": "^1.104.0"
}
}

View File

@ -0,0 +1,30 @@
import './utils/linq'
import DataCache from "./server/dataCache/dataCache";
import {TimeSpan, UnixTime} from "./server/dataCache/time";
import {filter} from "rxjs";
const resolution = TimeSpan.fromSeconds(2);
const cache = new DataCache(x => (Promise.resolve({foo: 0})), resolution)
const sampleTimes = [1, 2, 3, 4, 5, 6].select(e => UnixTime.fromTicks(e)).toArray();
const series = cache.getSeries(sampleTimes)
console.log(series)
console.log("done")

View File

@ -0,0 +1,79 @@
import {sha1Hmac} from "./Sha1";
import {Utf8} from "./Utf8";
import {toBase64} from "./UInt8Utils";
export class S3Access
{
constructor
(
readonly bucket: string,
readonly region: string,
readonly provider: string,
readonly key: string,
readonly secret: string,
readonly contentType: string
)
{}
get host() : string { return `${this.bucket}.${this.region}.${this.provider}` }
get url() : string { return `https://${this.host}` }
public get(s3Path : string): Promise<Response>
{
const method = "GET";
const auth = this.createAuthorizationHeader(method, s3Path, "");
const url = this.url + "/" + s3Path
const headers = {"Host": this.host, "Authorization": auth};
try
{
return fetch(url, {method: method, mode: "cors", headers: headers})
}
catch
{
return Promise.reject()
}
}
private createAuthorizationHeader(method: string,
s3Path: string,
date: string)
{
return createAuthorizationHeader
(
method,
this.bucket,
s3Path,
date,
this.key,
this.secret,
this.contentType
);
}
}
function createAuthorizationHeader(method: string,
bucket: string,
s3Path: string,
date: string,
s3Key: string,
s3Secret: string,
contentType: string,
md5Hash: string = "")
{
// StringToSign = HTTP-Verb + "\n" +
// Content-MD5 + "\n" +
// Content-Type + "\n" +
// Date + "\n" +
// CanonicalizedAmzHeaders +
// CanonicalizedResource;
const payload = Utf8.encode(`${method}\n${md5Hash}\n${contentType}\n${date}\n/${bucket}/${s3Path}`)
//console.log(`${method}\n${md5Hash}\n${contentType}\n${date}\n/${bucket}/${s3Path}`)
const secret = Utf8.encode(s3Secret)
const signature = toBase64(sha1Hmac(payload, secret));
return `AWS ${s3Key}:${signature}`
}

View File

@ -0,0 +1,125 @@
import {concat, pad} from "./UInt8Utils";
const BigEndian = false
export function sha1Hmac(msg: Uint8Array, key: Uint8Array): Uint8Array
{
if (key.byteLength > 64)
key = sha1(key)
if (key.byteLength < 64)
key = pad(key, 64)
const oKey = key.map(b => b ^ 0x5C);
const iKey = key.map(b => b ^ 0x36);
const iData = concat(iKey, msg);
const iHash = sha1(iData);
const oData = concat(oKey, iHash);
return sha1(oData);
}
export function sha1(data: Uint8Array): Uint8Array
{
const paddedData: DataView = initData(data)
const H = new Uint32Array([0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0])
const S = new Uint32Array(5) // State
const round = new Uint32Array(80);
function initRound(startOffset: number)
{
for (let i = 0; i < 16; i++)
round[i] = paddedData.getUint32((startOffset + i) * 4, BigEndian);
for (let i = 16; i < 80; i++)
{
const int32 = round[i - 3] ^ round[i - 8] ^ round[i - 14] ^ round[i - 16];
round[i] = rotate1(int32); // SHA0 has no rotate
}
}
const functions =
[
() => (S[1] & S[2] | ~S[1] & S[3]) + 0x5A827999,
() => (S[1] ^ S[2] ^ S[3]) + 0x6ED9EBA1,
() => (S[1] & S[2] | S[1] & S[3] | S[2] & S[3]) + 0x8F1BBCDC,
() => (S[1] ^ S[2] ^ S[3]) + 0xCA62C1D6
]
for (let startOffset = 0; startOffset < paddedData.byteLength / 4; startOffset += 16)
{
initRound(startOffset);
S.set(H)
for (let r = 0, i = 0; r < 4; r++)
{
const f = functions[r]
const end = i + 20;
do
{
const S0 = rotate5(S[0]) + f() + S[4] + round[i];
S[4] = S[3];
S[3] = S[2];
S[2] = rotate30(S[1]);
S[1] = S[0];
S[0] = S0;
}
while (++i < end)
}
for (let i = 0; i < 5; i++)
H[i] += S[i]
}
swapEndianness(H);
return new Uint8Array(H.buffer)
}
function rotate5(int32: number)
{
return (int32 << 5) | (int32 >>> 27); // >>> for unsigned shift
}
function rotate30(int32: number)
{
return (int32 << 30) | (int32 >>> 2);
}
function rotate1(int32: number)
{
return (int32 << 1) | (int32 >>> 31);
}
function initData(data: Uint8Array): DataView
{
const dataLength = data.length
const extendedLength = dataLength + 9; // add 8 bytes for UInt64 length + 1 byte for "stop-bit" (0x80)
const paddedLength = Math.ceil(extendedLength / 64) * 64; // pad to 512 bits block
const paddedData = new Uint8Array(paddedLength)
paddedData.set(data)
paddedData[dataLength] = 0x80 // append single 1 bit at end of data
const dataView = new DataView(paddedData.buffer)
// append UInt64 length
dataView.setUint32(paddedData.length - 4, dataLength << 3 , BigEndian) // dataLength in *bits* LO, (<< 3: x8 bits per byte)
dataView.setUint32(paddedData.length - 8, dataLength >>> 29, BigEndian) // dataLength in *bits* HI
return dataView
}
function swapEndianness(uint32Array: Uint32Array)
{
const dv = new DataView(uint32Array.buffer)
for (let i = 0; i < uint32Array.byteLength; i += 4)
{
const uint32 = dv.getUint32(i, false)
dv.setUint32(i, uint32, true)
}
}

View File

@ -0,0 +1,56 @@
export function pad(data: Uint8Array, length: number): Uint8Array
{
if (length < data.byteLength)
throw new RangeError("length")
const padded = new Uint8Array(length)
padded.set(data)
return padded;
}
export function concat(left: Uint8Array, right: Uint8Array): Uint8Array
{
const c = new Uint8Array(left.length + right.length);
c.set(left);
c.set(right, left.length);
return c
}
export function toHexString(data: Uint8Array)
{
return [...data].map(byteToHex).join('');
}
function byteToHex(b: number)
{
return b.toString(16).padStart(2, "0");
}
const b64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
export function toBase64(data : Uint8Array) : string
{
const byteLength = data.byteLength
const base64LengthPadded = 4 * Math.ceil(byteLength / 3)
const base64Length = Math.ceil(byteLength / 3 * 4);
const base64 = new Array<String>(base64LengthPadded)
for (let i = 0, o = 0; i < byteLength;)
{
const x = data[i++]
const y = data[i++] ?? 0
const z = data[i++] ?? 0
base64[o++] = b64Chars[x >>> 2]
base64[o++] = b64Chars[(x << 4 | y >>> 4) & 63]
base64[o++] = b64Chars[(y << 2 | z >>> 6) & 63]
base64[o++] = b64Chars[z & 63]
}
for (let i = base64LengthPadded; i > base64Length ;)
base64[--i] = "="
return base64.join('')
}

View File

@ -0,0 +1,10 @@
export namespace Utf8
{
const encoder = new TextEncoder()
const decoder = new TextDecoder()
export const encode = (text: string): Uint8Array => encoder.encode(text);
export const decode = (data: Uint8Array): string => decoder.decode(data);
}

View File

@ -0,0 +1,20 @@
import {Timestamped} from "./types";
import {isDefined, Maybe} from "../../utils/maybe";
export type DataRecord = Record<string, number>
export type DataPoint = Timestamped<Maybe<DataRecord>>
export type RecordSeries = Array<DataPoint>
export type PointSeries = Array<Timestamped<Maybe<number>>>
export type DataSeries = Array<Maybe<number>>
export function getPoints(recordSeries: RecordSeries, series: keyof DataRecord): PointSeries
{
return recordSeries.map(p => ({time: p.time, value: isDefined(p.value) ? p.value[series] : undefined}))
}
export function getData(recordSeries: RecordSeries, series: keyof DataRecord): DataSeries
{
return recordSeries.map(p => (isDefined(p.value) ? p.value[series] : undefined))
}

View File

@ -0,0 +1,167 @@
import {TimeSpan, UnixTime} from "./time";
import {Observable, Subject} from "rxjs";
import {SkipList} from "./skipList/skipList";
import {createDispatchQueue} from "./promiseQueue";
import {SkipListNode} from "./skipList/skipListNode";
import {RecordSeries} from "./data";
import {isUndefined, Maybe} from "../../utils/maybe";
export const FetchResult =
{
notAvailable : "N/A",
tryLater : "Try Later"
} as const
export type FetchResult<T> =
| T
| typeof FetchResult.notAvailable
| typeof FetchResult.tryLater
function reverseBits(x : number): number
{
// https://stackoverflow.com/a/60227327/141397
x = (x & 0x55555555) << 1 | (x & 0xAAAAAAAA) >> 1;
x = (x & 0x33333333) << 2 | (x & 0xCCCCCCCC) >> 2;
x = (x & 0x0F0F0F0F) << 4 | (x & 0xF0F0F0F0) >> 4;
x = (x & 0x00FF00FF) << 8 | (x & 0xFF00FF00) >> 8;
x = (x & 0x0000FFFF) << 16 | (x & 0xFFFF0000) >> 16;
return x >>> 0;
}
export default class DataCache<T extends Record<string, number>>
{
private readonly cache: SkipList<Maybe<T>> = new SkipList<Maybe<T>>()
private readonly resolution: TimeSpan;
readonly #fetch: (t: UnixTime) => Promise<FetchResult<T>>;
private readonly fetchQueue = createDispatchQueue(6)
private readonly fetching: Set<number> = new Set<number>()
public readonly gotData: Observable<UnixTime>;
constructor(fetch: (t: UnixTime) => Promise<FetchResult<T>>, resolution: TimeSpan)
{
this.#fetch = fetch;
this.resolution = resolution;
this.gotData = new Subject<UnixTime>()
}
public prefetch(times: Array<UnixTime>, clear = true)
{
if (clear)
{
this.fetching.clear()
this.fetchQueue.clear()
}
const timesWithPriority = times.map((time, index) => ({time, priority: reverseBits(index)}))
timesWithPriority.sort((x, y) => x.priority - y.priority)
for (let i = 0; i < timesWithPriority.length; i++)
{
const time = timesWithPriority[i].time.round(this.resolution)
const t = time.ticks;
const node = this.cache.find(t);
if (node.index !== t)
this.fetchData(time);
}
}
public get(timeStamp: UnixTime, fetch = true): Maybe<T>
{
const time = timeStamp.round(this.resolution)
const t = time.ticks;
const node = this.cache.find(t);
if (node.index === t)
return node.value
if (fetch)
this.fetchData(time);
return this.interpolate(node, t)
}
public getSeries(sampleTimes: UnixTime[]): RecordSeries
{
this.prefetch(sampleTimes)
return sampleTimes.map(time => ({time, value: this.get(time, false)}))
}
private interpolate(before: SkipListNode<Maybe<T>>, t: number): Maybe<T>
{
const dataBefore = before.value
const after = before.next[0];
const dataAfter = after.value
if (isUndefined(dataBefore) && isUndefined(dataAfter))
return undefined
if (isUndefined(dataBefore))
return dataAfter
if (isUndefined(dataAfter))
return dataBefore
const p = t - before.index
const n = after.index - t
const pn = p + n
let interpolated: Partial<Record<string, number>> = {}
for (const k of Object.keys(dataBefore))
{
interpolated[k] = (dataBefore[k] * n + dataAfter[k] * p) / pn
}
return interpolated as T
}
private fetchData(time: UnixTime)
{
const t = time.ticks;
if (this.fetching.has(t)) // we are already fetching t
return
const fetchTask = () =>
{
const onSuccess = (data: FetchResult<T>) =>
{
if (data === FetchResult.tryLater)
{
console.warn(FetchResult.tryLater)
return
}
const value = data === FetchResult.notAvailable ? undefined : data;
this.cache.insert(value, t)
}
const onFailure = (_: unknown) =>
{
console.error(time.ticks + " FAILED!") // should not happen
}
const dispatch = () =>
{
this.fetching.delete(time.ticks);
(this.gotData as Subject<UnixTime>).next(time);
}
return this.#fetch(time)
.then(d => onSuccess(d), f => onFailure(f))
.finally(() => dispatch())
};
this.fetching.add(t)
this.fetchQueue.dispatch(() => fetchTask());
}
}

View File

@ -0,0 +1,167 @@
import {isDefined} from "../../utils/maybe";
export class GraphCanvas
{
private constructor(private readonly canvasElement: HTMLCanvasElement,
private readonly ctx: CanvasRenderingContext2D,
private readonly scaleX: number = 1,
private readonly scaleY: number = 1,
private readonly translateX: number = 0,
private readonly translateY: number = 0,
)
{
}
public static fromCanvasElement(canvasElement: HTMLCanvasElement): GraphCanvas
{
const ctx = canvasElement.getContext("2d") as CanvasRenderingContext2D
return new GraphCanvas(canvasElement, ctx)
}
public transform(scaleX: number, scaleY: number, translateX: number, translateY: number): GraphCanvas
{
return new GraphCanvas(this.canvasElement,
this.ctx,
this.scaleX * scaleX,
this.scaleY * scaleY,
this.translateX + translateX * this.scaleX,
this.translateY + translateY * this.scaleY)
}
public clear(bgStyle?: string | CanvasGradient | CanvasPattern)
{
if (isDefined(bgStyle))
{
this.ctx.fillStyle = bgStyle
this.ctx.fillRect(0, 0, this.canvasElement.width, this.canvasElement.height)
}
else
{
this.ctx.clearRect(0, 0, this.canvasElement.width, this.canvasElement.height)
}
}
public beginPath()
{
this.ctx.beginPath()
}
public closePath()
{
this.ctx.closePath()
}
public moveTo(x: number, y: number)
{
this.ctx.moveTo(this.transformX(x), this.transformY(y))
}
public lineTo(x: number, y: number)
{
this.ctx.lineTo(this.transformX(x), this.transformY(y))
}
public fill()
{
this.ctx.fill();
}
public stroke()
{
this.ctx.stroke();
}
public save()
{
this.ctx.save();
}
public restore()
{
this.ctx.restore();
}
public set fillStyle(style: string | CanvasGradient | CanvasPattern)
{
this.ctx.fillStyle = style
}
public get fillStyle(): string | CanvasGradient | CanvasPattern
{
return this.ctx.fillStyle
}
public set strokeStyle(style: string | CanvasGradient | CanvasPattern)
{
this.ctx.strokeStyle = style
}
public get strokeStyle(): string | CanvasGradient | CanvasPattern
{
return this.ctx.strokeStyle
}
public set lineWidth(width: number)
{
this.ctx.lineWidth = width
}
public get lineWidth(): number
{
return this.ctx.lineWidth
}
public strokeCircle(x: number, y: number, radius: number, stroke?: string | CanvasGradient | CanvasPattern)
{
if (isDefined(stroke))
this.ctx.strokeStyle = stroke
this.ctx.beginPath();
this.ctx.ellipse(this.transformX(x), this.transformY(y), radius, radius, 0, 0, 2 * Math.PI)
this.ctx.stroke();
}
public fillCircle(x: number, y: number, radius: number, fill?: string | CanvasGradient | CanvasPattern)
{
if (isDefined(fill))
this.ctx.fillStyle = fill
this.ctx.beginPath();
this.ctx.ellipse(this.transformX(x), this.transformY(y), radius, radius, 0, 0, 2 * Math.PI)
this.ctx.fill();
}
public fillRect(x: number, y: number, w: number, h: number)
{
this.ctx.fillRect(this.transformX(x), this.transformY(y), w, h)
}
public strokeRect(x: number, y: number, w: number, h: number)
{
this.ctx.strokeRect(this.transformX(x), this.transformY(y), w, h)
}
public transformX(x: number)
{
return x * this.scaleX + this.translateX
}
public transformY(y: number)
{
return y * this.scaleY + this.translateY
}
public transformBackX(x: number)
{
return (x - this.translateX) / this.scaleX
}
public transformBackY(y: number)
{
return (y - this.translateY) / this.scaleY
}
}

View File

@ -0,0 +1,20 @@
// 0. Import Module
import { initializeLinq, IEnumerable } from "linq-to-typescript"
// 1. Declare that the JS types implement the IEnumerable interface
declare global {
interface Array<T> extends IEnumerable<T> { }
interface Uint8Array extends IEnumerable<number> { }
interface Uint8ClampedArray extends IEnumerable<number> { }
interface Uint16Array extends IEnumerable<number> { }
interface Uint32Array extends IEnumerable<number> { }
interface Int8Array extends IEnumerable<number> { }
interface Int16Array extends IEnumerable<number> { }
interface Int32Array extends IEnumerable<number> { }
interface Float32Array extends IEnumerable<number> { }
interface Float64Array extends IEnumerable<number> { }
interface Map<K, V> extends IEnumerable<[K, V]> { }
interface Set<T> extends IEnumerable<T> { }
interface String extends IEnumerable<string> { }
}
// 2. Bind Linq Functions to Array, Map, etc
initializeLinq()

View File

@ -0,0 +1,63 @@
import {map, MonoTypeOperatorFunction, Observable, tap} from "rxjs";
import {fastHash} from "./utils";
import {HslColor, toCssColor} from "./color";
type ConcatX<T extends readonly (readonly any[])[]> = [
...T[0], ...T[1], ...T[2], ...T[3], ...T[4],
...T[5], ...T[6], ...T[7], ...T[8], ...T[9],
...T[10], ...T[11], ...T[12], ...T[13], ...T[14],
...T[15], ...T[16], ...T[17], ...T[18], ...T[19]
];
type Flatten<T extends readonly any[]> =
ConcatX<[...{ [K in keyof T]: T[K] extends any[] ? T[K] : [T[K]] }, ...[][]]>
export function flatten()
{
return function<T extends Array<unknown>>(source: Observable<T>)
{
return source.pipe
(
map(a => a.flat() as Flatten<T>)
)
}
}
type RecursiveObject<T> = T extends object ? T : never;
type Terminals<TModel, T> =
{
[Key in keyof TModel]: TModel[Key] extends RecursiveObject<TModel[Key]>
? Terminals<TModel[Key], T>
: T;
};
export function debug<T>(tag: string = "debug"): MonoTypeOperatorFunction<T>
{
const hslColor: HslColor =
{
h: fastHash(tag),
s: (fastHash('s' + tag) % 30 + 30) / 100,
l: (fastHash('l' + tag) % 30 + 30) / 100
}
const color = toCssColor(hslColor)
const css = `background: ${color}; color: #fff; padding: 3px; font-size: 9px;`
return tap({
next(value)
{
console.log(`%c[${tag}]`, css, value)
},
error(error)
{
console.log(`%c[${tag}]`, css, error)
},
complete()
{
console.log(`%c[${tag}]`, css)
}
})
}

View File

@ -0,0 +1,51 @@
export function createDispatchQueue(maxInflight: number, debug = false): { dispatch: (task: () => Promise<void>) => number; clear: () => void }
{
const queue: Array<() => Promise<void>> = []
let inflight = 0;
function done()
{
inflight--
if (debug && inflight + queue.length === 0)
console.log("queue empty")
if (inflight < maxInflight && queue.length > 0)
{
const task = queue.pop()!
inflight++
task().finally(() => done())
}
}
function dispatch(task: () => Promise<void>) : number
{
if (inflight < maxInflight)
{
inflight++;
task().finally(() => done())
}
else
{
if (debug && queue.length === 0)
console.log("queue in use")
queue.push(task)
}
return queue.length
}
function clear()
{
// https://stackoverflow.com/questions/1232040/how-do-i-empty-an-array-in-javascript
queue.length = 0
if (debug)
console.log("queue cleared")
}
return {dispatch, clear}
}

View File

@ -0,0 +1,80 @@
import {find, findPath, insert, Path, SkipListNode} from "./skipListNode";
export class SkipList<T>
{
public readonly head: SkipListNode<T>;
public readonly tail: SkipListNode<T>;
private readonly nLevels: number;
private _length = 0
constructor(nLevels: number = 20)
{
// TODO: auto-levels
this.tail =
{
index: Number.MAX_VALUE,
next: [],
value: undefined!
};
this.head =
{
index: Number.MIN_VALUE,
next: Array(nLevels).fill(this.tail),
value: undefined!
};
this.nLevels = nLevels
}
public find(index: number, startNode = this.head, endNode = this.tail): SkipListNode<T>
{
return find(index, startNode, endNode)
}
private findPath(index: number, startNode = this.head, endNode = this.tail): Path<T>
{
return findPath(index, startNode, endNode)
}
public insert(value: T, index: number): SkipListNode<T>
{
const path = this.findPath(index)
const node = path[0];
if (node.index === index) // overwrite
{
node.value = value
return node
}
const nodeToInsert = {value, index, next: []} as SkipListNode<T>
const rnd = (Math.random() * (1 << this.nLevels)) << 0;
for (let level = 0; level < this.nLevels; level++)
{
insert(nodeToInsert, path[level], level)
if ((rnd & (1 << level)) === 0)
break
}
this._length += 1
return nodeToInsert;
}
get length(): number
{
return this._length;
}
// public remove(index: number): void
// {
// // TODO
// }
}

View File

@ -0,0 +1,52 @@
import {asMutableArray} from "../types";
export type Next<T> = { readonly next: ReadonlyArray<SkipListNode<T>> }
export type Index = { readonly index: number }
export type Indexed<T> = Index & { value: T }
export type SkipListNode<T> = Next<T> & Indexed<T>
export type Path<T> = SkipListNode<T>[];
export function find<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>): SkipListNode<T>
{
let node = startNode
for (let level = startNode.next.length - 1; level >= 0; level--)
node = findOnLevel(index, node, endNode, level)
return node
}
export function findOnLevel<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>, level: number): SkipListNode<T>
{
let node: SkipListNode<T> = startNode
while (true)
{
const next = node.next[level]
if (index < next.index || endNode.index < next.index)
return node
node = next
}
}
export function findPath<T>(index: number, startNode: SkipListNode<T>, endNode: SkipListNode<T>): Path<T>
{
const path = Array(startNode.next.length - 1)
let node = startNode
for (let level = startNode.next.length - 1; level >= 0; level--)
{
node = findOnLevel(index, node, endNode, level)
path[level] = node
}
return path
}
export function insert<T>(nodeToInsert: SkipListNode<T>, after: SkipListNode<T>, onLevel: number): void
{
asMutableArray(nodeToInsert.next)[onLevel] = after.next[onLevel]
asMutableArray(after.next)[onLevel] = nodeToInsert
}

View File

@ -0,0 +1,5 @@
export function trim(str: string, string: string = " "): string
{
const pattern = '^[' + string + ']*(.*?)[' + string + ']*$';
return str.replace(new RegExp(pattern), '$1')
}

View File

@ -0,0 +1,302 @@
import {trim} from "./stringUtils";
export class UnixTime
{
private constructor(readonly ticks: number)
{
}
public static readonly Epoch = new UnixTime(0)
public static now(): UnixTime
{
return UnixTime.fromTicks(Date.now() / 1000)
}
public static fromDate(date: Date): UnixTime
{
return UnixTime.fromTicks(date.getTime() / 1000)
}
public toDate(): Date
{
return new Date(this.ticks * 1000)
}
public static fromTicks(ticks: number): UnixTime
{
return new UnixTime(ticks)
}
public later(timeSpan: TimeSpan): UnixTime
{
return new UnixTime(this.ticks + timeSpan.ticks)
}
public move(ticks: number): UnixTime
{
return new UnixTime(this.ticks + ticks)
}
public earlier(timeSpan: TimeSpan): UnixTime
{
return new UnixTime(this.ticks - timeSpan.ticks)
}
public isEarlierThan(time: UnixTime): boolean
{
return this.ticks < time.ticks
}
public isEarlierThanOrEqual(time: UnixTime): boolean
{
return this.ticks <= time.ticks
}
public isLaterThan(time: UnixTime): boolean
{
return this.ticks > time.ticks
}
public isLaterThanOrEqual(time: UnixTime): boolean
{
return this.ticks >= time.ticks
}
public isEqual(time: UnixTime): boolean
{
return this.ticks === time.ticks
}
public isInTheFuture(): boolean
{
return this.isLaterThan(UnixTime.now())
}
public isInThePast(): boolean
{
return this.ticks < UnixTime.now().ticks
}
public round(ticks:number) : UnixTime
public round(duration: TimeSpan) : UnixTime
public round(durationOrTicks: TimeSpan | number) : UnixTime
{
const ticks = (typeof durationOrTicks === "number") ? durationOrTicks : durationOrTicks.ticks
return new UnixTime(Math.round(this.ticks / ticks) * ticks)
}
public rangeTo(time: UnixTime): TimeRange
{
return TimeRange.fromTimes(this, time);
}
public rangeBefore(timeSpan: TimeSpan): TimeRange
{
return TimeRange.fromTimes(this.earlier(timeSpan), this);
}
public rangeAfter(timeSpan: TimeSpan): TimeRange
{
return TimeRange.fromTimes(this, this.later(timeSpan));
}
public toString() : string
{
return this.ticks.toString()
}
}
export class TimeSpan
{
private constructor(readonly ticks: number) {}
get milliSeconds(): number { return this.ticks * 1000 }
get seconds() : number { return this.ticks }
get minutes() : number { return this.ticks / 60 }
get hours() : number { return this.minutes / 60 }
get days() : number { return this.hours / 24 }
get weeks() : number { return this.days / 7 }
public static fromTicks (t: number): TimeSpan { return new TimeSpan(t) }
public static fromSeconds(t: number): TimeSpan { return TimeSpan.fromTicks(t) }
public static fromMinutes(t: number): TimeSpan { return TimeSpan.fromSeconds(t*60) }
public static fromHours (t: number): TimeSpan { return TimeSpan.fromMinutes(t*60) }
public static fromDays (t: number): TimeSpan { return TimeSpan.fromHours(t*24) }
public static fromWeeks (t: number): TimeSpan { return TimeSpan.fromDays(t*7) }
public static span(from: UnixTime, to: UnixTime) : TimeSpan
{
return TimeSpan.fromTicks(Math.abs(to.ticks - from.ticks))
}
public add(timeSpan: TimeSpan) : TimeSpan
{
return TimeSpan.fromTicks(this.ticks + timeSpan.ticks)
}
public subtract(timeSpan: TimeSpan) : TimeSpan
{
return TimeSpan.fromTicks(this.ticks - timeSpan.ticks)
}
public divide(n: number) : TimeSpan
{
if (n <= 0)
throw 'n must be positive';
return TimeSpan.fromTicks(this.ticks/n)
}
public multiply(n: number) : TimeSpan
{
if (n < 0)
throw 'n cannot be negative';
return TimeSpan.fromTicks(this.ticks * n)
}
public round(ticks:number) : TimeSpan
public round(duration: TimeSpan) : TimeSpan
public round(durationOrTicks: TimeSpan | number) : TimeSpan
{
const ticks = (typeof durationOrTicks === "number")
? durationOrTicks
: durationOrTicks.ticks
return TimeSpan.fromTicks(Math.round(this.ticks / ticks) * ticks)
}
public toString() : string
{
let dt = 60*60*24*7
let ticks = this.ticks;
if (ticks === 0)
return "0s"
ticks = Math.abs(ticks)
const nWeeks = Math.floor(ticks / dt)
ticks -= nWeeks * dt
dt /= 7
const nDays = Math.floor(ticks / dt)
ticks -= nDays * dt
dt /= 24
const nHours = Math.floor(ticks / dt)
ticks -= nHours * dt
dt /= 60
const nMinutes = Math.floor(ticks / dt)
ticks -= nMinutes * dt
dt /= 60
const nSeconds = Math.floor(ticks / dt)
let s = ""
if (nWeeks > 0) s += nWeeks .toString() + "w "
if (nDays > 0) s += nDays .toString() + "d "
if (nHours > 0) s += nHours .toString() + "h "
if (nMinutes > 0) s += nMinutes.toString() + "m "
if (nSeconds > 0) s += nSeconds.toString() + "s"
return trim(s);
}
}
export class TimeRange
{
private constructor(private readonly from: number, private readonly to: number)
{
}
public get start(): UnixTime
{
return UnixTime.fromTicks(this.from)
}
public get mid(): UnixTime
{
return UnixTime.fromTicks((this.from + this.to) / 2)
}
public get end(): UnixTime
{
return UnixTime.fromTicks(this.to)
}
public get duration(): TimeSpan
{
return TimeSpan.fromTicks(this.to - this.from)
}
public static fromTimes(from: UnixTime, to: UnixTime): TimeRange
{
return from.isLaterThan(to)
? new TimeRange(to.ticks, from.ticks)
: new TimeRange(from.ticks, to.ticks)
}
public isInside(time: number) : boolean;
public isInside(time: UnixTime) : boolean;
public isInside(time: UnixTime | number)
{
const t = time instanceof UnixTime ? time.ticks : time
return t >= this.from && t < this.to
}
public sample(period: TimeSpan): UnixTime[]
{
const samples = []
for (let t = this.from; t < this.to; t += period.ticks)
samples.push(UnixTime.fromTicks(t));
return samples
}
public subdivide(n: number) : TimeRange[]
{
if (n <= 0)
throw 'n must be positive';
const period = TimeSpan.fromTicks(this.duration.ticks / n);
if (period === this.duration)
return [this];
const samples = this.sample(period);
const ranges : TimeRange[] = []
for (let i = 0; i < samples.length;)
ranges.push(TimeRange.fromTimes(samples[i], samples[++i]))
return ranges
}
public earlier(dt: TimeSpan) : TimeRange
{
return new TimeRange(this.from - dt.ticks, this.to - dt.ticks)
}
public later(dt: TimeSpan) : TimeRange
{
return new TimeRange(this.from + dt.ticks, this.to + dt.ticks)
}
public move(ticks: number) : TimeRange
{
return new TimeRange(this.from + ticks, this.to + ticks)
}
}

View File

@ -0,0 +1,21 @@
import {UnixTime} from "./time";
export type Timestamped<T> = { time: UnixTime, value: T }
export type Pair<T1, T2 = T1> = [T1, T2]
export type Position = { readonly x: number, readonly y: number }
export type Direction = { readonly dx: number, readonly dy: number }
export type Size = { readonly width: number, readonly height: number }
export type Rect = Position & Size
export type Mutable<T> = { -readonly [P in keyof T]: T[P] };
export type FieldKey<T> = { [P in keyof T]: T[P] extends (...args: any) => any ? never : P }[keyof T];
export type AllFields<T> = Pick<T, FieldKey<T>>;
export type SomeFields<T> = Partial<AllFields<T>>
export const asMutable = <T>(t: T) => (t as Mutable<T>);
export const asMutableArray = <T>(t: ReadonlyArray<T>) => (t as Array<T>);
export const cast = <T>(t: unknown) => (t as T);
export type Rename<T, K extends keyof T, N extends string> = Pick<T, Exclude<keyof T, K>> & { [P in N]: T[K] }

View File

@ -0,0 +1,119 @@
import {IEnumerable} from "linq-to-typescript";
import {isDefined} from "../../utils/maybe";
//export type UnionToIntersection<U> = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I : never
export type Nothing = Record<string, never>
// eslint-disable-next-line @typescript-eslint/no-empty-function
export function fastHash(str: string): number
{
const signed = str
.split('')
.reduce((p, c) => ((p << 5) - p) + c.charCodeAt(0) | 0, 0);
return Math.abs(signed);
}
// export function flattenObject(obj: object) : object
// {
// const flattened = {}
//
// for (const key of Object.keys(obj))
// {
// // @ts-ignore
// const value = obj[key]
//
// if (typeof value === 'object' && value !== null && !Array.isArray(value))
// {
// Object.assign(flattened, flattenObject(value))
// }
// else
// {
// // @ts-ignore
// flattened[key] = value
// }
// }
//
// return flattened
// }
//return function<TTerminal,TTree extends Terminals<TTree, TTerminal>>(source: Observable<TTerminal>)
export function* pairwise<T>(iterable: Iterable<T>, init?: T): Generator<[T, T]>
{
const it = iterable[Symbol.iterator]()
let first : T;
if (isDefined(init))
{
first = init
}
else
{
const f = it.next()
if (f.done)
return
first = f.value
}
let second = it.next()
while(!second.done)
{
yield [first, second.value]
first = second.value
second = it.next()
}
}
export function arraysEqual<T>(a: Array<T>, b: Array<T>)
{
if (a === b) return true;
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; ++i)
{
if (a[i] !== b[i]) return false;
}
return true;
}
export function mod(a:number, b:number)
{
return ((a % b) + b) % b;
}
export function clamp(a: number, min: number, max: number)
{
return a > max ? max
: a < min ? min
: a
}
export function isDST(d : Date)
{
const jan = new Date(d.getFullYear(), 0, 1).getTimezoneOffset();
const jul = new Date(d.getFullYear(), 6, 1).getTimezoneOffset();
return Math.max(jan, jul) != d.getTimezoneOffset();
}
export function Transpose<T>(src: IEnumerable<IEnumerable<T>>): IEnumerable<IEnumerable<T>>
{
return src
.selectMany(line => line.select((element, column) => ({element, column})))
.groupBy(i => i.column)
.select(g => g.select(e => e.element));
}

View File

@ -0,0 +1,46 @@
import fs from "fs";
export function doesFileExist(path: string): boolean
{
try
{
fs.accessSync(path, fs.constants.F_OK);
return true;
}
catch (e)
{
return false;
}
}
export function doesDirExist(path: string): boolean
{
try
{
fs.readdirSync(path)
return true;
}
catch (e)
{
return false;
}
}
export function readJsonFile<T>(path: string)
{
const data = fs.readFileSync(path, "utf-8")
return JSON.parse(data) as T
}
export function writeJsonFile<T>(path: string, contents: T)
{
const data = JSON.stringify(contents)
return fs.writeFileSync(path, data, "utf-8")
}
export function writeJsonFilePretty<T>(path: string, contents: T)
{
const data = JSON.stringify(contents,undefined,2)
return fs.writeFileSync(path, data, "utf-8")
}

View File

@ -0,0 +1,124 @@
import {isDefined, isUndefined, Maybe} from "../../utils/maybe";
import {Dictionary} from "../../utils/utilityTypes";
import MimeType from "./mime";
import {promisify} from "util";
import fs from "fs";
import http, {IncomingMessage, ServerResponse} from "http";
import {entries} from "../../utils/utils";
import {getLogger} from "../../utils/logging";
const log = getLogger("HTTP")
const readFile = promisify(fs.readFile)
export type HttpResponse = {
body: Maybe<Buffer | string>
headers : Dictionary<string>
statusCode: number
}
function contentTypeHeader(mimeType: string)
{
return {['Content-type']: mimeType};
}
function forbidden(message = "403 : forbidden", headers: Dictionary<string> = {}): HttpResponse
{
return text(message, 403, headers)
}
function notFound(message ="404 : not found", headers: Dictionary<string> = {}): HttpResponse
{
return text(message, 404, headers)
}
function text(text: string, statusCode = 200, headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: statusCode,
headers : {...headers, ...contentTypeHeader('text/plain')},
body : text
};
}
function json(json: Dictionary,
replacer?: (k: string, v: unknown) => unknown,
headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: 200,
headers : {...headers, ['Content-type']: MimeType.json},
body: JSON.stringify(json, replacer)
}
}
function empty(headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: 200,
headers,
body: undefined
}
}
function ok(body: Maybe<Buffer | string>, headers: Dictionary<string> = {}): HttpResponse
{
return {
statusCode: 200,
headers,
body
}
}
async function file(localRootPath: string, urlPath: string, headers: Dictionary<string> = {}, defaultPath = "/"): Promise<HttpResponse>
{
if (urlPath.contains('..'))
return HTTP.forbidden();
const localPath = localRootPath + (urlPath === "/" ? defaultPath : urlPath);
const body = await readFile(localPath).catch(_ => undefined)
if (isUndefined(body))
return HTTP.notFound();
if (!('Content-type' in headers))
{
headers = {...headers, ...contentTypeHeader(MimeType.guessFromPath(localPath))}
}
return HTTP.ok(body, headers)
}
function createServer(serve: (request: IncomingMessage) => Promise<HttpResponse>)
{
async function wrapServe(request: IncomingMessage, response: ServerResponse): Promise<void>
{
const r = await serve(request)
entries(r.headers).forEach(([k, v]) => response.setHeader(k, v))
response.statusCode = r.statusCode
if (isDefined(r.body))
response.end(r.body)
else
response.end()
}
return http.createServer(wrapServe);
}
const HTTP =
{
contentTypeHeader,
forbidden,
notFound,
ok,
json,
empty,
file,
createServer
}
export default HTTP;

View File

@ -0,0 +1,32 @@
import PlatformPath from "path";
import {isDefined, isUndefined} from "../../utils/maybe";
function guessFromPath(path: string) : string
{
const ext = PlatformPath.parse(path).ext?.substring(1) as keyof typeof MimeType;
const mimeType = MimeType[ext]
return isDefined(mimeType) && typeof mimeType === "string"
? mimeType
: 'application/octet-stream'
}
const MimeType =
{
ico : 'image/x-icon',
html: 'text/html; charset=UTF-8',
js : 'text/javascript',
json: 'application/json; charset=UTF-8',
css : 'text/css; charset=UTF-8',
png : 'image/png',
jpg : 'image/jpeg',
wav : 'audio/wav',
mp3 : 'audio/mpeg',
svg : 'image/svg+xml; charset=UTF-8',
pdf : 'application/pdf',
guessFromPath
};
export default MimeType

View File

@ -0,0 +1,45 @@
import {IncomingMessage} from "http";
import {firstValueFrom, map, Observable, startWith, toArray} from "rxjs";
export function observeData(request: IncomingMessage, maxLength: number = Number.POSITIVE_INFINITY): Observable<Uint8Array>
{
let nBytes = 0;
return new Observable<Uint8Array>(subscriber =>
{
request.on('end', () => subscriber.complete());
request.on('data', (data: Uint8Array) =>
{
nBytes += data.byteLength
if (nBytes <= maxLength)
subscriber.next(data);
else
{
const error = `too much data: expected ${maxLength} bytes or less, got ${nBytes} bytes.`;
subscriber.error(error);
request.destroy(new Error(error))
}
});
});
}
export async function getRequestJson<T = unknown>(request: IncomingMessage, maxLength = 500000): Promise<T>
{
const data = await getData(request, maxLength)
return JSON.parse(data.toString())
}
const noData = new Uint8Array(0);
export function getData(request: IncomingMessage, maxLength: number = Number.POSITIVE_INFINITY): Promise<Buffer>
{
const data = observeData(request, maxLength).pipe
(
startWith(noData),
toArray(),
map(b => Buffer.concat(b)), // cannot inline!
)
return firstValueFrom(data);
}

View File

@ -0,0 +1,22 @@
export function toLowercaseAscii(string: string)
{
return string
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase();
}
export function containsIgnoringAccents(string: string, substring: string)
{
if (substring === "") return true;
if (string === "") return false;
substring = "" + substring;
if (substring.length > string.length)
return false;
return toLowercaseAscii(string).includes(toLowercaseAscii(substring));
}

View File

@ -0,0 +1,37 @@
import {IncomingMessage} from "http";
import {Dictionary} from "../../utils/utilityTypes";
import {from} from "linq-to-typescript";
import {isUndefined, Maybe} from "../../utils/maybe";
type StringValued<T> =
{
[Key in keyof T]: T[Key] extends number ? Maybe<string>
: T[Key] extends string ? Maybe<string>
: T[Key] extends boolean ? Maybe<string>
: never
}
export function getQueryParams<T>(request: IncomingMessage): Maybe<StringValued<T>>
{
if (isUndefined(request.url))
return undefined
const url = new URL(request.url, `https://${request.headers.host}/`);
const query: Dictionary = {}
const urlSearchParams = new URLSearchParams(url.search);
if (!from(urlSearchParams.entries()).any())
return undefined
for (const [key, value] of urlSearchParams.entries())
query[key] = value;
return query as StringValued<T>;
}
export function getPath(req: IncomingMessage)
{
return new URL(req.url!, `https://${req.headers.host}/`).pathname;
}

View File

@ -0,0 +1,114 @@
import fs from "fs";
import fetch from "node-fetch";
import {getLogger} from "../../utils/logging";
import {doesDirExist, doesFileExist} from "../utils/fileSystem";
import {Dictionary} from "../../utils/utilityTypes";
const log = getLogger('VPM')
const vpnIp = "http://10.2.0.1"
const rxIp = /ifconfig-push +10\.2\.\d+\.\d+/g;
const ccdDir = "/etc/openvpn/server/Salino/ccd";
const vpnStatusFile = "/var/log/openvpn/status";
const useFs = doesFileExist(vpnStatusFile) && doesDirExist(ccdDir)
export type VpnIps = Dictionary<string>;
function vpn(vpnName: string): Promise<string>
{
return fetch(`${vpnIp}/vpn/${vpnName}`)
.then(r => r.text())
.then(t => t.match(rxIp)?.firstOrDefault()?.replace("ifconfig-push", "").trim() ?? "")
}
function getVpnIpFromFs(vpnName: string): string
{
return fs
.readFileSync(`${ccdDir}/${vpnName}`, 'utf-8')
.match(rxIp)
?.firstOrDefault()
?.replace("ifconfig-push", "")
.trim() ?? ""
}
// function getVpnOnlineStatusFromHttp(): Promise<string[]>
// {
// return fetch(`${vpnIp}/vpnstatus.txt`)
// .then(r => r.text())
// .then(s => s.split('\n'));
// }
//
// function getVpnOnlineStatusFromFs(): string[]
// {
// return fs
// .readFileSync(vpnStatusFile, 'utf8')
// .split('\n');
// }
function getVpnNamesFromHttp(): Promise<string[]>
{
return fetch(`${vpnIp}/vpn/`)
.then(r => r.json() as Promise<{ name: string, type: string }[]>)
.then(fs => fs
.where(n => n.type === "file")
.select(n => n.name)
.toArray()
);
}
function getCcdFilesFromFs()
{
return fs
.readdirSync(ccdDir)
.where(p => !p.endsWith(".bak"))
.where(f => fs.lstatSync(`${ccdDir}/${f}`).isFile())
.toArray();
}
function getVpnIpsFromFs(): VpnIps
{
const ccdFiles = getCcdFilesFromFs();
const vpnIps : VpnIps = {}
for (const vpnName of ccdFiles)
vpnIps[vpnName] = getVpnIpFromFs(vpnName)
return vpnIps
}
export async function getVpnIpsFromHttp(): Promise<VpnIps>
{
log(`getting VPN status (http)`)
const vpnNames = await getVpnNamesFromHttp();
const vpnIps : VpnIps = {}
for (const vpnName of vpnNames)
vpnIps[vpnName] = await vpn(vpnName)
return vpnIps
}
// export function updateVpnStats(installation: VrmInstallation, vpnStats: VpnStatus[])
// {
// if (!isDefined(installation.machineSerialNumber))
// return;
//
// const stat = vpnStats.firstOrDefault(s => s.vpnName === installation.machineSerialNumber)
// if (stat === null)
// return;
//
// installation.vpnIp = stat.vpnIp;
// installation.vpnOnline = stat.vpnOnline;
// installation.vpnName = stat.vpnName;
//
// }
export async function getVpnIps(): Promise<VpnIps>
{
return useFs
? getVpnIpsFromFs()
: await getVpnIpsFromHttp()
}

View File

@ -0,0 +1,25 @@
import {Milliseconds} from "../../utils/milliseconds";
import {getLogger} from "../../utils/logging";
import {exhaustMap, Observable, of, repeat, share, tap} from "rxjs";
import {getVpnIps, VpnIps} from "./vpn";
const updateDelay = Milliseconds.fromMinutes(5)
const log = getLogger('VPN_SYNC')
const vpnIps: Observable<VpnIps> = of(0).pipe
(
tap(_ => log('fetching VPN info')),
exhaustMap(_ => getVpnIps()),
repeat({delay: updateDelay}),
tap(_ => log("VPN info updated")),
share()
)
export default vpnIps

View File

@ -0,0 +1,222 @@
import fetch, {Request} from 'node-fetch';
import {delay} from "../../../utils/utils";
import {parseCreateTokenResponse, parseDevicesResponse, parseDiagnosticsResponse, parseGenericResponse, parseInstallationDataResponse, parseLoginResponse, parseTagsResponse} from "./parser";
import {createTokenAuth} from "./auth";
import {getLogger} from "../../../utils/logging";
import {isDefined, isUndefined, Maybe} from "../../../utils/maybe";
import {VrmApiAuth, VrmApiDevice, VrmApiDiagnostic, VrmApiError, VrmApiInstallation, VrmApiInstallationData} from "./apiTypes";
const vrmApiPrefix = 'https://vrmapi.victronenergy.com/v2/' // https://docs.victronenergy.com/vrmapi/overview.html
const nAttempts = 7;
const initialRetryDelayMs = 250;
const timeoutMs = 5000;
const log = getLogger('VRMAPI')
const tokenAuth = createTokenAuth(55450, "ccf470115ca6b729475b9f6a8722a7eb4036df043e7cbaf5002c2c18ccd2e4ee")
export function login(username: string, password: string): Promise<VrmApiAuth>
{
log(`logging into VRM, user: ${username}`)
const auth = undefined;
return vrmPost("auth/login", auth, {username, password}, parseLoginResponse)
}
export async function createToken(auth: VrmApiAuth, tokenName: string): Promise<VrmApiAuth>
{
log(`creating token "${tokenName}"`)
const path = `users/${auth.idUser}/accesstokens/create`;
const token = await vrmPost(path, auth, {name: tokenName}, parseCreateTokenResponse);
return createTokenAuth(auth.idUser, token)
}
function createVrmRequest(path: string, method: "GET" | "POST", auth: Maybe<VrmApiAuth>, payload?: unknown)
{
const url = path.startsWith(vrmApiPrefix) ? path
: path.startsWith('/') ? vrmApiPrefix + path.substring(1)
: vrmApiPrefix + path
const body = isDefined(payload)
? JSON.stringify(payload)
: undefined;
const headers = isDefined(auth)
? auth.header
: undefined
return new Request(url, {method, body, headers});
}
function vrmGet<T>(path: string, auth: Maybe<VrmApiAuth>, parseResponse: (response: unknown) => T)
{
const request = createVrmRequest(path, "GET", auth)
return callVrmApi(request, parseResponse)
}
function vrmPost<T>(path: string, auth: Maybe<VrmApiAuth>, payload: unknown, parseResponse: (response: unknown) => T): Promise<T>
{
const request = createVrmRequest(path, "POST", auth, payload)
return callVrmApi(request, parseResponse)
}
//https://vrmapi.victronenergy.com/v2/installations/34807/settings
//{"description":"Graber, Zürich/ZH | InnovEnergy","phonenumber":""}
export function editInstallation(idSite: number,
name?: string,
tags?: string[],
auth: VrmApiAuth = tokenAuth): Promise<boolean>
{
if (isUndefined(name) && isUndefined(tags))
return Promise.resolve(true) // nothing to do
return vrmPost(`installations/${idSite}/settings`,
auth,
{
description: name,
tags: isDefined(tags) ? tags.map(t => t.toUpperCase()) : undefined
},
parseGenericResponse)
}
export function deleteInstallation(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<boolean>
{
return vrmPost(`installations/${idSite}/remove`, auth, undefined, parseGenericResponse)
}
export function addInstallation(installationName: string, vrmPortalId: string, auth: VrmApiAuth = tokenAuth): Promise<boolean>
{
const payload =
{
installation_identifier: vrmPortalId,
description: installationName
}
return vrmPost(`users/${auth.idUser}/addsite`, auth, payload, parseGenericResponse)
}
export function listTokens(auth: VrmApiAuth): Promise<unknown>
{
log(`listing tokens of user: ${auth.idUser}`)
return vrmGet(`users/${auth.idUser}/accesstokens/list`, auth, (r: unknown) => r)
}
export function getInstallationSummaries(auth: VrmApiAuth = tokenAuth): Promise<VrmApiInstallation[]>
{
return vrmGet(`users/${auth.idUser}/installations`, auth, parseInstallationDataResponse)
}
function getDiagnostics(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<VrmApiDiagnostic[]>
{
return vrmGet(`installations/${idSite}/diagnostics?count=1000`, auth, parseDiagnosticsResponse)
}
export async function getDevices(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<VrmApiDevice[]>
{
const allDevices = await vrmGet(`installations/${idSite}/system-overview`, auth, parseDevicesResponse);
allDevices.devices.forEach(d => d.configured = true)
allDevices.unconfigured_devices.forEach(d => d.configured = false)
return [...allDevices.devices, ...allDevices.unconfigured_devices];
}
export function getTags(idSite: number, auth: VrmApiAuth = tokenAuth): Promise<string[]>
{
return vrmGet(`installations/${idSite}/tags`, auth, parseTagsResponse);
}
const isVrmError = (e: unknown): e is VrmApiError => e instanceof VrmApiError;
function error(msg: string, error?: unknown): Promise<never>
{
if (isVrmError(error))
return Promise.reject(error); // do not repackage existing errors (multiple catch clauses)
if (isDefined(error))
{
if (error instanceof Error)
return Promise.reject(new VrmApiError(`${msg}: ${error}`));
}
return Promise.reject(new VrmApiError(msg));
}
export async function getAllInstallationData(auth: VrmApiAuth = tokenAuth): Promise<VrmApiInstallationData[]>
{
const installationSummaries = await getInstallationSummaries(auth)
const installationData: VrmApiInstallationData[] = []
for (const installationSummary of installationSummaries/*.take(70)*/) // TODO: remove take
{
log(`fetching installation "${installationSummary.name}"`)
//await delay(1000)
const tags = await getTags(installationSummary.idSite, auth);
const devices = await getDevices(installationSummary.idSite, auth);
installationData.push({installation: installationSummary, devices, tags})
}
return installationData;
}
export async function getInstallationData(vrmPortalId: string, auth: VrmApiAuth = tokenAuth): Promise<Maybe<VrmApiInstallationData>>
{
const installationSummaries = await getInstallationSummaries(auth)
const installation = installationSummaries.firstOrDefault(s => s.identifier === vrmPortalId)
if (!isDefined(installation))
return undefined
const tags = await getTags(installation.idSite, auth);
const devices = await getDevices(installation.idSite, auth);
return {installation, devices, tags}
}
async function callVrmApi<T>(request: Request, parseVrmResponse: (r: unknown) => T): Promise<T>
{
let attempt = 1;
let retryDelayMs = initialRetryDelayMs;
// eslint-disable-next-line no-constant-condition
while (true)
{
const response = fetch(request)
.catch(e => error("fetch failed", e))
.then(r => r.ok ? r : error(`request failed: [${r.status}] ${r.statusText}`))
.then(r => r.json())
.catch(e => error("json parsing failed", e))
.then(parseVrmResponse)
.catch(e => error("vrm parser failed", e))
.catch(e => isVrmError(e) ? e : new VrmApiError(`unexpected error: ${e}`)) // gotta catch 'em all
const timeout = delay(timeoutMs)
.then(_ => new VrmApiError(`the request timed out after ${timeoutMs}ms`));
const result: T | VrmApiError = await Promise.race([response, timeout])
if (!isVrmError(result))
return result
if (attempt >= nAttempts)
throw `Giving up after ${attempt} attempts`
//log(`attempt ${attempt} failed: "${result.message}". retrying in ${retryDelayMs}ms`);
attempt++
await delay(retryDelayMs) // wait before retrying
retryDelayMs *= 2; // exponential backoff
}
}

View File

@ -0,0 +1,130 @@
import {Maybe} from "../../../utils/maybe";
export type VrmApiInstallationData =
{
installation : VrmApiInstallation,
tags : string[],
devices : VrmApiDevice[]
}
// "idSite":171797,
// "accessLevel":1,
// "owner":true,
// "is_admin":true,
// "name":"_ IBN tbd Weisshaubt, Neunkirch\/SH | Lutz Bodenm\u00fcller AG (2022-00070)",
// "identifier":"48e7da8755b5",
// "idUser":55450,
// "pvMax":0,
// "timezone":"Europe\/Berlin",
// "phonenumber":null,
// "notes":null,
// "geofence":null,
// "geofenceEnabled":false,
// "realtimeUpdates":true,
// "hasMains":1,
// "hasGenerator":0,
// "noDataAlarmTimeout":null,
// "alarmMonitoring":1,
// "invalidVRMAuthTokenUsedInLogRequest":0,
// "syscreated":1650888746,
// "grafanaEnabled":0,
// "shared":false,
// "device_icon":"battery",
// "alarm":false,
// "last_timestamp":1650964098,
// "current_time":"15:40",
// "timezone_offset":7200,
// "images":false,
// "view_permissions"
// "demo_mode":false,
// "mqtt_webhost":"webmqtt61.victronenergy.com",
// "high_workload":false,
export type VrmApiInstallation =
{
accessLevel: number,
name: string,
identifier: string,
hasMains: number,
hasGenerator: number,
noDataAlarmTimeout: Maybe<number>,
syscreated: number,
tags: string[],
vrmLink: Maybe<string>,
inverter: string,
inverterFw: string,
nbMppts: number,
nbPvInverters: number,
idSite: number,
}
// "name":"PV Inverter",
// "customName":"",
// "productCode":"",
// "idSite":98487,
// "productName":"Fronius PV Inverter",
// "lastConnection":1652968325,
// "class":"device-pv-inverter device-icon-fronius-pv-inverter",
// "froniusDeviceType":"Fronius Symo 10.0-3-M",
// "pL":"AC input 1",
// "connection":"AC current sensor",
// "instance":21,
// "idDeviceType":7,
// "settings"
export type VrmApiDevice =
{
name: string,
firmwareVersion: Maybe<string>,
instance: number,
lastConnection: number,
productName: string,
froniusDeviceType: Maybe<string>,
pL: Maybe<string>,
configured : boolean,
}
export type VrmApiGateway =
{
autoUpdate: string,
updateTo: string,
lastConnection: number,
identifier: string,
lastPowerUpOrRestart: number,
twoWayCommunication: boolean,
machineSerialNumber: string,
};
export type VrmApiGatewayDevice = VrmApiDevice & VrmApiGateway
export type VrmApiDevices = { devices: VrmApiDevice[], unconfigured_devices: VrmApiDevice[] };
export type VrmApiDiagnostic =
{
idSite: number,
timestamp: number,
Device: string,
instance: number,
idDataAttribute: number,
description: string,
formatWithUnit: string,
dbusServiceType: string,
dbusPath: string,
code: string,
bitmask: number,
formattedValue: string,
rawValue: string | number | boolean,
id: number
}
export type VrmApiSuccess = { success: boolean }
export type VrmApiLoginResponse = VrmApiSuccess & { idUser: number, token: string }
export type VrmApiCreateTokenResponse = VrmApiSuccess & { idAccessToken: number, token: string }
export type VrmApiTagsResponse = VrmApiSuccess & { tags: string[] }
export type VrmApiRecordsResponse<T> = VrmApiSuccess & { records: T }
export type VrmApiAuthHeader = { 'X-Authorization': string }
export type VrmApiAuth = { idUser: number, header: VrmApiAuthHeader }
export class VrmApiError extends Error {constructor(message: string) { super(message); }}

View File

@ -0,0 +1,13 @@
import {VrmApiAuth} from "./apiTypes";
export function createUserAuth(idUser: number, token: string): VrmApiAuth
{
const header = {"X-Authorization": `Bearer ${token}`};
return {idUser, header}
}
export function createTokenAuth(idUser: number, token: string): VrmApiAuth
{
const header = {"X-Authorization": `Token ${token}`};
return {idUser, header}
}

View File

@ -0,0 +1,68 @@
import {createUserAuth} from "./auth";
import {isUndefined} from "../../../utils/maybe";
import {VrmApiAuth, VrmApiCreateTokenResponse, VrmApiDevices, VrmApiDiagnostic, VrmApiInstallation, VrmApiLoginResponse, VrmApiRecordsResponse, VrmApiSuccess, VrmApiTagsResponse} from "./apiTypes";
export function parseLoginResponse(response: unknown): VrmApiAuth
{
const r = response as VrmApiLoginResponse
if (isUndefined(r.idUser) || isUndefined(r.token))
throw "failed to parse VrmTokenResponse"
return createUserAuth(r.idUser, r.token);
}
export function parseCreateTokenResponse(response: unknown) : string
{
const r = response as VrmApiCreateTokenResponse
if (isUndefined(r.token) || isUndefined(r.idAccessToken) || isUndefined(r.success))
throw "failed to parse VrmCreateTokenResponse"
if (!r.success)
throw "VrmCreateToken failed"
return r.token
}
export function parseTagsResponse(response: unknown): string[]
{
const tagsResponse = response as VrmApiTagsResponse
if (isUndefined(tagsResponse.tags) || isUndefined(tagsResponse.success))
throw "failed to parse VrmTagsResponse"
if (!tagsResponse.success)
throw "VrmTagsResponse failed"
return tagsResponse.tags
}
function parseRecordsResponse<T>(response: unknown): T
{
const recordsResponse = response as VrmApiRecordsResponse<T>
if (isUndefined(recordsResponse.records) || isUndefined(recordsResponse.success))
throw "failed to parse VrmRecordsResponse"
if (!recordsResponse.success)
throw "VrmRecordsResponse failed"
return recordsResponse.records
}
export function parseGenericResponse(response: unknown): boolean
{
const recordsResponse = response as VrmApiSuccess
if (isUndefined(recordsResponse.success))
throw "failed to parse VrmRecordsResponse"
return recordsResponse.success
}
export const parseInstallationDataResponse = (r: unknown) => parseRecordsResponse<VrmApiInstallation[]>(r);
export const parseDiagnosticsResponse = (r: unknown) => parseRecordsResponse<VrmApiDiagnostic[]>(r);
export const parseDevicesResponse = (r: unknown) => parseRecordsResponse<VrmApiDevices>(r);

View File

@ -0,0 +1,174 @@
import {VrmApiGatewayDevice, VrmApiInstallationData} from "./api/apiTypes";
import {VpnIps} from "../vpn/vpn";
import {unwrap, Unwrap} from "../../utils/match";
import {Database, Installation, VrmInstallation} from "../../api/data/types";
import {getDescendantsWithPath} from "../../api/data/query";
import {isDefined, isUndefined, Maybe} from "../../utils/maybe";
export function getGatewayDevice(iVrm: VrmApiInstallationData)
{
return iVrm
.devices
.firstOrDefault(d => d.name === "Gateway") as Maybe<VrmApiGatewayDevice>;
}
export function getInverter(iVrm: VrmApiInstallationData)
{
return iVrm
.devices
.where(d => d.name === "VE.Bus System")
.select(d => `${d.productName ?? "unknown"} [${d.firmwareVersion ?? "unknown"}]`)
.firstOrDefault() ?? "unknown";
}
export function getPvOnAcIn(iVrm: VrmApiInstallationData)
{
return iVrm
.devices
.where(d => d.name === "PV Inverter" && isDefined(d.pL))
.where(d => d.pL!.startsWith("AC input"))
.where(d => isDefined(d.productName))
.select(d => `${d.productName} [${d.firmwareVersion ?? "unknown"}]`)
.orderBy(d => d)
.toArray();
}
export function getPvOnAcOut(iVrm: VrmApiInstallationData)
{
return iVrm
.devices
.where(d => d.name === "PV Inverter" && isDefined(d.pL))
.where(d => d.pL!.startsWith("AC output"))
.where(d => isDefined(d.productName))
.select(d => `${d.productName} [${d.firmwareVersion ?? "unknown"}]`)
.orderBy(d => d)
.toArray();
}
export function getPvOnDc(iVrm: VrmApiInstallationData)
{
return iVrm
.devices
.where(d => d.name === "Solar Charger")
.where(d => isDefined(d.productName))
.select(d => `${d.productName} [${d.firmwareVersion ?? "unknown"}]`)
.orderBy(d => d)
.toArray();
}
function unmangleVrmInstallationName(installationName: string)
{
// "name": "_IBN Frischknecht, Waldstatt/AR | St.Gallisch-Appenzellisch Kraftwerke AG (2022-00128)",
//const rx = / *(_IBN)? *([^,]+?),([^/]+?) *\/ *(.+?)( *\/ *.+?)? *| * (\(.+\)) */
const [ibnName, rest1] = installationName.split(/,(.*)/s)
if (isUndefined(rest1) || isUndefined(ibnName))
return undefined
const [locality, rest2] = rest1.split(/\/(.*)/s)
if (isUndefined(rest2))
return undefined
const [regionCountry, rest3] = rest2.split(/\|(.*)/s)
if (isUndefined(rest3))
return undefined
const [region, country] = regionCountry.split(/\/(.*)/s)
const [folder, orderNumber] = rest3.split(/\((.*)/s)
if (isUndefined(orderNumber))
return undefined
const orderNumbers = orderNumber.replace(")", "").split(',')
const rxIbn = /^\s*_\s*IBN\s*/;
return {
ibn : rxIbn.test(ibnName),
name : ibnName.replace(rxIbn, "").trim(),
locality: locality.trim(),
region : region.trim(),
country : country?.trim(),
folder : folder.trim(),
orderNumbers
}
}
export function toVrmInstallation(vrmData: VrmApiInstallationData, vpnIps: VpnIps = {}): VrmInstallation
{
// TODO: VRM users
const tags = vrmData
.tags
.select(t => t.toUpperCase())
.where(t => t !== "ALARM" && t !== "NO-ALARM" && t !== "NODATA")
.orderBy(t => t)
.toArray();
const gateway = getGatewayDevice(vrmData)
const parsed = unmangleVrmInstallationName(vrmData.installation.name);
if (parsed?.ibn === true && !tags.includes("IBN"))
tags.push("IBN")
const vrmInstallation : Unwrap<VrmInstallation> = {
id : `vrm${vrmData.installation.idSite}`,
name : parsed?.name ?? vrmData.installation.name,
tags : tags,
orderNumbers: parsed?.orderNumbers ?? [],
vpnName : gateway?.machineSerialNumber ?? "unknown",
vpnIp : vpnIps?.[gateway!.machineSerialNumber] ?? "unknown",
country : parsed?.country,
region : parsed?.region,
locality : parsed?.locality,
children : [],
controllerType: gateway?.productName ?? "unknown",
firmware : gateway?.firmwareVersion ?? "unknown",
accessLevel : vrmData.installation.accessLevel, // TODO
autoUpdate : gateway?.autoUpdate ?? "unknown",
updateTo : gateway?.updateTo ?? "unknown",
vrmId : vrmData.installation.idSite,
vrmPortalId : vrmData.installation.identifier,
pvOnAcIn : getPvOnAcIn(vrmData),
pvOnAcOut : getPvOnAcOut(vrmData),
pvOnDc : getPvOnDc(vrmData),
inverter : getInverter(vrmData),
vrmName : vrmData.installation.name
};
return {vrmInstallation};
}
function getMangledVrmInstallationName(db: Database, installation: Unwrap<Installation>)
{
const path = getDescendantsWithPath(db.rootFolder)
.where(e => unwrap(e[0]).id === installation.id)
.take(1)
.selectMany(e => e)
.skip(1) // skip installation name
.select(e => unwrap(e).name)
.toArray()
path.pop() // remove root "InnovEnergy" folder
const {country, locality, region} = installation;
const lrc = [locality, region, country].filter(isDefined).join("/")
path.unshift(`${installation.name}, ${lrc}`)
const ibn = installation.tags.includes("IBN") ? "_IBN " : ""
const orderNumbers = installation.orderNumbers.any() ? ` (${installation.orderNumbers.join(",")})` : "";
return `${ibn + path.join(" | ") + orderNumbers}`
}

View File

@ -0,0 +1,42 @@
import {Milliseconds} from "../../utils/milliseconds";
import {getAllInstallationData} from "./api/api";
import {getLogger} from "../../utils/logging";
import {exhaustMap, map, Observable, of, repeat, share, tap, withLatestFrom} from "rxjs";
import {VrmApiInstallationData} from "./api/apiTypes";
import vpnIps from "../vpn/vpnSync";
import {VpnIps} from "../vpn/vpn";
import {toVrmInstallation} from "./convert";
import {VrmInstallation} from "../../api/data/types";
const updateDelay = Milliseconds.fromMinutes(5)
const log = getLogger('VRM_SYNC')
export const vrmInstallations: Observable<VrmInstallation[]> = of(0).pipe
(
tap(_ => log('iteration started')),
exhaustMap(_ => getAllInstallationData()),
withLatestFrom(vpnIps),
map(([installationData, vpnIps]) => toVrmInstallations(installationData, vpnIps)),
repeat({delay: updateDelay}),
tap(_ => log(`iteration finished. Waiting for ${Milliseconds.toMinutes(updateDelay)} minutes`)),
share()
)
export function toVrmInstallations(vrmData: VrmApiInstallationData[], vpnIps: VpnIps) : VrmInstallation[]
{
return vrmData.map(d => toVrmInstallation(d, vpnIps))
}
// const vrmUser = "victron@innov.energy";
// const vrmPassword = "NnoVctr201002";

View File

@ -0,0 +1,161 @@
import {from, IEnumerable} from "linq-to-typescript";
import {compactArray, hasOwnProperty, keys} from "./utils";
export type Change<T = any> = {path: string[], value: T}
const primitiveTypes = ["number", "string", "boolean", "null", "function", "bigint"]
const _keep = {} as const
export function keep<T>()
{
return _keep as T
}
export function getAt(target: any, path: IEnumerable<string>): any
{
let t = target
for (const p of path)
{
if (!hasOwnProperty(t, p))
throw "invalid path" // TODO: improve error msg
t = t[p]
}
return t
}
export function getParent(target: any, path: IEnumerable<string>): any
{
let t = target
const parentPath = path.toArray()
parentPath.pop()
for (const p of parentPath)
{
if (!hasOwnProperty(t, p))
throw "invalid path" // TODO: improve error msg
t = t[p]
}
return t
}
export function applyChanges(target: any, changes : IEnumerable<Change>)
{
for (const change of changes)
{
const path = [...change.path] // copy
const head = path.pop()!
const parent = getAt(target, path)
if (change.value === undefined)
delete parent[head]
else
{
console.log(`${change.path.join("/")}: ${parent[head]} => ${change.value}`)
parent[head] = change.value
}
if (Array.isArray(parent))
compactArray(parent) // TODO: review
}
}
export function getChanges(old_: any, new_: any): IEnumerable<Change>
{
return from(diff(old_, new_))
}
export function isArrayOp(p: Change)
{
return !isNaN(parseInt(p.path.lastOrDefault()!, 10));
}
// function observe<T extends object>(src: T, subject: Subject<Patch>, path: Path = []): T
// {
// function get(target: T, p: string ): any
// {
// return observe(target, subject, [...path, p])
// }
//
// function set(target: T, p: string, newValue: any, receiver: any): boolean
// {
//
// return true
// }
//
// return new Proxy(src,
// {
// get(target, prop, receiver)
// {
// alert(`GET ${prop}`);
// return Reflect.get(target, prop, receiver); // (1)
// },
// set(target, prop:string, val, receiver)
// {
// alert(`SET ${prop}=${val}`);
// const success = Reflect.set(target, prop, val, receiver);
//
// if (success)
// {
// const patch : Patch = { path: [...path, prop], }
// subject.next()
// }
//
// return success; // (2)
// }
// });
//
// //return new Proxy<T>(src, {get, set})
// }
function* diff(old_: any, new_: any, path: string[] = []): Generator<Change>
{
const oldKeys = keys(old_);
const newKeys = keys(new_);
const allKeys = oldKeys
.concatenate(newKeys)
.distinct()
for (const k of allKeys)
{
const p = [...path, k] as string[];
const oldValue = old_[k]
const newValue = new_[k]
if (oldValue === newValue /*|| newValue === _keep*/)
continue // "reference equal": easy out
const oldType = typeof oldValue
const newType = typeof newValue
if (oldType !== newType || primitiveTypes.contains(newType))
{
// oldValue !== newValue because of test already made above
yield {path: p, value: newValue}
continue
}
if (newType !== "object")
throw "unsupported data type!"
if (keys(oldValue).any() && !keys(oldValue).intersect(keys(newValue)).any())
{
// no common keys: replace whole object
yield {path: p, value: newValue}
continue
}
yield* diff(oldValue, newValue, p)
}
}

View File

@ -0,0 +1,21 @@
// 0. Import Module
import {IEnumerable, initializeLinq} from "linq-to-typescript"
// 1. Declare that the JS types implement the IEnumerable interface
declare global {
interface Array<T> extends IEnumerable<T> { }
interface Uint8Array extends IEnumerable<number> { }
interface Uint8ClampedArray extends IEnumerable<number> { }
interface Uint16Array extends IEnumerable<number> { }
interface Uint32Array extends IEnumerable<number> { }
interface Int8Array extends IEnumerable<number> { }
interface Int16Array extends IEnumerable<number> { }
interface Int32Array extends IEnumerable<number> { }
interface Float32Array extends IEnumerable<number> { }
interface Float64Array extends IEnumerable<number> { }
interface Map<K, V> extends IEnumerable<[K, V]> { }
interface Set<T> extends IEnumerable<T> { }
interface String extends IEnumerable<string> { }
}
// 2. Bind Linq Functions to Array, Map, etc
initializeLinq()

View File

@ -0,0 +1,11 @@
let subsystemPadding = 0
export function getLogger(subsystem: string): (msg: string) => void
{
subsystemPadding = Math.max(subsystem.length, subsystemPadding)
// eslint-disable-next-line no-console
return (msg: string) => console.log(`${new Date().toLocaleString()} | ${(subsystem.padEnd(subsystemPadding))} | ${msg}`);
}

View File

@ -0,0 +1,176 @@
import {Dictionary, Func, Normalize1} from "./utilityTypes";
import {keys, valueToFunction} from "./utils";
import {isUndefined} from "./maybe";
import {UnionToIntersection} from "simplytyped";
import {current} from "immer";
// Type Compatibility
// https://www.typescriptlang.org/docs/handbook/type-compatibility.html
//TODO: review
export type IsUnionCase<T> =
T extends Dictionary
? [UnionToIntersection<keyof T>] extends [keyof T]
? [keyof T] extends [UnionToIntersection<keyof T>]
? true
: false
: false
: false
//TODO: review
export type IsTaggedUnion<T> = true extends UnionToIntersection<IsUnionCase<T>> ? Dictionary : never
export type Unwrap<U extends IsTaggedUnion<U>> = UnionToIntersection<U>[keyof UnionToIntersection<U>] ;
export function update<U extends IsTaggedUnion<U>>(u: U, e: Partial<Unwrap<U>>)
{
const v = u as UnionToIntersection<U>
const o = current(v)
const ks = keys(v)
if (ks.length != 1)
throw new Error("not a valid union case")
const tag = ks[0]
const before = v[tag];
const before2 = current(before);
const newVar = {...before, ...e};
v[tag] = newVar
}
export function unwrap<U extends IsTaggedUnion<U>>(u: U) : Normalize1<Unwrap<U>>
{
const v = u as UnionToIntersection<U>
const ks = keys(v)
if (ks.length != 1)
throw new Error("not a valid union case")
const key = ks[0]
return v[key] as any;
}
export function base<U extends IsTaggedUnion<U>>(u: U): Normalize1<Unwrap<U> & Partial<UnionToIntersection<Unwrap<U>>>>
{
return unwrap(u) as Normalize1<Unwrap<U> & Partial<UnionToIntersection<Unwrap<U>>>>
}
export function tag<U extends IsTaggedUnion<U>>(u: U) : keyof UnionToIntersection<U>
{
const v = u as UnionToIntersection<U>
const ks = keys(v)
if (ks.length != 1)
throw new Error("not a valid union case")
return ks[0]
}
export function tagsEqual<U extends IsTaggedUnion<U>>(u: U, v: U) : v is U
{
return tag(u) === tag(v)
}
type MapFuncs<U> = { [k in keyof UnionToIntersection<U>]: Func<UnionToIntersection<U>[k]> }
type OtherwiseKeys<U,M> = Exclude<keyof UnionToIntersection<U>, keyof M>;
type OtherwiseArg<U,M> = {
[k in keyof UnionToIntersection<U>]: Record<k, UnionToIntersection<U>[k]>
}[OtherwiseKeys<U,M>]
type OtherwiseFunc<U, M extends Partial<MapFuncs<U>>, R> = Func<OtherwiseArg<U,M> extends never ? unknown : OtherwiseArg<U,M>, R>;
export function match<U extends IsTaggedUnion<U>, M extends Partial<MapFuncs<U>>, R>(uCase: U, matchFuncs: M, otherwise: OtherwiseFunc<U, M, R> | R):{ [k in keyof M]: M[k] extends Func<any, infer O> ? O : never }[keyof M] | R
{
const otw = valueToFunction(otherwise)
const c = uCase as UnionToIntersection<U>
const ks = keys(c)
if (ks.length != 1)
return otw(c)
const key = ks[0]
const arg = c[key]
const matchFunc = matchFuncs[key]
if (isUndefined(matchFunc))
return otw(c);
return matchFunc(arg as any) as any;
}
export function dispatch<U extends IsTaggedUnion<U>>()
{
// type Intersection = UnionToIntersection<U>;
//
// type MapFuncs = { [k in keyof Intersection]: Func<Intersection[k]> }
// type OtherwiseKeys<M> = Exclude<keyof Intersection, keyof M>;
//
// type OtherwiseArg<M> = {
// [k in keyof Intersection]: Record<k, Intersection[k]>
// }[OtherwiseKeys<M>]
//
// type OtherwiseFunc<M extends Partial<MapFuncs>, R> = Func<OtherwiseArg<M> extends never ? unknown : OtherwiseArg<M>, R>;
return <M extends Partial<MapFuncs<U>>, R>(matchFuncs: M, otherwise: OtherwiseFunc<U,M, R> | R) =>
{
const otw = valueToFunction(otherwise)
return (uCase: U): { [k in keyof M]: M[k] extends Func<any, infer O> ? O : never }[keyof M] | R =>
{
const c = uCase as UnionToIntersection<U>
const ks = keys(c)
if (ks.length != 1)
return otw(c)
const key = ks[0]
const arg = c[key]
const matchFunc = matchFuncs[key]
if (isUndefined(matchFunc))
return otw(c);
return matchFunc(arg as any) as any;
}
};
}
export function concat<R extends Record<keyof any, any>, T extends Dictionary>(rec: R, t:T)
{
const result = {} as {
[k in keyof UnionToIntersection<R>]: Record<k, UnionToIntersection<R>[k] & T>
}[keyof UnionToIntersection<R>]
for (const k in rec)
{
// @ts-ignore
result[k] = { ...rec[k], ...t}
}
return result
}

View File

@ -0,0 +1,16 @@
export type Maybe<T> = T | undefined | null;
export function isDefined<T>(e: Maybe<T>): e is T
{
return e != undefined // != by design to include null
}
export function isUndefined<T>(e: Maybe<T>): e is undefined | null
{
return e == undefined // == by design to include null
}
export function toArray<T>(e: Maybe<T>): T[]
{
return isDefined(e) ? [e] : []
}

View File

@ -0,0 +1,18 @@
export type Milliseconds = number
export const Milliseconds =
{
fromSeconds: (count: number): Milliseconds => count * 1000,
fromMinutes: (count: number): Milliseconds => count * 1000 * 60,
fromHours : (count: number): Milliseconds => count * 1000 * 60 * 60,
fromDays : (count: number): Milliseconds => count * 1000 * 60 * 60 * 24,
fromWeeks : (count: number): Milliseconds => count * 1000 * 60 * 60 * 24 * 7,
toSeconds: (count: Milliseconds): number => count / 1000,
toMinutes: (count: Milliseconds): number => count / 1000 / 60,
toHours : (count: Milliseconds): number => count / 1000 / 60 / 60,
toDays : (count: Milliseconds): number => count / 1000 / 60 / 60 / 24,
toWeeks : (count: Milliseconds): number => count / 1000 / 60 / 60 / 24 / 7,
} as const

View File

@ -0,0 +1,81 @@
import {isUndefined} from "./maybe";
import {from, IEnumerable} from "linq-to-typescript";
import {isBoolean, isNumber, isPlainObject, isString} from "./runtimeTypeChecking";
function getAt(root: any, path: (keyof any)[])
{
return path.reduce((v, p) => v[p], root)
}
function iterate(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
{
if (isUndefined(root))
return []
return from(iterate(root))
function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
{
if (isString(node) || isNumber(node) || isBoolean(node))
yield {path, node}
else if (isPlainObject(node))
for (const key in node)
{
path.push(key)
yield {path, node}
yield* iterate(node[key], path)
path.pop()
}
}
}
function iterateLeafs(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
{
if (isUndefined(root))
return []
return from(iterate(root))
function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
{
if (isString(node) || isNumber(node) || isBoolean(node))
yield {path, node}
else if (isPlainObject(node))
for (const key in node)
{
path.push(key)
yield* iterate(node[key], path)
path.pop()
}
}
}
function iterateBranches(root: unknown): IEnumerable<{ path: string[]; node: unknown }>
{
if (isUndefined(root))
return []
return from(iterate(root))
function* iterate(node: unknown, path: string[] = []): Generator<{ path: string[]; node: unknown }>
{
if (isPlainObject(node))
for (const key in node)
{
path.push(key)
yield {path, node}
yield* iterate(node[key], path)
path.pop()
}
}
}
export const Path =
{
iterate,
iterateLeafs,
iterateBranches,
getAt
} as const

View File

@ -0,0 +1,66 @@
export type TypeCode =
| "undefined"
| "object"
| "boolean"
| "number"
| "string"
| "function"
| "symbol"
| "bigint";
export type PlainObject<K extends keyof any = keyof any, V = unknown> = Record<K, V>
export function isObject(thing: unknown) : thing is object
{
return typeof thing === "object"
}
export function isDate(thing: unknown) : thing is Date
{
return thing instanceof Date
}
export function isPlainObject(thing: unknown) : thing is PlainObject
{
return isObject(thing) && !isDate(thing)
}
export function isArray(thing: unknown) : thing is Array<unknown>
{
return Array.isArray(thing)
}
export function isNumber(thing: unknown) : thing is number
{
return typeof thing === "number"
}
export function isBoolean(thing: unknown) : thing is boolean
{
return typeof thing === "boolean"
}
export function isString(thing: unknown) : thing is string
{
return typeof thing === "string"
}
// export function isFunction(thing: unknown): thing is (...args: unknown[]) => unknown
// {
// return typeof thing === "function"
// }
export function isFunction(obj: unknown): obj is (...args: any[]) => any
{
return obj instanceof Function;
}
export function isSymbol(thing: unknown) : thing is symbol
{
return typeof thing === "symbol"
}
export function isBigint(thing: unknown) : thing is bigint
{
return typeof thing === "bigint"
}

View File

@ -0,0 +1,71 @@
import {from, IEnumerable} from "linq-to-typescript";
import {isDefined, isUndefined, Maybe} from "./maybe";
export function Tree<T>(getChildren: (t: T) => IEnumerable<T>)
{
function iterate(root: Maybe<T>): IEnumerable<T>
{
if (isUndefined(root))
return []
return from(iterateTree())
function* iterateTree()
{
const queue: T[] = [root!]
do
{
const element = queue.shift()!
yield element
for (const child of getChildren(element))
queue.push(child)
}
while (queue.length > 0)
}
}
function iterateWithPath(root: Maybe<T>): IEnumerable<T[]>
{
return isDefined(root)
? from(iterateTreeWithPath())
: [];
function* iterateTreeWithPath()
{
const stack: Array<Array<T>> = [[root!]]
while (true)
{
const head = stack[0];
if (head.length > 0)
{
yield stack
.select(l => l[0])
.toArray()
const children = getChildren(head[0]).toArray()
stack.unshift(children)
}
else
{
stack.shift() // remove empty array in front
if(stack.length > 0)
stack[0].shift()
else
break;
}
}
}
}
return {
iterate,
iterateWithPath
} as const
}

View File

@ -0,0 +1,115 @@
export {}
// export type Type =
// | "number"
// | "object"
// | "string"
// | "never"
// | "any"
// | "unknown"
// | "undefined"
// | "boolean"
// | "bigint"
// | "symbol"
// | Property[]
// | Func
//
// export type Key = "string" | "number" | "symbol"
//
// export type Property = Func |
// {
// key: Key,
// type: Type,
// readonly? : boolean,
// nullable? : boolean
// }
//
// export type Arg =
// {
// name: string,
// type: Type,
// nullable? : boolean
// }
//
// export type Func =
// {
// args: Arg[],
// returnType: Type,
// }
//
//
// type X = Partial<any>
//
// export function render(t: Type, indent = 0)
// {
// if (typeof t === "string")
// return t
//
//
// return "ERROR"
// }
//
type DeviceType =
| "Pv"
| "Load"
| "Battery"
| "Grid"
| "Inverter"
| "AcInToAcOut"
| "DcDc"
| "AcInBus"
| "AcOutBus"
| "DcBus"
| "Dc48Bus" // low voltage DC Bus, to be eliminated in later versions
type Phase =
{
voltage : number // U, non-negative
current : number // I, sign depends on device type, see sign convention below
}
type AcPhase = Phase &
{
phi : number // [0,2pi)
}
type Device =
{
Type: DeviceType,
Name?: string,
}
type Stack =
{
Top? : Device[], // 0 to N
Right? : Device // 0 or 1
Bottom? : Device[] // 0 to N
Disconnected?: boolean // not present = false
}
/// A DC device must have a field denoting its DC connection
type DcDevice = Device &
{
Dc : Phase
}
/// An AC device can have 1 to 3 AC phases
/// An AC device also needs a Frequency measurement
/// Total power can be obtained by summing the power of the phases
type AcDevice = Device &
{
Ac: AcPhase[]
Frequency: number
}
/// A low voltage 48V DC device
/// Needed to distinguish the two sides of the DCDC
/// Will be dropped once we get HV batteries
type Dc48Device = Device &
{
dc48 : Phase
}

View File

@ -0,0 +1,52 @@
import {UnionToIntersection} from "simplytyped";
export type Dictionary<T = unknown> = Record<string, T>
export type Nothing = Dictionary<never>
export type IsUnion<T> = [T] extends [UnionToIntersection<T>] ? false : true;
export type UnionToDeepPartialIntersection<U> = DeepPartial<UnionToIntersection<U>>
export type UnionToPartialIntersection<U> = Partial<UnionToIntersection<U>>
export type Func<T = unknown, R = unknown> = (arg: T) => R
export type AsyncFunc<T = unknown, R = unknown> = (arg: T) => Promise<R>
export type SyncAction<T> = (arg: T) => void
export type AsyncAction<T> = (arg: T) => Promise<void>
export type Action<T> = SyncAction<T> | AsyncAction<T>
export type Lazy<T> = () => T
export type Base64 = string
export type ValueOf<T> = T[keyof T];
export type DeepPartial<T> = T extends object ? { [P in keyof T]?: DeepPartial<T[P]>; } : T;
export type DeepMutable<T> = { -readonly [P in keyof T]: DeepMutable<T[P]> };
export type Mutable<T> = { -readonly [P in keyof T]: T[P] };
export type NumberLiteralToStringLiteral<T> = T extends number ? `${T}` : T
export type KeyedChildren<T> = { children?: Dictionary<T> }
export type Union<K extends string, V extends string = string> = { [S in K] : V}
export type IntersectionToUnion<T extends Dictionary> = { [Prop in keyof T]: Record<Prop, T[Prop]> }[keyof T] // not sure if this is aptly named
// helper to flatten (instantiate) types in editor popups
// eslint-disable-next-line @typescript-eslint/ban-types
export type Normalize<T> = T extends (...args: infer A) => infer R ? (...args: Normalize<A>) => Normalize<R>
: [T] extends [any] ? { [K in keyof T]: Normalize<T[K]> }
: never
export type Normalize1<T> = T extends (...args: infer A) => infer R ? (...args: A) => R
: [T] extends [any] ? { [K in keyof T]: T[K] }
: T
export type Normalize2<T> = T extends (...args: infer A) => infer R ? (...args: Normalize1<A>) => Normalize1<R>
: [T] extends [any] ? { [K in keyof T]: Normalize1<T[K]> }
: never
export function mutable<T>(t: T)
{
return t as Mutable<T>
}

View File

@ -0,0 +1,281 @@
import {Dictionary, Func, Lazy} from "./utilityTypes";
import {isUndefined, Maybe} from "./maybe";
import {IEnumerable} from "linq-to-typescript";
import {Builder} from "../client/njsx/njsx";
export const id = <T>(e:T) => e;
// eslint-disable-next-line @typescript-eslint/no-empty-function
export const nop = (_?:unknown) => {};
export function sleep(ms: number): Promise<never>
{
return new Promise(resolve => setTimeout(resolve, ms));
}
export function fail<T>(error?: string) : T
{
throw new Error(error)
}
export function first<T>(a: Array<T>) : Maybe<T>
{
return a[0]
}
export function last<T>(array: T[]) : Maybe<T>
{
const last = array.length - 1
return last < 0
? undefined
: array[last]
}
export function compactArray<T>(array: T[])
{
let j = 0;
array.forEach((e, i) =>
{
if (i !== j)
array[j] = e;
j++;
});
array.length = j;
return array;
}
export const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(() => resolve(), ms));
export function hasOwnProperty(thing: unknown, key : keyof any)
{
return Object.prototype.hasOwnProperty.call(thing, key)
}
export function toDictionary<T>(ts: IEnumerable<T>, keySelector: (element: T) => string): Dictionary<T>
{
function addEntry(d: Dictionary<T>, t: T)
{
const key = keySelector(t);
d[key] = t;
return d
}
return ts.aggregate({}, addEntry)
}
export function keys<T>(t: T): (keyof T)[]
{
return Object.keys(t as object) as (keyof T)[]
}
export function values<V>(t: Record<keyof any, V>): V[]
{
return Object.values(t)
}
type Entry<T> =
{
[P in keyof T]: P extends number ? [`${P}`, T[P]]
: P extends string ? [P, T[P]]
: never
};
export function entries<T>(t: T)
{
return Object.entries(t as Dictionary)
}
export function shallowEqual(left: any, right: any)
{
const keysLeft = keys(left) ;
const keysRight = keys(right);
return keysLeft.length === keysRight.length &&
keysLeft.every(key => Object.prototype.hasOwnProperty.call(right, key) && left[key] === right[key]);
}
export function objectEquals(x: any, y: any): boolean
{
if (isUndefined(x) || isUndefined(y))
return x === y;
// after this just checking type of one would be enough
if (x.constructor !== y.constructor)
return false;
// if they are functions, they should exactly refer to same one (because of closures)
// if they are regexps, they should exactly refer to same one (it is hard to better equality check on current ES)
if (x instanceof Function || x instanceof RegExp)
return x === y;
if (x === y || x.valueOf() === y.valueOf())
return true;
// optimization
if (Array.isArray(x) && x.length !== y.length)
return false;
// optimization: if they are dates, they must have had equal valueOf (above)
if (x instanceof Date)
return false;
// if they are strictly equal, they both need to be an object at least
if (!(x instanceof Object && y instanceof Object)) return false;
const xKeys = Object.keys(x);
const yKeys = Object.keys(y);
// recursive object equality check
return xKeys.length === yKeys.length
&& yKeys.every(yKey => yKey in x)
&& xKeys.every(xKey => objectEquals(x[xKey], y[xKey]));
}
export function structurallyEqual(left: any, right: any)
{
if (left === right)
return true;
if (typeof left !== typeof right)
return false;
if (typeof left !== 'object')
return left === right;
if (keys(left).length !== keys(right).length)
return false;
for (const key in left)
{
if (!(key in right))
return false;
if (!structurallyEqual(left[key], right[key]))
return false;
}
return true;
}
type FuncDict<T> ={ [P in keyof T]: (a: T[P]) => any };
type Out<T extends FuncDict<any>> = { [P in keyof T]: ReturnType<T[P]> };
// TODO: use Ramda ?
export function mapObject<S, M extends FuncDict<S>>(src: S, map: M): Out<FuncDict<S>>
{
const result : Partial<Out<FuncDict<S>>> = {}
for (const k in src)
result[k] = map[k](src[k])
return result as Out<FuncDict<S>>
}
export function mapDict<T, R>(src: Dictionary<T>, map: (arg: T) => R): Dictionary<R>
{
const result : Partial<Dictionary<R>> = {}
for (const k in src)
result[k] = map(src[k])
return result as Dictionary<R>
}
export function valueToFunction<T,R>(tr: Func<T,R> | R) : Func<T,R>
{
if (typeof tr === "function")
return tr as Func<T,R>
return (_: T) => tr
}
// export function mapEntries<V, R>(src: Dictionary<V>, map: (k: string, v: V) => [string, R]): Dictionary<R>
// {
// const result: Dictionary<R> = {}
//
// entries(src).forEach(([key, value]) =>
// {
// const [k, v] = map(key, value)
// result[k] = v;
// })
//
// return result
// }
export function once<T>(action: Builder<T>) : Builder<T>
export function once<T>(action: Lazy<T>) : Lazy<Maybe<T>>
{
let done = false;
return () =>
{
if (done)
return undefined
done = true
return action()
}
}
// export function deepMerge<K extends keyof any, V>(...objects: Record<K, V>[])
// {
// function extracted(prev: Record<K, V>, cur: Record<K, V>)
// {
// for (const key of keys(cur))
// {
// const k = key as K
//
// const pVal = prev[k];
// const oVal = cur[k];
//
// if (isArray(pVal) && isArray(oVal))
// {
// prev[k] = pVal.concat(...oVal);
// }
// else if (isObject(pVal) && isObject(oVal))
// {
// prev[k] = deepMerge(pVal, oVal);
// }
// else
// {
// prev[k] = oVal;
// }
// }
//
// return prev;
// }
//
// return objects.reduce((prev, cur) => extracted(prev, cur), {} as Record<K, V>);
// }
// // Test objects
// const obj1 = {
// a: 1,
// b: 1,
// c: { x: 1, y: 1 },
// d: [ 1, 1 ]
// }
// const obj2 = {
// b: 2,
// c: { y: 2, z: 2 },
// d: [ 2, 2 ],
// e: 2
// }
// const obj3 = deepMerge(obj1, obj2);

View File

@ -0,0 +1,82 @@
{
"compilerOptions":
{
"lib": ["es2021"],
"module": "ESNext",
"target": "es2021",
"strict": true,
"esModuleInterop": true,
//"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"useDefineForClassFields": true,
"outDir": "dist",
"allowSyntheticDefaultImports": true,
// d.ts files
// "declaration": true,
// "emitDeclarationOnly": true,
/* Basic Options */
"incremental": true, /* Enable incremental compilation */
//"module": "ES6", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
"allowJs": false, /* Allow javascript files to be compiled. */
// "checkJs": true, /* Report errors in .js files. */
"jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
// "outDir": "./", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
"tsBuildInfoFile": "./node_modules/tsconfig.tsbuildinfo", /* Specify file to store incremental compilation information */
"removeComments": true, /* Do not emit comments to output. */
"noEmit": true, /* Do not emit outputs. */
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
"downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
"isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
"strictNullChecks": true, /* Enable strict null checks. */
"strictFunctionTypes": true, /* Enable strict checking of function types. */
"strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
"strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
"noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
"alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
/* Additional Checks */
//"noUnusedLocals": true, /* Report errors on unused locals. */
// "noUnusedParameters": true, /* Report errors on unused parameters. */
"noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
"noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
//"noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
// "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
/* Module Resolution Options */
"moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
// "typeRoots": [], /* List of folders to include type definitions from. */
// "types": [], /* Type declaration files to be included in compilation. */
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
// "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
},
"include": ["src/**/*.ts","src/**/*.tsx"],
"exclude": ["node_modules", "dist"]
}

File diff suppressed because it is too large Load Diff