refactor: works towards dynamic flatpack (#6264)

Author: Jason3S

packages/flatpack-json/src/Flatpack.mts

@@ -1,4 +1,7 @@
+import assert from 'node:assert';
+
 import { FlatpackedWrapper } from './flatpackUtil.mjs';
+import { proxyDate, proxyObject, proxySet } from './proxy.mjs';
 import {
     ArrayRefElement,
     BigIntRefElement,
@@ -8,6 +11,7 @@ import {
     ObjectRefElement,
     ObjectWrapperRefElement,
     PrimitiveRefElement,
+    PrimitiveRefElementBase,
     RefElements,
     RegExpRefElement,
     SetRefElement,
@@ -76,6 +80,8 @@ export class FlatpackStore {
     private cachedSets = new Map<ArrayRefElement | undefined, SetRefElement>();
     private cachedMaps = new Map<ArrayRefElement | undefined, Map<ArrayRefElement | undefined, MapRefElement>>();
 
+    private cachedProxies = new WeakMap<RefElements, Serializable>();
+
     /**
      * Cache of strings that have been deduped and stored in the data array.
      */
@@ -275,7 +281,7 @@ export class FlatpackStore {
 
     private dedupeSetRefs(value: Set<Serializable>, element: SetRefElement): SetRefElement {
         if (!this.dedupe) return element;
-        const values = element.values();
+        const values = element.valueRefs();
         const found = this.cachedSets.get(values);
         if (!found) {
             this.cachedSets.set(values, element);
@@ -287,6 +293,10 @@ export class FlatpackStore {
         return found;
     }
 
+    private proxySetRef(ref: SetRefElement): Set<Serializable> {
+        return proxySet(new Set(this.#toValue(ref.valueRefs()) as Serializable[]), () => {});
+    }
+
     private createUniqueKeys(keys: Serializable[]): ArrayRefElement {
         const cacheValue = false;
         let k = this.arrToRef(keys, cacheValue);
@@ -320,8 +330,8 @@ export class FlatpackStore {
 
     private dedupeMapRefs(value: Map<Serializable, Serializable>, element: MapRefElement): MapRefElement {
         if (!this.dedupe) return element;
-        const keys = element.keys();
-        const values = element.values();
+        const keys = element.keyRefs();
+        const values = element.valueRefs();
         let found = this.cachedMaps.get(keys);
         if (!found) {
             found = new Map();
@@ -340,6 +350,10 @@ export class FlatpackStore {
         return element;
     }
 
+    private proxyMapRef(_ref: MapRefElement): Map<Serializable, Serializable> {
+        return new Map();
+    }
+
     private cvtRegExpToRef(value: RegExp): RegExpRefElement {
         const found = this.cache.get(value);
         if (found !== undefined) {
@@ -361,6 +375,10 @@ export class FlatpackStore {
         return this.addValueAndElement(value, new DateRefElement(value.getTime()));
     }
 
+    private proxyDateRef(ref: DateRefElement): Date {
+        return proxyDate(ref.value, (date) => ref.setTime(date.getTime()));
+    }
+
     private cvtBigintToRef(value: bigint): BigIntRefElement {
         const found = this.cache.get(value);
         if (found !== undefined) {
@@ -415,8 +433,8 @@ export class FlatpackStore {
 
     private dedupeObject(value: PrimitiveObject | ObjectWrapper, element: ObjectRefElement): ObjectRefElement {
         if (!this.dedupe) return element;
-        const keys = element.keys();
-        const values = element.values();
+        const keys = element.keyRefs();
+        const values = element.valueRefs();
         let found = this.cachedObjects.get(keys);
         if (!found) {
             found = new Map();
@@ -435,6 +453,18 @@ export class FlatpackStore {
         return element;
     }
 
+    private proxyObjectRef(ref: ObjectRefElement): PrimitiveObject {
+        const keys = this.#toValue(ref.keyRefs()) as string[] | undefined;
+        const values = this.#toValue(ref.valueRefs()) as Serializable[] | undefined;
+        const obj = keys && values ? Object.fromEntries(keys.map((key, i) => [key, values[i]])) : {};
+        return proxyObject(obj, (_value) => {});
+    }
+
+    private proxyObjectWrapperRef(ref: ObjectWrapperRefElement): PrimitiveObject {
+        const value = Object(this.#toValue(ref.valueRef())) as PrimitiveObject;
+        return proxyObject(value, (_value) => {});
+    }
+
     /**
      *
      * @param value - The array converted to an ArrayRefElement.
@@ -482,6 +512,11 @@ export class FlatpackStore {
         return this.dedupeArray(value, element, cacheValue);
     }
 
+    private proxyArrayRef(ref: ArrayRefElement): PrimitiveArray {
+        const arr = ref.valueRefs().map((v) => this.#toValue(v));
+        return proxyObject(arr, (_value) => {});
+    }
+
     private valueToRef(value: Serializable): RefElements {
         if (value === null) {
             return this.primitiveToRef(value);
@@ -619,6 +654,26 @@ export class FlatpackStore {
         }
     }
 
+    #resolveToValueProxy(ref: RefElements | undefined): Unpacked {
+        if (!ref) return undefined;
+        if (ref instanceof ArrayRefElement) return this.proxyArrayRef(ref);
+        if (ref instanceof ObjectRefElement) return this.proxyObjectRef(ref);
+        if (ref instanceof PrimitiveRefElementBase) return ref.value;
+        if (isStringRefElements(ref)) return ref.value;
+        if (ref instanceof MapRefElement) return this.proxyMapRef(ref);
+        if (ref instanceof SetRefElement) return this.proxySetRef(ref);
+        if (ref instanceof BigIntRefElement) return ref.value;
+        if (ref instanceof RegExpRefElement) return ref.value;
+        if (ref instanceof DateRefElement) return this.proxyDateRef(ref);
+        if (ref instanceof ObjectWrapperRefElement) return this.proxyObjectWrapperRef(ref);
+        assert(false, 'Unknown ref type');
+    }
+
+    #toValue(ref: RefElements | undefined): Unpacked {
+        if (!ref) return undefined;
+        return getOrResolve(this.cachedProxies, ref, (ref) => this.#resolveToValueProxy(ref));
+    }
+
     toJSON(): Flatpacked {
         const data = [dataHeader] as Flatpacked;
         const idxLookup = this.assignedElements;
@@ -654,6 +709,10 @@ export class FlatpackStore {
     toValue(): Unpacked {
         return fromJSON(this.toJSON());
     }
+
+    _toValueProxy(): Unpacked {
+        return this.#toValue(this.root);
+    }
 }
 
 type TrieData = StringRefElements;
@@ -686,3 +745,15 @@ export function toJSON<V extends Serializable>(json: V, options?: FlatpackOption
 export function stringify(data: Unpacked, pretty = true): string {
     return pretty ? stringifyFlatpacked(toJSON(data)) : JSON.stringify(toJSON(data));
 }
+
+type WeakOrNever<K, V> = K extends WeakKey ? WeakMap<K, V> : never;
+type SupportedMap<K, V> = Map<K, V> | WeakOrNever<K, V>;
+
+function getOrResolve<K, V>(map: SupportedMap<K, V>, key: K, resolver: (key: K) => V): V {
+    let value = map.get(key);
+    if (value === undefined && !map.has(key)) {
+        value = resolver(key);
+        map.set(key, value);
+    }
+    return value as V;
+}

packages/flatpack-json/src/Flatpack.test.mts

@@ -7,6 +7,7 @@ import { describe, expect, test } from 'vitest';
 import { FlatpackStore, stringify, toJSON } from './Flatpack.mjs';
 import { stringifyFlatpacked } from './stringify.mjs';
 import { fromJSON } from './unpack.mjs';
+import { deepEqual } from './proxy.mts';
 
 const urlFileList = new URL('../fixtures/fileList.txt', import.meta.url);
 const baseFilename = new URL(import.meta.url).pathname.split('/').slice(-1).join('').split('.').slice(0, -2).join('.');
@@ -189,6 +190,23 @@ describe('Flatpack', async () => {
         expect(fromJSON(fp.toJSON())).toEqual(data);
         expect(fp.toJSON()).not.toEqual(v);
     });
+
+    test.each`
+        data
+        ${undefined}
+        ${'string'}
+        ${1}
+        ${1.1}
+        ${null}
+        ${true}
+        ${false}
+        ${new Date()}
+        ${/[a-z]+/}
+    `('toValue $data', ({ data }) => {
+        const fp = new FlatpackStore(data);
+        expect(fp.toValue()).toEqual(data);
+        expect(fp._toValueProxy()).toEqual(data);
+    });
 });
 
 async function sampleFileList() {
@@ -249,3 +267,34 @@ function sampleNestedData() {
         cValues,
     };
 }
+
+describe('Flatpack value proxy', () => {
+    test.each`
+        value
+        ${undefined}
+        ${'string'}
+        ${1}
+        ${1.1}
+        ${null}
+        ${true}
+        ${false}
+        ${[]}
+        ${[1, 2]}
+        ${['a', 'b', 'a', 'b']}
+        ${{}}
+        ${{ a: 1 }}
+        ${{ a: { b: 1 } }}
+        ${{ a: { a: 'a', b: 42 } }}
+        ${{ a: [1] }}
+        ${new Set(['apple', 'banana', 'pineapple'])}
+        ${new Map([['apple', 1], ['banana', 2], ['pineapple', 3]])}
+        ${/[\p{L}\p{M}]+/gu}
+        ${new Date('2024-01-01')}
+    `('identity $value', ({ value }) => {
+        const fp = new FlatpackStore(value);
+        const proxy = fp._toValueProxy();
+        expect(deepEqual(proxy, value)).toBe(true);
+        !(proxy instanceof Map || proxy instanceof Set) && expect(proxy).toEqual(value);
+        expect(fp._toValueProxy()).toBe(proxy);
+    });
+});

packages/flatpack-json/src/RefElements.mts

@@ -124,11 +124,11 @@ export class ObjectRefElement extends BaseRefElement implements RefElement<Objec
         this.#v = values;
     }
 
-    keys(): ArrayRefElement | undefined {
+    keyRefs(): ArrayRefElement | undefined {
         return this.#k;
     }
 
-    values(): ArrayRefElement | undefined {
+    valueRefs(): ArrayRefElement | undefined {
         return this.#v;
     }
 
@@ -174,6 +174,10 @@ export class ObjectWrapperRefElement extends BaseRefElement implements RefElemen
         this.#v = value;
     }
 
+    valueRef(): RefElements | undefined {
+        return this.#v;
+    }
+
     toElement(lookup: FnIndexLookup): ObjectWrapperElement {
         return [ElementType.Object, 0, lookup(this.#v)];
     }
@@ -216,7 +220,7 @@ export class SetRefElement extends BaseRefElement implements RefElement<SetEleme
         return this.#v ? [this.#v] : undefined;
     }
 
-    values(): ArrayRefElement | undefined {
+    valueRefs(): ArrayRefElement | undefined {
         return this.#v;
     }
 
@@ -255,11 +259,11 @@ export class MapRefElement extends BaseRefElement implements RefElement<MapEleme
         return [this.#k, this.#v].filter((r) => !!r);
     }
 
-    keys(): ArrayRefElement | undefined {
+    keyRefs(): ArrayRefElement | undefined {
         return this.#k;
     }
 
-    values(): ArrayRefElement | undefined {
+    valueRefs(): ArrayRefElement | undefined {
         return this.#v;
     }
 
@@ -286,6 +290,10 @@ export class RegExpRefElement extends BaseRefElement implements RefElement<RegEx
         return [ElementType.RegExp, lookup(this.#p), lookup(this.#f)];
     }
 
+    get value(): RegExp {
+        return new RegExp(this.#p.value, this.#f.value);
+    }
+
     clone(): RegExpRefElement {
         return new RegExpRefElement(this.#p, this.#f);
     }
@@ -315,6 +323,14 @@ export class DateRefElement extends BaseRefElement implements RefElement<DateEle
         return [ElementType.Date, this.#v];
     }
 
+    get value(): Date {
+        return new Date(this.#v);
+    }
+
+    setTime(time: number): void {
+        this.#v = time;
+    }
+
     clone(): DateRefElement {
         return new DateRefElement(this.#v);
     }
@@ -336,6 +352,10 @@ export class BigIntRefElement extends BaseRefElement implements RefElement<BigIn
         this.#v = value;
     }
 
+    get value(): bigint {
+        return BigInt(this.#v.value);
+    }
+
     toElement(lookup: FnIndexLookup): BigIntElement {
         return [ElementType.BigInt, lookup(this.#v)];
     }