diff --git a/lib/KeyValue/newsrc/Data.ts b/lib/KeyValue/newsrc/Data.ts new file mode 100755 index 0000000..96f2d4e --- /dev/null +++ b/lib/KeyValue/newsrc/Data.ts @@ -0,0 +1,127 @@ +import { Optional } from "../../typings/type.js"; +import { KeyValueTypeList } from "../typings/type.js"; + +import { + KeyValueDataInterface, + KeyValueJSONOption, +} from "../typings/interface.js"; +import { types } from "util"; +export default class Data { + file: string; + key: string; + value: any; + type: KeyValueTypeList; + deleted: boolean = false; + + /** + * @description create data + * @param data data to create + * + * @memberof Data + * + * @example + * ```js + * const data = new Data({ + * file:"file", + * key:"key", + * value:"value", + * type:"string" + * }) + * ``` + */ + + constructor(data: Optional) { + this.file = data.file; + this.key = data.key; + this.type = data.type ?? this.#getType(data.value); + this.value = this.#parseValue(data); + } + /** + * @private + * @description get type of value + * @param value value to get type + * @returns + */ + #getType(value: any): KeyValueTypeList { + return value instanceof Date ? "date" : typeof value; + } + /** + * @private + * @description parse value to correct type + * @param data data to parse + * @returns + */ + #parseValue(data: Optional) { + return data.type === "date" && + (typeof data.value === "string" || + typeof data.value === "number" || + types.isDate(data.value)) + ? // @ts-ignore + new Date(data.value) + : data.type === "bigint" && + (typeof data.value === "string" || typeof data.value === "number") + ? BigInt(data.value) + : typeof data.value === "number" && + data.value > Number.MAX_SAFE_INTEGER + ? BigInt(data.value) + : data.type === "boolean" + ? Boolean(data.value) + : data.type === "object" + ? (typeof data.value === "string" ? JSON.parse(data.value) : data.value) + : data.value; + } + /** + * @description convert data to json + * @returns + * @memberof Data + * @example + * ```js + * .toJSON() + * ``` + */ + toJSON(): KeyValueJSONOption { + return { + value: types.isDate(this.value) + ? this.value.toISOString() + : typeof this.value === "bigint" + ? this.value.toString() + : this.value, + type: this.type, + key: this.key, + }; + } + + get size() { + return Buffer.byteLength(JSON.stringify(this.toJSON())); + } + /** + * @description create empty data + * @static + * @returns + */ + static emptyData() { + return new Data({ + file: "", + key: "", + value: "", + type: "undefined", + }); + } + + static deletedData(key: string,file:string) { + const data = Data.emptyData(); + data.key = key; + data.deleted = true; + data.file = file; + return data; + } + + static fromJSON(data: KeyValueJSONOption,file:string) { + return new Data({ + file, + key: data.key, + value: data.value, + type: data.type, + }); + } +} diff --git a/lib/KeyValue/newsrc/File.ts b/lib/KeyValue/newsrc/File.ts new file mode 100755 index 0000000..89e5b38 --- /dev/null +++ b/lib/KeyValue/newsrc/File.ts @@ -0,0 +1,252 @@ +/** + ** json file + ** max keys 10k + ** + */ + +import { constants, FileHandle, open, rename } from "node:fs/promises"; +import { IJSONOptions, KeyValueJSONOption } from "./typings/interface.js"; +import Data from "./Data.js"; +import Mutex from "./Mutex.js"; +import { PriorityQueue } from "@akarui/structures"; + +export default class JSONFile { + #options: IJSONOptions; + #data: Record | null = null; + #tmpData: Record = {}; + #fileHandle: FileHandle | null = null; + #mutex: Mutex; + #maxTries = 10; + constructor(options: IJSONOptions) { + this.#options = options; + this.#mutex = new Mutex(); + } + + async #load() { + const data = await this.#fileHandle?.readFile(); + if (!data) return; + this.#data = JSON.parse(data.toString()) as Record< + string, + KeyValueJSONOption + >; + } + + async #readAllDataFromFile() { + await this.#mutex.lock(); + return JSON.parse( + ((await this.#fileHandle?.readFile()) ?? "{}").toString() + ); + } + + async #atomicWrite() { + const tmpPath = `${this.#options.filePath}.tmp`; + const tmpHandle = await open( + tmpPath, + constants.O_RDWR | constants.O_CREAT | constants.O_TRUNC + ); + await tmpHandle.writeFile(JSON.stringify(this.#data)); + await tmpHandle.close(); + await this.#fileHandle?.close(); + try { + await rename(tmpPath, this.#options.filePath); + } catch (error) { + if (this.#maxTries === 0) { + this.#maxTries = 10; + throw error; + } + await this.#atomicWrite(); + this.#maxTries--; + } + + this.#fileHandle = await open( + this.#options.filePath, + constants.O_RDWR | constants.O_CREAT + ); + } + + async #atomicFlush(data: Data[]) { + const tmpPath = `${this.#options.filePath}.tmp`; + const tmpHandle = await open( + tmpPath, + constants.O_RDWR | constants.O_CREAT | constants.O_TRUNC + ); + const allFileData = await this.#readAllDataFromFile() + for (const item of data) { + if (item.deleted) { + delete allFileData[item.key]; + continue; + } + allFileData[item.key] = item.toJSON(); + } + await tmpHandle.writeFile(JSON.stringify(allFileData)); + await tmpHandle.close(); + await this.#fileHandle?.close(); + try { + await rename(tmpPath, this.#options.filePath); + } catch (error) { + if (this.#maxTries === 0) { + this.#maxTries = 10; + throw error; + } + await this.#atomicFlush(data); + this.#maxTries--; + } + + this.#fileHandle = await open( + this.#options.filePath, + constants.O_RDWR | constants.O_CREAT + ); + this.#data = this.#tmpData; + this.#tmpData = {}; + } + + async open() { + this.#fileHandle = await open( + this.#options.filePath, + constants.O_RDWR | constants.O_CREAT + ); + + if (this.#options.loadInMemory) { + await this.#load(); + } + } + + async set(data: Data[]) { + await this.#mutex.lock(); + if (this.#data !== null) { + for (const item of data) { + if (item.deleted) { + delete this.#data[item.key]; + continue; + } + this.#data[item.key] = item.toJSON(); + } + + await this.#atomicWrite(); + } else { + for (const item of data) { + this.#tmpData[item.key] = item; + } + await this.#atomicFlush(data); + } + + this.#mutex.unlock(); + } + + async get(key: string) { + await this.#mutex.lock(); + + if (this.#data) { + const data = this.#data[key]; + this.#mutex.unlock(); + return data + ? Data.fromJSON(data, this.#options.filePath) + : Data.emptyData(); + } + + if (this.#tmpData[key]) { + this.#mutex.unlock(); + if (this.#tmpData[key].deleted) { + return null; + } + return Data.fromJSON(this.#tmpData[key], this.#options.filePath); + } + + const allData = await this.#readAllDataFromFile(); + this.#mutex.unlock(); + const data = allData[key]; + return data ? Data.fromJSON(data, this.#options.filePath) : null; + } + + async findMany(query: (data: KeyValueJSONOption) => boolean) { + await this.#mutex.lock(); + const list: Data[] = []; + if (this.#data) { + for (const key in this.#data) { + if (query(this.#data[key])) { + list.push( + Data.fromJSON(this.#data[key], this.#options.filePath) + ); + } + } + + this.#mutex.unlock(); + return list; + } else { + for (const key in this.#tmpData) { + if (query(this.#tmpData[key]) && !this.#tmpData[key].deleted) { + list.push(this.#tmpData[key]); + } + } + + const allData = await this.#readAllDataFromFile(); + this.#mutex.unlock(); + + for (const key in allData) { + if (query(allData[key])) { + list.push( + Data.fromJSON(allData[key], this.#options.filePath) + ); + } + } + + return list; + } + } + + async findOne(query: (data: KeyValueJSONOption) => boolean) { + await this.#mutex.lock(); + if (this.#data) { + for (const key in this.#data) { + if (query(this.#data[key])) { + this.#mutex.unlock(); + return Data.fromJSON( + this.#data[key], + this.#options.filePath + ); + } + } + + this.#mutex.unlock(); + return null; + } else { + for (const key in this.#tmpData) { + if (query(this.#tmpData[key]) && !this.#tmpData[key].deleted) { + this.#mutex.unlock(); + return this.#tmpData[key]; + } + } + + const allData = await this.#readAllDataFromFile(); + this.#mutex.unlock(); + + for (const key in allData) { + if (query(allData[key])) { + return Data.fromJSON(allData[key], this.#options.filePath); + } + } + + return null; + } + } + + async all( + query: (data: KeyValueJSONOption) => boolean, + order: "asc" | "desc" | "firstN" = "asc", + start = 0, + length = 10 + ): Promise { + const list = await this.findMany(query); + if (order === "asc") { + return list + .sort((a, b) => a.value - b.value) + .slice(start, start + length); + } else if (order === "desc") { + return list + .sort((a, b) => b.value - a.value) + .slice(start, start + length); + } else { + return list.slice(start, start + length); + } + } +} diff --git a/lib/KeyValue/newsrc/Mutex.ts b/lib/KeyValue/newsrc/Mutex.ts new file mode 100755 index 0000000..9d79afb --- /dev/null +++ b/lib/KeyValue/newsrc/Mutex.ts @@ -0,0 +1,33 @@ +export default class Mutex { + #lock: boolean; + #queue: (() => void)[]; + + constructor() { + this.#lock = false; + this.#queue = []; + } + + async lock() { + return new Promise((resolve) => { + if (!this.#lock) { + this.#lock = true; + resolve(); + return; + } + this.#queue.push(resolve); + }); + } + + unlock() { + if (this.#queue.length > 0) { + const resolve = this.#queue.shift()!; + resolve(); + return; + } + this.#lock = false; + } + + isLocked() { + return this.#lock; + } +} diff --git a/lib/KeyValue/newsrc/typings/interface.ts b/lib/KeyValue/newsrc/typings/interface.ts new file mode 100755 index 0000000..3a8e931 --- /dev/null +++ b/lib/KeyValue/newsrc/typings/interface.ts @@ -0,0 +1,81 @@ + +import { KeyValueDataValueType, KeyValueTypeList } from "./type.js"; +import { + CacheType, + DatabaseMethod, + ReferenceType, +} from "../../../typings/enum.js"; + +export interface IJSONOptions { + maxKeys: number; + filePath: string; + loadInMemory: boolean; +} + + +export interface KeyValueOptions { + dataConfig?: KeyValueDataConfig; + fileConfig?: KeyValueFileConfig; + encryptionConfig: KeyValueEncryptionConfig; + cacheConfig?: KeyValueCacheConfig; + debug?: boolean; +} + +export interface KeyValueDataConfig { + path?: string; + tables?: string[]; + referencePath?: string; +} + +export interface KeyValueFileConfig { + extension?: string; + transactionLogPath?: string; + maxSize?: number; + reHashOnStartup?: boolean; + staticRehash?: boolean; + minFileCount?: number; +} + +export interface KeyValueEncryptionConfig { + securityKey: string; + encriptData?: boolean; +} + +export interface KeyValueCacheConfig { + cache: CacheType; + reference: ReferenceType; + limit: number; + sorted: boolean; + sortFunction?: (a: any, b: any) => number; +} + +export interface KeyValueTableOptions { + name: string; +} + +export interface KeyValueDataInterface { + file: string; + value: KeyValueDataValueType; + key: string; + type: KeyValueTypeList; +} + +export interface LogBlock { + key: string; + value: string | null; + type: KeyValueTypeList; + method: DatabaseMethod; +} + +export interface KeyValueJSONOption { + value: KeyValueDataValueType; + key: string; + type: KeyValueTypeList; +} + +export interface CacherOptions { + cache: CacheType; + limit: number; + sorted: boolean; + sortFunction?: (a: any, b: any) => number; +} diff --git a/lib/KeyValue/newsrc/typings/type.ts b/lib/KeyValue/newsrc/typings/type.ts new file mode 100755 index 0000000..f7b5f2a --- /dev/null +++ b/lib/KeyValue/newsrc/typings/type.ts @@ -0,0 +1,34 @@ +export type DeepRequired = { + [K in keyof T]: DeepRequired; +} & Required; +export type KeyValueTypeList = + | "string" + | "bigint" + | "number" + | "null" + | "boolean" + | "object" + | "date" + | "symbol" + | "undefined" + | "function"; +export type KeyValueDataValueType = + | string + | bigint + | number + | null + | boolean + | Array + | ValidJSON + | Date; + +export type ValidJSON = { + [x: string | number | symbol]: + | ValidJSON + | number + | string + | Array + | null + | boolean + | (unknown & { toJSON(): ValidJSON }); +}; diff --git a/package-lock.json b/package-lock.json index 11ddc53..512f690 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@akarui/aoi.db", - "version": "2.3.2", + "version": "2.3.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@akarui/aoi.db", - "version": "2.3.2", + "version": "2.3.3", "license": "MIT", "dependencies": { "@akarui/structures": "github:akaruidevelopment/structures#v2", diff --git a/package.json b/package.json index 5ccc033..d5e9bef 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@akarui/aoi.db", - "version": "2.3.2", + "version": "2.3.3", "description": "@akarui/aoi.db - Database Management System with different types of databases.", "main": "dist/cjs/index.js", "module": "dist/esm/index.js",