From bab43f26ea572f6b2217aa43748fd491cc9e12f7 Mon Sep 17 00:00:00 2001 From: Winzlieb Date: Fri, 26 Jul 2024 12:16:30 +0200 Subject: [PATCH] put import functionality to cli + fixes --- apps/edb-cli/package.json | 1 + apps/edb-cli/src/dataStore.ts | 50 +++++++++++++++++-- apps/edb-cli/src/index.ts | 9 +++- packages/edb-cli-creator/src/makeEdbCli.ts | 49 +++++++++++++++++- .../src/import/startBulkImport.ts | 13 ++++- 5 files changed, 112 insertions(+), 10 deletions(-) diff --git a/apps/edb-cli/package.json b/apps/edb-cli/package.json index c1e831ad..318f7e45 100644 --- a/apps/edb-cli/package.json +++ b/apps/edb-cli/package.json @@ -28,6 +28,7 @@ "@slub/edb-core-utils": "workspace:*", "@slub/edb-graph-traversal": "workspace:*", "@slub/remote-query-implementations": "workspace:*", + "@slub/prisma-db-impl": "workspace:*", "@slub/sparql-schema": "workspace:*", "@slub/edb-authorities": "workspace:*", "@slub/edb-data-mapping": "workspace:*", diff --git a/apps/edb-cli/src/dataStore.ts b/apps/edb-cli/src/dataStore.ts index 5333acc5..f5835116 100644 --- a/apps/edb-cli/src/dataStore.ts +++ b/apps/edb-cli/src/dataStore.ts @@ -4,6 +4,35 @@ import { getSPARQLFlavour, } from "@slub/remote-query-implementations"; import { initSPARQLDataStoreFromConfig } from "@slub/sparql-db-impl"; +import { extendSchemaShortcut } from "@slub/json-schema-utils"; +import { primaryFields, schema } from "@slub/exhibition-schema"; +import { JSONSchema7 } from "json-schema"; +import { initPrismaStore } from "@slub/prisma-db-impl"; +import { + typeIRItoTypeName, + typeNameToTypeIRI, +} from "@slub/edb-api/src/dataStore"; +import { PrismaClient } from "@prisma/edb-exhibition-client"; + +const initPrisma = async () => { + const rootSchema = extendSchemaShortcut(schema as JSONSchema7, "type", "id"); + const prisma = new PrismaClient(); + //bun only runs if we call it here: why?? + //find first object that can be counted: + for (const key of Object.keys(prisma)) { + if (prisma[key]?.count) { + const c = await prisma[key].count(); + console.log(c); + break; + } + } + return initPrismaStore(prisma, rootSchema, primaryFields, { + jsonldContext: config.defaultJsonldContext, + defaultPrefix: config.defaultPrefix, + typeIRItoTypeName: typeIRItoTypeName, + typeNameToTypeIRI: typeNameToTypeIRI, + }); +}; export const provider = getProviderOrDefault(config.sparqlEndpoint); if (!provider) { @@ -11,8 +40,19 @@ if (!provider) { } export const crudFunctions = provider(config.sparqlEndpoint); -export const dataStore = initSPARQLDataStoreFromConfig( - config, - crudFunctions, - getSPARQLFlavour(config.sparqlEndpoint), -); +export const dataStore = + process.env.DATABASE_PROVIDER === "sparql" + ? initSPARQLDataStoreFromConfig( + config, + crudFunctions, + getSPARQLFlavour(config.sparqlEndpoint), + ) + : await initPrisma(); + +export const importStores = { + oxigraph: initSPARQLDataStoreFromConfig( + config, + crudFunctions, + getSPARQLFlavour(config.sparqlEndpoint), + ), +}; diff --git a/apps/edb-cli/src/index.ts b/apps/edb-cli/src/index.ts index 1bd66d79..4096387b 100644 --- a/apps/edb-cli/src/index.ts +++ b/apps/edb-cli/src/index.ts @@ -1,10 +1,15 @@ import { run, subcommands } from "cmd-ts"; -import { dataStore } from "./dataStore"; +import { dataStore, importStores } from "./dataStore"; import { flatImportHandler } from "./flatImportHandler"; import { availableFlatMappings } from "@slub/exhibition-schema"; import { makeEdbCli } from "@slub/edb-cli-creator"; -const cli = makeEdbCli(dataStore, [], availableFlatMappings, flatImportHandler); +const cli = makeEdbCli( + dataStore, + importStores, + availableFlatMappings, + flatImportHandler, +); run( subcommands({ diff --git a/packages/edb-cli-creator/src/makeEdbCli.ts b/packages/edb-cli-creator/src/makeEdbCli.ts index 48d92928..4192b945 100644 --- a/packages/edb-cli-creator/src/makeEdbCli.ts +++ b/packages/edb-cli-creator/src/makeEdbCli.ts @@ -26,7 +26,7 @@ export type FlatImportHandler = (option: { }) => Promise; export const makeEdbCli = ( dataStore: AbstractDatastore, - importStores: AbstractDatastore[], + importStores: Record, flatMappings?: AvailableFlatMappings, flatImportHandler?: FlatImportHandler, ) => { @@ -100,6 +100,7 @@ export const makeEdbCli = ( console.log(formatJSONResult(item, pretty, noJsonld)); return Promise.resolve(); }); + process.exit(0); }, }); @@ -155,5 +156,49 @@ export const makeEdbCli = ( handler: flatImportHandler, }); - return { list, get, flatImport }; + const importCommand = command({ + name: "import", + description: "Recursively import data from another data store", + args: { + typeName: positional({ + type: string, + displayName: "type", + description: "The Type of the document", + }), + importStoreName: option({ + type: oneOf(Object.keys(importStores)), + long: "importFrom", + short: "i", + description: `the store where data should be imported (${Object.keys(importStores).join(" , ")})`, + }), + entityIRI: option({ + type: optional(string), + long: "entityIRI", + short: "e", + description: "only import a single entity identified by the entityIRI", + }), + limit: option({ + type: optional(number), + description: "The number of documents to import", + defaultValue: () => 10000, + long: "limit", + short: "l", + }), + }, + handler: async ({ entityIRI, typeName, importStoreName, limit }) => { + const importStore = importStores[importStoreName]; + if (!importStore) { + throw new Error(`import store ${importStoreName} not found in config`); + process.exit(1); + } + if (entityIRI) { + await dataStore.importDocument(typeName, entityIRI, importStore); + } else { + await dataStore.importDocuments(typeName, importStore, limit || 10); + } + process.exit(0); + }, + }); + + return { list, get, flatImport, import: importCommand }; }; diff --git a/packages/prisma-db-impl/src/import/startBulkImport.ts b/packages/prisma-db-impl/src/import/startBulkImport.ts index 0dbdcf80..267c72ea 100644 --- a/packages/prisma-db-impl/src/import/startBulkImport.ts +++ b/packages/prisma-db-impl/src/import/startBulkImport.ts @@ -26,7 +26,18 @@ export const startBulkImport = async ( const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); bar.start(amount, 0); for await (let doc of docs) { - await importDocument(typeName, doc, importStore, prisma, visited, errored); + try { + await importDocument( + typeName, + doc, + importStore, + prisma, + visited, + errored, + ); + } catch (e) { + console.error(e); + } bar.increment(); } bar.stop();