[DO NOT MERGE]Main based model registry

.github/workflows/experimental-release.yml

@@ -0,0 +1,65 @@
+name: experimental-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      opensearch_dashboards_version:
+        description: 'Which version of OpenSearch-Dashboards that this build is targeting'
+        required: false
+        default: '2.x'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_TAG: ${{ github.event.inputs.opensearch_dashboards_version }}-experimental.${{ github.run_id }}
+
+    steps:
+      - run: echo Target OpenSearch-Dashboards version ${{ github.event.inputs.opensearch_dashboards_version }}
+      - name: Checkout OpenSearch Dashboards
+        uses: actions/checkout@v2
+        with:
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ github.event.inputs.opensearch_dashboards_version }}
+          path: OpenSearch-Dashboards
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: './OpenSearch-Dashboards/.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install Yarn
+        # Need to use bash to avoid having a windows/linux specific step
+        shell: bash
+        run: |
+          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
+          echo "Installing yarn@$YARN_VERSION"
+          npm i -g yarn@$YARN_VERSION
+      - run: node -v
+      - run: yarn -v
+      - name: Checkout ML Commons OpenSearch Dashboards plugin
+        uses: actions/checkout@v2
+        with:
+          path: OpenSearch-Dashboards/plugins/ml-commons-dashboards
+      - name: Bootstrap plugin/opensearch-dashboards
+        run: |
+          cd OpenSearch-Dashboards/plugins/ml-commons-dashboards
+          yarn osd bootstrap
+      - name: Run build
+        run: |
+          cd OpenSearch-Dashboards/plugins/ml-commons-dashboards
+          yarn run build
+      - name: Create Release Tag
+        run: |
+          cd OpenSearch-Dashboards/plugins/ml-commons-dashboards
+          git tag ${{ env.RELEASE_TAG }}
+          git push origin ${{ env.RELEASE_TAG }}
+      - name: Release
+        id: release_step
+        uses: softprops/action-gh-release@v1
+        with:
+          files: OpenSearch-Dashboards/plugins/ml-commons-dashboards/build/*.zip
+          tag_name: ${{ env.RELEASE_TAG }}
+      - name: Update Artifact URL
+        run: |
+          echo ${{ steps.release_step.outputs.assets }}

.github/workflows/lint-workflow.yml

@@ -2,10 +2,10 @@ name: Lint workflow
 on:
   push:
     branches:
-      - '*'
+      - '**'
   pull_request:
     branches:
-      - '*'
+      - '**'
 env:
   OPENSEARCH_DASHBOARDS_VERSION: 'main'
 jobs:

.github/workflows/unit-tests-workflow.yml

@@ -2,10 +2,10 @@ name: Unit tests workflow
 on:
   push:
     branches:
-      - '*'
+      - '**'
   pull_request:
     branches:
-      - '*'
+      - '**'
 env:
   OPENSEARCH_DASHBOARDS_VERSION: 'main'
 jobs:

common/constant.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export const ONE_MB = 1000 * 1000;
+export const ONE_GB = 1000 * ONE_MB;
+
+export const MAX_MODEL_CHUNK_SIZE = 10 * ONE_MB;
+
+export const DATE_FORMAT = 'MMM D, yyyy @ HH:mm:ss.SSS';

common/index.ts

@@ -7,5 +7,9 @@ export const PLUGIN_ID = 'ml-commons-dashboards';
 export const PLUGIN_NAME = 'Machine Learning';
 export const PLUGIN_DESC = `ML Commons for OpenSearch eases the development of machine learning features by providing a set of common machine learning (ML) algorithms through transport and REST API calls. Those calls choose the right nodes and resources for each ML request and monitors ML tasks to ensure uptime. This allows you to leverage existing open-source ML algorithms and reduce the effort required to develop new ML features.`;
 
+export * from './constant';
 export * from './status';
+export * from './model_version';
+export * from './router_paths';
 export * from './model';
+export * from './model_aggregate';

common/model.ts

@@ -3,55 +3,29 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-// TODO: rename the enum keys accordingly
-export enum MODEL_STATE {
-  loaded = 'DEPLOYED',
-  trained = 'TRAINED',
-  unloaded = 'UNDEPLOYED',
-  uploaded = 'REGISTERED',
-  uploading = 'REGISTERING',
-  loading = 'DEPLOYING',
-  partiallyLoaded = 'PARTIALLY_DEPLOYED',
-  loadFailed = 'DEPLOY_FAILED',
-}
-
-export interface OpenSearchModelBase {
-  name: string;
-  model_id: string;
-  model_state: MODEL_STATE;
-  model_version: string;
-}
-
-export interface OpenSearchSelfTrainedModel extends OpenSearchModelBase {
-  algorithm: string;
-}
-
-export interface OpenSearchCustomerModel extends OpenSearchModelBase {
-  chunk_number: number;
-  created_time: number;
-  description: string;
-  last_loaded_time?: number;
-  last_unloaded_time?: number;
-  last_uploaded_time: number;
-  model_config: {
-    all_config?: string;
-    embedding_dimension: number;
-    framework_type: string;
-    model_type: string;
+export interface OpenSearchModel {
+  id: string;
+  owner?: {
+    backend_roles: string[];
+    roles: string[];
+    name: string;
   };
-  model_content: string;
-  model_content_hash_value: string;
-  model_content_size_in_bytes: string;
-  model_format: string;
-  total_chunks: number;
-  version: number;
-  planning_worker_nodes: string[];
+  latest_version: number;
+  created_time: number;
+  last_updated_time: number;
+  name: string;
+  description?: string;
+  access: 'public' | 'restricted' | 'private';
 }
 
-export type ModelSearchSort =
+export type ModelSort =
   | 'name-asc'
   | 'name-desc'
-  | 'id-asc'
-  | 'model_state-asc'
-  | 'model_state-desc'
-  | 'id-desc';
+  | 'latest_version-asc'
+  | 'latest_version-desc'
+  | 'description-asc'
+  | 'description-desc'
+  | 'owner.name-asc'
+  | 'owner.name-desc'
+  | 'last_updated_time-asc'
+  | 'last_updated_time-desc';

common/model_aggregate.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface ModelAggregateItem {
+  id: string;
+  name: string;
+  description?: string;
+  latest_version: number;
+  deployed_versions: string[];
+  owner_name: string;
+  created_time?: number;
+  last_updated_time: number;
+}
+
+export type ModelAggregateSort =
+  | 'name-asc'
+  | 'name-desc'
+  | 'latest_version-asc'
+  | 'latest_version-desc'
+  | 'description-asc'
+  | 'description-desc'
+  | 'owner_name-asc'
+  | 'owner_name-desc'
+  | 'last_updated_time-asc'
+  | 'last_updated_time-desc';

common/model_version.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// TODO: rename the enum keys accordingly
+export enum MODEL_VERSION_STATE {
+  deployed = 'DEPLOYED',
+  trained = 'TRAINED',
+  undeployed = 'UNDEPLOYED',
+  registered = 'REGISTERED',
+  registering = 'REGISTERING',
+  deploying = 'DEPLOYING',
+  partiallyDeployed = 'PARTIALLY_DEPLOYED',
+  deployFailed = 'DEPLOY_FAILED',
+  registerFailed = 'REGISTER_FAILED',
+}
+
+export const isModelDeployable = (state: MODEL_VERSION_STATE) => {
+  if (
+    state === MODEL_VERSION_STATE.undeployed ||
+    state === MODEL_VERSION_STATE.registered ||
+    state === MODEL_VERSION_STATE.deployFailed
+  ) {
+    return true;
+  }
+  return false;
+};
+
+export const isModelUndeployable = (state: MODEL_VERSION_STATE) => {
+  if (state === MODEL_VERSION_STATE.deployed || state === MODEL_VERSION_STATE.partiallyDeployed) {
+    return true;
+  }
+  return false;
+};

common/profile.ts

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { MODEL_STATE } from './model';
+import { MODEL_VERSION_STATE } from './model';
 
 export interface OpenSearchMLCommonsProfile {
   nodes: {
@@ -12,7 +12,7 @@ export interface OpenSearchMLCommonsProfile {
         [key: string]: {
           worker_nodes: string[];
           predictor: string;
-          model_state: MODEL_STATE;
+          model_state: MODEL_VERSION_STATE;
           predict_request_stats: {
             count: number;
             max: number;

common/router.ts

@@ -11,12 +11,17 @@ interface RouteConfig {
   Component: React.ComponentType<any>;
   label: string;
   exact?: boolean;
+  /**
+   * true: display route in nav bar
+   */
+  nav: boolean;
 }
 
 export const ROUTES: RouteConfig[] = [
   {
     path: routerPaths.overview,
     Component: Monitoring,
     label: 'Overview',
+    nav: true,
   },
 ];

common/router_paths.ts

@@ -6,4 +6,9 @@
 export const routerPaths = {
   root: '/',
   overview: '/overview',
+  monitoring: '/monitoring',
+  registerModel: '/model-registry/register-model/:id?',
+  modelList: '/model-registry/model-list',
+  model: '/model-registry/model/:id',
+  modelVersion: '/model-registry/model-version/:id',
 };

common/security.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface OpenSearchSecurityAccount {
+  user_name: string;
+  is_reserved: boolean;
+  is_hidden: boolean;
+  is_interval_user: boolean;
+  user_required_tenant: null;
+  backed_roles: string[];
+  custom_attribute_names: string[];
+  tenants: {
+    global_tenant: boolean;
+    admin_tenant: true;
+    admin: true;
+  };
+  roles: string[];
+}

package.json

@@ -10,13 +10,26 @@
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "osd": "node ../../scripts/osd",
     "lint:es": "node ../../scripts/eslint",
-    "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
+    "test:jest": "TZ=UTC ../../node_modules/.bin/jest --config ./test/jest.config.js",
+    "test:watch": "TZ=UTC ../../node_modules/.bin/jest --config ./test/jest.config.js --watch",
     "prepare": "husky install"
   },
-  "dependencies": {},
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  },
+  "dependencies": {
+    "hash-wasm": "^4.9.0",
+    "papaparse": "^5.3.2",
+    "react-hook-form": "^7.39.4",
+    "rxjs": "^6.5.5"
+  },
   "devDependencies": {
     "@testing-library/user-event": "^14.4.3",
     "husky": "^8.0.0",
-    "lint-staged": "^10.0.0"
+    "@types/papaparse": "^5.3.5",
+    "lint-staged": "^10.0.0",
+    "msw": "^1.1.0"
   }
-}
+}

public/ace-themes/sql_console.js

@@ -0,0 +1,18 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as ace from 'brace';
+
+ace.define('ace/theme/sql_console', ['require', 'exports', 'module', 'ace/lib/dom'], function (
+  acequire,
+  exports
+) {
+  exports.isDark = false;
+  exports.cssClass = 'ace-sql-console';
+  exports.cssText = require('../index.scss');
+
+  const dom = acequire('../lib/dom');
+  dom.importCssString(exports.cssText, exports.cssClass);
+});