Update PluginStats with new field in response optional_extended_plugi…

…ns and upgrade to 2.19.0 (#814) * Update PluginStats with new field in response optional_extended_plugins Signed-off-by: Craig Perkins <cwperx@amazon.com> * Fix lint error and add to CHANGELOG Signed-off-by: Craig Perkins <cwperx@amazon.com> * Update test-spec Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add test for /_cluster/stats/{metric}/nodes/{node_id} Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add mechanism to assert that response contains given a list of target values Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add additional unit test Signed-off-by: Craig Perkins <cwperx@amazon.com> * Revert "Add additional unit test" This reverts commit e555bc2. Signed-off-by: Craig Perkins <cwperx@amazon.com> * Revert "Add mechanism to assert that response contains given a list of target values" This reverts commit e9c9eb1. Signed-off-by: Craig Perkins <cwperx@amazon.com> * Remove test Signed-off-by: Craig Perkins <cwperx@amazon.com> * Remove from required and update sha Signed-off-by: Craig Perkins <cwperx@amazon.com> * Update FlowFrameworkDeleteResponse to use WriteResponseBase Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix k-NN model training tests to work with small training sample set Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Add `id` and `group_by` to `TopQuery` Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Even smaller k-NN model training Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix k-NN tests Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix flow framework test Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Split k-NN plugin tests out of default Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Simplify flow framework delete response Signed-off-by: Thomas Farr <tsfarr@amazon.com> --------- Signed-off-by: Craig Perkins <cwperx@amazon.com> Signed-off-by: Thomas Farr <tsfarr@amazon.com> Co-authored-by: Thomas Farr <tsfarr@amazon.com>
opensearch-project · Feb 28, 2025 · fcbfc10 · fcbfc10
1 parent 595fb05
commit fcbfc10
Show file tree

Hide file tree

Showing 15 changed files with 157 additions and 82 deletions.
diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
@@ -28,40 +28,42 @@ jobs:
             admin_password: admin
           - version: 2.0.0
             admin_password: admin
-          - version: 2.18.0
-          - version: 2.18.0
+          - version: 2.19.0
+          - version: 2.19.0
             tests: plugins/index_state_management
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/ml
-          - version: 2.18.0
+          - version: 2.19.0
             tests: routing
-          - version: 2.18.0
+          - version: 2.19.0
             tests: snapshot
-          - version: 2.18.0
+          - version: 2.19.0
             tests: remote_store
-          - version: 2.18.0
+          - version: 2.19.0
             tests: dangling
             url: http://localhost:9200
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/replication
             url: http://localhost:9200
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/streaming
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/notifications
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/query_insights
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/workload-management
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/analysis
-          - version: 2.18.0
+          - version: 2.19.0
+            tests: plugins/knn
+          - version: 2.19.0
             tests: plugins/security
             cert: tests/plugins/security/.kirk.pem
             key: tests/plugins/security/.kirk-key.pem
-          - version: 2.19.0
+          - version: 2.20.0
             hub: opensearchstaging
-            ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
+            ref: '@sha256:037a2eb5a4f48b9a9a29950be5e5fae4ebd82d41de1066e00bb8d8b0ce7871b1'
           - version: 3.0.0
             hub: opensearchstaging
             ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de'

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -61,6 +61,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `POST /_plugins/_ml/_execute/{algorithm_name}` ([#811](https://github.com/opensearch-project/opensearch-api-specification/pull/811))
 - Added search suggester types ([#817](https://github.com/opensearch-project/opensearch-api-specification/pull/817))
 - Added `SimplePatternTokenizer` and `SimplePatternSplitTokenizer` ([#820](https://github.com/opensearch-project/opensearch-api-specification/pull/820))
+- Added `optional_extended_plugins` field to `PluginStats` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814))
+- Added `id` & `group_by` fields to `TopQuery` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
@@ -113,6 +115,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed GeoDistanceSort schema ([#805](https://github.com/opensearch-project/opensearch-api-specification/pull/805))
 - Fixed Aggregation schemas ([#801](https://github.com/opensearch-project/opensearch-api-specification/pull/801))
 - Fixed FilterQueryRequestProcessor to use correct query type ([#821](https://github.com/opensearch-project/opensearch-api-specification/pull/821))
+- Fixed `knn.train_model`'s request body `method` field to accept an object rather than a string ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814))
 
 ### Changed
 - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683))
@@ -121,6 +124,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730))
 - Changed `ml.get_memory` and `ml.get_message` to split out `get_all` variants ([#796](https://github.com/opensearch-project/opensearch-api-specification/pull/796))
 - Changed `ml.get_tools` to have two different operation groups `ml.get_all_tools` and `ml.get_tool` ([#799](https://github.com/opensearch-project/opensearch-api-specification/pull/799))
+- Changed `FlowFrameworkDeleteResponse` to utilize `WriteResponseBase` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814))
 
 ## [0.1.0] - 2024-10-25
 

diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml
@@ -444,7 +444,7 @@ components:
       content:
         application/json:
           schema:
-            $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkDeleteResponse'
+            $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase'
     flow_framework.get@200:
       content:
         application/json:

diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml
@@ -390,7 +390,7 @@ components:
       in: query
       description: The default operator for query string query (AND or OR).
       schema:
-        $ref: '../schemas/knn._common.yaml#/components/schemas/DefaultOperator'
+        $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/Operator'
     knn.search_models::query.df:
       name: df
       in: query
@@ -518,7 +518,7 @@ components:
       in: query
       description: Search operation type.
       schema:
-        $ref: '../schemas/knn._common.yaml#/components/schemas/SearchType'
+        $ref: '../schemas/_common.yaml#/components/schemas/SearchType'
     knn.search_models::query.seq_no_primary_term:
       name: seq_no_primary_term
       in: query
@@ -580,7 +580,7 @@ components:
       in: query
       description: Specify suggest mode.
       schema:
-        $ref: '../schemas/knn._common.yaml#/components/schemas/SuggestMode'
+        $ref: '../schemas/_common.yaml#/components/schemas/SuggestMode'
     knn.search_models::query.suggest_size:
       name: suggest_size
       in: query

diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml
@@ -1516,6 +1516,11 @@ components:
           type: ['null', string]
         opensearch_version:
           $ref: '#/components/schemas/VersionString'
+        optional_extended_plugins:
+          type: array
+          items:
+            type: string
+          x-version-added: '2.19'
       required:
         - classname
         - description

diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml
@@ -76,30 +76,6 @@ components:
         version:
           $ref: '#/components/schemas/version'
       minProperties: 1
-    FlowFrameworkDeleteResponse:
-      type: object
-      properties:
-        _index:
-          type: string
-          description: The name of the index where the document was stored.
-        _id:
-          type: string
-          description: The unique identifier of the deleted document.
-        _version:
-          type: integer
-          description: The version of the document after the deletion.
-        result:
-          type: string
-          description: The result of the deletion operation.
-          enum: [deleted, not_found]
-        _shards:
-          $ref: '#/components/schemas/shards'
-        _seq_no:
-          type: integer
-          description: The sequence number assigned to the document after the deletion.
-        _primary_term:
-          type: integer
-          description: The primary term assigned to the document after the deletion.
     FlowFrameworkGetResponse:
       type: object
       properties:
@@ -148,14 +124,6 @@ components:
         skipped:
           type: integer
           description: The number of shards where the deletion was skipped.
-    query:
-      type: object
-      description: The search query to match workflows. Use `match_all` to retrieve all workflows, or `match` to search by specific fields like `use_case`.
-      properties:
-        match:
-          type: object
-        match_all:
-          type: object
     total:
       type: object
       properties:
@@ -224,6 +192,23 @@ components:
           type: string
         user:
           $ref: '#/components/schemas/user'
+        provision_start_time:
+          $ref: '_common.yaml#/components/schemas/DateTime'
+        provision_end_time:
+          $ref: '_common.yaml#/components/schemas/DateTime'
+        resources_created:
+          $ref: '#/components/schemas/ResourcesCreated'
+    ResourcesCreated:
+      type: object
+      properties:
+        workflow_step_name:
+          type: string
+        workflow_step_id:
+          type: string
+        resource_type:
+          type: string
+        resource_id:
+          type: string
     user:
       type: object
       properties:
@@ -367,7 +352,7 @@ components:
       type: object
       properties:
         query:
-          $ref: '#/components/schemas/query'
+          $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer'
     WorkflowSearchResponse:
       type: object
       properties:

diff --git a/spec/schemas/insights._common.yaml b/spec/schemas/insights._common.yaml
@@ -19,6 +19,10 @@ components:
     TopQuery:
       type: object
       properties:
+        id:
+          type: string
+        group_by:
+          $ref: '#/components/schemas/GroupingType'
         timestamp:
           type: integer
           description: The timestamp of the query execution.
@@ -56,6 +60,13 @@ components:
         measurements:
           type: object
           $ref: '#/components/schemas/Measurements'
+    GroupingType:
+      type: string
+      enum:
+        - NONE
+        - SIMILARITY
+        - none
+        - similarity
     TaskResourceUsages:
       type: object
       properties:

diff --git a/spec/schemas/knn._common.yaml b/spec/schemas/knn._common.yaml
@@ -6,25 +6,6 @@ info:
 paths: {}
 components:
   schemas:
-    DefaultOperator:
-      type: string
-      description: The default operator for query string query (AND or OR).
-      enum:
-        - AND
-        - OR
-    SearchType:
-      type: string
-      description: Search operation type.
-      enum:
-        - dfs_query_then_fetch
-        - query_then_fetch
-    SuggestMode:
-      type: string
-      description: Specify suggest mode.
-      enum:
-        - always
-        - missing
-        - popular
     Stats:
       type: object
       properties:
@@ -143,6 +124,20 @@ components:
           $ref: '_common.yaml#/components/schemas/ByteCount'
         current_size_in_bytes:
           $ref: '_common.yaml#/components/schemas/ByteCount'
+    KnnMethod:
+      type: object
+      properties:
+        name:
+          type: string
+        space_type:
+          type: string
+        engine:
+          type: string
+        parameters:
+          type: object
+          additionalProperties: true
+      required:
+        - name
     TrainedModel:
       type: object
       properties:
@@ -166,7 +161,7 @@ components:
         compression_level:
           type: string
         method:
-          type: string
+          $ref: '#/components/schemas/KnnMethod'
         spaceType:
           type: string
       required:

diff --git a/tests/default/flow_framework/workflow/state/search.yaml b/tests/default/flow_framework/workflow/state/search.yaml
@@ -18,19 +18,20 @@ prologues:
     output:
       workflow_id: payload.workflow_id
 chapters:
-  - synopsis: Search for a workflow.
+  - synopsis: Search for a workflow state.
     path: /_plugins/_flow_framework/workflow/state/_search
     method: GET
     request:
       payload:
         query:
-          match_all: {}
+          term:
+            workflow_id:
+              value: ${workflow.workflow_id}
     response:
       status: 200
       payload:
         hits:
           hits:
             - _index: .plugins-flow-framework-state
               _source:
-                state: NOT_STARTED
-
+                workflow_id: ${workflow.workflow_id}
diff --git a/tests/plugins/knn/docker-compose.yml b/tests/plugins/knn/docker-compose.yml
@@ -0,0 +1,12 @@
+version: '3'
+
+services:
+  opensearch-cluster:
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
+    ports:
+      - 9200:9200
+      - 9600:9600
+    environment:
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - discovery.type=single-node
diff --git a/tests/default/knn/models.yaml → tests/plugins/knn/knn/models.yaml b/tests/default/knn/models.yaml → tests/plugins/knn/knn/models.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Delete a k-NN model.
 version: '>= 2.17'
@@ -41,6 +41,15 @@ prologues:
         mode: on_disk
         compression_level: 32x
         spaceType: l2
+        method:
+          name: ivf
+          engine: faiss
+          parameters:
+            nlist: 2
+            encoder:
+              name: pq
+              parameters:
+                code_size: 1
     status: [200]
 epilogues:
   - path: /movies

diff --git a/tests/default/knn/models/search.yaml → tests/plugins/knn/knn/models/search.yaml b/tests/default/knn/models/search.yaml → tests/plugins/knn/knn/models/search.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../../json_schemas/test_story.schema.yaml
 
 description: Search for a k-NN model.
 version: '>= 2.17'
@@ -41,6 +41,25 @@ prologues:
         mode: on_disk
         compression_level: 32x
         spaceType: l2
+        method:
+          name: ivf
+          engine: faiss
+          parameters:
+            nlist: 2
+            encoder:
+              name: pq
+              parameters:
+                code_size: 1
+    status: [200]
+  - method: GET
+    path: /_plugins/_knn/models/model-1
+    retry:
+      count: 3
+    response:
+      status: 200
+      payload:
+        model_id: model-1
+        state: created
     status: [200]
 epilogues:
   - path: /movies