Update PluginStats with new field in response optional_extended_plugins and upgrade to 2.19.0

.github/workflows/test-spec.yml

@@ -28,38 +28,38 @@ jobs:
             admin_password: admin
           - version: 2.0.0
             admin_password: admin
-          - version: 2.18.0
-          - version: 2.18.0
+          - version: 2.19.0
+          - version: 2.19.0
             tests: plugins/index_state_management
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/ml
-          - version: 2.18.0
+          - version: 2.19.0
             tests: routing
-          - version: 2.18.0
+          - version: 2.19.0
             tests: snapshot
-          - version: 2.18.0
+          - version: 2.19.0
             tests: remote_store
-          - version: 2.18.0
+          - version: 2.19.0
             tests: dangling
             url: http://localhost:9200
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/replication
             url: http://localhost:9200
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/streaming
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/notifications
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/query_insights
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/workload-management
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/analysis
-          - version: 2.18.0
+          - version: 2.19.0
             tests: plugins/security
             cert: tests/plugins/security/.kirk.pem
             key: tests/plugins/security/.kirk-key.pem
-          - version: 2.19.0
+          - version: 2.20.0
             hub: opensearchstaging
             ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
           - version: 3.0.0

CHANGELOG.md

@@ -117,6 +117,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730))
 - Changed `ml.get_memory` and `ml.get_message` to split out `get_all` variants ([#796](https://github.com/opensearch-project/opensearch-api-specification/pull/796))
 - Changed `ml.get_tools` to have two different operation groups `ml.get_all_tools` and `ml.get_tool` ([#799](https://github.com/opensearch-project/opensearch-api-specification/pull/799))
+- Update `PluginStats` with new field in response `optional_extended_plugins` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814))
 
 ## [0.1.0] - 2024-10-25
 

json_schemas/test_story.schema.yaml

@@ -196,6 +196,10 @@ definitions:
       content_type:
         type: string
         default: application/json
+      contains:
+        type: array
+        items:
+          type: string
       payload:
         $ref: '#/definitions/Payload'
     required: [status]

spec/schemas/_common.yaml

@@ -1516,6 +1516,10 @@ components:
           type: ['null', string]
         opensearch_version:
           $ref: '#/components/schemas/VersionString'
+        optional_extended_plugins:
+          type: array
+          items:
+            type: string
       required:
         - classname
         - description
@@ -1524,6 +1528,7 @@ components:
         - java_version
         - name
         - opensearch_version
+        - optional_extended_plugins
         - version
     NodeStatistics:
       type: object

tests/plugins/analysis/indices/cluster/stats.yaml

@@ -0,0 +1,16 @@
+$schema: ../../../../../json_schemas/test_story.schema.yaml
+
+description: Test cluster stats.
+chapters:
+  - synopsis: Returns statistics about a cluster.
+    path: /_cluster/stats/{metric}/nodes/{node_id}
+    parameters:
+      metric:
+        - plugins
+      node_id: _cluster_manager
+    method: GET
+    response:
+      status: 200
+      contains:
+        - analysis-phonenumber
+        - optional_extended_plugins

tools/src/tester/ChapterEvaluator.ts

@@ -70,7 +70,8 @@ export default class ChapterEvaluator {
     const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED }
     const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } }
     const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
-    const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED }
+    const response_contains: string[] | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.contains) : chapter.response?.contains
+    const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload, response_contains) : { result: Result.SKIPPED }
 
     if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)
 

tools/src/tester/ResponsePayloadEvaluator.ts

@@ -21,19 +21,35 @@
     this.logger = logger
   }
 
-  evaluate(response: ActualResponse, expected_payload?: Payload): Evaluation {
-    if (expected_payload == null) return { result: Result.PASSED }
+  evaluate(response: ActualResponse, expected_payload?: Payload, response_contains?: string[]): Evaluation {
     const payload = response.payload
     this.logger.info(`${to_json(payload)}`)
-    const delta = atomizeChangeset(diff(expected_payload, payload))
-    const messages: string[] = _.compact(delta.map((value, _index, _array) => {
-      switch (value.type) {
-        case Operation.UPDATE:
-          return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
-        case Operation.REMOVE:
-          return `missing ${value.path.replace('$.', '')}='${value.value}'`
-      }
-    }))
+
+    let messages: string[] = [];
+
+    if (response_contains && response_contains.length > 0) {
+        const payloadStr = JSON.stringify(payload);
+        const missingValues = response_contains.filter(value => !payloadStr.includes(value));
+
+        if (missingValues.length > 0) {
+            messages.push(`Response payload is missing required values: ${missingValues.join(', ')}`);
+        }
+    }
+
+    if (!!expected_payload) {
+      const delta = atomizeChangeset(diff(expected_payload, payload))
+      const diffMessages: string[] = _.compact(delta.map((value, _index, _array) => {
+        switch (value.type) {
+          case Operation.UPDATE:
+            return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
+          case Operation.REMOVE:
+            return `missing ${value.path.replace('$.', '')}='${value.value}'`
+        }
+      }))
+
+      messages = [...messages, ...diffMessages];
+    }
+
     return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
   }
 }

tools/src/tester/SupplementalChapterEvaluator.ts

@@ -53,7 +53,8 @@ export default class SupplementalChapterEvaluator {
     const status = chapter.status ?? [200, 201]
     const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error }
     const response_payload: Payload | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
-    const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED }
+    const response_contains: string[] | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.contains) : chapter.response?.contains
+    const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload, response_contains) : { result: Result.SKIPPED }
     const result: Result = overall_result(_.compact([overall, payload_body_evaluation, output_values_evaluation.evaluation]))
 
     var evaluation_result: EvaluationWithOutput = { evaluation: { result } }

tools/src/tester/types/story.types.ts

@@ -192,6 +192,7 @@ export interface ExpectedResponse {
   status: number;
   content_type?: string;
   payload?: Payload;
+  contains?: string[];
 }
 /**
  * This interface was referenced by `Story`'s JSON-Schema

tools/tests/tester/ResponsePayloadEvaluator.test.ts

@@ -25,15 +25,23 @@ describe('ResponsePayloadEvaluator', () => {
 
   describe('evaluate', () => {
     test('succeeds without an expected payload', () => {
-      expect(evaluator.evaluate(create_response({}), undefined)).toEqual({ result: Result.PASSED })
+      expect(evaluator.evaluate(create_response({}), undefined, undefined)).toEqual({ result: Result.PASSED })
     })
 
     test('fails with a non-matching payload', () => {
-      expect(evaluator.evaluate(create_response({}), { x: 1 })).toEqual({ result: Result.FAILED, message: "missing x='1'" })
+      expect(evaluator.evaluate(create_response({}), { x: 1 }, undefined)).toEqual({ result: Result.FAILED, message: "missing x='1'" })
     })
 
     test('succeeds with a matching payload', () => {
-      expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 })).toEqual({ result: Result.PASSED })
+      expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 }, undefined)).toEqual({ result: Result.PASSED })
+    })
+
+    test('succeeds with a matching payload using contains', () => {
+      expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["x"])).toEqual({ result: Result.PASSED })
+    })
+
+    test('fails with a non-matching payload using container', () => {
+      expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["y"])).toEqual({ result: Result.FAILED, message: "Response payload is missing required values: y" })
     })
   })
 })