From d75819d938077d7473ab4801e36b1c0224a5970c Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 8 Aug 2024 15:26:13 -0400 Subject: [PATCH] Added support for testing multiple distributions. Signed-off-by: dblock --- .cspell | 1 - CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 1 + TESTING_GUIDE.md | 34 +++++++++++++- json_schemas/test_story.schema.yaml | 11 +++++ spec/namespaces/_core.yaml | 3 ++ spec/schemas/nodes._common.yaml | 18 +++++++- spec/schemas/nodes.info.yaml | 17 +------ tests/default/_core/info.yaml | 3 ++ .../_core/search/rest_total_hits_as_int.yaml | 35 ++------------ tests/default/cat/health.yaml | 10 ++-- tests/default/cat/indices.yaml | 4 ++ tests/default/cat/nodeattrs.yaml | 2 + tests/default/indices/cache.yaml | 2 + tests/default/indices/dangling.yaml | 2 + tests/default/indices/forcemerge.yaml | 2 + tests/default/indices/segments.yaml | 3 +- tests/default/indices/settings.yaml | 4 ++ tests/default/ml/model_groups.yaml | 2 + tests/default/ml/models.yaml | 2 + tools/src/OpenSearchHttpClient.ts | 6 +++ tools/src/merger/OpenApiVersionExtractor.ts | 30 ++++++++++-- tools/src/tester/MergedOpenApiSpec.ts | 8 ++-- tools/src/tester/StoryEvaluator.ts | 19 ++++++-- tools/src/tester/TestRunner.ts | 16 +++++-- tools/src/tester/test.ts | 6 ++- tools/src/tester/types/story.types.ts | 15 +++++- .../merger/OpenApiVersionExtractor.test.ts | 6 +-- tools/tests/tester/MergedOpenApiSpec.test.ts | 46 +++++++++++++++---- .../fixtures/evals/skipped/distributions.yaml | 6 +++ .../specs/complete/namespaces/index.yaml | 14 ++++++ .../stories/skipped/distributions.yaml | 9 ++++ tools/tests/tester/helpers.ts | 2 +- .../tests/tester/integ/StoryEvaluator.test.ts | 6 +++ 34 files changed, 258 insertions(+), 88 deletions(-) create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions.yaml diff --git a/.cspell b/.cspell index 1542876a8..1d8417dee 100644 --- a/.cspell +++ b/.cspell @@ -130,7 +130,6 @@ readingform rebalance Rebalance recoverysource -Refn reindex Reindex relo diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ecf4f75..11ae3755e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) +- Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Changed diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 951f00cdd..7f8b21261 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -146,6 +146,7 @@ This repository includes several OpenAPI Specification Extensions to fill in any - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them. - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml). - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema. +- `x-distributions`: Contains a list of distributions known to include the API. Use `opensearch.org` for the official distribution, `aos` for Amazon Managed OpenSearch, and `aoss` for Amazon OpenSearch Serverless. ## Writing Spec Tests diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 5c066549e..3155fd4f9 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -10,6 +10,7 @@ - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) - [Managing Versions](#managing-versions) + - [Managing Distributions](#managing-distributions) - [Waiting for Tasks](#waiting-for-tasks) - [Warnings](#warnings) - [multiple-paths-detected](#multiple-paths-detected) @@ -205,7 +206,38 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne status: 200 ``` -The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. +The test tool will fetch the server version when it starts and use it automatically. The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. + +### Managing Distributions + +OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not. + +```yaml +/: + get: + operationId: info.0 + x-distributions: + - opensearch.org + - aos + description: Returns basic information about the cluster. +``` + +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. + +```yaml +description: Test root endpoint. +distributions: + - opensearch.org + - aos +chapters: + - synopsis: Get server info. + path: / + method: GET + response: + status: 200 +``` + +To test a particular distribution pass `--opensearch-distribution` to the test tool. ### Waiting for Tasks diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 6d1916509..928ba4bad 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -21,6 +21,8 @@ properties: $ref: '#/definitions/Chapter' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' required: [chapters,description] additionalProperties: false @@ -85,6 +87,8 @@ definitions: $ref: '#/definitions/Output' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' retry: $ref: '#/definitions/Retry' required: [method, path] @@ -106,6 +110,13 @@ definitions: The semver range to execute the story or chapter against. type: string + Distributions: + description: | + The list of distributions that support this API. + type: array + items: + type: string + Retry: description: | Number of times to retry on error. diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 9aa5dbeaa..4b5a55cc8 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -9,6 +9,9 @@ paths: operationId: info.0 x-operation-group: info x-version-added: '1.0' + x-distributions: + - aos + - opensearch.org description: Returns basic information about the cluster. externalDocs: url: https://opensearch.org/docs/latest diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index 59378e88a..995e76ea5 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -562,6 +562,14 @@ components: write_operations: description: The total number of write operations for the device completed since starting OpenSearch. type: number + read_time: + type: number + write_time: + type: number + queue_size: + type: number + io_time_in_millis: + $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis' Jvm: type: object properties: @@ -1068,8 +1076,10 @@ components: type: boolean enforced: type: boolean + total_rejections_breakup: + $ref: '#/components/schemas/TotalRejectionsBreakup' total_rejections_breakup_shadow_mode: - $ref: '#/components/schemas/TotalRejectionsBreakupShadowMode' + $ref: '#/components/schemas/TotalRejectionsBreakup' ShardSearchBackpressureStats: type: object properties: @@ -1140,7 +1150,11 @@ components: type: number cancellation_limit_reached_count: type: number - TotalRejectionsBreakupShadowMode: + cancelled_task_percentage: + type: number + current_cancellation_eligible_tasks_count: + type: number + TotalRejectionsBreakup: type: object properties: node_limits: diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml index 7086c17ca..6fbcefdc2 100644 --- a/spec/schemas/nodes.info.yaml +++ b/spec/schemas/nodes.info.yaml @@ -102,14 +102,10 @@ components: search_pipelines: $ref: '#/components/schemas/NodeInfoSearchPipelines' required: - - attributes - build_hash - build_type - - host - - ip - name - roles - - transport_address - version NodeInfoHttp: type: object @@ -156,7 +152,7 @@ components: bundled_jdk: type: boolean using_bundled_jdk: - type: boolean + type: [boolean, 'null'] using_compressed_ordinary_object_pointers: oneOf: - type: boolean @@ -167,16 +163,9 @@ components: type: string required: - bundled_jdk - - gc_collectors - - input_arguments - mem - - memory_pools - pid - start_time_in_millis - - version - - vm_name - - vm_vendor - - vm_version NodeInfoJvmMemory: type: object properties: @@ -256,12 +245,8 @@ components: swap: $ref: '#/components/schemas/NodeInfoMemory' required: - - arch - available_processors - - name - - pretty_name - refresh_interval_in_millis - - version NodeInfoOSCPU: type: object properties: diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 67e245235..9de580a2f 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -2,6 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. +distributions: + - aos + - opensearch.org chapters: - synopsis: Get server info. path: / diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 3d01d28da..fedcf4b2d 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -12,16 +12,6 @@ prologues: title: Moneyball year: 2011 status: [201] - - path: /movies/_doc - method: POST - parameters: - refresh: true - request: - payload: - director: Nicolas Winding Refn - title: Drive - year: 2011 - status: [201] epilogues: - path: /movies method: DELETE @@ -42,7 +32,7 @@ chapters: payload: timed_out: false hits: - total: 2 + total: 1 max_score: 1 hits: - _index: movies @@ -51,12 +41,6 @@ chapters: director: Bennett Miller title: Moneyball year: 2011 - - _index: movies - _score: 1 - _source: - director: Nicolas Winding Refn - title: Drive - year: 2011 - synopsis: Search with rest_total_hits_as_int=false. path: /{index}/_search parameters: @@ -73,7 +57,7 @@ chapters: timed_out: false hits: total: - value: 2 + value: 1 relation: eq max_score: 1 hits: @@ -83,12 +67,6 @@ chapters: director: Bennett Miller title: Moneyball year: 2011 - - _index: movies - _score: 1 - _source: - director: Nicolas Winding Refn - title: Drive - year: 2011 - synopsis: Search with rest_total_hits_as_int=false track_total_hits=1. path: /{index}/_search parameters: @@ -107,7 +85,7 @@ chapters: hits: total: value: 1 - relation: gte + relation: eq max_score: 1 hits: - _index: movies @@ -116,10 +94,3 @@ chapters: director: Bennett Miller title: Moneyball year: 2011 - - _index: movies - _score: 1 - _source: - director: Nicolas Winding Refn - title: Drive - year: 2011 - diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 38865e9e5..196cff4b3 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -53,7 +53,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat with master response (format=json). method: GET @@ -66,7 +65,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_master: 'true' - synopsis: Cat with cluster_manager response (format=json). @@ -80,7 +78,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_cluster_manager: 'true' - synopsis: Cat in different formats (format=yaml). @@ -93,9 +90,10 @@ chapters: content_type: application/yaml payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -105,9 +103,10 @@ chapters: content_type: application/cbor payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -117,5 +116,4 @@ chapters: content_type: application/smile payload: - node.total: '1' - status: yellow node.data: '1' diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index c20df6fe8..6b6130823 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -71,6 +71,8 @@ chapters: status: 200 content_type: application/yaml - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: @@ -79,6 +81,8 @@ chapters: status: 200 content_type: application/cbor - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index cb9dc328c..01dba1cae 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -3,6 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. chapters: - synopsis: Cat with a json response. + distributions: + - opensearch.org path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index db4db9dd5..2e056a526 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 94087d494..0b1b0c784 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. +distributions: + - opensearch.org chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index d537e3a7c..cc1608cb6 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -2,6 +2,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index da87fec7b..0445ebd64 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -1,7 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. - +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 5bfada4b9..80906a7e3 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -11,6 +11,8 @@ epilogues: status: [200, 404] chapters: - synopsis: Get global settings. + distributions: + - opensearch.org path: /_settings method: GET parameters: @@ -23,6 +25,8 @@ chapters: response: status: 200 - synopsis: Get global settings (cluster_manager_timeout). + distributions: + - opensearch.org path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index a08efa0ce..8c036ae6e 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index ac386c0a4..8c2b7359b 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index f81656074..7bc09d1f7 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -22,6 +22,9 @@ export const OPENSEARCH_URL_OPTION = new Option('--opensearch-url ', 'URL a .default(DEFAULT_URL) .env('OPENSEARCH_URL') +export const OPENSEARCH_DISTRIBUTION_OPTION = new Option('--opensearch-distribution ', 'OpenSearch distribution') + .env('OPENSEARCH_DISTRIBUTION') + export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username ', 'username to use when authenticating with OpenSearch') .default(DEFAULT_USER) .env('OPENSEARCH_USERNAME') @@ -64,6 +67,7 @@ export interface AwsAuth { export interface OpenSearchHttpClientOptions { url?: string + distribution?: string, insecure?: boolean responseType?: ResponseType logger?: Logger, @@ -73,6 +77,7 @@ export interface OpenSearchHttpClientOptions { export type OpenSearchHttpClientCliOptions = { opensearchUrl?: string + opensearchDistribution?: string, opensearchUsername?: string opensearchPassword?: string opensearchInsecure?: boolean @@ -88,6 +93,7 @@ export type OpenSearchHttpClientCliOptions = { export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptions): OpenSearchHttpClientOptions { return { url: opts.opensearchUrl, + distribution: opts.opensearchDistribution, insecure: opts.opensearchInsecure, basic_auth: opts.opensearchUsername !== undefined && opts.opensearchPassword !== undefined ? { username: opts.opensearchUsername, diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 5d4164b1c..ff32c8857 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -17,12 +17,14 @@ import semver from 'semver' export default class OpenApiVersionExtractor { private _spec?: Record private _source_spec: OpenAPIV3.Document - private _target_version: string + private _target_version?: string + private _target_distribution?: string private _logger: Logger - constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { + constructor(source_spec: OpenAPIV3.Document, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this._source_spec = source_spec - this._target_version = semver.coerce(target_version)?.toString() ?? target_version + this._target_version = target_version !== undefined ? (semver.coerce(target_version)?.toString() ?? target_version) : target_version + this._target_distribution = target_distribution this._logger = logger this._spec = undefined } @@ -45,10 +47,13 @@ export default class OpenApiVersionExtractor { #extract() : void { this._logger.info(`Extracting version ${this._target_version} ...`) this.#remove_keys_not_matching_semver() + this.#remove_keys_not_matching_distribution() this.#remove_unused() } #exclude_per_semver(obj: any): boolean { + if (this._target_version == undefined) return false + const x_version_added = semver.coerce(obj['x-version-added'] as string) const x_version_removed = semver.coerce(obj['x-version-removed'] as string) @@ -61,11 +66,30 @@ export default class OpenApiVersionExtractor { return false } + #exclude_per_distribution(obj: any): boolean { + if (this._target_distribution == undefined) return false + + const x_distributions = obj['x-distributions'] as string[] + + if (x_distributions?.length > 0 && !x_distributions.includes(this._target_distribution)) { + return true + } + + return false + } + // Remove any elements that are x-version-added/removed incompatible with the target server version. #remove_keys_not_matching_semver(): void { + if (this._target_version == undefined) return delete_matching_keys(this._spec, this.#exclude_per_semver.bind(this)) } + // Remove any elements that are x-distributions incompatible with the target distribution. + #remove_keys_not_matching_distribution(): void { + if (this._target_distribution === undefined) return + delete_matching_keys(this._spec, this.#exclude_per_distribution.bind(this)) + } + #remove_unused(): void { if (this._spec === undefined) return diff --git a/tools/src/tester/MergedOpenApiSpec.ts b/tools/src/tester/MergedOpenApiSpec.ts index ee21f55ac..d4a3e2604 100644 --- a/tools/src/tester/MergedOpenApiSpec.ts +++ b/tools/src/tester/MergedOpenApiSpec.ts @@ -20,21 +20,23 @@ export default class MergedOpenApiSpec { logger: Logger file_path: string target_version?: string + target_distribution?: string protected _spec: OpenAPIV3.Document | undefined - constructor (spec_path: string, target_version?: string, logger: Logger = new Logger()) { + constructor (spec_path: string, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this.logger = logger this.file_path = spec_path this.target_version = target_version + this.target_distribution = target_distribution } spec (): OpenAPIV3.Document { if (this._spec) return this._spec const merger = new OpenApiMerger(this.file_path, this.logger) var spec = merger.spec() - if (this.target_version !== undefined) { - const version_extractor = new OpenApiVersionExtractor(spec, this.target_version) + if (this.target_version !== undefined || this.target_distribution !== undefined) { + const version_extractor = new OpenApiVersionExtractor(spec, this.target_version, this.target_distribution) spec = version_extractor.extract() } const ctx = new SpecificationContext(this.file_path) diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 5db454a5c..5c0675f13 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -26,7 +26,7 @@ export default class StoryEvaluator { this._supplemental_chapter_evaluator = supplemental_chapter_evaluator } - async evaluate({ story, display_path, full_path }: StoryFile, version?: string, dry_run: boolean = false): Promise { + async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise { if (version != undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { return { result: Result.SKIPPED, @@ -37,13 +37,23 @@ export default class StoryEvaluator { } } + if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + } + } + const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error } const story_outputs = new StoryOutputs() const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) - const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) return { display_path, @@ -76,7 +86,7 @@ export default class StoryEvaluator { } } - async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string): Promise { + async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { if (dry_run) { @@ -85,6 +95,9 @@ export default class StoryEvaluator { } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) + } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) } else { try { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index fdde53791..d2e43f360 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -38,19 +38,25 @@ export default class TestRunner { this._result_logger = result_logger } - async run (story_path: string, version?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { + async run (story_path: string, version?: string, distribution?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { let failed = false const story_files = this.story_files(story_path) const results: StoryEvaluations = { evaluations: [] } if (!dry_run) { - const info = await this._http_client.wait_until_available() - console.log(`OpenSearch ${ansi.green(info.version.number)}\n`) - version = info.version.number + if (distribution === 'aoss') { + // TODO: Fetch OpenSearch version when Amazon Serverless OpenSearch supports multiple. + version = '2.1' + } else { + const info = await this._http_client.wait_until_available() + version = info.version.number + } + + console.log(`OpenSearch ${ansi.green(version)}\n`) } for (const story_file of story_files) { - const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, dry_run) + const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run) results.evaluations.push(evaluation) this._result_logger.log(evaluation) if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 134888d7f..fbd775310 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -17,6 +17,7 @@ import { AWS_SERVICE_OPTION, AWS_SESSION_TOKEN_OPTION, get_opensearch_opts_from_cli, + OPENSEARCH_DISTRIBUTION_OPTION, OPENSEARCH_INSECURE_OPTION, OPENSEARCH_PASSWORD_OPTION, OPENSEARCH_URL_OPTION, @@ -48,6 +49,7 @@ const command = new Command() .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) .addOption(new Option('--opensearch-version ', 'target OpenSearch schema version').default(undefined)) .addOption(OPENSEARCH_URL_OPTION) + .addOption(OPENSEARCH_DISTRIBUTION_OPTION) .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) .addOption(OPENSEARCH_INSECURE_OPTION) @@ -63,7 +65,7 @@ const command = new Command() const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) -const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) +const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, opts.opensearchDistribution, new Logger(LogLevel.error)) const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) @@ -73,7 +75,7 @@ const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapt const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger) -runner.run(opts.testsPath, spec.api_version(), opts.dryRun) +runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts.dryRun) .then( ({ results, failed }) => { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e418ff762..83d307b44 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -50,6 +50,14 @@ export type Payload = {} | any[] | string | number | boolean; * via the `definition` "Version". */ export type Version = string; +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export type Distributions = string[]; /** * Number of times to retry on error. * @@ -94,6 +102,7 @@ export interface Story { epilogues?: SupplementalChapter[]; chapters: Chapter[]; version?: Version; + distributions?: Distributions; } /** * This interface was referenced by `Story`'s JSON-Schema @@ -112,6 +121,7 @@ export interface ChapterRequest { request?: Request; output?: Output; version?: Version; + distributions?: Distributions; retry?: Retry; } /** @@ -131,7 +141,7 @@ export interface Request { * The values are paths to the values in the response. * The values should be in the form: * - `payload.` for the payload - * - `headers.` for the headers + * - `headers.` for the headers. * * * This interface was referenced by `Story`'s JSON-Schema @@ -157,6 +167,9 @@ export interface ExpectedResponse { * via the `definition` "Warnings". */ export interface Warnings { + /** + * Enable/disable warnings about multiple paths being tested in the same story. + */ 'multiple-paths-detected'?: boolean; } /** diff --git a/tools/tests/merger/OpenApiVersionExtractor.test.ts b/tools/tests/merger/OpenApiVersionExtractor.test.ts index 61bc42622..0b62493ae 100644 --- a/tools/tests/merger/OpenApiVersionExtractor.test.ts +++ b/tools/tests/merger/OpenApiVersionExtractor.test.ts @@ -17,7 +17,7 @@ describe('extract() from a merged API spec', () => { const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete') describe('1.3', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3') + const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3', 'ignore') describe('write_to', () => { var temp: tmp.DirResult @@ -49,7 +49,7 @@ describe('extract() from a merged API spec', () => { }) describe('2.0', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0', 'ignore') test('has matching responses', () => { const spec = extractor.extract() @@ -81,7 +81,7 @@ describe('extract() from a merged API spec', () => { }) describe('2.1', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'ignore') test('has matching responses', () => { const spec = extractor.extract() diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index 7e6c64641..57d75f429 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -13,7 +13,7 @@ import MergedOpenApiSpec from "tester/MergedOpenApiSpec" describe('merged API spec', () => { describe('defaults', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, undefined, new Logger()) test('has an api version', () => { expect(spec.api_version()).toEqual('1.2.3') @@ -30,7 +30,7 @@ describe('merged API spec', () => { test('has all responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1' + '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', 'distributed-aos', 'distributed-all' ]) }) @@ -65,31 +65,61 @@ describe('merged API spec', () => { }) describe('1.3', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-aos', 'distributed-all' + ]) + }) + }) + + describe('another', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'another', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', 'distributed-all' ]) }) }) describe('2.0', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', undefined, new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-aos', 'distributed-all' + ]) + }) + }) + + describe('2.0 aos', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'aos', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-aos' + ]) + }) + }) + + describe('2.0 another', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'another', new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-all' ]) }) }) describe('2.1', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-aos', 'distributed-all' ]) }) }) diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml new file mode 100644 index 000000000..6c59b7792 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml + +result: SKIPPED +description: This story should be skipped because of distributions. +message: Skipped because distribution opensearch.org is not one of some, another. diff --git a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml index 68337dc67..972cb6af9 100644 --- a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml +++ b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml @@ -28,6 +28,16 @@ paths: $ref: '#/components/responses/info@500' '503': $ref: '#/components/responses/info@503' + distributed-aos: + $ref: '#/components/responses/info@distributed-aos' + x-distributions: + - aos + distributed-all: + $ref: '#/components/responses/info@distributed-all' + x-distributions: + - another + - distribution + - opensearch.org components: responses: info@200: @@ -76,6 +86,10 @@ components: info@added-2.1: description: Added in 2.1 via attribute in response body. x-version-added: '2.1' + info@distributed-aos: + description: Distributed only in AOS. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@500: content: application/json: diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml new file mode 100644 index 000000000..8efebbb2f --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml @@ -0,0 +1,9 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions. +distributions: + - another + - some +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 9e4cf2d46..843128c25 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.15.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.15.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) } diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index 20d22136c..f5be5df1e 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -63,6 +63,12 @@ test('skipped/semver', async () => { expect(actual).toEqual(expected) }) +test('skipped/distributions', async () => { + const actual = await load_actual_evaluation(story_evaluator, 'skipped/distributions') + const expected = load_expected_evaluation('skipped/distributions') + expect(actual).toEqual(expected) +}) + test('with an unexpected error', async () => { chapter_evaluator.evaluate = jest.fn().mockImplementation(() => { throw new Error('This was unexpected.');