Skip to content

Commit 33fec3c

Browse files
peterzhuamazonpeterzhuamazon
authored
Generate new input manifest with previous latest version (#4523)
Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
1 parent e55396b commit 33fec3c

7 files changed

+366
-155
lines changed

‎src/manifests_workflow/README.md

+5-2
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@
44

55
This workflow reacts to version increments in OpenSearch and its components by extracting Gradle properties from project branches. Currently OpenSearch `main`, and `x.y` branches are checked out one-by-one, published to local maven, and their versions extracted using `./gradlew properties`. When a new version is found, a new input manifest is added to [manifests](../../manifests), and [a pull request is opened](../../.github/workflows/manifests.yml) (e.g. [opensearch-build#491](https://github.com/opensearch-project/opensearch-build/pull/491)).
66

7-
Show information about existing manifests.
7+
### Show information about existing manifests
88

99
```bash
1010
./manifests.sh list
1111
```
1212

13-
Check for updates and create any new manifests.
13+
### Check for updates and create any new manifests
14+
* If there are no existing manifests in either the `manifests` or `legacy-manifests` folder, use the manifest templates located in `manifests/templates` as a base to generate new manifests.
15+
* If the requested new manifest version (e.g., 0.10.1) is lower than the lowest existing manifest version (e.g., 1.0.0), again, use the manifest templates located in `manifests/templates` as a base to generate new manifests.
16+
* In other scenarios, always use the latest existing manifest version that is lower than the requested new manifest version as the base to generate new manifests (e.g., Use 2.12.0 to generate 2.12.1, even if 2.13.0 exists).
1417

1518
```bash
1619
./manifests.sh update

‎src/manifests_workflow/input_manifests.py

+44-60
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@
1313
from typing import Dict, List, Type, Union
1414

1515
import ruamel.yaml
16+
from packaging.version import Version
17+
from packaging.version import parse as version_parse
1618

17-
from manifests.component_manifest import ComponentFromSource
18-
from manifests.input_manifest import InputComponents, InputManifest
19+
from manifests.input_manifest import InputManifest
1920
from manifests.manifests import Manifests
2021
from manifests_workflow.component_opensearch import ComponentOpenSearch
2122
from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin
@@ -84,92 +85,75 @@ def update(
8485
component_klass: Type[ComponentOpenSearch],
8586
keep: bool = False,
8687
) -> None:
87-
known_versions = self.versions
88+
known_versions = sorted(self.versions, key=version_parse)
89+
branch_versions: Dict = {}
8890
logging.info(f"Known versions: {known_versions}")
89-
main_versions: Dict = {}
9091
with TemporaryDirectory(keep=keep, chdir=True) as work_dir:
9192
logging.info(f"Checking out components into {work_dir.name}")
9293

9394
# check out and build #main, 1.x, etc.
94-
branches = min_klass.branches()
95+
branches = sorted(min_klass.branches())
9596

9697
logging.info(f"Checking {self.name} {branches} branches")
9798
for branch in branches:
98-
c = min_klass.checkout(
99+
min_component_klass = min_klass.checkout(
99100
path=os.path.join(work_dir.name, self.name.replace(" ", ""), branch),
100101
branch=branch,
101102
)
102103

103-
version = c.version
104+
version = min_component_klass.version
104105
logging.info(f"{self.name}#{branch} is version {version}")
105-
if version not in main_versions.keys():
106-
main_versions[version] = [c]
107-
108-
if component_klass is not None:
109-
# components can increment their own version first without incrementing min
110-
manifest = self.latest
111-
logging.info(f"Examining components in the latest manifest of {manifest.build.name} ({manifest.build.version})")
112-
for component in manifest.components.values():
113-
if component.name == self.name:
114-
continue
115-
116-
if type(component) is ComponentFromSource:
117-
logging.info(f"Checking out {component.name}#main")
118-
component = component_klass.checkout(
119-
name=component.name,
120-
path=os.path.join(work_dir.name, component.name),
121-
opensearch_version=manifest.build.version,
122-
repo_url=component.repository,
123-
branch="main",
124-
)
125-
126-
component_version = component.version
127-
if component_version:
128-
release_version = ".".join(component_version.split(".")[:3])
129-
if release_version not in main_versions.keys():
130-
main_versions[release_version] = []
131-
main_versions[release_version].append(component)
132-
logging.info(f"{component.name}#main is version {release_version} (from {component_version})")
133-
134-
# summarize
135-
logging.info("Found versions on main:")
136-
for main_version in main_versions.keys():
137-
for component in main_versions[main_version]:
138-
logging.info(f" {component.name}={main_version}")
106+
if version not in branch_versions:
107+
branch_versions[version] = branch
139108

140109
# generate new manifests
141-
for release_version in sorted(main_versions.keys() - known_versions):
142-
self.write_manifest(release_version, main_versions[release_version])
143-
self.add_to_cron(release_version)
144-
self.add_to_versionincrement_workflow(release_version)
145-
146-
def create_manifest(self, version: str, components: List = []) -> InputManifest:
147-
templates_base_path = os.path.join(self.manifests_path(), "templates")
148-
template_version_folder = version.split(".")[0] + ".x"
149-
template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml")
150-
if not os.path.exists(template_full_path):
151-
template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml")
110+
new_versions = sorted(branch_versions.keys() - known_versions, key=version_parse)
111+
logging.info(f"New Versions: {new_versions}")
112+
for new_version_entry in new_versions:
113+
self.write_manifest(new_version_entry, branch_versions[new_version_entry], known_versions)
114+
self.add_to_cron(new_version_entry)
115+
self.add_to_versionincrement_workflow(new_version_entry)
116+
known_versions.append(new_version_entry)
117+
118+
def create_manifest(self, version: str, branch: str, known_versions: List[str]) -> InputManifest:
119+
# If : No known_versions manifests exist or new version smaller than the min(known_versions), create new manifests from the templates
120+
# (1.0.0-3.0.0 based on template 1.x-3.x, 4.0.0+ from default.x, previous behavior)
121+
# Else: Create new manifests based on the latest version before the new version
122+
# (2.12.1 from 2.12.0, 2.13.0 from 2.12.1, 3.0.0 from 2.13.0, 4.0.0 from 3.0.0, etc.)
123+
if not known_versions or Version(version) < Version(min(known_versions, key=version_parse)):
124+
logging.info("No previous version exist before {version}, create with templates")
125+
templates_base_path = os.path.join(self.manifests_path(), "templates")
126+
template_version_folder = version.split(".")[0] + ".x"
127+
template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml")
128+
if not os.path.exists(template_full_path):
129+
template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml")
130+
else:
131+
previous_versions = [v for v in known_versions if Version(v) < Version(version)]
132+
base_version = max(previous_versions, key=version_parse)
133+
logging.info(f"Base Version: {base_version} is the highest version before {version}")
134+
template_full_path = os.path.join(self.manifests_path(), base_version, f"{self.prefix}-{base_version}.yml")
135+
if not os.path.exists(template_full_path):
136+
template_full_path = os.path.join(self.legacy_manifests_path(), base_version, f"{self.prefix}-{base_version}.yml")
137+
138+
logging.info(f"Using {template_full_path} as the base manifest")
152139

153140
manifest = InputManifest.from_file(open(template_full_path))
154141

155142
manifest.build.version = version
156-
manifests_components = []
157143

158-
for component in components:
159-
logging.info(f" Adding {component.name}")
160-
manifests_components.append(component.to_dict())
144+
for component in manifest.components.select():
145+
component.ref = branch # type: ignore
161146

162-
manifest.components = InputComponents(manifests_components) # type: ignore
163147
return manifest
164148

165-
def write_manifest(self, version: str, components: List = []) -> None:
149+
def write_manifest(self, version: str, branch: str, known_versions: List[str]) -> None:
166150
logging.info(f"Creating new version: {version}")
167-
manifest = self.create_manifest(version, components)
151+
manifest = self.create_manifest(version, branch, known_versions)
168152
manifest_dir = os.path.join(self.manifests_path(), version)
169153
os.makedirs(manifest_dir, exist_ok=True)
170154
manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}.yml")
171155
manifest.to_file(manifest_path)
172-
logging.info(f"Wrote {manifest_path}")
156+
logging.info(f"Wrote {manifest_path} as the new manifest")
173157

174158
def add_to_cron(self, version: str) -> None:
175159
logging.info(f"Adding new version to cron: {version}")

‎tests/tests_manifests_workflow/data/opensearch-2.12.1000.yml

+185
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
---
2+
schema-version: '1.1'
3+
build:
4+
name: OpenSearch
5+
version: 2.12.1000
6+
ci:
7+
image:
8+
name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3
9+
args: -e JAVA_HOME=/opt/java/openjdk-21
10+
components:
11+
- name: OpenSearch
12+
repository: https://github.com/opensearch-project/OpenSearch.git
13+
ref: '2.12'
14+
- name: common-utils
15+
repository: https://github.com/opensearch-project/common-utils.git
16+
ref: '2.12'
17+
platforms:
18+
- linux
19+
- windows
20+
- name: job-scheduler
21+
repository: https://github.com/opensearch-project/job-scheduler.git
22+
ref: '2.12'
23+
platforms:
24+
- linux
25+
- windows
26+
- name: security
27+
repository: https://github.com/opensearch-project/security.git
28+
ref: '2.12'
29+
platforms:
30+
- linux
31+
- windows
32+
- name: k-NN
33+
repository: https://github.com/opensearch-project/k-NN.git
34+
ref: '2.12'
35+
platforms:
36+
- linux
37+
- windows
38+
- name: geospatial
39+
repository: https://github.com/opensearch-project/geospatial.git
40+
ref: '2.12'
41+
platforms:
42+
- linux
43+
- windows
44+
depends_on:
45+
- job-scheduler
46+
- name: cross-cluster-replication
47+
repository: https://github.com/opensearch-project/cross-cluster-replication.git
48+
ref: '2.12'
49+
platforms:
50+
- linux
51+
- windows
52+
depends_on:
53+
- common-utils
54+
- name: ml-commons
55+
repository: https://github.com/opensearch-project/ml-commons.git
56+
ref: '2.12'
57+
platforms:
58+
- linux
59+
- windows
60+
depends_on:
61+
- common-utils
62+
- name: neural-search
63+
repository: https://github.com/opensearch-project/neural-search.git
64+
ref: '2.12'
65+
platforms:
66+
- linux
67+
- windows
68+
depends_on:
69+
- ml-commons
70+
- k-NN
71+
- name: notifications-core
72+
repository: https://github.com/opensearch-project/notifications.git
73+
ref: '2.12'
74+
working_directory: notifications
75+
platforms:
76+
- linux
77+
- windows
78+
depends_on:
79+
- common-utils
80+
- name: notifications
81+
repository: https://github.com/opensearch-project/notifications.git
82+
ref: '2.12'
83+
working_directory: notifications
84+
platforms:
85+
- linux
86+
- windows
87+
depends_on:
88+
- common-utils
89+
- name: opensearch-observability
90+
repository: https://github.com/opensearch-project/observability.git
91+
ref: '2.12'
92+
platforms:
93+
- linux
94+
- windows
95+
depends_on:
96+
- common-utils
97+
- name: opensearch-reports
98+
repository: https://github.com/opensearch-project/reporting.git
99+
ref: '2.12'
100+
platforms:
101+
- linux
102+
- windows
103+
depends_on:
104+
- common-utils
105+
- job-scheduler
106+
- name: sql
107+
repository: https://github.com/opensearch-project/sql.git
108+
ref: '2.12'
109+
platforms:
110+
- linux
111+
- windows
112+
depends_on:
113+
- ml-commons
114+
- name: asynchronous-search
115+
repository: https://github.com/opensearch-project/asynchronous-search.git
116+
ref: '2.12'
117+
platforms:
118+
- linux
119+
- windows
120+
depends_on:
121+
- common-utils
122+
- name: anomaly-detection
123+
repository: https://github.com/opensearch-project/anomaly-detection.git
124+
ref: '2.12'
125+
platforms:
126+
- linux
127+
- windows
128+
depends_on:
129+
- common-utils
130+
- job-scheduler
131+
- name: alerting
132+
repository: https://github.com/opensearch-project/alerting.git
133+
ref: '2.12'
134+
platforms:
135+
- linux
136+
- windows
137+
depends_on:
138+
- common-utils
139+
- name: security-analytics
140+
repository: https://github.com/opensearch-project/security-analytics.git
141+
ref: '2.12'
142+
platforms:
143+
- linux
144+
- windows
145+
depends_on:
146+
- common-utils
147+
- name: index-management
148+
repository: https://github.com/opensearch-project/index-management.git
149+
ref: '2.12'
150+
platforms:
151+
- linux
152+
- windows
153+
depends_on:
154+
- common-utils
155+
- job-scheduler
156+
- name: performance-analyzer
157+
repository: https://github.com/opensearch-project/performance-analyzer.git
158+
ref: '2.12'
159+
platforms:
160+
- linux
161+
- name: custom-codecs
162+
repository: https://github.com/opensearch-project/custom-codecs.git
163+
ref: '2.12'
164+
platforms:
165+
- linux
166+
- windows
167+
- name: flow-framework
168+
repository: https://github.com/opensearch-project/flow-framework.git
169+
ref: '2.12'
170+
platforms:
171+
- linux
172+
- windows
173+
depends_on:
174+
- common-utils
175+
- name: skills
176+
repository: https://github.com/opensearch-project/skills.git
177+
ref: '2.12'
178+
platforms:
179+
- linux
180+
- windows
181+
depends_on:
182+
- job-scheduler
183+
- anomaly-detection
184+
- sql
185+
- ml-commons

0 commit comments

Comments
 (0)