initial release tooling for buck2 builds (#297)

Some light docs and this kinda seems to work so fire in the hole.

Author: nathanaelhuffman

.gitignore

@@ -34,6 +34,9 @@ vivado*.log
 /clockinfo.txt
 **/*clockInfo.txt
 *.txt
+*.zip
+*.fst
+*.vcd
 # Yep, even ignore Vivado core dump logs!
 *.log 
 

README.md

@@ -109,6 +109,22 @@ and use the shell to execute each line:
 
 `buck2 bxl //tools/vunit-sims.bxl:vunit_sim_gen | while IFS= read -r line; do eval "$line" ; done`
 
+## Release tooling
+There's a rudimentary FPGA releaser tool that can be run as follows:
+`buck2 run //tools/fpga_releaser:cli -- --fpga <fpga-name> --hubris <path to root of hubris checkout>`
+This tool will fetch the latest main build of your chosen FPGA, download the build archive,
+collect the relevant files, create a GitHub releaes, copy the hubris-impacting files into the 
+appropriate location in hubris (for you to manually commit), write out the hubris README.md
+for these files and print some timing and utilization statistics.
+`--skip-gh` will skip doing the github release (mostly for testing)
+`--token` you can pass your github token in, your env will be checked for `GITHUB_TOKEN` if this is
+empty.
+
+Config information is stored in `tools/fpga_releaser/config.toml` which controls the fpga name
+to build image mapping, toolchain info, and hubris subdirectory information for each build.
+
+Currently, only buck2-based build archives are able to be processed, cobble-based stuff is
+not implemented.
 
 ## multitool
 multitool is a collection of quality of live utilities built in-tree for regular use, but whose

tools/fpga_releaser/BUCK

@@ -0,0 +1,13 @@
+python_binary(
+    name = 'cli',
+    main = 'cli.py',
+    deps = [':fpga_releaser_lib'],
+    visibility = ["PUBLIC"],
+)
+
+python_library(
+    name = 'fpga_releaser_lib',
+    srcs = glob(["*.py"]),
+    base_module = "fpga_releaser",
+    resources = glob(["*.toml"])
+)

tools/fpga_releaser/__init__.py

@@ -0,0 +1,4 @@
+import importlib.resources as resources
+
+files = resources.files(__name__)
+config_toml = files / "config.toml"

tools/fpga_releaser/archive_parser.py

@@ -0,0 +1,31 @@
+# We're going to be handed a zip archive of the buck2 build
+# It contains a directory with some hash name
+# For a yosys build we're going to need the following things:
+# the bz2 file
+# the nextpnr log at that same directory called nextpnr.log
+# all the stuff in the _maps_ directory at that same level
+
+def get_relevant_files_from_buck_zip(zip):
+    zip_names = []
+    for item in zip.infolist():
+        if item.filename.endswith(".bz2"):
+            zip_names.append(item.filename)
+        if "/maps/" in item.filename and (item.filename.endswith(".json") or item.filename.endswith(".html")):
+            zip_names.append(item.filename)
+        if item.filename.endswith("nextpnr.log"):
+           zip_names.append(item.filename)
+        # Vivado stuff
+        if item.filename.endswith("route_timing.rpt"):
+            zip_names.append(item.filename)
+        if item.filename.endswith("place_optimize_utilization.rpt"):
+            zip_names.append(item.filename)
+        if item.filename.endswith("synthesize.log"):
+            zip_names.append(item.filename)
+        if item.filename.endswith("route.log"):
+            zip_names.append(item.filename)
+        if item.filename.endswith("place.log"):
+            zip_names.append(item.filename)
+        if item.filename.endswith("optimize.log"):
+            zip_names.append(item.filename)
+
+    return zip_names

tools/fpga_releaser/cli.py

@@ -0,0 +1,110 @@
+import sys
+import os
+import io
+import argparse
+import zipfile
+
+from pathlib import Path
+import arrow
+import requests
+from ghapi.all import GhApi
+from fastcore.xtras import obj2dict
+
+import fpga_releaser.project as project
+import fpga_releaser.gh_releaser as gh_releaser
+
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--token", default=None, help="Pass in your GitHub personal access token (will also use from GH_ACCESS_TOKEN env variable")
+parser.add_argument("--branch", default="main", help="Quartz branch to use for the artifact")
+parser.add_argument("--fpga", help="Name of FPGA project to release")
+parser.add_argument("--hubris", default=None, help="Path to hubris git checkout as target for copying files. Will skip if None")
+parser.add_argument("--skip-gh", default=False, action="store_true", help="Skip doing GH release. Note that doing this still generates release metadata that just will be wrong")
+parser.add_argument("--zip", default=None, help="Path to zip file to use instead of downloading from GitHub")
+
+hubris_ignore = [".html", ".log", ".rpt"]
+
+def main():
+    """
+    Main function to process the command line arguments and run the release process.
+    """
+    args = parser.parse_args()
+    if args.fpga is None:
+        print("Please specify an FPGA project with --fpga")
+        sys.exit(1)
+
+    if args.token is not None:
+          token = args.token
+    else:
+        token = os.getenv("GITHUB_TOKEN", None)
+    if token is None:
+        print("the --token option or an env_variable GITHUB_TOKEN is required to use this tool")
+        sys.exit(1)
+
+    repo = "quartz"
+    api = GhApi(owner='oxidecomputer', repo=repo, token=token)
+    
+    project_info = project.get_config(args.fpga)
+    print(f"Processing {args.fpga} with builder {project_info.builder}")
+
+    # Get build archive
+    zip_file = process_gh_build(args, api, project_info.job_name)
+    project_info.add_archive(zip_file)
+    
+    # Do build reports
+    project_info.report_timing()
+    project_info.report_utilization()
+
+    # extract files for GH release
+    # do GH release
+    if args.skip_gh:
+        print("Skipping GH release per command line request")
+    gh_releaser.do_gh_release(api, project_info, args.skip_gh)
+
+    # put files in hubris location
+    hubris_path = Path(args.hubris) / Path(project_info.hubris_path)
+    print(f"Materializing files at {hubris_path} for hubris commit")
+    project_info.materialize_relevant_files(hubris_path, exclusions=hubris_ignore)
+
+    
+
+def process_gh_build(args, api, name: str):
+
+    # Get build archive zip file
+    # Get the latest artifact from the repo since we didn't specify a zip file
+    if args.zip is None:
+        print("Fetching latest from GitHub")
+        # Download the artifact from github
+        artifact_inf = get_latest_artifact_info(api, name, branch=args.branch)
+        zip_file = download_artifact(api, artifact_inf)
+    else:
+        print("Using local zip file")
+        # Use the zip file from the command line
+        zip_file = zipfile.ZipFile(args.zip)
+
+    return zip_file
+
+
+def get_latest_artifact_info(api, fpga_name: str, branch: str = "main") -> dict:
+    """
+    Get the latest artifact from the specified branch.
+    """
+    artifacts = api.actions.list_artifacts_for_repo(name=fpga_name)
+    artifacts = obj2dict(artifacts)
+    artifacts = list(filter(lambda x: x["workflow_run"]["head_branch"] == branch, artifacts["artifacts"]))
+    if len(artifacts) == 0:
+        print(f"No artifacts found for {fpga_name} on {branch}")
+        return None
+    artifacts = sorted(artifacts, key=lambda x: arrow.get(x["created_at"]), reverse=True)
+    return artifacts[0]
+
+def download_artifact(api: GhApi, artifact_inf: dict):
+    print(f"Downloading artifact {artifact_inf['name']} from GH: {artifact_inf['workflow_run']['head_branch']}")
+    r = requests.get(artifact_inf["archive_download_url"], auth=("oxidecomputer", os.getenv("GITHUB_TOKEN", None)))
+    return zipfile.ZipFile(io.BytesIO(r.content))
+
+if __name__ == '__main__':
+    main()
+    
+    

tools/fpga_releaser/config.toml

@@ -0,0 +1,17 @@
+[cosmo-hp]
+job_name = "cosmo-hp-image"
+hubris_path = "drv/ice40-loader/cosmo-hp"
+builder = "buck2"
+toolchain = "yosys"
+
+[cosmo-seq]
+job_name = "cosmo-seq-image"
+hubris_path = "drv/spartan7-loader/cosmo-seq"
+builder = "buck2"
+toolchain = "vivado"
+
+[grapefruit]
+job_name = "gfruit-image"
+hubris_path = "drv/spartan7-loader/grapefruit"
+builder = "buck2"
+toolchain = "vivado"

tools/fpga_releaser/gh_releaser.py

@@ -0,0 +1,37 @@
+# create a release with image-name-TIMESTAMP?
+# push up logs, binary, docs and readme
+from ghapi.all import GhApi
+from pathlib import Path
+import tempfile
+
+
+def do_gh_release(api: GhApi, info, skip_mods=False):
+
+    name = f"{info.name}-{info.timestamp}"
+    body = f"FPGA release for {info.name} made on {info.timestamp} UTC"
+    # TODO: should we add something about timing failure here?
+    # TODO: info about this being a locally generated file?
+    tag_name = name
+    if info.gh_build_sha == "":
+        # Built locally? Or not from a GH release
+        sha = api.git.get_ref("heads/main")["object"].get("sha")
+    else:
+        sha = info.gh_build_sha
+
+    info.add_build_sha(sha)  # in case local build
+
+    # Create a tag
+    if not skip_mods:
+        api.git.create_tag(tag_name, body, sha, "commit")
+
+    # Stick the release files somewhere temporary
+    with tempfile.TemporaryDirectory() as temp_dir:
+        files = info.materialize_relevant_files(temp_dir)
+        full_files = [Path(temp_dir) / Path(x) for x in files]
+        # Create a Release
+        if not skip_mods:
+            rel = api.create_release(tag_name,sha,name,body, files=full_files)
+            info.add_release_url(rel.get('url'))
+        else:
+            info.add_release_url('None: gh release was skipped')
+