Skip to content

Commit d1d063f

Browse files
thesayynthesayyn
authored
feat: introduce zstd toolchain (#831)
1 parent 977f27f commit d1d063f

9 files changed

+276
-1
lines changed

‎MODULE.bazel

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ bazel_lib_toolchains.jq()
2121
bazel_lib_toolchains.yq()
2222
bazel_lib_toolchains.coreutils()
2323
bazel_lib_toolchains.tar()
24+
bazel_lib_toolchains.zstd()
2425
bazel_lib_toolchains.expand_template()
2526
bazel_lib_toolchains.bats()
26-
use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains")
27+
use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains", "zstd_toolchains")
2728

2829
register_toolchains(
2930
"@copy_directory_toolchains//:all",

‎docs/repositories.md

+19
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/BUILD.bazel

+5
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ toolchain_type(
6464
name = "tar_toolchain_type",
6565
)
6666

67+
toolchain_type(
68+
name = "zstd_toolchain_type",
69+
)
70+
6771
toolchain_type(
6872
name = "bats_toolchain_type",
6973
)
@@ -254,6 +258,7 @@ bzl_library(
254258
"//lib/private:source_toolchains_repo",
255259
"//lib/private:tar_toolchain",
256260
"//lib/private:yq_toolchain",
261+
"//lib/private:zstd_toolchain",
257262
],
258263
)
259264

‎lib/extensions.bzl

+12
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ load(
1414
"DEFAULT_TAR_REPOSITORY",
1515
"DEFAULT_YQ_REPOSITORY",
1616
"DEFAULT_YQ_VERSION",
17+
"DEFAULT_ZSTD_REPOSITORY",
1718
"register_bats_toolchains",
1819
"register_copy_directory_toolchains",
1920
"register_copy_to_directory_toolchains",
@@ -22,6 +23,7 @@ load(
2223
"register_jq_toolchains",
2324
"register_tar_toolchains",
2425
"register_yq_toolchains",
26+
"register_zstd_toolchains",
2527
)
2628
load("//lib/private:extension_utils.bzl", "extension_utils")
2729
load("//lib/private:host_repo.bzl", "host_repo")
@@ -89,6 +91,15 @@ def _toolchains_extension_impl(mctx):
8991
get_version_fn = lambda attr: None,
9092
)
9193

94+
extension_utils.toolchain_repos_bfs(
95+
mctx = mctx,
96+
get_tag_fn = lambda tags: tags.zstd,
97+
toolchain_name = "zstd",
98+
default_repository = DEFAULT_ZSTD_REPOSITORY,
99+
toolchain_repos_fn = lambda name, version: register_zstd_toolchains(name = name, register = False),
100+
get_version_fn = lambda attr: None,
101+
)
102+
92103
extension_utils.toolchain_repos_bfs(
93104
mctx = mctx,
94105
get_tag_fn = lambda tags: tags.expand_template,
@@ -115,6 +126,7 @@ toolchains = module_extension(
115126
"yq": tag_class(attrs = {"name": attr.string(default = DEFAULT_YQ_REPOSITORY), "version": attr.string(default = DEFAULT_YQ_VERSION)}),
116127
"coreutils": tag_class(attrs = {"name": attr.string(default = DEFAULT_COREUTILS_REPOSITORY), "version": attr.string(default = DEFAULT_COREUTILS_VERSION)}),
117128
"tar": tag_class(attrs = {"name": attr.string(default = DEFAULT_TAR_REPOSITORY)}),
129+
"zstd": tag_class(attrs = {"name": attr.string(default = DEFAULT_ZSTD_REPOSITORY)}),
118130
"expand_template": tag_class(attrs = {"name": attr.string(default = DEFAULT_EXPAND_TEMPLATE_REPOSITORY)}),
119131
"bats": tag_class(attrs = {
120132
"name": attr.string(default = DEFAULT_BATS_REPOSITORY),

‎lib/private/BUILD.bazel

+6
Original file line numberDiff line numberDiff line change
@@ -356,3 +356,9 @@ bzl_library(
356356
srcs = ["strings.bzl"],
357357
visibility = ["//lib:__subpackages__"],
358358
)
359+
360+
bzl_library(
361+
name = "zstd_toolchain",
362+
srcs = ["zstd_toolchain.bzl"],
363+
visibility = ["//lib:__subpackages__"],
364+
)

‎lib/private/zstd_toolchain.bzl

+177
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
"Provide access to ZSTD"
2+
3+
ZSTD_PLATFORMS = {
4+
"darwin_amd64": struct(
5+
compatible_with = [
6+
"@platforms//os:osx",
7+
"@platforms//cpu:x86_64",
8+
],
9+
),
10+
"darwin_arm64": struct(
11+
compatible_with = [
12+
"@platforms//os:osx",
13+
"@platforms//cpu:aarch64",
14+
],
15+
),
16+
"linux_amd64": struct(
17+
compatible_with = [
18+
"@platforms//os:linux",
19+
"@platforms//cpu:x86_64",
20+
],
21+
),
22+
"linux_arm64": struct(
23+
compatible_with = [
24+
"@platforms//os:linux",
25+
"@platforms//cpu:aarch64",
26+
],
27+
),
28+
}
29+
30+
ZSTD_PREBUILT = {
31+
"darwin_amd64": (
32+
"https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_darwin_amd64",
33+
"e4d517212005cf26f8b8d657455d1380318b071cb52a3ffd9dfbdf4c2ba71a13",
34+
),
35+
"darwin_arm64": (
36+
"https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_darwin_arm64",
37+
"6e210eeae08fb6ba38c3ac2d1857075c28113aef68296f7e396f1180f7e894b9",
38+
),
39+
"linux_amd64": (
40+
"https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_linux_amd64",
41+
"0f0bd1193509a598629d7fa745c4b0b6d5fa6719e0c94c01ef0f20e466d801a7",
42+
),
43+
"linux_arm64": (
44+
"https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_linux_arm64",
45+
"82aacf8f1c67ff3c94e04afb0721a848bbba70fbf8249ee4bc4c9085afb84548",
46+
),
47+
}
48+
49+
def _zstd_binary_repo(rctx):
50+
(url, sha256) = ZSTD_PREBUILT[rctx.attr.platform]
51+
rctx.download(
52+
url = url,
53+
output = "zstd",
54+
executable = True,
55+
sha256 = sha256,
56+
)
57+
binary = "zstd"
58+
59+
rctx.file("BUILD.bazel", """\
60+
# @generated by @aspect_bazel_lib//lib/private:zstd_toolchain.bzl
61+
62+
load("@aspect_bazel_lib//lib/private:zstd_toolchain.bzl", "zstd_toolchain")
63+
64+
package(default_visibility = ["//visibility:public"])
65+
66+
zstd_toolchain(name = "zstd_toolchain", binary = "{}")
67+
""".format(binary))
68+
69+
zstd_binary_repo = repository_rule(
70+
implementation = _zstd_binary_repo,
71+
attrs = {
72+
"platform": attr.string(mandatory = True, values = ZSTD_PLATFORMS.keys()),
73+
},
74+
)
75+
76+
ZstdInfo = provider(
77+
doc = "Provide info for executing zstd",
78+
fields = {
79+
"binary": "zstd executable",
80+
},
81+
)
82+
83+
def _zstd_toolchain_impl(ctx):
84+
binary = ctx.executable.binary
85+
86+
# Make the $(ZSTD_BIN) variable available in places like genrules.
87+
# See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables
88+
template_variables = platform_common.TemplateVariableInfo({
89+
"ZSTD_BIN": binary.path,
90+
})
91+
92+
default_info = DefaultInfo(
93+
files = depset(ctx.files.binary + ctx.files.files),
94+
)
95+
zstdinfo = ZstdInfo(
96+
binary = binary,
97+
)
98+
99+
# Export all the providers inside our ToolchainInfo
100+
# so the resolved_toolchain rule can grab and re-export them.
101+
toolchain_info = platform_common.ToolchainInfo(
102+
zstdinfo = zstdinfo,
103+
template_variables = template_variables,
104+
default = default_info,
105+
)
106+
107+
return [toolchain_info, template_variables, default_info]
108+
109+
zstd_toolchain = rule(
110+
implementation = _zstd_toolchain_impl,
111+
attrs = {
112+
"binary": attr.label(
113+
doc = "a command to find on the system path",
114+
allow_files = True,
115+
executable = True,
116+
cfg = "exec",
117+
),
118+
"files": attr.label_list(allow_files = True),
119+
},
120+
)
121+
122+
def _zstd_toolchains_repo_impl(rctx):
123+
# Expose a concrete toolchain which is the result of Bazel resolving the toolchain
124+
# for the execution or zstdget platform.
125+
# Workaround for https://github.com/bazelbuild/bazel/issues/14009
126+
starlark_content = """\
127+
# @generated by @aspect_bazel_lib//lib/private:zstd_toolchain.bzl
128+
129+
# Forward all the providers
130+
def _resolved_toolchain_impl(ctx):
131+
toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:zstd_toolchain_type"]
132+
return [
133+
toolchain_info,
134+
toolchain_info.default,
135+
toolchain_info.zstdinfo,
136+
toolchain_info.template_variables,
137+
]
138+
139+
# Copied from java_toolchain_alias
140+
# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
141+
resolved_toolchain = rule(
142+
implementation = _resolved_toolchain_impl,
143+
toolchains = ["@aspect_bazel_lib//lib:zstd_toolchain_type"],
144+
incompatible_use_toolchain_transition = True,
145+
)
146+
"""
147+
rctx.file("defs.bzl", starlark_content)
148+
149+
build_content = """# @generated by @aspect_bazel_lib//lib/private:zstd_toolchain.bzl
150+
load(":defs.bzl", "resolved_toolchain")
151+
load("@local_config_platform//:constraints.bzl", "HOST_CONSTRAINTS")
152+
153+
resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])"""
154+
155+
for [platform, meta] in ZSTD_PLATFORMS.items():
156+
build_content += """
157+
toolchain(
158+
name = "{platform}_toolchain",
159+
exec_compatible_with = {compatible_with},
160+
toolchain = "@{user_repository_name}_{platform}//:zstd_toolchain",
161+
toolchain_type = "@aspect_bazel_lib//lib:zstd_toolchain_type",
162+
)
163+
""".format(
164+
platform = platform,
165+
user_repository_name = rctx.attr.user_repository_name,
166+
compatible_with = meta.compatible_with,
167+
)
168+
169+
rctx.file("BUILD.bazel", build_content)
170+
171+
zstd_toolchains_repo = repository_rule(
172+
_zstd_toolchains_repo_impl,
173+
doc = """Creates a repository that exposes a zstd_toolchain_type zstdget.""",
174+
attrs = {
175+
"user_repository_name": attr.string(doc = "Base name for toolchains repository"),
176+
},
177+
)

‎lib/repositories.bzl

+25
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_host_alias_repo", "jq
1010
load("//lib/private:source_toolchains_repo.bzl", "source_toolchains_repo")
1111
load("//lib/private:tar_toolchain.bzl", "BSDTAR_PLATFORMS", "bsdtar_binary_repo", "tar_toolchains_repo")
1212
load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_host_alias_repo", "yq_platform_repo", "yq_toolchains_repo", _DEFAULT_YQ_VERSION = "DEFAULT_YQ_VERSION")
13+
load("//lib/private:zstd_toolchain.bzl", "ZSTD_PLATFORMS", "zstd_binary_repo", "zstd_toolchains_repo")
1314
load("//tools:version.bzl", "IS_PRERELEASE")
1415

1516
# buildifier: disable=unnamed-macro
@@ -104,6 +105,29 @@ def register_tar_toolchains(name = DEFAULT_TAR_REPOSITORY, register = True):
104105
user_repository_name = name,
105106
)
106107

108+
DEFAULT_ZSTD_REPOSITORY = "zstd"
109+
110+
def register_zstd_toolchains(name = DEFAULT_ZSTD_REPOSITORY, register = True):
111+
"""Registers zstd toolchain and repositories
112+
113+
Args:
114+
name: override the prefix for the generated toolchain repositories
115+
register: whether to call through to native.register_toolchains.
116+
Should be True for WORKSPACE users, but false when used under bzlmod extension
117+
"""
118+
for [platform, _] in ZSTD_PLATFORMS.items():
119+
zstd_binary_repo(
120+
name = "%s_%s" % (name, platform),
121+
platform = platform,
122+
)
123+
if register:
124+
native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform))
125+
126+
zstd_toolchains_repo(
127+
name = "%s_toolchains" % name,
128+
user_repository_name = name,
129+
)
130+
107131
DEFAULT_BATS_REPOSITORY = "bats"
108132

109133
DEFAULT_BATS_CORE_VERSION = "v1.10.0"
@@ -324,4 +348,5 @@ def aspect_bazel_lib_register_toolchains():
324348
register_jq_toolchains()
325349
register_yq_toolchains()
326350
register_tar_toolchains()
351+
register_zstd_toolchains()
327352
register_bats_toolchains()

‎lib/tests/zstd/BUILD.bazel

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains")
2+
3+
# Case 1: Can decompress gzip archive
4+
genrule(
5+
name = "tar",
6+
srcs = [
7+
"srcfile",
8+
],
9+
outs = ["1.tar.gz"],
10+
cmd = "$(BSDTAR_BIN) --create --gzip --dereference --file $@ -s '#$(BINDIR)##' $(execpath srcfile)",
11+
toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"],
12+
)
13+
14+
genrule(
15+
name = "decompress_tar",
16+
srcs = [
17+
":tar",
18+
],
19+
outs = ["1.tar"],
20+
cmd = "$(ZSTD_BIN) -f --decompress $(execpath :tar) --stdout > $@",
21+
toolchains = ["@zstd_toolchains//:resolved_toolchain"],
22+
)
23+
24+
assert_archive_contains(
25+
name = "test_decompressed",
26+
archive = "1.tar",
27+
expected = [
28+
"lib/tests/zstd/srcfile",
29+
],
30+
)

‎lib/tests/zstd/srcfile

Whitespace-only changes.

0 commit comments

Comments
 (0)