diff --git a/.bazelrc b/.bazelrc index a690bef..deba416 100644 --- a/.bazelrc +++ b/.bazelrc @@ -6,10 +6,11 @@ common --enable_bzlmod # Build using platforms by default build --incompatible_enable_cc_toolchain_resolution -# Remote builds -build:remote --genrule_strategy=remote -build:remote --remote_download_minimal -build:remote --remote_executor=grpc://localhost:8980 +# Remote builds (nativelink) +build:remote --remote_instance_name=main +build:remote --remote_cache=grpc://127.0.0.1:50051 +build:remote --remote_executor=grpc://127.0.0.1:50051 +build:remote --remote_default_exec_properties=cpu_count=1 # Linux host (used for remote builds) build:linux --extra_execution_platforms=//platforms/host:linux diff --git a/.bazelversion b/.bazelversion index 66ce77b..9fe9ff9 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -7.0.0 +7.0.1 diff --git a/.github/bre/basic_cas.json b/.github/bre/basic_cas.json new file mode 100644 index 0000000..c26d815 --- /dev/null +++ b/.github/bre/basic_cas.json @@ -0,0 +1,161 @@ +{ + "stores": { + "AC_MAIN_STORE": { + "memory": { + "eviction_policy": { + // 100mb. + "max_bytes": 100000000, + } + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + // "fast" must be a "filesystem" store because the worker uses it to make + // hardlinks on disk to a directory where the jobs are running. + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + // 10gb. + "max_bytes": 10000000000, + } + } + }, + "slow": { + /// Discard data. + /// This example usage has the CAS and the Worker live in the same place, + /// so they share the same underlying CAS. Since workers require a fast_slow + /// store, we use the fast store as our primary data store, and the slow store + /// is just a noop, since there's no shared storage in this config. + "noop": {} + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority", + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": "priority", + } + } + } + }, + "workers": [{ + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50061", + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE", + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": ["16"], + }, + "memory_kb": { + "values": ["500000"], + }, + "network_kbps": { + "values": ["100000"], + }, + "cpu_arch": { + "values": ["x86_64"], + }, + "OSFamily": { + "values": [""] + }, + "container-image": { + "values": [""] + }, + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": { + // "query_cmd": "docker images --format {{.Repository}}:{{.Tag}}", + // } + } + } + }], + "servers": [{ + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE", + } + } + } + }, { + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + }, + "admin": {} + } + }], + "global": { + "max_open_files": 512 + } +} diff --git a/.github/workflows/remote.yml b/.github/workflows/remote.yml index d3de70e..44c16b3 100644 --- a/.github/workflows/remote.yml +++ b/.github/workflows/remote.yml @@ -15,6 +15,6 @@ jobs: - uses: actions/checkout@v2 - name: remote execution run: | - git clone https://github.com/bazelbuild/bazel-buildfarm - cd bazel-buildfarm && ./examples/bf-run start && cd .. + cargo install --git https://github.com/TraceMachina/nativelink --tag v0.2.0 + (nativelink .github/bre/basic_cas.json &) bazelisk build --config=remote --config=linux //examples/... diff --git a/MODULE.bazel b/MODULE.bazel index 3be74e0..6a06198 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -4,10 +4,12 @@ module( compatibility_level = 1, ) -arm_none_eabi = use_extension("@arm_none_eabi//:extensions.bzl", "arm_none_eabi") -arm_none_eabi.toolchain(version = "9.2.1") +bazel_dep(name = "platforms", version = "0.0.8") + +arm_none_eabi_gcc = use_extension("@arm_none_eabi//:extensions.bzl", "arm_none_eabi") +arm_none_eabi_gcc.toolchain(version = "9.2.1") use_repo( - arm_none_eabi, + arm_none_eabi_gcc, "arm_none_eabi_darwin_x86_64", "arm_none_eabi_linux_aarch64", "arm_none_eabi_linux_x86_64", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7b5252d..736217c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 3, - "moduleFileHash": "07168ca38367d53a6365a4db6723b342d096cf51d9250d67fa94a7a09009b31f", + "moduleFileHash": "34ee30e2562d5f7543514f26a6e41c227768a67f47808745f814f6724de2b8cc", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -33,8 +33,8 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 7, - "column": 30 + "line": 9, + "column": 34 }, "imports": { "arm_none_eabi_darwin_x86_64": "arm_none_eabi_darwin_x86_64", @@ -52,8 +52,8 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 8, - "column": 24 + "line": 10, + "column": 28 } } ], @@ -62,12 +62,41 @@ } ], "deps": { + "platforms": "platforms@0.0.8", "aspect_bazel_lib": "aspect_bazel_lib@2.0.0", "bazel_skylib": "bazel_skylib@1.5.0", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" } }, + "platforms@0.0.8": { + "name": "platforms", + "version": "0.0.8", + "key": "platforms@0.0.8", + "repoName": "platforms", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [], + "deps": { + "rules_license": "rules_license@0.0.7", + "bazel_tools": "bazel_tools@_", + "local_config_platform": "local_config_platform@_" + }, + "repoSpec": { + "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "name": "platforms", + "urls": [ + "https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz" + ], + "integrity": "sha256-gVBAZgU4ns7LbaB8vLUJ1WN6OrmiS8abEQFTE2fYnXQ=", + "strip_prefix": "", + "remote_patches": {}, + "remote_patch_strip": 0 + } + } + }, "aspect_bazel_lib@2.0.0": { "name": "aspect_bazel_lib", "version": "2.0.0", @@ -184,7 +213,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "io_bazel_stardoc": "stardoc@0.5.4", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -219,7 +248,7 @@ ], "extensionUsages": [], "deps": { - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -365,7 +394,7 @@ "rules_license": "rules_license@0.0.7", "rules_proto": "rules_proto@4.0.0", "rules_python": "rules_python@0.4.0", - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "com_google_protobuf": "protobuf@3.19.6", "zlib": "zlib@1.3", "build_bazel_apple_support": "apple_support@1.5.0", @@ -381,20 +410,19 @@ "toolchainsToRegister": [], "extensionUsages": [], "deps": { - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "bazel_tools": "bazel_tools@_" } }, - "platforms@0.0.7": { - "name": "platforms", + "rules_license@0.0.7": { + "name": "rules_license", "version": "0.0.7", - "key": "platforms@0.0.7", - "repoName": "platforms", + "key": "rules_license@0.0.7", + "repoName": "rules_license", "executionPlatformsToRegister": [], "toolchainsToRegister": [], "extensionUsages": [], "deps": { - "rules_license": "rules_license@0.0.7", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -402,11 +430,11 @@ "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "name": "platforms", + "name": "rules_license~0.0.7", "urls": [ - "https://github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz" + "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" ], - "integrity": "sha256-OlYcmee9vpFzqmU/1Xn+hJ8djWc5V4CrR3Cx84FDHVE=", + "integrity": "sha256-RTHezLkTY5ww5cdRKgVNXYdWmNrrddjPkPKEN1/nw2A=", "strip_prefix": "", "remote_patches": {}, "remote_patch_strip": 0 @@ -472,7 +500,7 @@ } ], "deps": { - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -571,7 +599,7 @@ } ], "deps": { - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "rules_cc": "rules_cc@0.0.9", "bazel_skylib": "bazel_skylib@1.5.0", "rules_proto": "rules_proto@4.0.0", @@ -594,33 +622,6 @@ } } }, - "rules_license@0.0.7": { - "name": "rules_license", - "version": "0.0.7", - "key": "rules_license@0.0.7", - "repoName": "rules_license", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_license~0.0.7", - "urls": [ - "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" - ], - "integrity": "sha256-RTHezLkTY5ww5cdRKgVNXYdWmNrrddjPkPKEN1/nw2A=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, "rules_proto@4.0.0": { "name": "rules_proto", "version": "4.0.0", @@ -754,7 +755,7 @@ "toolchainsToRegister": [], "extensionUsages": [], "deps": { - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "rules_cc": "rules_cc@0.0.9", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -808,7 +809,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", - "platforms": "platforms@0.0.7", + "platforms": "platforms@0.0.8", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -831,7 +832,7 @@ "moduleExtensions": { "//:extensions.bzl%arm_none_eabi": { "general": { - "bzlTransitiveDigest": "aatlmXDDJLdcw47Yap9VcGiNsKeZfMCZQljYq8cjMb4=", + "bzlTransitiveDigest": "w6FoxDH5UMqUd/+Aj661LOkG5eu1KFX2uSnJSNy3qvE=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -878,7 +879,19 @@ "url": "https://developer.arm.com/-/media/Files/downloads/gnu-rm/9-2019q4/gcc-arm-none-eabi-9-2019-q4-major-aarch64-linux.tar.bz2?revision=4583ce78-e7e7-459a-ad9f-bff8e94839f1&la=en&hash=550DB9C0184B7C70B6C020A5DCBB9D1E156264B7" } } - } + }, + "recordedRepoMappingEntries": [ + [ + "", + "arm_none_eabi", + "" + ], + [ + "", + "bazel_tools", + "bazel_tools" + ] + ] } }, "@@apple_support~1.5.0//crosstool:setup.bzl%apple_cc_configure_extension": { @@ -901,12 +914,13 @@ "name": "apple_support~1.5.0~apple_cc_configure_extension~local_config_apple_cc_toolchains" } } - } + }, + "recordedRepoMappingEntries": [] } }, "@@aspect_bazel_lib~2.0.0//lib:extensions.bzl%toolchains": { "general": { - "bzlTransitiveDigest": "Ke4Q/FAvwgVP2axnXO5hNDIudZ2yG2ztShHSIr3KYEQ=", + "bzlTransitiveDigest": "Q7YSENIWWVzKmOANBFx5jBHdcaDhZbxWCn8zl2mREm8=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -1309,12 +1323,29 @@ "version": "4.25.2" } } - } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib~2.0.0", + "aspect_bazel_lib", + "aspect_bazel_lib~2.0.0" + ], + [ + "aspect_bazel_lib~2.0.0", + "bazel_skylib", + "bazel_skylib~1.5.0" + ], + [ + "aspect_bazel_lib~2.0.0", + "bazel_tools", + "bazel_tools" + ] + ] } }, "@@bazel_tools//tools/cpp:cc_configure.bzl%cc_configure_extension": { "general": { - "bzlTransitiveDigest": "O9sf6ilKWU9Veed02jG9o2HM/xgV/UAyciuFBuxrFRY=", + "bzlTransitiveDigest": "mcsWHq3xORJexV5/4eCvNOLxFOQKV6eli3fkr+tEaqE=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -1332,7 +1363,14 @@ "name": "bazel_tools~cc_configure_extension~local_config_cc_toolchains" } } - } + }, + "recordedRepoMappingEntries": [ + [ + "bazel_tools", + "bazel_tools", + "bazel_tools" + ] + ] } }, "@@bazel_tools//tools/osx:xcode_configure.bzl%xcode_configure_extension": { @@ -1350,7 +1388,8 @@ "remote_xcode": "" } } - } + }, + "recordedRepoMappingEntries": [] } }, "@@bazel_tools//tools/sh:sh_configure.bzl%sh_configure_extension": { @@ -1366,12 +1405,13 @@ "name": "bazel_tools~sh_configure_extension~local_config_sh" } } - } + }, + "recordedRepoMappingEntries": [] } }, "@@rules_java~7.1.0//java:extensions.bzl%toolchains": { "general": { - "bzlTransitiveDigest": "iUIRqCK7tkhvcDJCAfPPqSd06IHG0a8HQD0xeQyVAqw=", + "bzlTransitiveDigest": "D02GmifxnV/IhYgspsJMDZ/aE8HxAjXgek5gi6FSto4=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -1906,7 +1946,19 @@ "build_file": "\nconfig_setting(\n name = \"prefix_version_setting\",\n values = {\"java_runtime_version\": \"remotejdk_21\"},\n visibility = [\"//visibility:private\"],\n)\nconfig_setting(\n name = \"version_setting\",\n values = {\"java_runtime_version\": \"21\"},\n visibility = [\"//visibility:private\"],\n)\nalias(\n name = \"version_or_prefix_version_setting\",\n actual = select({\n \":version_setting\": \":version_setting\",\n \"//conditions:default\": \":prefix_version_setting\",\n }),\n visibility = [\"//visibility:private\"],\n)\ntoolchain(\n name = \"toolchain\",\n target_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:runtime_toolchain_type\",\n toolchain = \"@remotejdk21_win//:jdk\",\n)\ntoolchain(\n name = \"bootstrap_runtime_toolchain\",\n # These constraints are not required for correctness, but prevent fetches of remote JDK for\n # different architectures. As every Java compilation toolchain depends on a bootstrap runtime in\n # the same configuration, this constraint will not result in toolchain resolution failures.\n exec_compatible_with = [\"@platforms//os:windows\", \"@platforms//cpu:x86_64\"],\n target_settings = [\":version_or_prefix_version_setting\"],\n toolchain_type = \"@bazel_tools//tools/jdk:bootstrap_runtime_toolchain_type\",\n toolchain = \"@remotejdk21_win//:jdk\",\n)\n" } } - } + }, + "recordedRepoMappingEntries": [ + [ + "rules_java~7.1.0", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_java~7.1.0", + "remote_java_tools", + "rules_java~7.1.0~toolchains~remote_java_tools" + ] + ] } } } diff --git a/README.md b/README.md index eb8c74a..6f30313 100644 --- a/README.md +++ b/README.md @@ -85,17 +85,24 @@ register_default_arm_none_eabi_toolchains() ## Bzlmod -Bazel mod support is experimental, add this to your module to use the compiler +Bazel mod support is experimental, add this to your module to use the compiler. +Eventually this module will be released to BCR, but for now it requires a `git_override`` ```python # MODULE.bazel -bazel_dep(name = "arm_none_eabi", version = "0.0.1") +git_override( + module_name = "arm_none_eabi", + remote = "https://github.com/hexdae/bazel-arm-none-eabi", + commit = "", +) + +bazel_dep(name = "arm_none_eabi", version = "1.0.0") -arm_none_eabi = use_extension("@arm_none_eabi//:extensions.bzl", "arm_none_eabi") -arm_none_eabi.toolchain(version = "9.2.1") +arm_none_eabi_gcc = use_extension("@arm_none_eabi//:extensions.bzl", "arm_none_eabi") +arm_none_eabi_gcc.toolchain(version = "9.2.1") use_repo( - arm_none_eabi, + arm_none_eabi_gcc, "arm_none_eabi_darwin_x86_64", "arm_none_eabi_linux_aarch64", "arm_none_eabi_linux_x86_64", diff --git a/examples/simple/BUILD b/examples/simple/BUILD index b880276..9cac8fe 100644 --- a/examples/simple/BUILD +++ b/examples/simple/BUILD @@ -1,7 +1,7 @@ # project/BUILD.bazel -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") cc_library( name = "arm_library", diff --git a/extensions.bzl b/extensions.bzl index 5187539..e6dc0fe 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -2,8 +2,9 @@ load("@arm_none_eabi//:deps.bzl", "arm_none_eabi_deps") def _arm_none_eabi_impl(ctx): for mod in ctx.modules: - for attr in mod.tags.toolchain: - arm_none_eabi_deps(attr.version) + if mod.name == "arm_none_eabi": + for attr in mod.tags.toolchain: + arm_none_eabi_deps(attr.version) _toolchain = tag_class(attrs = { "version": attr.string(),