bb_clientd.jsonnet

local os = std.extVar('OS');
local cacheDirectory = std.extVar('XDG_CACHE_HOME') + '/bb_clientd';

{
  // List of clusters to which bb_clientd is permitted to connect.
  clusters:: {
    'mycluster-prod.example.com': 'api.mycluster-prod.example.com',
    'mycluster-qa.example.com': 'api.mycluster-qa.example.com',
    'mycluster-dev.example.com': 'api.mycluster-dev.example.com',
  },

  // Options that users can override.
  casBlocksSizeBytes:: 100 * 1024 * 1024 * 1024,
  filePoolSizeBytes:: 100 * 1024 * 1024 * 1024,

  // The key location map entries are 66 bytes and calculate with 50% fill ratio.
  averageCasBlobSizeBytes:: 5 * 1024,
  casKeyLocationMapSizeBytes:: std.ceil((self.casBlocksSizeBytes * 66) / (self.averageCasBlobSizeBytes * 0.5)),

  // The action cache is used as a system local cache if the instance name
  // 'local' is choosen.
  acBlocksSizeBytes:: 1024 * 1024 * 1024,
  // The key location map entries are 66 bytes and calculate with 50% fill ratio.
  averageAcBlobSizeBytes:: 1024,
  acKeyLocationMapSizeBytes:: std.ceil((self.acBlocksSizeBytes * 66) / (self.averageAcBlobSizeBytes * 0.5)),

  // Maximum supported Protobuf message size.
  maximumMessageSizeBytes: 16 * 1024 * 1024,
  maximumTreeSizeBytes: 256 * 1024 * 1024,

  // When set, don't forward credentials from Bazel to clusters and
  // office caches. Instead, use a static credentials in the form of a
  // HTTP "Authorization" header.
  authorizationHeader:: null,

  // HTTP proxy to use for all outgoing requests not going to office caches.
  proxyURL:: '',

  // If enabled, use NFSv4 instead of FUSE.
  useNFSv4:: os == 'Darwin',

  grpcClient:: function(hostname, authorizationHeader, proxyURL) {
    address: hostname + ':443',
    tls: {},
    [if authorizationHeader != null then 'addMetadata']: [
      { header: 'authorization', values: [authorizationHeader] },
    ],
    addMetadataJmespathExpression: '{"build.bazel.remote.execution.v2.requestmetadata-bin": incomingGRPCMetadata."build.bazel.remote.execution.v2.requestmetadata-bin"}',
    // Enable gRPC keepalives. Make sure to tune these settings based on
    // what your cluster permits.
    keepalive: {
      time: '60s',
      timeout: '30s',
    },
    proxyUrl: proxyURL,
  },

  // Route requests to one of the clusters listed above by parsing the
  // prefix of the instance name. This prefix will be stripped on outgoing
  // requests.
  blobstoreConfig:: function(authorizationHeader, proxyURL) {
    demultiplexing: {
      instanceNamePrefixes: {
        [cluster]: { backend: {
          grpc: $.grpcClient($.clusters[cluster], authorizationHeader, proxyURL),
        } }
        for cluster in std.objectFields($.clusters)
      },
    },
  },

  // Backends for the Action Cache and Content Addressable Storage.
  blobstore: {
    actionCache: { demultiplexing: { instanceNamePrefixes: {
      // Instance names starting with 'local' can be used to set up
      // bb_clientd as a system-local cache.
      'local': { backend: { 'local': {
        keyLocationMapOnBlockDevice: { file: {
          path: cacheDirectory + '/ac/key_location_map',
          sizeBytes: $.acKeyLocationMapSizeBytes,
        } },
        keyLocationMapMaximumGetAttempts: 16,
        keyLocationMapMaximumPutAttempts: 64,
        oldBlocks: 1,
        currentBlocks: 5,
        newBlocks: 1,
        blocksOnBlockDevice: {
          source: { file: {
            path: cacheDirectory + '/ac/blocks',
            sizeBytes: $.acBlocksSizeBytes,
          } },
          spareBlocks: 1,
        },
        persistent: {
          stateDirectoryPath: cacheDirectory + '/ac/persistent_state',
          minimumEpochInterval: '300s',
        },
      } } },
      // Other instance names are assumed to correspond to remote clusters.
      '': { backend: $.blobstoreConfig($.authorizationHeader, $.proxyURL) },
    } } },
    contentAddressableStorage: { withLabels: {
      backend: { demultiplexing: { instanceNamePrefixes: {
        // Instance names starting with 'local' can be used to use
        // bb_clientd as a system-local cache.
        'local': { backend: { label: 'localCAS' } },
        // Other instance names are assumed to correspond to remote clusters.
        '': { backend: { readCaching: {
          slow: {
            existenceCaching: {
              backend: { label: 'clustersCAS' },
              // Assume that if FindMissingBlobs() reports a blob as being
              // present, it's going to stay around for five more minutes.
              // This significantly reduces the combined size of
              // FindMissingBlobs() calls generated by Bazel.
              existenceCache: {
                cacheSize: 1000 * 1000,
                cacheDuration: '300s',
                cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
              },
            },
          },
          // On-disk cache to speed up access to recently used objects.
          fast: { label: 'localCAS' },
          replicator: {
            deduplicating: {
              // Bazel's -j flag not only affects the number of actions
              // executed concurrently, it also influences the concurrency
              // of ByteStream requests. Prevent starvation by limiting
              // the number of requests that are forwarded when cache
              // misses occur.
              concurrencyLimiting: {
                base: { 'local': {} },
                maximumConcurrency: 100,
              },
            },
          },
        } } },
      } } },
      labels: {
        // Let the local CAS consume up to 100 GiB of disk space. A 64
        // MiB index is large enough to accommodate approximately one
        // million objects.
        localCAS: { 'local': {
          keyLocationMapOnBlockDevice: { file: {
            path: cacheDirectory + '/cas/key_location_map',
            sizeBytes: $.casKeyLocationMapSizeBytes,
          } },
          keyLocationMapMaximumGetAttempts: 16,
          keyLocationMapMaximumPutAttempts: 64,
          oldBlocks: 1,
          currentBlocks: 5,
          newBlocks: 1,
          blocksOnBlockDevice: {
            source: { file: {
              path: cacheDirectory + '/cas/blocks',
              sizeBytes: $.casBlocksSizeBytes,
            } },
            spareBlocks: 1,
            dataIntegrityValidationCache: {
              cacheSize: 100000,
              cacheDuration: '14400s',
              cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
            },
          },
          persistent: {
            stateDirectoryPath: cacheDirectory + '/cas/persistent_state',
            minimumEpochInterval: '300s',
          },
        } },
        clustersCAS: $.blobstoreConfig($.authorizationHeader, $.proxyURL),
      },
    } },
  },

  // Schedulers to which to route execution requests. This uses the same
  // routing policy as the storage configuration above.
  schedulers: {
    [cluster]: {
      endpoint: $.grpcClient($.clusters[cluster], $.authorizationHeader, $.proxyURL),
    }
    for cluster in std.objectFields($.clusters)
  },

  // A gRPC server to which Bazel can send requests, as opposed to
  // contacting clusters directly. This allows bb_clientd to capture
  // credentials.
  grpcServers: [{
    listenPaths: [cacheDirectory + '/grpc'],
    authenticationPolicy: { allow: {} },
  }],

  // The FUSE or NFSv4 file system through which data stored in the
  // Content Addressable Storage can be loaded lazily. This file system
  // relies on credentials captured through gRPC.
  mount: {
    mountPath: std.extVar('HOME') + '/bb_clientd',
  } + if $.useNFSv4 then {
    nfsv4: {
      enforcedLeaseTime: '120s',
      announcedLeaseTime: '60s',
    } + {
      // OS specific configuration options for NFSv4.
      Darwin: { darwin: { socketPath: cacheDirectory + '/nfsv4' } },
      Linux: { linux: { mountOptions: ['vers=4.1'] } },
    }[os],
  } else {
    fuse: {
      directoryEntryValidity: '300s',
      inodeAttributeValidity: '300s',
      // Enabling this option may be necessary if you want to permit
      // super-user access to the FUSE file system. It is strongly
      // recommended that the permissions on the parent directory of the
      // FUSE file system are locked down before enabling this option.
      allowOther: true,
    },
  },

  // The location where locally created files in the "scratch" and
  // "outputs" directories of the FUSE file system are stored. These
  // files are not necessarily backed by remote storage.
  filePool: { blockDevice: { file: {
    path: cacheDirectory + '/filepool',
    sizeBytes: $.filePoolSizeBytes,
  } } },

  // The location where contents of the "outputs" are stored, so that
  // they may be restored after restarts of bb_clientd. Because data is
  // stored densely, and only the metadata of files is stored (i.e.,
  // REv2 digests), these files tend to be small.
  outputPathPersistency: {
    stateDirectoryPath: cacheDirectory + '/outputs',
    maximumStateFileSizeBytes: 1024 * 1024 * 1024,
    maximumStateFileAge: '604800s',
  },

  // Keep a small number of unmarshaled REv2 Directory objects in memory
  // to speed up their instantiation under "outputs".
  directoryCache: {
    maximumCount: 10000,
    maximumSizeBytes: 1024 * self.maximumCount,
    cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
  },

  // Retry read operations performed through the virtual file system.
  // This prevents EIO errors in case of transient network issues.
  maximumFileSystemRetryDelay: '300s',

  global: {
    // Multiplex logs into a file. That way they remain accessible, even
    // if bb_clientd is run through a system that doesn't maintain logs
    // for us.
    logPaths: [cacheDirectory + '/log'],

    // Attach credentials provided by Bazel to all outgoing gRPC calls.
    [if $.authorizationHeader == null then 'grpcForwardAndReuseMetadata']: ['authorization'],

    // Optional: create a HTTP server that exposes Prometheus metrics
    // and allows debugging using pprof. Make sure to only enable it
    // when you need it, or at least make sure that access is limited.
    /*
    diagnosticsHttpServer: {
      httpServers: [{
        listenAddresses: ['127.0.0.1:12345'],
        authenticationPolicy: { allow: {} },
      }],
      enablePrometheus: true,
      enablePprof: true,
      enableActiveSpans: true,
    },
    */
  },
}