Integrate bb-browser web interface into bb-portal

Co-authored-by: Oscar Falk <oscar@meroton.com>

Author: isakstenstrom

BUILD.bazel

@@ -9,12 +9,14 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve proto go build/bazel/remote/execution/v2/remote_execution.proto @bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto
 # gazelle:resolve proto go google/rpc/status.proto @org_golang_google_genproto_googleapis_rpc//status
 # gazelle:resolve proto go pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth
+# gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore
 # gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global
 # gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc
 # gazelle:resolve proto go pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http
 # gazelle:resolve proto build/bazel/remote/execution/v2/remote_execution.proto @bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto
 # gazelle:resolve proto google/rpc/status.proto @googleapis//google/rpc:status_proto
 # gazelle:resolve proto pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth:auth_proto
+# gazelle:resolve proto pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto
 # gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
 # gazelle:resolve proto pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:grpc_proto
 # gazelle:resolve proto pkg/proto/configuration/http/http.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/http:http_proto

MODULE.bazel

@@ -101,6 +101,7 @@ use_repo(
     "io_entgo_ent",
     "org_golang_google_api",
     "org_golang_google_genproto",
+    "org_golang_google_genproto_googleapis_bytestream",
     "org_golang_google_grpc",
     "org_golang_google_protobuf",
     "org_golang_x_lint",

README.md

@@ -106,6 +106,17 @@ Once you have BEP files produced by Bazel, you can upload them via the applicati
 
 BB-portal can show the same information as the web interface from BB-scheduler. To do this, you need to configure the `buildQueueStateProxy` in the portal configuration file. The interface can be found under the `Remote Execution Scheduler` tab in the menu.
 
+### BB-browser
+
+BB-portal can show the same information as BB-browser. Everything it can show is visible under the tab `Browser`. To make the browser functionality work, you need to configure
+
+- `actionCacheProxy`
+- `contentAddressableStorageProxy`
+- (optional) `initialSizeClassCacheProxy`
+- (optional) `fileSystemAccessCacheProxy`
+
+in the portal configuration file. Despite having the name "browser", it is not possible to browse through the content. Instead other parts of Buildbarn will generate links to the browser. To open the content in bb-portal, the the prefix for the links should be `http://url-to-bb-portal/browser/`. After the `/browser/` prefix, the rest of the URL is compatible with urls for bb-browser.
+
 ## Using GraphiQL To Explore the GraphQL API
 
 The GraphiQL explorer is available via http://localhost:8081/graphiql.

cmd/bb_portal/BUILD.bazel

@@ -14,18 +14,28 @@ go_library(
         "//ent/gen/ent/migrate",
         "//internal/api",
         "//internal/api/grpc/bes",
+        "//internal/api/grpcweb/actioncacheproxy",
         "//internal/api/grpcweb/buildqueuestateproxy",
+        "//internal/api/grpcweb/casproxy",
+        "//internal/api/grpcweb/fsacproxy",
+        "//internal/api/grpcweb/isccproxy",
+        "//internal/api/servefiles:servefiles_lib",
         "//internal/graphql",
         "//pkg/processing",
         "//pkg/proto/configuration/bb_portal",
+        "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
         "@com_github_99designs_gqlgen//graphql/handler",
         "@com_github_99designs_gqlgen//graphql/playground",
         "@com_github_buildbarn_bb_remote_execution//pkg/proto/buildqueuestate",
         "@com_github_buildbarn_bb_storage//pkg/auth",
+        "@com_github_buildbarn_bb_storage//pkg/blobstore",
+        "@com_github_buildbarn_bb_storage//pkg/blobstore/configuration",
         "@com_github_buildbarn_bb_storage//pkg/global",
         "@com_github_buildbarn_bb_storage//pkg/grpc",
         "@com_github_buildbarn_bb_storage//pkg/http",
         "@com_github_buildbarn_bb_storage//pkg/program",
+        "@com_github_buildbarn_bb_storage//pkg/proto/fsac",
+        "@com_github_buildbarn_bb_storage//pkg/proto/iscc",
         "@com_github_buildbarn_bb_storage//pkg/util",
         "@com_github_fsnotify_fsnotify//:fsnotify",
         "@com_github_gorilla_mux//:mux",
@@ -36,6 +46,7 @@ go_library(
         "@io_entgo_ent//dialect",
         "@io_entgo_ent//dialect/sql",
         "@org_golang_google_genproto//googleapis/devtools/build/v1:build",
+        "@org_golang_google_genproto_googleapis_bytestream//:bytestream",
         "@org_golang_google_grpc//:grpc",
     ],
 )

cmd/bb_portal/grpcweb_proxy_server.go

@@ -4,14 +4,22 @@ import (
 	"log"
 	"slices"
 
+	remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/actioncacheproxy"
 	"github.com/buildbarn/bb-portal/internal/api/grpcweb/buildqueuestateproxy"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/casproxy"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/fsacproxy"
+	"github.com/buildbarn/bb-portal/internal/api/grpcweb/isccproxy"
 	"github.com/buildbarn/bb-portal/pkg/proto/configuration/bb_portal"
 	"github.com/buildbarn/bb-remote-execution/pkg/proto/buildqueuestate"
 	"github.com/buildbarn/bb-storage/pkg/auth"
 	bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc"
 	bb_http "github.com/buildbarn/bb-storage/pkg/http"
 	"github.com/buildbarn/bb-storage/pkg/program"
+	"github.com/buildbarn/bb-storage/pkg/proto/fsac"
+	"github.com/buildbarn/bb-storage/pkg/proto/iscc"
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
+	"google.golang.org/genproto/googleapis/bytestream"
 	go_grpc "google.golang.org/grpc"
 )
 
@@ -67,4 +75,48 @@ func StartGrpcWebProxyServer(
 			buildqueuestate.RegisterBuildQueueStateServer(grpcServer, buildqueuestateproxy.NewBuildQueueStateServerImpl(c, instanceNameAuthorizer))
 		},
 	)
+
+	registerAndStartServer(
+		configuration.ActionCacheProxy,
+		siblingsGroup,
+		grpcClientFactory,
+		"ActionCacheProxy",
+		func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+			c := remoteexecution.NewActionCacheClient(grpcClient)
+			remoteexecution.RegisterActionCacheServer(grpcServer, actioncacheproxy.NewAcctionCacheServerImpl(c, instanceNameAuthorizer))
+		},
+	)
+
+	registerAndStartServer(
+		configuration.ContentAddressableStorageProxy,
+		siblingsGroup,
+		grpcClientFactory,
+		"ContentAddressableStorageProxy",
+		func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+			c := bytestream.NewByteStreamClient(grpcClient)
+			bytestream.RegisterByteStreamServer(grpcServer, casproxy.NewCasServerImpl(c, instanceNameAuthorizer))
+		},
+	)
+
+	registerAndStartServer(
+		configuration.InitialSizeClassCacheProxy,
+		siblingsGroup,
+		grpcClientFactory,
+		"InitialSizeClassCacheProxy",
+		func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+			c := iscc.NewInitialSizeClassCacheClient(grpcClient)
+			iscc.RegisterInitialSizeClassCacheServer(grpcServer, isccproxy.NewIsccServerImpl(c, instanceNameAuthorizer))
+		},
+	)
+
+	registerAndStartServer(
+		configuration.FileSystemAccessCacheProxy,
+		siblingsGroup,
+		grpcClientFactory,
+		"FileSystemAccessCacheProxy",
+		func(grpcServer *go_grpc.Server, grpcClient go_grpc.ClientConnInterface) {
+			c := fsac.NewFileSystemAccessCacheClient(grpcClient)
+			fsac.RegisterFileSystemAccessCacheServer(grpcServer, fsacproxy.NewFsacServerImpl(c, instanceNameAuthorizer))
+		},
+	)
 }

cmd/bb_portal/main.go

@@ -30,10 +30,13 @@ import (
 	"github.com/buildbarn/bb-portal/ent/gen/ent/migrate"
 	"github.com/buildbarn/bb-portal/internal/api"
 	"github.com/buildbarn/bb-portal/internal/api/grpc/bes"
+	"github.com/buildbarn/bb-portal/internal/api/servefiles"
 	"github.com/buildbarn/bb-portal/internal/graphql"
 	"github.com/buildbarn/bb-portal/pkg/processing"
 	"github.com/buildbarn/bb-portal/pkg/proto/configuration/bb_portal"
 	"github.com/buildbarn/bb-storage/pkg/auth"
+	"github.com/buildbarn/bb-storage/pkg/blobstore"
+	blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration"
 	"github.com/buildbarn/bb-storage/pkg/global"
 	bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc"
 	bb_http "github.com/buildbarn/bb-storage/pkg/http"
@@ -124,8 +127,22 @@ func main() {
 			log.Fatalf("Failed to create InstanceNameAuthorizer: %v", err)
 		}
 
+		// Storage access.
+		contentAddressableStorage, _, err := blobstore_configuration.NewCASAndACBlobAccessFromConfiguration(
+			dependenciesGroup,
+			configuration.Blobstore,
+			grpcClientFactory,
+			int(configuration.MaximumMessageSizeBytes))
+		if err != nil {
+			return err
+		}
+
+		authorizedCas := blobstore.NewAuthorizingBlobAccess(contentAddressableStorage, instanceNameAuthorizer, nil, nil)
+
+		serveFileService := servefiles.NewFileServerService(authorizedCas, int(configuration.MaximumMessageSizeBytes))
+
 		router := mux.NewRouter()
-		newPortalService(blobArchiver, dbClient, router)
+		newPortalService(blobArchiver, dbClient, serveFileService, router)
 		bb_http.NewServersFromConfigurationAndServe(
 			configuration.HttpServers,
 			bb_http.NewMetricsHandler(router, "PortalUI"),
@@ -206,13 +223,16 @@ func fatal(msg string, args ...any) {
 	os.Exit(1)
 }
 
-func newPortalService(archiver processing.BlobMultiArchiver, dbClient *ent.Client, router *mux.Router) {
+func newPortalService(archiver processing.BlobMultiArchiver, dbClient *ent.Client, serveFilesService *servefiles.FileServerService, router *mux.Router) {
 	srv := handler.NewDefaultServer(graphql.NewSchema(dbClient))
 	srv.Use(entgql.Transactioner{TxOpener: dbClient})
 
 	router.PathPrefix("/graphql").Handler(srv)
 	router.Handle("/graphiql", playground.Handler("GraphQL Playground", "/graphql"))
 	router.Handle("/api/v1/bep/upload", api.NewBEPUploadHandler(dbClient, archiver)).Methods("POST")
+	router.HandleFunc("/api/servefile/{instanceName:(?:.*?/)?}blobs/{digestFunction}/file/{hash}-{sizeBytes}/{name}", serveFilesService.HandleFile).Methods("GET")
+	router.HandleFunc("/api/servefile/{instanceName:(?:.*?/)?}blobs/{digestFunction}/command/{hash}-{sizeBytes}/", serveFilesService.HandleCommand).Methods("GET")
+	router.HandleFunc("/api/servefile/{instanceName:(?:.*?/)?}blobs/{digestFunction}/directory/{hash}-{sizeBytes}/", serveFilesService.HandleDirectory).Methods("GET")
 	router.PathPrefix("/").Handler(frontendServer())
 }
 

config/portal.jsonnet

@@ -4,11 +4,36 @@
 // [bb-deployments](https://github.com/buildbarn/bb-deployments), i.e. it
 // assumes that the following services are running:
 // - A Buildbarn scheduler, accessible at localhost:8984
+// - A Buildbarn frontend, accessible at localhost:8980
 
 {
+  blobstore: {
+    contentAddressableStorage: {
+      grpc: { address: 'localhost:8980' },
+    },
+    actionCache: {
+      completenessChecking: {
+        backend: { grpc: { address: 'localhost:8980' } },
+        maximumTotalTreeSizeBytes: 64 * 1024 * 1024,
+      },
+    },
+  },
+  maximumMessageSizeBytes: 2 * 1024 * 1024,
+
   httpServers: [{
     listenAddresses: [':8081'],
-    authenticationPolicy: { allow: {} },
+    authenticationPolicy: {
+      allow: {
+        public: {
+          user: 'FooBar',
+        },
+        private: {
+          groups: ['admin'],
+          instances: ['fuse', 'testingQueue'],
+          email: 'foo@example.com',
+        },
+      },
+    },
   }],
   grpcServers: [{
     listenAddresses: [':8082'],
@@ -44,4 +69,91 @@
       },
     }],
   },
+
+  actionCacheProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9434'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
+
+  contentAddressableStorageProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9435'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
+
+  initialSizeClassCacheProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9436'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
+  fileSystemAccessCacheProxy: {
+    client: {
+      address: 'localhost:8980',
+    },
+    allowedOrigins: ['http://localhost:8081'],
+    httpServers: [{
+      listenAddresses: [':9437'],
+      authenticationPolicy: {
+        allow: {
+          public: {
+            user: 'FooBar',
+          },
+          private: {
+            groups: ['admin'],
+            instances: ['fuse', 'testingQueue'],
+            email: 'foo@example.com',
+          },
+        },
+      },
+    }],
+  },
 }

frontend/.env

@@ -1,6 +1,10 @@
 NEXT_PUBLIC_BES_BACKEND_URL=http://localhost:8081
 NEXT_PUBLIC_BES_GRPC_BACKEND_URL=grpc://localhost:8082
 NEXT_PUBLIC_BB_BUILDQUEUESTATE_GRPC_BACKEND_URL=http://localhost:9433
+NEXT_PUBLIC_BB_ACTIONCACHE_GRPC_BACKEND_URL=http://localhost:9434
+NEXT_PUBLIC_BB_CAS_GRPC_BACKEND_URL=http://localhost:9435
+NEXT_PUBLIC_BB_ISCC_GRPC_BACKEND_URL=http://localhost:9436
+NEXT_PUBLIC_BB_FSAC_GRPC_BACKEND_URL=http://localhost:9437
 NEXT_PUBLIC_BROWSER_URL=https://browser.example.com
 NEXT_PUBLIC_COMPANY_NAME="Example Co"
 NEXT_PUBLIC_COMPANY_SLACK_CHANNEL_NAME=ExampleCoBuild

frontend/scripts/grpc-client-gen.sh

@@ -19,4 +19,11 @@ generate_grpc_client() {
     ${proto_file}
 }
 
+generate_grpc_client "${PROTO_DIR}/buildbarn/auth/auth.proto"
 generate_grpc_client "${PROTO_DIR}/buildbarn/buildqueuestate/buildqueuestate.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/cas/cas.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/resourceusage/resourceusage.proto"
+generate_grpc_client "${PROTO_DIR}/google/bytestream/bytestream.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/iscc/iscc.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/fsac/fsac.proto"
+generate_grpc_client "${PROTO_DIR}/buildbarn/query/query.proto"