Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable v1 APIs for Disk, App, and Runtime for Azure hosted Leonardo instances #4737

Merged
merged 21 commits into from
Aug 13, 2024
Merged

Enable v1 APIs for Disk, App, and Runtime for Azure hosted Leonardo instances #4737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
87fc9b2
add base v1 routes to azure
bennettn4 Jul 15, 2024
de9477a
added diskservice to azure dependency
bennettn4 Jul 15, 2024
9d487bc
finish diskservice options
bennettn4 Jul 15, 2024
2669ad1
temp remove runtime routes
bennettn4 Jul 15, 2024
1c834e5
Update azure routes
bennettn4 Jul 17, 2024
bd9e457
temp remove approutes
bennettn4 Jul 17, 2024
abd7803
add runtimeservice to service registry
bennettn4 Jul 17, 2024
7979684
update oidcconfig
bennettn4 Jul 17, 2024
4d71d9c
readd routes
bennettn4 Jul 18, 2024
5ea28b9
update approutes
bennettn4 Jul 18, 2024
6937324
Merge branch 'develop' into azure-v1-apis
bennettn4 Aug 5, 2024
737c944
cleanup
bennettn4 Aug 6, 2024
b8a2635
Merge branch 'develop' into azure-v1-apis
bennettn4 Aug 6, 2024
d06805c
update tests
bennettn4 Aug 6, 2024
150e65d
scalafmt
bennettn4 Aug 6, 2024
52eced7
clean
bennettn4 Aug 6, 2024
37d0b38
Merge branch 'develop' into azure-v1-apis
bennettn4 Aug 6, 2024
e44b8ae
Update AzureDependenciesBuilder.scala
bennettn4 Aug 7, 2024
34d956a
Merge branch 'develop' into azure-v1-apis
bennettn4 Aug 12, 2024
b0db8fe
retrigger checks
bennettn4 Aug 12, 2024
a50a8f3
retrigger checks
bennettn4 Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion ...main/scala/org/broadinstitute/dsde/workbench/leonardo/http/AzureDependenciesBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@ import akka.actor.ActorSystem
import cats.effect.{IO, Resource}
import org.broadinstitute.dsde.workbench.leonardo.config.Config.{appServiceConfig, gkeCustomAppConfig}
import org.broadinstitute.dsde.workbench.leonardo.db.DbReference
import org.broadinstitute.dsde.workbench.leonardo.http.service.LeoAppServiceInterp
import org.broadinstitute.dsde.workbench.leonardo.http.service.{
DiskService,
DiskServiceInterp,
LeoAppServiceInterp,
RuntimeService
}
import org.broadinstitute.dsde.workbench.leonardo.monitor.MonitorAtBoot
import org.broadinstitute.dsde.workbench.leonardo.util.ServicesRegistry
import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics
Expand Down Expand Up @@ -85,9 +90,32 @@ class AzureDependenciesBuilder extends CloudDependenciesBuilder {
baselineDependencies.wsmClientProvider
)

// Needed for v1 APIs
val diskService = new DiskServiceInterp[IO](
ConfigReader.appConfig.persistentDisk,
baselineDependencies.authProvider,
baselineDependencies.serviceAccountProvider,
baselineDependencies.publisherQueue,
None,
None
)

val runtimeService = RuntimeService(
baselineDependencies.runtimeServicesConfig,
ConfigReader.appConfig.persistentDisk,
baselineDependencies.authProvider,
baselineDependencies.serviceAccountProvider,
baselineDependencies.dockerDAO,
None,
None,
baselineDependencies.publisherQueue
)

var servicesRegistry = ServicesRegistry()

servicesRegistry.register[LeoAppServiceInterp[IO]](leoKubernetesService)
servicesRegistry.register[DiskService[IO]](diskService)
servicesRegistry.register[RuntimeService[IO]](runtimeService)

Resource.make(IO(servicesRegistry))(_ => IO.unit)
}
Expand Down
8 changes: 4 additions & 4 deletions ...c/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/GcpDependenciesBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,8 @@ class GcpDependencyBuilder extends CloudDependenciesBuilder {
baselineDependencies.authProvider,
baselineDependencies.serviceAccountProvider,
baselineDependencies.publisherQueue,
gcpDependencies.googleDiskService,
gcpDependencies.googleProjectDAO
Some(gcpDependencies.googleDiskService),
Some(gcpDependencies.googleProjectDAO)
)

val runtimeService = RuntimeService(
Expand All @@ -294,8 +294,8 @@ class GcpDependencyBuilder extends CloudDependenciesBuilder {
baselineDependencies.authProvider,
baselineDependencies.serviceAccountProvider,
baselineDependencies.dockerDAO,
gcpDependencies.googleStorageService,
gcpDependencies.googleComputeService,
Some(gcpDependencies.googleStorageService),
Some(gcpDependencies.googleComputeService),
baselineDependencies.publisherQueue
)

Expand Down
10 changes: 9 additions & 1 deletion http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/HttpRoutes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class HttpRoutes(
private val statusRoutes = new StatusRoutes(statusService)
private val corsSupport = new CorsSupport(contentSecurityPolicy, refererConfig)
private val kubernetesRoutes = new AppRoutes(kubernetesService, userInfoDirectives)
private val appRoutes = createAppRoutesUsingServicesRegistry
private val appV2Routes = new AppV2Routes(kubernetesService, userInfoDirectives)
private val runtimeV2Routes = new RuntimeV2Routes(refererConfig, azureService, userInfoDirectives)
private val diskV2Routes = new DiskV2Routes(diskV2Service, userInfoDirectives)
Expand Down Expand Up @@ -133,7 +134,9 @@ class HttpRoutes(
oidcConfig
.swaggerRoutes("swagger/api-docs.yaml") ~ oidcConfig.oauth2Routes ~ statusRoutes.route ~
pathPrefix("api") {
runtimeV2Routes.routes ~ appV2Routes.routes ~ diskV2Routes.routes ~ adminRoutes.routes
runtimeRoutes.get.routes ~ runtimeV2Routes.routes ~
diskRoutes.get.routes ~ diskV2Routes.routes ~
appRoutes.get.routes ~ appV2Routes.routes ~ adminRoutes.routes
}
)
}
Expand All @@ -153,6 +156,11 @@ class HttpRoutes(
.lookup[RuntimeService[IO]]
.map(runtimeService => new RuntimeRoutes(refererConfig, runtimeService, userInfoDirectives))

private def createAppRoutesUsingServicesRegistry =
gcpOnlyServicesRegistry
.lookup[LeoAppServiceInterp[IO]]
.map(appService => new AppRoutes(appService, userInfoDirectives))

private def createProxyRoutesUsingServicesRegistry =
gcpOnlyServicesRegistry
.lookup[ProxyService]
Expand Down
10 changes: 5 additions & 5 deletions ...ain/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/DiskServiceInterp.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ class DiskServiceInterp[F[_]: Parallel](config: PersistentDiskConfig,
authProvider: LeoAuthProvider[F],
serviceAccountProvider: ServiceAccountProvider[F],
publisherQueue: Queue[F, LeoPubsubMessage],
googleDiskService: GoogleDiskService[F],
googleProjectDAO: GoogleProjectDAO
googleDiskService: Option[GoogleDiskService[F]],
googleProjectDAO: Option[GoogleProjectDAO]
)(implicit
F: Async[F],
log: StructuredLogger[F],
Expand Down Expand Up @@ -122,10 +122,10 @@ class DiskServiceInterp[F[_]: Parallel](config: PersistentDiskConfig,
): F[Unit] =
for {
ctx <- as.ask
sourceAncestry <- F.fromFuture(F.delay(googleProjectDAO.getAncestry(sourceGoogleProject.value)))
sourceAncestry <- F.fromFuture(F.delay(googleProjectDAO.get.getAncestry(sourceGoogleProject.value)))
Copy link
Collaborator

@rtitle rtitle Aug 8, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My concern with sprinkling .get in this code is that the API will fail with a HTTP 500 scala.MatchError if these APIs are called in a Leo deployment with no GCP dependencies.

A cleaner option might be doing something like this in the appropriate endpoints (e.g. create/delete runtime/disk):

projectDAO <- F.fromOption(googleProjectDAO, BadRequestException("GCP is not enabled"))
// then use projectDAO as needed

Then you would get a sensible error message and HTTP status code if a GCP-specific API is called.

(I realize that this only affects the VA deployment and v1 create/delete runtime/disk will never actually be called. So just a minor suggestion.)

sourceAncestor <- immediateAncestor(sourceAncestry)

targetAncestry <- F.fromFuture(F.delay(googleProjectDAO.getAncestry(targetGoogleProject.value)))
targetAncestry <- F.fromFuture(F.delay(googleProjectDAO.get.getAncestry(targetGoogleProject.value)))
targetAncestor <- immediateAncestor(targetAncestry)

_ <- F.raiseWhen(
Expand Down Expand Up @@ -160,7 +160,7 @@ class DiskServiceInterp[F[_]: Parallel](config: PersistentDiskConfig,
case _: DiskNotFoundException =>
F.raiseError(BadRequestException("source disk does not exist", Option(ctx.traceId)))
}
maybeGoogleDisk <- googleDiskService.getDisk(sourceDiskReq.googleProject, sourceDisk.zone, sourceDisk.name)
maybeGoogleDisk <- googleDiskService.get.getDisk(sourceDiskReq.googleProject, sourceDisk.zone, sourceDisk.name)
googleDisk <- maybeGoogleDisk.toOptionT.getOrElseF(
F.raiseError(
LeoInternalServerError(s"Source disk $sourceDiskReq does not exist in google", Option(ctx.traceId))
Expand Down
4 changes: 2 additions & 2 deletions ...c/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeService.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ object RuntimeService {
authProvider: LeoAuthProvider[F],
serviceAccountProvider: ServiceAccountProvider[F],
dockerDAO: DockerDAO[F],
googleStorageService: GoogleStorageService[F],
googleComputeService: GoogleComputeService[F],
googleStorageService: Option[GoogleStorageService[F]],
googleComputeService: Option[GoogleComputeService[F]],
publisherQueue: Queue[F, LeoPubsubMessage]
)(implicit
F: Async[F],
Expand Down
8 changes: 4 additions & 4 deletions .../scala/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterp.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,8 @@ class RuntimeServiceInterp[F[_]: Parallel](
authProvider: LeoAuthProvider[F],
serviceAccountProvider: ServiceAccountProvider[F],
dockerDAO: DockerDAO[F],
googleStorageService: GoogleStorageService[F],
googleComputeService: GoogleComputeService[F],
googleStorageService: Option[GoogleStorageService[F]],
googleComputeService: Option[GoogleComputeService[F]],
publisherQueue: Queue[F, LeoPubsubMessage]
)(implicit
F: Async[F],
Expand Down Expand Up @@ -369,7 +369,7 @@ class RuntimeServiceInterp[F[_]: Parallel](
disk <- F.fromEither(
diskOpt.toRight(new RuntimeException(s"Can't find ${diskId} in PERSISTENT_DISK table"))
)
detachOp <- googleComputeService.detachDisk(
detachOp <- googleComputeService.get.detachDisk(
req.googleProject,
disk.zone,
InstanceName(runtime.runtimeName.asString),
Expand Down Expand Up @@ -765,7 +765,7 @@ class RuntimeServiceInterp[F[_]: Parallel](
)

val res = for {
blob <- googleStorageService
blob <- googleStorageService.get
.getBlob(
gcsPath.bucketName,
GcsBlobName(gcsPath.objectName.value),
Expand Down
4 changes: 2 additions & 2 deletions http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/TestLeoRoutes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@ trait TestLeoRoutes {
allowListAuthProvider,
serviceAccountProvider,
new MockDockerDAO,
FakeGoogleStorageInterpreter,
FakeGoogleComputeService,
Some(FakeGoogleStorageInterpreter),
Some(FakeGoogleComputeService),
QueueFactory.makePublisherQueue()
)

Expand Down
14 changes: 7 additions & 7 deletions ...scala/org/broadinstitute/dsde/workbench/leonardo/http/service/DiskServiceInterpSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ trait DiskServiceInterpSpec extends AnyFlatSpec with LeonardoTestSuite with Test
allowListAuthProvider,
serviceAccountProvider,
publisherQueue,
MockGoogleDiskService,
googleProjectDAO
Some(MockGoogleDiskService),
Some(googleProjectDAO)
)
(diskService, publisherQueue)
}
Expand Down Expand Up @@ -166,13 +166,13 @@ class DiskServiceInterpTest
allowListAuthProvider,
serviceAccountProvider,
publisherQueue,
new MockGoogleDiskService {
Some(new MockGoogleDiskService {
override def getDisk(project: GoogleProject, zone: ZoneName, diskName: DiskName)(implicit
ev: Ask[IO, TraceId]
): IO[Option[Disk]] =
IO.pure(Some(Disk.newBuilder().setSelfLink(dummyDiskLink).build()))
},
new MockGoogleProjectDAOWithCustomAncestors(projectToFolder)
}),
Some(new MockGoogleProjectDAOWithCustomAncestors(projectToFolder))
)

val userInfo = UserInfo(OAuth2BearerToken(""),
Expand Down Expand Up @@ -291,8 +291,8 @@ class DiskServiceInterpTest
authProviderMock,
serviceAccountProvider,
publisherQueue,
googleDiskServiceMock,
new MockGoogleProjectDAO
Some(googleDiskServiceMock),
Some(new MockGoogleProjectDAO)
)
val userInfoCreator =
UserInfo(OAuth2BearerToken(""), WorkbenchUserId("creator"), WorkbenchEmail("creator@example.com"), 0)
Expand Down
4 changes: 2 additions & 2 deletions ...la/org/broadinstitute/dsde/workbench/leonardo/http/service/RuntimeServiceInterpSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ trait RuntimeServiceInterpSpec extends AnyFlatSpec with LeonardoTestSuite with T
authProvider,
serviceAccountProvider,
new MockDockerDAO,
FakeGoogleStorageInterpreter,
computeService,
Some(FakeGoogleStorageInterpreter),
Some(computeService),
publisherQueue
)

Expand Down
Loading