pass out the file sizes of jars in the metadata, so we can construct …

…a notion of bazels digest
ianoc · Jul 28, 2022 · 24b3083 · 24b3083
1 parent fd4a295
commit 24b3083
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 76 deletions.
diff --git a/src/scala/com/github/johnynek/bazel_deps/AetherResolver.scala b/src/scala/com/github/johnynek/bazel_deps/AetherResolver.scala
@@ -205,6 +205,7 @@ class AetherResolver(servers: List[MavenServer], resolverCachePath: Path)
                         )
                         .toString
                     ),
+                    fileSizeBytes = Some(f.length()),
                     sha1 = Some(sha1),
                     sha256 = Some(sha256),
                     serverId = r.getRepository.getId

diff --git a/src/scala/com/github/johnynek/bazel_deps/CoursierResolver.scala b/src/scala/com/github/johnynek/bazel_deps/CoursierResolver.scala
@@ -125,47 +125,10 @@ class CoursierResolver(
     type N[x] = Nested[Task, L, x]
 
     def lookup(c: MavenCoordinate): N[ResolvedShasValue] = {
-
-      def downloadShas(
-          digestType: DigestType,
-          as: List[Artifact]
-      ): Task[Option[(Artifact, ShaValue)]] =
-        as.foldM(Option.empty[(Artifact, ShaValue)]) {
-          case (s @ Some(r), _) => Task.point(s)
-          case (None, a)        => downloadSha(digestType, a)
-        }
-
-      def downloadSha(
-          digestType: DigestType,
-          a: Artifact
-      ): Task[Option[(Artifact, ShaValue)]] = {
-        // Because Cache.file is hijacked to download SHAs directly (rather than signed artifacts) checksum verification
-        // is turned off. Checksums don't themselves have checksum files.
-        FileCache()
-          .withChecksums(Seq(None))
-          .withCachePolicies(Seq(CachePolicy.FetchMissing))
-          .withPool(CoursierResolver.downloadPool)
-          .file(a)
-          .run
-          .map {
-            case Left(error) =>
-              logger.info(s"failure to download ${a.url}, ${error.describe}")
-              None
-            case Right(file) =>
-              val o = ShaValue.parseFile(digestType, file).toOption
-              o.foreach { r =>
-                logger.info(
-                  s"$digestType for ${c.asString} downloaded from ${a.url} (${r.toHex})"
-                )
-              }
-              o.map { sha => (a, sha) }
-          }
-      }
-
       def computeSha(
           digestType: DigestType,
           artifact: Artifact
-      ): Task[(Artifact, ShaValue)] = {
+      ): Task[(Artifact, ShaValue, Long)] = {
         FileCache()
           .withCachePolicies(Seq(CachePolicy.FetchMissing))
           .withPool(CoursierResolver.downloadPool)
@@ -178,7 +141,7 @@ class CoursierResolver(
                 Failure(FileErrorException(error))
               case Right(file) =>
                 ShaValue.computeShaOf(digestType, file).map { sha =>
-                  (artifact, sha)
+                  (artifact, sha, file.length())
                 }
             })
           }
@@ -187,8 +150,8 @@ class CoursierResolver(
       def computeShas(
           digestType: DigestType,
           as: NonEmptyList[Artifact]
-      ): Task[(Artifact, ShaValue)] = {
-        val errorFn: Throwable => Task[(Artifact, ShaValue)] = as.tail match {
+      ): Task[(Artifact, ShaValue, Long)] = {
+        val errorFn: Throwable => Task[(Artifact, ShaValue, Long)] = as.tail match {
           case Nil => { e: Throwable =>
             resolverMonad.raiseError(
               new RuntimeException(
@@ -203,36 +166,6 @@ class CoursierResolver(
         resolverMonad.handleErrorWith(computeSha(digestType, as.head))(errorFn)
       }
 
-      def fetchOrComputeShas(
-          artifacts: NonEmptyList[Artifact],
-          digestType: DigestType
-      ): Task[(Artifact, ShaValue)] = {
-        val checksumArtifacts = artifacts.toList.flatMap { a =>
-          a.checksumUrls
-            .get(digestType.name)
-            .map(url =>
-              Artifact(
-                url,
-                Map.empty,
-                Map.empty,
-                a.changing,
-                false,
-                a.authentication
-              )
-            )
-        }
-
-        downloadShas(digestType, checksumArtifacts).flatMap {
-          case Some(s) => Task.point(s)
-          case None => {
-            logger.info(
-              s"Preforming cached fetch to execute $digestType calculation for ${artifacts.head.url}"
-            )
-            computeShas(digestType, artifacts)
-          }
-        }
-      }
-
       def processArtifact(
           src: coursier.core.ArtifactSource,
           dep: Dependency,
@@ -311,19 +244,21 @@ class CoursierResolver(
           .fromList(maybeArtifacts)
           .map { artifacts =>
             for {
-              foundSha1Data <- fetchOrComputeShas(artifacts, DigestType.Sha1)
-              (sha1Artifact, sha1) = foundSha1Data
-              // I could not find any example of artifacts that actually have a SHA256 checksum, so don't bother
-              // trying to fetch them. Save on network latency and just calculate.
+              // No artifacts actually have Sha256's available
+              // so don't bother trying to fetch anything.
+              // Once we download the jar at all, calculating sha's is ~cheap.
+              foundSha1Data <- computeShas(DigestType.Sha1, artifacts)
+              (sha1Artifact, sha1, _) = foundSha1Data
               foundShaData <- computeShas(DigestType.Sha256, artifacts)
-              (artifact, sha256) = foundShaData
+              (artifact, sha256, fileSizeBytes) = foundShaData
             } yield {
               val serverId = serverFor(artifact).fold("")(_.id)
 
               Some(
                 JarDescriptor(
                   sha1 = Some(sha1),
                   sha256 = Some(sha256),
+                  fileSizeBytes = Some(fileSizeBytes),
                   serverId = serverId,
                   url = Some(artifact.url)
                 )

diff --git a/src/scala/com/github/johnynek/bazel_deps/DepsModel.scala b/src/scala/com/github/johnynek/bazel_deps/DepsModel.scala
@@ -358,6 +358,7 @@ object JarDescriptor {
         url = url,
         sha1 = Some(sha1),
         sha256 = Some(sha256),
+        fileSizeBytes = Some(f.length()),
         serverId = serverId
       )
     }
@@ -367,6 +368,7 @@ case class JarDescriptor(
     url: Option[String],
     sha1: Option[ShaValue],
     sha256: Option[ShaValue],
+    fileSizeBytes: Option[Long],
     serverId: String
 )
 

diff --git a/src/scala/com/github/johnynek/bazel_deps/Writer.scala b/src/scala/com/github/johnynek/bazel_deps/Writer.scala
@@ -12,6 +12,7 @@ import scala.util.{Failure, Success}
 case class DataSource(
     sha1: Option[String],
     sha256: Option[String],
+    file_size_bytes: Option[Long],
     repository: Option[String],
     urls: List[String]
 )
@@ -104,6 +105,7 @@ object Writer {
       DataSource(
         sha1 = sha.binaryJar.sha1.map(_.toHex),
         sha256 = sha.binaryJar.sha256.map(_.toHex),
+        file_size_bytes = sha.binaryJar.fileSizeBytes,
         repository = servers.get(sha.binaryJar.serverId),
         urls = List(sha.binaryJar.url.toList).flatten
       )
@@ -113,6 +115,7 @@ object Writer {
       DataSource(
         sha1 = sourceJar.sha1.map(_.toHex),
         sha256 = sourceJar.sha256.map(_.toHex),
+        file_size_bytes = sourceJar.fileSizeBytes,
         repository = servers.get(sourceJar.serverId),
         urls = List(sourceJar.url.toList).flatten
       )