add logging for bytes processed (#1132)

DataBiosphere · Jan 16, 2025 · d864f89 · d864f89
1 parent 8b7c5ce
commit d864f89
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 2 deletions.
diff --git a/underlay/src/main/java/bio/terra/tanagra/query/bigquery/BQExecutor.java b/underlay/src/main/java/bio/terra/tanagra/query/bigquery/BQExecutor.java
@@ -52,7 +52,6 @@ public SqlQueryResult run(SqlQueryRequest queryRequest) {
               '@' + paramName,
               toSql(toQueryParameterValue(queryRequest.getSqlParams().getParamValue(paramName))));
     }
-    LOGGER.info("SQL no parameters: {}", sqlNoParams);
 
     // Build a BQ parameter value object for each SQL query parameter.
     Map<String, QueryParameterValue> bqQueryParams =

diff --git a/underlay/src/main/java/bio/terra/tanagra/utils/GoogleBigQuery.java b/underlay/src/main/java/bio/terra/tanagra/utils/GoogleBigQuery.java
@@ -258,7 +258,17 @@ public TableResult runQuery(
           TableResult tableResult =
               job.getQueryResults(
                   queryJobConfig.getRight().toArray(new BigQuery.QueryResultsOption[0]));
-          LOGGER.info("SQL query returns {} rows across all pages", tableResult.getTotalRows());
+          Job completedJob = job.waitFor();
+          JobStatistics.QueryStatistics stats = completedJob.getStatistics();
+          Long totalBytesProcessed = stats.getTotalBytesProcessed();
+          String jobId = completedJob.getJobId().getJob();
+          LOGGER.info("job: {}, query: {}", jobId, sql);
+          LOGGER.info(
+              "job: {}, total rows: {}, cache hit: {}, total megabytes processed: {}MB",
+              jobId,
+              tableResult.getTotalRows(),
+              stats.getCacheHit(),
+              totalBytesProcessed / 1_048_576);
           return tableResult;
         },
         "Error running query: " + queryJobConfig.getLeft().getQuery());