Merge branch 'darwin_sprint' into issue72

Author: ginberg

R/ConceptSetUtils.R

@@ -256,65 +256,7 @@ mergeTempTables <-
     }
   }
 
-instantiateUniqueConceptSets <- function(uniqueConceptSets,
-                                         connection,
-                                         vocabularyDatabaseSchema,
-                                         tempEmulationSchema,
-                                         conceptSetsTable = "#inst_concept_sets") {
-  ParallelLogger::logInfo("Instantiating concept sets")
-  
-  if (nrow(uniqueConceptSets) > 0) {
-    sql <- sapply(
-      split(uniqueConceptSets, 1:nrow(uniqueConceptSets)),
-      function(x) {
-        sub(
-          "SELECT [0-9]+ as codeset_id",
-          sprintf("SELECT %s as codeset_id", x$uniqueConceptSetId),
-          x$conceptSetSql
-        )
-      }
-    )
-    
-    batchSize <- 100
-    tempTables <- c()
-    pb <- utils::txtProgressBar(style = 3)
-    for (start in seq(1, length(sql), by = batchSize)) {
-      utils::setTxtProgressBar(pb, start / length(sql))
-      tempTable <-
-        paste("#", paste(sample(letters, 20, replace = TRUE), collapse = ""), sep = "")
-      tempTables <- c(tempTables, tempTable)
-      end <- min(start + batchSize - 1, length(sql))
-      sqlSubset <- sql[start:end]
-      sqlSubset <- paste(sqlSubset, collapse = "\n\n  UNION ALL\n\n")
-      sqlSubset <-
-        sprintf(
-          "SELECT *\nINTO %s\nFROM (\n %s\n) tmp;",
-          tempTable,
-          sqlSubset
-        )
-      sqlSubset <-
-        SqlRender::render(sqlSubset, vocabulary_database_schema = vocabularyDatabaseSchema)
-      sqlSubset <- SqlRender::translate(sqlSubset,
-                                        targetDialect = connection@dbms,
-                                        tempEmulationSchema = tempEmulationSchema
-      )
-      DatabaseConnector::executeSql(connection,
-                                    sqlSubset,
-                                    progressBar = FALSE,
-                                    reportOverallTime = FALSE
-      )
-    }
-    utils::setTxtProgressBar(pb, 1)
-    close(pb)
-    
-    mergeTempTables(
-      connection = connection,
-      tableName = conceptSetsTable,
-      tempTables = tempTables,
-      tempEmulationSchema = tempEmulationSchema
-    )
-  }
-}
+
 
 getCodeSetId <- function(criterion) {
   if (is.list(criterion)) {

R/ResultsDataModel.R

@@ -28,7 +28,7 @@ getResultsDataModelSpecifications <- function(tableName = NULL) {
   readr::local_edition(1)
   pathToCsv <- system.file("settings", "resultsDataModelSpecification.csv", package = "CohortDiagnostics")
 
-  resultsDataModelSpecifications <- readr::read_csv(file = pathToCsv, col_types = "cccccccccc")
+  resultsDataModelSpecifications <- readr::read_csv(file = pathToCsv, col_types = "ccccccccccc")
 
   colnames(resultsDataModelSpecifications) <- SqlRender::snakeCaseToCamelCase(colnames(resultsDataModelSpecifications))
 

R/runBreakdownIndexEvents.R

@@ -51,9 +51,6 @@ runBreakdownIndexEvents <- function(connection,
                                     vocabularyDatabaseSchema = cdmDatabaseSchema,
                                     databaseId,
                                     cohorts,
-                                    runIncludedSourceConcepts,
-                                    runOrphanConcepts,
-                                    runBreakdownIndexEvents,
                                     exportFolder,
                                     minCellCount,
                                     conceptCountsDatabaseSchema = NULL,
@@ -526,151 +523,6 @@ runBreakdownIndexEvents <- function(connection,
     }
   }
 
-  if (runOrphanConcepts) {
-    # Orphan concepts ---------------------------------------------------------
-    ParallelLogger::logInfo("Finding orphan concepts")
-    if (incremental && (nrow(cohorts) - nrow(subsetOrphans)) > 0) {
-      ParallelLogger::logInfo(sprintf(
-        "Skipping %s cohorts in incremental mode.",
-        nrow(cohorts) - nrow(subsetOrphans)
-      ))
-    }
-    if (nrow(subsetOrphans > 0)) {
-      start <- Sys.time()
-      
-      if (!useExternalConceptCountsTable) {
-        ParallelLogger::logTrace("Using internal concept count table.")
-      } else {
-        stop("Use of external concept count table is not supported")
-      }
-      
-      # [OPTIMIZATION idea] can we modify the sql to do this for all uniqueConceptSetId in one query using group by?
-      data <- list()
-      for (i in (1:nrow(uniqueConceptSets))) {
-        conceptSet <- uniqueConceptSets[i, ]
-        ParallelLogger::logInfo(
-          "- Finding orphan concepts for concept set '",
-          conceptSet$conceptSetName,
-          "'"
-        )
-        
-        timeExecution(
-          exportFolder,
-          taskName = "orphanConcepts",
-          parent = "runConceptSetDiagnostics",
-          cohortIds = paste("concept_set-", conceptSet$conceptSetName),
-          expr = {
-            data[[i]] <- .findOrphanConcepts(
-              connection = connection,
-              cdmDatabaseSchema = cdmDatabaseSchema,
-              tempEmulationSchema = tempEmulationSchema,
-              useCodesetTable = TRUE,
-              codesetId = conceptSet$uniqueConceptSetId,
-              conceptCountsDatabaseSchema = conceptCountsDatabaseSchema,
-              conceptCountsTable = conceptCountsTable,
-              conceptCountsTableIsTemp = conceptCountsTableIsTemp,
-              instantiatedCodeSets = "#inst_concept_sets",
-              orphanConceptTable = "#orphan_concepts"
-            )
-            
-            if (!is.null(conceptIdTable)) {
-              sql <- "INSERT INTO @concept_id_table (concept_id)
-                  SELECT DISTINCT concept_id
-                  FROM @orphan_concept_table;"
-              DatabaseConnector::renderTranslateExecuteSql(
-                connection = connection,
-                sql = sql,
-                tempEmulationSchema = tempEmulationSchema,
-                concept_id_table = conceptIdTable,
-                orphan_concept_table = "#orphan_concepts",
-                progressBar = FALSE,
-                reportOverallTime = FALSE
-              )
-            }
-          }
-        )
-        sql <-
-          "TRUNCATE TABLE @orphan_concept_table;\nDROP TABLE @orphan_concept_table;"
-        DatabaseConnector::renderTranslateExecuteSql(
-          connection = connection,
-          sql = sql,
-          tempEmulationSchema = tempEmulationSchema,
-          orphan_concept_table = "#orphan_concepts",
-          progressBar = FALSE,
-          reportOverallTime = FALSE
-        )
-      }
-      
-      data <- dplyr::bind_rows(data) %>%
-        dplyr::distinct() %>%
-        dplyr::rename("uniqueConceptSetId" = "codesetId") %>%
-        dplyr::inner_join(
-          conceptSets %>%
-            dplyr::select(
-              "uniqueConceptSetId",
-              "cohortId",
-              "conceptSetId"
-            ) %>% dplyr::distinct(),
-          by = "uniqueConceptSetId",
-          relationship = "many-to-many"
-        ) %>%
-        dplyr::select(-"uniqueConceptSetId") %>%
-        dplyr::select(
-          "cohortId",
-          "conceptSetId",
-          "conceptId",
-          "conceptCount",
-          "conceptSubjects"
-        ) %>%
-        dplyr::group_by(
-          .data$cohortId,
-          .data$conceptSetId,
-          .data$conceptId
-        ) %>%
-        dplyr::summarise(
-          conceptCount = max(.data$conceptCount),
-          conceptSubjects = max(.data$conceptSubjects)
-        ) %>%
-        dplyr::ungroup()
-      
-      
-      exportDataToCsv(
-        data = data,
-        tableName = "orphan_concept",
-        fileName = file.path(exportFolder, "orphan_concept.csv"),
-        minCellCount = minCellCount,
-        databaseId = databaseId,
-        incremental = incremental,
-        cohortId = subsetOrphans$cohortId
-      )
-      
-      recordTasksDone(
-        cohortId = subsetOrphans$cohortId,
-        task = "runOrphanConcepts",
-        checksum = subsetOrphans$checksum,
-        recordKeepingFile = recordKeepingFile,
-        incremental = incremental
-      )
-      
-      delta <- Sys.time() - start
-      
-      timeExecution(
-        exportFolder,
-        taskName = "allOrphanConcepts",
-        parent = "runConceptSetDiagnostics",
-        start = start,
-        execTime = delta
-      )
-      
-      ParallelLogger::logInfo(
-        "Finding orphan concepts took ",
-        signif(delta, 3),
-        " ",
-        attr(delta, "units")
-      )
-    }
-  }
-  
   # put all instantiated concepts into #concept_ids table
   # this is extracted with vocabulary tables
   # this will have more codes than included source concepts

R/runIncludedSourceConcepts.R

@@ -1,4 +1,12 @@
 
+
+getIncludedSourceConcepts <- function() {
+  
+}
+
+
+
+
 #' Title
 #'
 #' @param connection