Merge pull request #584 from mlr-org/refactor_bench_plots

refactor benchplots

Author: berndbischl

NAMESPACE

@@ -639,14 +639,12 @@ export(fp)
 export(fpr)
 export(friedmanPostHocTestBMR)
 export(friedmanTestBMR)
-export(generateBenchmarkSummaryData)
 export(generateCalibrationData)
 export(generateCritDifferencesData)
 export(generateFilterValuesData)
 export(generateLearningCurveData)
 export(generatePartialPredictionData)
 export(generateROCRCurvesData)
-export(generateRankMatrixAsBarData)
 export(generateThreshVsPerfData)
 export(getBMRAggrPerformances)
 export(getBMRFeatSelResults)
@@ -819,8 +817,9 @@ export(normalizeFeatures)
 export(npv)
 export(oversample)
 export(performance)
-export(plotBenchmarkResult)
-export(plotBenchmarkSummary)
+export(plotBMRBoxplots)
+export(plotBMRRanksAsBarChart)
+export(plotBMRSummary)
 export(plotCalibration)
 export(plotCritDifferences)
 export(plotFilterValues)
@@ -833,7 +832,6 @@ export(plotPartialPredictionGGVIS)
 export(plotROCCurves)
 export(plotROCRCurves)
 export(plotROCRCurvesGGVIS)
-export(plotRankMatrixAsBar)
 export(plotThreshVsPerf)
 export(plotThreshVsPerfGGVIS)
 export(plotTuneMultiCritResult)

R/BenchmarkResultOrderLevels.R

@@ -1,36 +1,27 @@
 # order levels of task.ids in a BenchmarkResult or a similar data.frame
-# usefull for plotting in ggplot2
-
+# useful for plotting in ggplot2
+# if order.tsks is NULL, just return the df
 orderBMRTasks = function(bmr, df = NULL, order.tsks) {
-  
-  assertClass(bmr, "BenchmarkResult")
-  assertVector(order.tsks, len = length(getBMRTaskIds(bmr)))
-  
   if (is.null(df))
     df = as.data.frame(bmr)
-  assertClass(df, "data.frame")
-  assertCharacter(order.tsks)
-  assertSetEqual(order.tsks, getBMRTaskIds(bmr), ordered = FALSE) 
-
-  df$task.id = factor(df$task.id, order.tsks)
+  if (!is.null(order.tsks)) {
+    assertCharacter(order.tsks, len = length(getBMRTaskIds(bmr)))
+    assertSetEqual(order.tsks, getBMRTaskIds(bmr), ordered = FALSE)
+    df$task.id = factor(df$task.id, order.tsks)
+  }
   return(df)
 }
 
-
-
 # order levels of learner.ids of a BenchmarkResult or similar data.frame
-# usefull for plotting in ggplot2
+# useful for plotting in ggplot2
+# if order.tsks is NULL, just return the df
 orderBMRLrns = function(bmr, df = NULL, order.lrns){
-  
-  assertClass(bmr, "BenchmarkResult")
-  assertVector(order.lrns, len = length(getBMRLearnerIds(bmr)))
-  
-  # create df and/or getLearnerIds
   if (is.null(df))
     df = as.data.frame(bmr)
-    
-  assertCharacter(order.lrns)
-  assertSetEqual(order.lrns, getBMRLearnerIds(bmr), ordered = FALSE) 
-  df$learner.id = factor(df$learner.id, order.lrns)
+  if (!is.null(order.lrns)) {
+    assertCharacter(order.lrns, len = length(getBMRLearnerIds(bmr)))
+    assertSetEqual(order.lrns, getBMRLearnerIds(bmr), ordered = FALSE)
+    df$learner.id = factor(df$learner.id, order.lrns)
+  }
   return(df)
 }

R/benchmark.R

@@ -26,6 +26,17 @@
 #' @return [\code{\link{BenchmarkResult}}].
 #' @family benchmark
 #' @export
+#' @examples
+#' lrns = list(makeLearner("classif.lda"), makeLearner("classif.rpart"))
+#' tasks = list(iris.task, sonar.task)
+#' rdesc = makeResampleDesc("CV", iters = 2L)
+#' meas = list(acc, ber)
+#' bmr = benchmark(lrns, tasks, rdesc, measures = meas)
+#' rmat = convertBMRToRankMatrix(bmr)
+#' print(rmat)
+#' plotBMRSummary(bmr)
+#' plotBMRBoxplots(bmr, ber, style = "violin")
+#' plotBMRRanksAsBarChart(bmr, pos = "stack")
 benchmark = function(learners, tasks, resamplings, measures, keep.pred = TRUE, models = TRUE, show.info = getMlrOption("show.info")) {
   learners = ensureVector(learners, 1L, "Learner")
   assertList(learners, min.len = 1L)

R/checkBMRMeasure.R

@@ -0,0 +1,11 @@
+# small arg checker for a selected measure for a BMR
+# if NULL, the 1st measure in the BMR is returned
+checkBMRMeasure = function(measure, bmr) {
+  if (is.null(measure)) {
+    measure = getBMRMeasures(bmr)[[1]]
+  } else {
+    assertClass(measure, "Measure")
+    assertChoice(measure$id, getBMRMeasureIds(bmr))
+  }
+  return(measure)
+}

R/convertBMRToRankMatrix.R

@@ -1,39 +1,30 @@
 #' @title Convert BenchmarkResult to a rank-matrix.
-#' 
-#' @description Computes a matrix of all the ranks of different algorithms
-#' over different datasets (tasks). Ranks are computed from aggregated 
-#' measures.
-#' 
+#'
+#' @description
+#' Computes a matrix of all the ranks of different algorithms over different datasets (tasks).
+#' Ranks are computed from aggregated measures.
+#' Smaller ranks imply better methods, so for measures that are minimized, small ranks imply small scores.
+#' for measures that are maximized, small ranks imply large scores.
+#'
 #' @template arg_bmr
 #' @template arg_measure
-#' @template arg_aggregation_method
 #' @param ties.method [\code{character(1)}]\cr
-#'   see \code{\link{rank}} for details.
+#'   See \code{\link{rank}} for details.
+#' @template arg_aggregation_method
 #' @return [\code{matrix}]
-#' Matrix, with measure ranks as entries. \cr  
-#' The matrix has one row for each \code{learner},
+#' Matrix, with rank of learners per task as entries. The matrix has one row for each \code{learner},
 #' and one column for each \code{task}.
-#' 
-#' 
-#' @examples 
-#' # see benchmark
-#' # convertBMRToRankMatrix(res, acc)
-#' 
 #' @family benchmark
 #' @export
-
-convertBMRToRankMatrix = function(bmr, measure = NULL, ties.method = "average",
-                                  aggregation = "default") {
-  
+#' @examples
+#' # see benchmark
+convertBMRToRankMatrix = function(bmr, measure = NULL, ties.method = "average", aggregation = "default") {
   assertClass(bmr, "BenchmarkResult")
-  if (is.null(measure))
-    measure = getBMRMeasures(bmr)[[1]]
-  assertClass(measure, "Measure")
-  assertChoice(measure$id, getBMRMeasureIds(bmr))
+  measure = checkBMRMeasure(measure, bmr)
   assertChoice(aggregation, c("mean", "default"))
-  
+
   # aggregate mean over iterations
-  if (aggregation == "mean") { 
+  if (aggregation == "mean") {
     df = as.data.frame(bmr)
     df = aggregate(df[[measure$id]],
                    by = list(task.id = df$task.id,
@@ -45,22 +36,20 @@ convertBMRToRankMatrix = function(bmr, measure = NULL, ties.method = "average",
     df = df[, c("task.id", "learner.id", aggr.meas)]
     names(df)[names(df) == aggr.meas] = c("x")
   }
-  
-  # calculate ranks, rank according to minimize option of the measure 
-  # due to a bug in plyr, eval(parse()) has to be called in order to 
-  # dynamically adjust the ties.method
+
+  # calculate ranks, rank according to minimize option of the measure
   if (!measure$minimize)
-    df$x = desc(df$x)
-  eval(parse(text = paste0(
-    "df = ddply(df, .(task.id), transform, alg.rank = rank(x, ties.method = '",
-    ties.method, "'))"
-  )))
-  
+    df$x = -df$x
+  df = ddply(df, "task.id", function(d) {
+    d$alg.rank = rank(d$x, ties.method = ties.method)
+    return(d)
+  })
+
   # convert into matrix, rows = leaner, cols = tasks
   df = melt(df, c("task.id", "learner.id"), "alg.rank")
   df = dcast(df, learner.id ~ task.id )
   rownames(df) = df$learner.id
   mat = as.matrix(df[,colnames(df) != "learner.id"])
-  
+
   return(mat)
 }