Merge pull request #270 from JdeRobot/269-allow-for-en-arbitrary-choice-of-splits-during-evaluation

Allow for an arbitrary choice of splits during evaluation

Author: dpascualhe

detectionmetrics/cli/evaluate.py

@@ -46,15 +46,15 @@ def get_dataset(
             raise ValueError("--dataset_ontology is required for 'rellis3d' format")
 
     elif dataset_format in ["goose", "generic"]:
-        if split == "train" and train_dataset_dir is None:
+        if "train" in split and train_dataset_dir is None:
             raise ValueError(
                 "--train_dataset_dir is required for 'train' split in 'goose' format"
             )
-        elif split == "val" and val_dataset_dir is None:
+        elif "val" in split and val_dataset_dir is None:
             raise ValueError(
                 "--val_dataset_dir is required for 'val' split in 'goose' format"
             )
-        elif split == "test" and test_dataset_dir is None:
+        elif "test" in split and test_dataset_dir is None:
             raise ValueError(
                 "--test_dataset_dir is required for 'test' split in 'goose' format"
             )
@@ -118,6 +118,17 @@ def get_dataset(
         )
     return datasets.REGISTRY[dataset_name](**dataset_args)
 
+def parse_split(ctx, param, value):
+    """Parse split argument"""
+    splits = value.split(",")
+    valid_splits = ["train", "val", "test"]
+    if not all(split in valid_splits for split in splits):
+        raise click.BadParameter(
+            f"Split must be one of {valid_splits} or a comma-separated list of them",
+            param_hint=value,
+        )
+
+    return splits
 
 @click.command(name="evaluate", help="Evaluate model on dataset")
 @click.argument("task", type=click.Choice(["segmentation"], case_sensitive=False))
@@ -219,10 +230,10 @@ def get_dataset(
 )
 @click.option(
     "--split",
-    type=click.Choice(["train", "val", "test"], case_sensitive=False),
     show_default=True,
     default="test",
-    help="Name of the split to be evaluated",
+    callback=parse_split,
+    help="Name of the split or splits separated by commas to be evaluated",
 )
 @click.option(
     "--ontology_translation",
@@ -260,6 +271,8 @@ def evaluate(
     out_fname,
 ):
     """Evaluate model on dataset"""
+    if isinstance(split, str):  # if evaluate has been called directly
+        split = parse_split(None, None, split)
 
     model = get_model(task, input_type, model_format, model, model_ontology, model_cfg)
     dataset = get_dataset(

detectionmetrics/datasets/dataset.py

@@ -44,7 +44,28 @@ def append(self, new_dataset: Self):
         :param new_dataset: Dataset to be appended
         :type new_dataset: Self
         """
-        assert self.ontology == new_dataset.ontology, "Ontologies don't match"
+        if not self.has_label_count:
+            assert self.ontology == new_dataset.ontology, "Ontologies don't match"
+        else:
+            # Check if classes match
+            assert (
+                self.ontology.keys() == new_dataset.ontology.keys()
+            ), "Ontologies don't match"
+            for class_name in self.ontology:
+                # Check if indices, and RGB values match
+                assert (
+                    self.ontology[class_name]["idx"]
+                    == new_dataset.ontology[class_name]["idx"]
+                ), "Ontologies don't match"
+                assert (
+                    self.ontology[class_name]["rgb"]
+                    == new_dataset.ontology[class_name]["rgb"]
+                ), "Ontologies don't match"
+
+                # Accumulate label count
+                self.ontology[class_name]["label_count"] += new_dataset.ontology[
+                    class_name
+                ]["label_count"]
 
         # Global filenames to avoid dealing with each dataset relative location
         self.make_fname_global()

detectionmetrics/models/model.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 import os
-from typing import Any, Optional, Union
+from typing import Any, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -67,15 +67,15 @@ def inference(
     def eval(
         self,
         dataset: dm_dataset.SegmentationDataset,
-        split: str = "all",
+        split: str | List[str] = "test",
         ontology_translation: Optional[str] = None,
     ) -> pd.DataFrame:
         """Perform evaluation for an image segmentation dataset
 
         :param dataset: Segmentation dataset for which the evaluation will be performed
         :type dataset: ImageSegmentationDataset
-        :param split: Split to be used from the dataset, defaults to "all"
-        :type split: str, optional
+        :param split: Split or splits to be used from the dataset, defaults to "test"
+        :type split: str | List[str], optional
         :param ontology_translation: JSON file containing translation between dataset and model output ontologies
         :type ontology_translation: str, optional
         :return: DataFrame containing evaluation results
@@ -158,15 +158,15 @@ def inference(self, image: Image.Image) -> Image.Image:
     def eval(
         self,
         dataset: dm_dataset.ImageSegmentationDataset,
-        split: str = "all",
+        split: str | List[str] = "test",
         ontology_translation: Optional[str] = None,
     ) -> pd.DataFrame:
         """Perform evaluation for an image segmentation dataset
 
         :param dataset: Image segmentation dataset for which the evaluation will be performed
         :type dataset: ImageSegmentationDataset
-        :param split: Split to be used from the dataset, defaults to "all"
-        :type split: str, optional
+        :param split: Split or splits to be used from the dataset, defaults to "test"
+        :type split: str | List[str], optional
         :param ontology_translation: JSON file containing translation between dataset and model output ontologies
         :type ontology_translation: str, optional
         :return: DataFrame containing evaluation results
@@ -215,15 +215,15 @@ def inference(self, points: np.ndarray) -> np.ndarray:
     def eval(
         self,
         dataset: dm_dataset.LiDARSegmentationDataset,
-        split: str = "all",
+        split: str | List[str] = "test",
         ontology_translation: Optional[str] = None,
     ) -> pd.DataFrame:
         """Perform evaluation for a LiDAR segmentation dataset
 
         :param dataset: LiDAR segmentation dataset for which the evaluation will be performed
         :type dataset: LiDARSegmentationDataset
-        :param split: Split to be used from the dataset, defaults to "all"
-        :type split: str, optional
+        :param split: Split or splits to be used from the dataset, defaults to "test"
+        :type split: str | List[str], optional
         :param ontology_translation: JSON file containing translation between dataset and model output ontologies
         :type ontology_translation: str, optional
         :return: DataFrame containing evaluation results

detectionmetrics/models/torch.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 import os
 import time
-from typing import Any, Optional, Tuple, Union
+from typing import Any, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -139,20 +139,19 @@ class ImageSegmentationTorchDataset(Dataset):
     :type transform: transforms.Compose
     :param target_transform: Transformation to be applied to labels
     :type target_transform: transforms.Compose
-    :param split: Split to be used from the dataset, defaults to "all"
-    :type split: str, optional
+    :param splits: Splits to be used from the dataset, defaults to ["test"]
+    :type splits: str, optional
     """
 
     def __init__(
         self,
         dataset: dm_dataset.ImageSegmentationDataset,
         transform: transforms.Compose,
         target_transform: transforms.Compose,
-        split: str = "all",
+        splits: List[str] = ["test"],
     ):
         # Filter split and make filenames global
-        if split != "all":
-            dataset.dataset = dataset.dataset[dataset.dataset["split"] == split]
+        dataset.dataset = dataset.dataset[dataset.dataset["split"].isin(splits)]
         self.dataset = dataset
         self.dataset.make_fname_global()
 
@@ -192,8 +191,8 @@ class LiDARSegmentationTorchDataset(Dataset):
     :type preprocess: callable
     :param n_classes: Number of classes estimated by the model
     :type n_classes: int
-    :param split: Split to be used from the dataset, defaults to "all"
-    :type split: str, optional
+    :param splits: Splits to be used from the dataset, defaults to ["test"]
+    :type splits: str, optional
     """
 
     def __init__(
@@ -202,11 +201,10 @@ def __init__(
         model_cfg: dict,
         preprocess: callable,
         n_classes: int,
-        split: str = "all",
+        splits: str = ["test"],
     ):
         # Filter split and make filenames global
-        if split != "all":
-            dataset.dataset = dataset.dataset[dataset.dataset["split"] == split]
+        dataset.dataset = dataset.dataset[dataset.dataset["split"].isin(splits)]
         self.dataset = dataset
         self.dataset.make_fname_global()
 
@@ -366,15 +364,15 @@ def inference(self, image: Image.Image) -> Image.Image:
     def eval(
         self,
         dataset: dm_dataset.ImageSegmentationDataset,
-        split: str = "all",
+        split: str | List[str] = "test",
         ontology_translation: Optional[str] = None,
     ) -> pd.DataFrame:
         """Perform evaluation for an image segmentation dataset
 
         :param dataset: Image segmentation dataset for which the evaluation will be performed
         :type dataset: ImageSegmentationDataset
-        :param split: Split to be used from the dataset, defaults to "all"
-        :type split: str, optional
+        :param split: Split or splits to be used from the dataset, defaults to "test"
+        :type split: str | List[str], optional
         :param ontology_translation: JSON file containing translation between dataset and model output ontologies
         :type ontology_translation: str, optional
         :return: DataFrame containing evaluation results
@@ -394,7 +392,7 @@ def eval(
             dataset,
             transform=self.transform_input,
             target_transform=self.transform_label,
-            split=split,
+            splits=[split] if isinstance(split, str) else split,
         )
 
         dataloader = DataLoader(
@@ -605,15 +603,15 @@ def inference(self, points: np.ndarray) -> np.ndarray:
     def eval(
         self,
         dataset: dm_dataset.LiDARSegmentationDataset,
-        split: str = "all",
+        split: str | List[str] = "test",
         ontology_translation: Optional[str] = None,
     ) -> pd.DataFrame:
         """Perform evaluation for a LiDAR segmentation dataset
 
         :param dataset: LiDAR segmentation dataset for which the evaluation will be performed
         :type dataset: LiDARSegmentationDataset
-        :param split: Split to be used from the dataset, defaults to "all"
-        :type split: str, optional
+        :param split: Split or splits to be used from the dataset, defaults to "test"
+        :type split: str | List[str], optional
         :param ontology_translation: JSON file containing translation between dataset and model output ontologies
         :type ontology_translation: str, optional
         :return: DataFrame containing evaluation results
@@ -634,7 +632,7 @@ def eval(
             model_cfg=self.model_cfg,
             preprocess=self.preprocess,
             n_classes=self.n_classes,
-            split=split,
+            splits=[split] if isinstance(split, str) else split,
         )
 
         # Init metrics

docs/py_docs/_build/doctrees/detectionmetrics.cli.doctree

@@ -89,6 +89,12 @@ <h2>Submodules<a class="headerlink" href="#submodules" title="Link to this headi
 <span class="sig-prename descclassname"><span class="pre">detectionmetrics.cli.evaluate.</span></span><span class="sig-name descname"><span class="pre">get_model</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">task</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_type</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">model_format</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">model</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ontology</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">model_cfg</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#detectionmetrics.cli.evaluate.get_model" title="Link to this definition"></a></dt>
 <dd></dd></dl>
 
+<dl class="py function">
+<dt class="sig sig-object py" id="detectionmetrics.cli.evaluate.parse_split">
+<span class="sig-prename descclassname"><span class="pre">detectionmetrics.cli.evaluate.</span></span><span class="sig-name descname"><span class="pre">parse_split</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ctx</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">param</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">value</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#detectionmetrics.cli.evaluate.parse_split" title="Link to this definition"></a></dt>
+<dd><p>Parse split argument</p>
+</dd></dl>
+
 </section>
 <section id="module-detectionmetrics.cli">
 <span id="module-contents"></span><h2>Module contents<a class="headerlink" href="#module-detectionmetrics.cli" title="Link to this heading"></a></h2>

docs/py_docs/_build/doctrees/detectionmetrics.models.doctree

@@ -177,6 +177,7 @@ <h2>Subpackages<a class="headerlink" href="#subpackages" title="Link to this hea
 </ul>
 </li>
 <li class="toctree-l3"><a class="reference internal" href="detectionmetrics.models.html#detectionmetrics.models.tensorflow.get_computational_cost"><code class="docutils literal notranslate"><span class="pre">get_computational_cost()</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal" href="detectionmetrics.models.html#detectionmetrics.models.tensorflow.resize_image"><code class="docutils literal notranslate"><span class="pre">resize_image()</span></code></a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="detectionmetrics.models.html#module-detectionmetrics.models.torch">detectionmetrics.models.torch module</a><ul>