feat(tier4_perception_launch): add option for new TL detector model (a…

…utowarefoundation#9731) * feat: add traffic_light_detector launch Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: tier4 perception launch Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: add multi tlr detector launch Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: tier4 launch Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: tl detector launch Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: data director Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: precision int8 Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> chore: revert to fp16 Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> feat: remove occlusion and add car ped classification merger Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: launch for multi camera Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> chore: pre-commit Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> fix: update matching score Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> feat: add max_iou_threshold Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> feat: add occlusion unknown classifier Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: tl detector launch Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * refactor: traffic_light_launch.xml Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: remove tl fine detector Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: refactor Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * chore: pre-commit Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: cspelling check Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: error after rename package Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: default tl model name Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * fix: new tlr for multi cameras Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> * modify args Signed-off-by: MasatoSaeki <masato.saeki@tier4.jp> * style(pre-commit): autofix * refactor Signed-off-by: MasatoSaeki <masato.saeki@tier4.jp> * add category_merger to container Signed-off-by: MasatoSaeki <masato.saeki@tier4.jp> * fix args Signed-off-by: MasatoSaeki <masato.saeki@tier4.jp> * run pre-commit Signed-off-by: MasatoSaeki <masato.saeki@tier4.jp> --------- Signed-off-by: badai-nguyen <dai.nguyen@tier4.jp> Signed-off-by: MasatoSaeki <masato.saeki@tier4.jp> Co-authored-by: Masato Saeki <78376491+MasatoSaeki@users.noreply.github.com> Co-authored-by: MasatoSaeki <masato.saeki@tier4.jp> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
tier4 · Feb 19, 2025 · d5e923e · d5e923e
1 parent 4a77f3d
commit d5e923e
Show file tree

Hide file tree

Showing 9 changed files with 147 additions and 22 deletions.
diff --git a/launch/tier4_perception_launch/launch/perception.launch.xml b/launch/tier4_perception_launch/launch/perception.launch.xml
@@ -100,7 +100,10 @@
 
   <!-- Traffic light recognition parameters -->
   <arg name="use_traffic_light_recognition" default="false"/>
-  <arg name="traffic_light_recognition/enable_fine_detection" default="true"/>
+  <arg name="traffic_light_recognition/use_ml_detector" default="false" description="where ml model is used for TL detection"/>
+  <arg name="traffic_light_recognition/ml_detection_model_type" default="fine_detection_model" description="select ml model for TL detection: fine_detection_model or whole_image_detection_model"/>
+  <arg name="traffic_light_recognition/whole_image_detector_model_path" default="$(var data_path)/tensorrt_yolox"/>
+  <arg name="traffic_light_recognition/whole_image_detector_model_name" default="yolox_s_car_ped_tl_detector_960_960_batch_1.onnx"/>
   <arg name="traffic_light_recognition/fusion_only" default="false"/>
   <arg name="all_traffic_light_camera" default="[camera6, camera7]" description="choose camera which use for traffic light recognition"/>
   <arg
@@ -307,7 +310,10 @@
     <group if="$(var use_traffic_light_recognition)">
       <push-ros-namespace namespace="traffic_light_recognition"/>
       <include file="$(find-pkg-share tier4_perception_launch)/launch/traffic_light_recognition/traffic_light.launch.xml">
-        <arg name="enable_fine_detection" value="$(var traffic_light_recognition/enable_fine_detection)"/>
+        <arg name="use_ml_detector" value="$(var traffic_light_recognition/use_ml_detector)"/>
+        <arg name="ml_detection_model_type" value="$(var traffic_light_recognition/ml_detection_model_type)"/>
+        <arg name="whole_image_detector_model_path" value="$(var traffic_light_recognition/whole_image_detector_model_path)"/>
+        <arg name="whole_image_detector_model_name" value="$(var traffic_light_recognition/whole_image_detector_model_name)"/>
         <arg name="fusion_only" value="$(var traffic_light_recognition/fusion_only)"/>
         <arg name="all_camera_namespaces" value="$(var all_traffic_light_camera)"/>
         <arg name="traffic_light_arbiter_param_path" value="$(var traffic_light_arbiter_param_path)"/>

diff --git a/launch/tier4_perception_launch/launch/traffic_light_recognition/traffic_light.launch.xml b/launch/tier4_perception_launch/launch/traffic_light_recognition/traffic_light.launch.xml
@@ -1,7 +1,8 @@
 <?xml version="1.0"?>
 <launch>
   <arg name="enable_image_decompressor" default="true" description="enable image decompressor"/>
-  <arg name="enable_fine_detection" default="true" description="enable fine position adjustment of traffic light"/>
+  <arg name="use_ml_detector" default="true" description="enable fine position adjustment of traffic light"/>
+  <arg name="ml_detection_model_type" default="fine_detection_model" description="select ml model for TL detection: fine_detection_model or whole_image_detection_model"/>
   <arg name="fusion_only" default="false" description="launch only occlusion_predictor and multi_camera_fusion"/>
   <arg name="input/cloud" default="/sensing/lidar/top/pointcloud_raw_ex" description="point cloud for occlusion prediction"/>
   <arg name="judged/traffic_signals" default="/perception/traffic_light_recognition/judged/traffic_signals"/>
@@ -13,6 +14,8 @@
   <arg name="all_camera_namespaces" default="[camera6, camera7]"/>
 
   <!-- ML parameters -->
+  <arg name="whole_image_detector_model_path" default="$(find-pkg-share autoware_tensorrt_yolox)/data"/>
+  <arg name="whole_image_detector_model_name" default="yolox_s_car_ped_tl_detector_960_960_batch_1.onnx"/>
   <arg name="fine_detector_param_path" default="$(find-pkg-share autoware_traffic_light_fine_detector)/config/traffic_light_fine_detector.param.yaml"/>
   <arg name="car_classifier_param_path" default="$(find-pkg-share autoware_traffic_light_classifier)/config/car_traffic_light_classifier.param.yaml"/>
   <arg name="pedestrian_classifier_param_path" default="$(find-pkg-share autoware_traffic_light_classifier)/config/pedestrian_traffic_light_classifier.param.yaml"/>
@@ -32,22 +35,25 @@
     </include>
     <include file="$(find-pkg-share tier4_perception_launch)/launch/traffic_light_recognition/traffic_light_map_based_detector.launch.py">
       <arg name="all_camera_namespaces" value="$(var all_camera_namespaces)"/>
-      <arg name="enable_fine_detection" value="$(var enable_fine_detection)"/>
+      <arg name="use_ml_detector" value="$(var use_ml_detector)"/>
     </include>
     <include file="$(find-pkg-share tier4_perception_launch)/launch/traffic_light_recognition/traffic_light_node_container.launch.py">
       <arg name="all_camera_namespaces" value="$(var all_camera_namespaces)"/>
       <arg name="enable_image_decompressor" value="$(var enable_image_decompressor)"/>
-      <arg name="enable_fine_detection" value="$(var enable_fine_detection)"/>
+      <arg name="ml_detection_model_type" value="$(var ml_detection_model_type)"/>
+      <arg name="use_ml_detector" value="$(var use_ml_detector)"/>
       <arg name="use_intra_process" value="true"/>
       <arg name="use_multithread" value="true"/>
+      <arg name="whole_image_detector_model_path" value="$(var whole_image_detector_model_path)"/>
+      <arg name="whole_image_detector_model_name" value="$(var whole_image_detector_model_name)"/>
       <arg name="fine_detector_param_path" value="$(var fine_detector_param_path)"/>
       <arg name="car_classifier_param_path" value="$(var car_classifier_param_path)"/>
       <arg name="pedestrian_classifier_param_path" value="$(var pedestrian_classifier_param_path)"/>
     </include>
   </group>
 
   <!-- traffic_light_occlusion_predictor on each camera -->
-  <group>
+  <group if="$(eval &quot;'$(var ml_detection_model_type)'=='fine_detection_model'&quot;)">
     <include file="$(find-pkg-share tier4_perception_launch)/launch/traffic_light_recognition/traffic_light_occlusion_predictor.launch.py">
       <arg name="all_camera_namespaces" value="$(var all_camera_namespaces)"/>
       <arg name="input/cloud" value="$(var input/cloud)"/>

diff --git a/...eption_launch/launch/traffic_light_recognition/traffic_light_map_based_detector.launch.py b/...eption_launch/launch/traffic_light_recognition/traffic_light_map_based_detector.launch.py
@@ -32,7 +32,7 @@ def create_traffic_light_map_based_detector(namespace, context):
 
     output_rois = (
         "rough/rois"
-        if IfCondition(LaunchConfiguration("enable_fine_detection")).evaluate(context)
+        if IfCondition(LaunchConfiguration("use_ml_detector")).evaluate(context)
         else f"/perception/traffic_light_recognition/{namespace}/detection/rois"
     )
 
@@ -87,9 +87,9 @@ def add_launch_arg(name: str, default_value=None, description=None):
 
     add_launch_arg("all_camera_namespaces", "[camera6, camera7]")
     add_launch_arg(
-        "enable_fine_detection",
+        "use_ml_detector",
         "True",
-        "If True, output_topic will be for fine detector, otherwise for classifier",
+        "If True, output_topic will be for ml detector, otherwise for classifier",
     )
 
     return launch.LaunchDescription(

diff --git a/...perception_launch/launch/traffic_light_recognition/traffic_light_node_container.launch.py b/...perception_launch/launch/traffic_light_recognition/traffic_light_node_container.launch.py
@@ -24,6 +24,8 @@
 from launch.conditions import IfCondition
 from launch.conditions import UnlessCondition
 from launch.substitutions import LaunchConfiguration
+from launch.substitutions import PathJoinSubstitution
+from launch.substitutions import PythonExpression
 from launch_ros.actions import ComposableNodeContainer
 from launch_ros.actions import LoadComposableNodes
 from launch_ros.actions import PushRosNamespace
@@ -60,11 +62,13 @@ def launch_setup(context, *args, **kwargs):
 
 def create_traffic_light_node_container(namespace, context, *args, **kwargs):
     camera_arguments = {
+        "input/camera_info": f"/sensing/camera/{namespace}/camera_info",
         "input/image": f"/sensing/camera/{namespace}/image_raw",
         "output/rois": f"/perception/traffic_light_recognition/{namespace}/detection/rois",
         "output/traffic_signals": f"/perception/traffic_light_recognition/{namespace}/classification/traffic_signals",
         "output/car/traffic_signals": f"/perception/traffic_light_recognition/{namespace}/classification/car/traffic_signals",
         "output/pedestrian/traffic_signals": f"/perception/traffic_light_recognition/{namespace}/classification/pedestrian/traffic_signals",
+        "output/debug": f"/perception/traffic_light_recognition/{namespace}/detection/rois/debug",
     }
 
     def create_parameter_dict(*args):
@@ -126,7 +130,7 @@ def create_parameter_dict(*args):
                 package="autoware_traffic_light_visualization",
                 plugin="autoware::traffic_light::TrafficLightRoiVisualizerNode",
                 name="traffic_light_roi_visualizer",
-                parameters=[create_parameter_dict("enable_fine_detection", "use_image_transport")],
+                parameters=[create_parameter_dict("use_ml_detector", "use_image_transport")],
                 remappings=[
                     ("~/input/image", camera_arguments["input/image"]),
                     ("~/input/rois", camera_arguments["output/rois"]),
@@ -192,13 +196,110 @@ def create_parameter_dict(*args):
             ),
         ],
         target_container=container,
-        condition=IfCondition(LaunchConfiguration("enable_fine_detection")),
+        condition=IfCondition(
+            PythonExpression(
+                [
+                    "'",
+                    LaunchConfiguration("ml_detection_model_type"),
+                    "' == 'fine_detection_model' ",
+                ]
+            )
+        ),
+    )
+
+    # cspell: ignore semseg
+    whole_img_detector_loader = LoadComposableNodes(
+        composable_node_descriptions=[
+            ComposableNode(
+                package="autoware_tensorrt_yolox",
+                plugin="autoware::tensorrt_yolox::TrtYoloXNode",
+                name="traffic_light_detector",
+                namespace=f"{namespace}/detection",
+                parameters=[
+                    LaunchConfiguration("whole_image_detector_param_path"),
+                    {
+                        "model_path": PathJoinSubstitution(
+                            [
+                                LaunchConfiguration("whole_image_detector_model_path"),
+                                LaunchConfiguration("whole_image_detector_model_name"),
+                            ]
+                        ),
+                        "label_path": PathJoinSubstitution(
+                            [
+                                LaunchConfiguration("whole_image_detector_model_path"),
+                                "car_ped_tl_detector_labels.txt",
+                            ]
+                        ),
+                        "build_only": False,
+                        "clip_value": 0.0,
+                    },
+                ],
+                remappings=[
+                    ("~/in/image", camera_arguments["input/image"]),
+                    ("~/out/objects", "ml_detected/rois"),
+                    ("~/out/image", camera_arguments["output/debug"] + "/image"),
+                    (
+                        "~/out/image/compressed",
+                        camera_arguments["output/debug"] + "/image/compressed",
+                    ),
+                    (
+                        "~/out/image/compressedDepth",
+                        camera_arguments["output/debug"] + "/image/compressedDepth",
+                    ),
+                    ("~/out/image/theora", camera_arguments["output/debug"] + "/image/theora"),
+                ],
+                extra_arguments=[
+                    {"use_intra_process_comms": LaunchConfiguration("use_intra_process")}
+                ],
+            ),
+            ComposableNode(
+                package="autoware_traffic_light_selector",
+                plugin="autoware::traffic_light::TrafficLightSelectorNode",
+                name="traffic_light_selector",
+                namespace=f"{namespace}/detection",
+                parameters=[
+                    {
+                        "max_iou_threshold": -0.5,
+                    }
+                ],
+                remappings=[
+                    ("input/detected_rois", "ml_detected/rois"),
+                    ("input/rough_rois", "rough/rois"),
+                    ("input/expect_rois", "expect/rois"),
+                    ("input/camera_info", camera_arguments["input/camera_info"]),
+                    ("output/traffic_rois", camera_arguments["output/rois"]),
+                ],
+            ),
+            ComposableNode(
+                package="autoware_traffic_light_category_merger",
+                plugin="autoware::traffic_light::TrafficLightCategoryMergerNode",
+                name="traffic_light_category_merger",
+                namespace=f"{namespace}/classification",
+                parameters=[],
+                remappings=[
+                    ("input/car_signals", "classified/car/traffic_signals"),
+                    ("input/pedestrian_signals", "classified/pedestrian/traffic_signals"),
+                    ("output/traffic_signals", camera_arguments["output/traffic_signals"]),
+                ],
+            ),
+        ],
+        target_container=container,
+        condition=IfCondition(
+            PythonExpression(
+                [
+                    "'",
+                    LaunchConfiguration("ml_detection_model_type"),
+                    "' == 'whole_image_detection_model' ",
+                ]
+            )
+        ),
     )
 
     return [
         GroupAction([PushRosNamespace(namespace), container]),
         decompressor_loader,
         fine_detector_loader,
+        whole_img_detector_loader,
     ]
 
 
@@ -211,11 +312,13 @@ def add_launch_arg(name: str, default_value=None, description=None):
             DeclareLaunchArgument(name, default_value=default_value, description=description)
         )
 
+    tensorrt_yolox_share_dir = get_package_share_directory("autoware_tensorrt_yolox")
     fine_detector_share_dir = get_package_share_directory("autoware_traffic_light_fine_detector")
     classifier_share_dir = get_package_share_directory("autoware_traffic_light_classifier")
     add_launch_arg("all_camera_namespaces", "[camera6, camera7]")
     add_launch_arg("enable_image_decompressor", "True")
-    add_launch_arg("enable_fine_detection", "True")
+    add_launch_arg("use_ml_detector", "True")
+    add_launch_arg("ml_detection_model_type", "fine_detection_model")
     add_launch_arg("use_image_transport", "True")
 
     # traffic_light_fine_detector
@@ -224,6 +327,16 @@ def add_launch_arg(name: str, default_value=None, description=None):
         os.path.join(fine_detector_share_dir, "config", "traffic_light_fine_detector.param.yaml"),
     )
 
+    # whole image (traffic light) detector by yolox
+    add_launch_arg(
+        "whole_image_detector_model_path", os.path.expandvars("$HOME/autoware_data/tensorrt_yolox")
+    )
+    add_launch_arg("whole_image_detector_model_name", "tlr_car_ped_yolox_s_960_960_batch_1")
+    add_launch_arg(
+        "whole_image_detector_param_path",
+        os.path.join(tensorrt_yolox_share_dir, "config", "yolox_traffic_light_detector.param.yaml"),
+    )
+
     # traffic_light_classifier
     add_launch_arg(
         "car_classifier_param_path",

diff --git a/...eption/autoware_traffic_light_visualization/config/traffic_light_visualization.param.yaml b/...eption/autoware_traffic_light_visualization/config/traffic_light_visualization.param.yaml
@@ -1,4 +1,4 @@
 /**:
   ros__parameters:
-    enable_fine_detection: false
+    use_ml_detector: false
     use_image_transport: true
diff --git a/...ption/autoware_traffic_light_visualization/launch/traffic_light_roi_visualizer.launch.xml b/...ption/autoware_traffic_light_visualization/launch/traffic_light_roi_visualizer.launch.xml
@@ -4,7 +4,7 @@
   <arg name="input/rough/rois" default="~/rough/rois"/>
   <arg name="input/traffic_signals" default="~/input/traffic_signals"/>
   <arg name="output/image" default="~/debug/rois"/>
-  <arg name="enable_fine_detection" default="false"/>
+  <arg name="use_ml_detector" default="false"/>
   <arg name="use_image_transport" default="true"/>
 
   <node pkg="autoware_traffic_light_visualization" exec="traffic_light_visualization_node" name="traffic_light_visualization">
@@ -13,7 +13,7 @@
     <remap from="~/input/rough/rois" to="$(var input/rough/rois)"/>
     <remap from="~/input/traffic_signals" to="$(var input/traffic_signals)"/>
     <remap from="~/output/image" to="$(var output/image)"/>
-    <param name="enable_fine_detection" value="$(var enable_fine_detection)"/>
+    <param name="use_ml_detector" value="$(var use_ml_detector)"/>
     <param name="use_image_transport" value="$(var use_image_transport)"/>
   </node>
 </launch>
diff --git a/...ption/autoware_traffic_light_visualization/schema/traffic_light_visualization.schema.json b/...ption/autoware_traffic_light_visualization/schema/traffic_light_visualization.schema.json
@@ -6,7 +6,7 @@
     "traffic_light_visualization": {
       "type": "object",
       "properties": {
-        "enable_fine_detection": {
+        "use_ml_detector": {
           "type": "boolean",
           "description": "whether to visualize result of the traffic light fine detection",
           "default": "false"
@@ -17,7 +17,7 @@
           "default": "true"
         }
       },
-      "required": ["enable_fine_detection", "use_image_transport"]
+      "required": ["use_ml_detector", "use_image_transport"]
     }
   },
   "properties": {

diff --git a/perception/autoware_traffic_light_visualization/src/traffic_light_roi_visualizer/node.cpp b/perception/autoware_traffic_light_visualization/src/traffic_light_roi_visualizer/node.cpp
@@ -30,10 +30,10 @@ TrafficLightRoiVisualizerNode::TrafficLightRoiVisualizerNode(const rclcpp::NodeO
   using std::placeholders::_2;
   using std::placeholders::_3;
   using std::placeholders::_4;
-  enable_fine_detection_ = this->declare_parameter<bool>("enable_fine_detection");
+  use_ml_detector_ = this->declare_parameter<bool>("use_ml_detector");
   use_image_transport_ = this->declare_parameter<bool>("use_image_transport");
 
-  if (enable_fine_detection_) {
+  if (use_ml_detector_) {
     sync_with_rough_roi_.reset(new SyncWithRoughRoi(
       SyncPolicyWithRoughRoi(10), image_sub_, roi_sub_, rough_roi_sub_, traffic_signals_sub_));
     sync_with_rough_roi_->registerCallback(
@@ -69,15 +69,15 @@ void TrafficLightRoiVisualizerNode::connectCb()
     image_sub_.unsubscribe();
     traffic_signals_sub_.unsubscribe();
     roi_sub_.unsubscribe();
-    if (enable_fine_detection_) {
+    if (use_ml_detector_) {
       rough_roi_sub_.unsubscribe();
     }
   } else if (!image_sub_.getSubscriber()) {
     image_sub_.subscribe(this, "~/input/image", "raw", rmw_qos_profile_sensor_data);
     roi_sub_.subscribe(this, "~/input/rois", rclcpp::QoS{1}.get_rmw_qos_profile());
     traffic_signals_sub_.subscribe(
       this, "~/input/traffic_signals", rclcpp::QoS{1}.get_rmw_qos_profile());
-    if (enable_fine_detection_) {
+    if (use_ml_detector_) {
       rough_roi_sub_.subscribe(this, "~/input/rough/rois", rclcpp::QoS{1}.get_rmw_qos_profile());
     }
   }

diff --git a/perception/autoware_traffic_light_visualization/src/traffic_light_roi_visualizer/node.hpp b/perception/autoware_traffic_light_visualization/src/traffic_light_roi_visualizer/node.hpp
@@ -180,7 +180,7 @@ class TrafficLightRoiVisualizerNode : public rclcpp::Node
   typedef message_filters::Synchronizer<SyncPolicyWithRoughRoi> SyncWithRoughRoi;
   std::shared_ptr<SyncWithRoughRoi> sync_with_rough_roi_;
 
-  bool enable_fine_detection_;
+  bool use_ml_detector_;
   bool use_image_transport_;
 };