Skip to content

Commit e16ebd8

Browse files
lexavtankepre-commit-ci[bot]
lexavtanke
and
pre-commit-ci[bot]
authored
build(tvm_utility): remove download logic from CMake and update documentation (#4923)
* add include tier4_autoware_utils and dependency Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * remove downloading logic from Cmake, update documentation Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): remove downloading logic from Cmake, update documentation Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): fix lint_cmake error Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): format warning message Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): add logic to work with autoware_data folder, add nn config header and test image Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * style(pre-commit): autofix * build(tvm_utility): refactor, update InferenceEngineTVM constructor Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): add lightweight model and test with it Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): make building yolo_v2_tiny disable by default Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): remove test artifact for yolo_v2_tiny Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): update docs Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): update docs Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): update namespace in abs_model test Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): rewrite yolo_v2_tiny as example Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): clean comments in yolo_v2_tiny example main.cpp Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): add launch file for yolo_v2_tiny example Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): update yolo_v2_tiny example readme Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): add model for arm based systems, need to be tested on device Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * style(pre-commit): autofix * build(tvm_utility): update config header for arm Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): remove debug output Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): add find_package conditional section Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): fix lint_cmake errors Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): remove coping model files during build Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): update readme with new data folder structure Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): fix spell check warnings Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): add no model files guard to get_neural_network Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): set back default paths in config headers Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): add param file, update launch file Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): add schema file, update node name Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): fix json-schema-check Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): fix json-schema-check Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * build(tvm_utility): add parameter table to example readme Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * build(tvm_utility): fix typo-error in description of schema.json Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> * style(pre-commit): autofix * buiild(tvm_utility): fix spell-check warning and typo Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> --------- Signed-off-by: Alexey Panferov <lexavtanke@gmail.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 27f28d6 commit e16ebd8

21 files changed

+649
-144
lines changed

‎.cspell-partial.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@
55
"perception/bytetrack/lib/**"
66
],
77
"ignoreRegExpList": [],
8-
"words": []
8+
"words": ["dltype", "tvmgen"]
99
}

‎common/tvm_utility/.gitignore

-2
Original file line numberDiff line numberDiff line change
@@ -1,2 +0,0 @@
1-
artifacts/**/*.jpg
2-
data/

‎common/tvm_utility/CMakeLists.txt

+67-13
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,19 @@ set(TVM_UTILITY_NODE_LIB_HEADERS
2929
ament_auto_add_library(${PROJECT_NAME} SHARED ${TVM_UTILITY_NODE_LIB_HEADERS})
3030
set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
3131

32-
if(BUILD_TESTING)
32+
set(BUILD_EXAMPLE OFF CACHE BOOL "enable build yolo_v2_tiny")
33+
34+
if(BUILD_TESTING OR BUILD_EXAMPLE)
35+
find_package(OpenCV REQUIRED)
36+
set(tvm_runtime_DIR ${tvm_vendor_DIR})
37+
find_package(tvm_runtime CONFIG REQUIRED)
3338
# Get target backend
3439
set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend")
40+
endif()
3541

42+
if(BUILD_TESTING)
3643
# compile each folder inside test/ as a test case
3744
find_package(ament_cmake_gtest REQUIRED)
38-
find_package(OpenCV REQUIRED)
39-
set(tvm_runtime_DIR ${tvm_vendor_DIR})
40-
find_package(tvm_runtime CONFIG REQUIRED)
4145

4246
set(TEST_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts")
4347
file(GLOB TEST_CASES test/*)
@@ -47,17 +51,11 @@ if(BUILD_TESTING)
4751
endif()
4852
# the folder name becomes the test case name
4953
file(RELATIVE_PATH TEST_CASE_NAME ${CMAKE_CURRENT_LIST_DIR}/test ${TEST_FOLDER})
50-
5154
# Get neural network.
5255
set(NN_DEPENDENCY "")
53-
get_neural_network(${TEST_CASE_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY)
56+
get_neural_network(${TEST_CASE_NAME}_${CMAKE_SYSTEM_PROCESSOR} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY)
5457

5558
if(NOT NN_DEPENDENCY STREQUAL "")
56-
if(TEST_CASE_NAME STREQUAL "yolo_v2_tiny" AND
57-
NOT EXISTS ${TEST_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg)
58-
message(WARNING "Missing image artifact for yolo_v2_tiny, skipping test")
59-
continue()
60-
endif()
6159
# add all cpp files in the folder to the target
6260
file(GLOB TEST_CASE_SOURCES ${TEST_FOLDER}/*.cpp)
6361
ament_add_gtest(${TEST_CASE_NAME} ${TEST_CASE_SOURCES})
@@ -75,7 +73,7 @@ if(BUILD_TESTING)
7573
target_include_directories("${TEST_CASE_NAME}" SYSTEM PUBLIC
7674
"${OpenCV_INCLUDE_DIRS}"
7775
"${tvm_utility_FOUND_INCLUDE_DIRS}"
78-
"data/models"
76+
"data/models/${TEST_CASE_NAME}_${CMAKE_SYSTEM_PROCESSOR}"
7977
"include"
8078
)
8179

@@ -93,5 +91,61 @@ if(BUILD_TESTING)
9391
endforeach()
9492
endif()
9593

94+
if(BUILD_EXAMPLE)
95+
# compile each folder inside example/ as an example
96+
find_package(rclcpp REQUIRED)
97+
98+
set(EXAMPLE_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts")
99+
file(GLOB EXAMPLE_CASES example/*)
100+
foreach(EXAMPLE_FOLDER ${EXAMPLE_CASES})
101+
if(NOT IS_DIRECTORY ${EXAMPLE_FOLDER})
102+
continue()
103+
endif()
104+
# the folder name becomes the example name
105+
file(RELATIVE_PATH EXAMPLE_NAME ${CMAKE_CURRENT_LIST_DIR}/example ${EXAMPLE_FOLDER})
106+
# Get neural network.
107+
set(NN_DEPENDENCY "")
108+
get_neural_network(${EXAMPLE_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY)
109+
110+
if(NOT NN_DEPENDENCY STREQUAL "")
111+
if(EXAMPLE_NAME STREQUAL "yolo_v2_tiny" AND
112+
NOT EXISTS ${EXAMPLE_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg)
113+
message(WARNING "Missing image artifact for yolo_v2_tiny, skipping example")
114+
continue()
115+
endif()
116+
# add all cpp files in the folder to the target
117+
file(GLOB EXAMPLE_SOURCES ${EXAMPLE_FOLDER}/*.cpp)
118+
ament_auto_add_executable(${EXAMPLE_NAME} ${EXAMPLE_SOURCES})
119+
ament_target_dependencies(${EXAMPLE_NAME}
120+
"ament_index_cpp"
121+
"tvm_vendor"
122+
"rclcpp"
123+
)
124+
add_dependencies(${EXAMPLE_NAME} ${NN_DEPENDENCY})
125+
126+
target_link_libraries("${EXAMPLE_NAME}"
127+
"${OpenCV_LIBRARIES}"
128+
"${tvm_runtime_LIBRARIES}"
129+
)
130+
131+
target_include_directories("${EXAMPLE_NAME}" SYSTEM PUBLIC
132+
"${OpenCV_INCLUDE_DIRS}"
133+
"${tvm_utility_FOUND_INCLUDE_DIRS}"
134+
"data/models"
135+
"include"
136+
)
137+
138+
else()
139+
message(WARNING "No model is generated for ${EXAMPLE_FOLDER} example")
140+
endif()
141+
142+
endforeach()
143+
endif()
144+
96145
list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake")
97-
ament_auto_package()
146+
# ament_auto_package()
147+
ament_auto_package(
148+
INSTALL_TO_SHARE
149+
launch
150+
config
151+
artifacts)

‎common/tvm_utility/README.md

+19-40
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ The earliest supported version depends on each package making use of the inferen
4141

4242
#### Models
4343

44-
Dependent packages are expected to use the `get_neural_network` cmake function from this package in order to get the compiled TVM models.
44+
Dependent packages are expected to use the `get_neural_network` cmake function from this package in order to build proper external dependency.
4545

4646
### Error detection and handling
4747

@@ -50,76 +50,55 @@ error description.
5050

5151
### Neural Networks Provider
5252

53-
This package also provides a utility to get pre-compiled neural networks to packages using them for their inference.
54-
5553
The neural networks are compiled as part of the
5654
[Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket.
57-
This package exports cmake variables and functions for ease of access to those neural networks.
5855

5956
The `get_neural_network` function creates an abstraction for the artifact management.
60-
The artifacts are saved under the source directory of the package making use of the function; under "data/".
61-
Priority is given to user-provided files, under "data/user/${MODEL_NAME}/".
62-
If there are no user-provided files, the function tries to reuse previously-downloaded artifacts.
63-
If there are no previously-downloaded artifacts, and if the `DOWNLOAD_ARTIFACTS` cmake variable is set, they will be downloaded from the bucket.
64-
Otherwise, nothing happens.
57+
Users should check if model configuration header file is under "data/user/${MODEL_NAME}/". Otherwise, nothing happens and compilation of the package will be skipped.
6558

6659
The structure inside of the source directory of the package making use of the function is as follow:
6760

6861
```{text}
6962
.
7063
├── data
71-
│ ├── downloads
72-
│ │ ├── ${MODEL 1}-${ARCH 1}-{BACKEND 1}-{VERSION 1}.tar.gz
73-
│ │ ├── ...
74-
│ │ └── ${MODEL ...}-${ARCH ...}-{BACKEND ...}-{VERSION ...}.tar.gz
75-
│ ├── models
76-
│ │ ├── ${MODEL 1}
77-
│ │ │ ├── ...
78-
│ │ │ └── inference_engine_tvm_config.hpp
79-
│ │ ├── ...
80-
│ │ └── ${MODEL ...}
81-
│ │ └── ...
82-
│ └── user
64+
│ └── models
8365
│ ├── ${MODEL 1}
84-
│ │ ├── deploy_graph.json
85-
│ │ ├── deploy_lib.so
86-
│ │ ├── deploy_param.params
8766
│ │ └── inference_engine_tvm_config.hpp
8867
│ ├── ...
8968
│ └── ${MODEL ...}
9069
│ └── ...
9170
```
9271

93-
The `inference_engine_tvm_config.hpp` file needed for compilation by dependent packages is made available under "data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp".
72+
The `inference_engine_tvm_config.hpp` file needed for compilation by dependent packages should be available under "data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp".
9473
Dependent packages can use the cmake `add_dependencies` function with the name provided in the `DEPENDENCY` output parameter of `get_neural_network` to ensure this file is created before it gets used.
9574

9675
The other `deploy_*` files are installed to "models/${MODEL_NAME}/" under the `share` directory of the package.
9776

98-
The target version to be downloaded can be overwritten by setting the `MODELZOO_VERSION` cmake variable.
99-
100-
#### Assumptions / Known limits
101-
102-
If several packages make use of the same neural network, it will be downloaded once per package.
103-
104-
In case a requested artifact doesn't exist in the S3 bucket, the error message from ExternalProject is not explicit enough for the user to understand what went wrong.
77+
The other model files should be stored in autoware_data folder under package folder with the structure:
10578

106-
In case the user manually sets `MODELZOO_VERSION` to "latest", the archive will not be re-downloaded when it gets updated in the S3 bucket (it is not a problem for tagged versions as they are not expected to be updated).
79+
```{text}
80+
$HOME/autoware_data
81+
| └──${package}
82+
| └──models
83+
| ├── ${MODEL 1}
84+
| | ├── deploy_graph.json
85+
| | ├── deploy_lib.so
86+
| | └── deploy_param.params
87+
| ├── ...
88+
| └── ${MODEL ...}
89+
| └── ...
90+
```
10791

10892
#### Inputs / Outputs
10993

110-
Inputs:
111-
112-
- `DOWNLOAD_ARTIFACTS` cmake variable; needs to be set to enable downloading the artifacts
113-
- `MODELZOO_VERSION` cmake variable; can be used to overwrite the default target version of downloads
114-
11594
Outputs:
11695

117-
- `get_neural_network` cmake function; can be used to get a neural network compiled for a specific backend
96+
- `get_neural_network` cmake function; create proper external dependency for a package with use of the model provided by the user.
11897

11998
In/Out:
12099

121100
- The `DEPENDENCY` argument of `get_neural_network` can be checked for the outcome of the function.
122-
It is an empty string when the neural network couldn't be made available.
101+
It is an empty string when the neural network wasn't provided by the user.
123102

124103
## Security considerations
125104

‎common/tvm_utility/config/yolo_v2_tiny_example.param.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/**:
2+
ros__parameters:
3+
image_filename: $(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/test_image_0.jpg
4+
label_filename: $(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/labels.txt
5+
anchor_filename: $(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/anchors.csv
6+
data_path: $(env HOME)/autoware_data

‎common/tvm_utility/data/models/abs_model_aarch64/deploy_graph.json

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"nodes": [
3+
{
4+
"op": "null",
5+
"name": "a",
6+
"inputs": []
7+
},
8+
{
9+
"op": "tvm_op",
10+
"name": "tvmgen_default_fused_abs",
11+
"attrs": {
12+
"num_outputs": "1",
13+
"num_inputs": "1",
14+
"flatten_data": "0",
15+
"func_name": "tvmgen_default_fused_abs",
16+
"hash": "1be44995aa501758"
17+
},
18+
"inputs": [[0, 0, 0]]
19+
}
20+
],
21+
"arg_nodes": [0],
22+
"heads": [[1, 0, 0]],
23+
"attrs": {
24+
"dltype": ["list_str", ["float32", "float32"]],
25+
"device_index": ["list_int", [1, 1]],
26+
"storage_id": ["list_int", [0, 1]],
27+
"shape": [
28+
"list_shape",
29+
[
30+
[2, 2],
31+
[2, 2]
32+
]
33+
]
34+
},
35+
"node_row_ptr": [0, 1, 2]
36+
}

‎common/tvm_utility/data/models/abs_model_aarch64/deploy_lib.so

14.5 KB
Binary file not shown.

‎common/tvm_utility/data/models/abs_model_aarch64/deploy_param.params

32 Bytes
Binary file not shown.

‎common/tvm_utility/data/models/abs_model_aarch64/inference_engine_tvm_config.hpp

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright 2021 Arm Limited and Contributors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "tvm_utility/pipeline.hpp"
16+
17+
#ifndef COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_AARCH64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ // NOLINT
18+
#define COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_AARCH64__INFERENCE_ENGINE_TVM_CONFIG_HPP_
19+
20+
namespace model_zoo
21+
{
22+
namespace inf_test
23+
{
24+
namespace engine_load
25+
{
26+
namespace abs_model
27+
{
28+
29+
static const tvm_utility::pipeline::InferenceEngineTVMConfig config{
30+
{0, 0, 0}, // modelzoo_version
31+
32+
// cspell: ignore mtriple
33+
"abs_model_aarch64", // network_name
34+
"llvm -mtriple=aarch64-linux-gnu", // network_backend
35+
36+
"deploy_lib.so", // network_module_path
37+
"deploy_graph.json", // network_graph_path
38+
"deploy_param.params", // network_params_path
39+
40+
// cspell: ignore DLCPU
41+
kDLCPU, // tvm_device_type
42+
0, // tvm_device_id
43+
44+
{{"a", kDLFloat, 32, 1, {2, 2}}}, // network_inputs
45+
46+
{{"output", kDLFloat, 32, 1, {2, 2}}} // network_outputs
47+
};
48+
49+
} // namespace abs_model
50+
} // namespace engine_load
51+
} // namespace inf_test
52+
} // namespace model_zoo
53+
#endif // COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_AARCH64__INFERENCE_ENGINE_TVM_CONFIG_HPP_
54+
// NOLINT

‎common/tvm_utility/data/models/abs_model_x86_64/deploy_graph.json

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"nodes": [
3+
{
4+
"op": "null",
5+
"name": "a",
6+
"inputs": []
7+
},
8+
{
9+
"op": "tvm_op",
10+
"name": "tvmgen_default_fused_abs",
11+
"attrs": {
12+
"num_outputs": "1",
13+
"num_inputs": "1",
14+
"flatten_data": "0",
15+
"func_name": "tvmgen_default_fused_abs",
16+
"hash": "1be44995aa501758"
17+
},
18+
"inputs": [[0, 0, 0]]
19+
}
20+
],
21+
"arg_nodes": [0],
22+
"heads": [[1, 0, 0]],
23+
"attrs": {
24+
"dltype": ["list_str", ["float32", "float32"]],
25+
"device_index": ["list_int", [1, 1]],
26+
"storage_id": ["list_int", [0, 1]],
27+
"shape": [
28+
"list_shape",
29+
[
30+
[2, 2],
31+
[2, 2]
32+
]
33+
]
34+
},
35+
"node_row_ptr": [0, 1, 2]
36+
}

‎common/tvm_utility/data/models/abs_model_x86_64/deploy_lib.so

17.7 KB
Binary file not shown.

‎common/tvm_utility/data/models/abs_model_x86_64/deploy_param.params

32 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)