Skip to content

Commit 8f8208f

Browse files
LucaFosLuca Foschiani
LucaFos
and
Luca Foschiani
authored
feat(tvm_utility): port tvm_utility (#893)
Signed-off-by: Ambroise Vincent <ambroise.vincent@arm.com> Co-authored-by: Luca Foschiani <luca.foschiani@arm.com>
1 parent 51ab335 commit 8f8208f

File tree

12 files changed

+998
-0
lines changed

12 files changed

+998
-0
lines changed

‎common/tvm_utility/.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
libs
2+
*.o
3+
*.so

‎common/tvm_utility/CMakeLists.txt

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
# Copyright 2021-2022 Arm Limited and Contributors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
#    http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
cmake_minimum_required(VERSION 3.14)
16+
project(tvm_utility)
17+
18+
find_package(autoware_cmake REQUIRED)
19+
autoware_package()
20+
21+
# Configure
22+
23+
set(tvm_utility_NETWORKS_DIR ${neural_networks_provider_NETWORKS_DIR})
24+
configure_file(tvm_utility-extras.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake @ONLY)
25+
26+
foreach(network_name ${neural_networks_provider_NAMES})
27+
list(APPEND MODEL_INCLUDES_LIST
28+
"#define INCLUDE <${network_name}/NETWORKS_BACKEND/inference_engine_tvm_config.hpp>"
29+
"#include INCLUDE"
30+
"#undef INCLUDE"
31+
)
32+
endforeach()
33+
list(JOIN MODEL_INCLUDES_LIST "\n" GENERATED_MODEL_INCLUDES)
34+
configure_file(model_zoo.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp)
35+
36+
# Library
37+
38+
set(TVM_UTILITY_NODE_LIB_HEADERS
39+
"include/${PROJECT_NAME}/pipeline.hpp"
40+
"${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp"
41+
)
42+
43+
ament_auto_add_library(${PROJECT_NAME} SHARED ${TVM_UTILITY_NODE_LIB_HEADERS})
44+
set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
45+
46+
ament_export_include_directories(${tvm_utility_NETWORKS_DIR})
47+
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/ DESTINATION include)
48+
49+
if(BUILD_TESTING)
50+
# Get target backend
51+
set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend")
52+
53+
# compile each folder inside test/ as a test case
54+
find_package(ament_cmake_gtest REQUIRED)
55+
find_package(OpenCV REQUIRED)
56+
find_package(tvm_vendor REQUIRED)
57+
set(tvm_runtime_DIR ${tvm_vendor_DIR})
58+
find_package(tvm_runtime CONFIG REQUIRED)
59+
include(${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake)
60+
61+
set(TEST_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts")
62+
file(GLOB TEST_CASES test/*)
63+
foreach(TEST_FOLDER ${TEST_CASES})
64+
if(NOT IS_DIRECTORY ${TEST_FOLDER})
65+
continue()
66+
endif()
67+
# the folder name becomes the test case name
68+
file(RELATIVE_PATH TEST_CASE_NAME ${CMAKE_CURRENT_LIST_DIR}/test ${TEST_FOLDER})
69+
70+
# Test if files exist. The result is set in ${TEST_CASE_NAME}_FOUND
71+
autoware_check_neural_network(${TEST_CASE_NAME} "${${PROJECT_NAME}_BACKEND}")
72+
73+
if(${TEST_CASE_NAME}_FOUND)
74+
if(TEST_CASE_NAME STREQUAL "yolo_v2_tiny" AND
75+
NOT EXISTS ${TEST_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg)
76+
message(WARNING "Missing image artifact for yolo_v2_tiny, skipping test")
77+
continue()
78+
endif()
79+
# add all cpp files in the folder to the target
80+
file(GLOB TEST_CASE_SOURCES ${TEST_FOLDER}/*.cpp)
81+
ament_add_gtest(${TEST_CASE_NAME} ${TEST_CASE_SOURCES})
82+
ament_target_dependencies(${TEST_CASE_NAME}
83+
"ament_index_cpp"
84+
"tvm_vendor"
85+
"sensor_msgs")
86+
87+
target_link_libraries("${TEST_CASE_NAME}"
88+
"${OpenCV_LIBRARIES}"
89+
"${tvm_runtime_LIBRARIES}"
90+
)
91+
92+
target_include_directories("${TEST_CASE_NAME}" SYSTEM PUBLIC
93+
"${OpenCV_INCLUDE_DIRS}"
94+
"${tvm_vendor_INCLUDE_DIRS}"
95+
"include"
96+
"${CMAKE_CURRENT_BINARY_DIR}/include"
97+
"${tvm_utility_NETWORKS_DIR}"
98+
)
99+
100+
target_compile_definitions(${TEST_CASE_NAME} PRIVATE NETWORKS_BACKEND=${${PROJECT_NAME}_BACKEND})
101+
102+
# Install test-specific files
103+
if(IS_DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME})
104+
install(DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}/
105+
DESTINATION ${CMAKE_BINARY_DIR}/${TEST_CASE_NAME}_artifacts
106+
)
107+
endif()
108+
109+
else()
110+
message(WARNING "No model is generated for ${TEST_FOLDER}, skipping test")
111+
endif()
112+
113+
endforeach()
114+
endif()
115+
116+
list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake")
117+
ament_auto_package()

‎common/tvm_utility/artifacts/README.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# TVM Utility Artifacts {#tvm-utility-artifacts-readme}
2+
3+
Place any test artifacts in subdirectories within this directory.
4+
5+
e.g.:
6+
./artifacts/yolo_v2_tiny

‎common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
0.57273, 0.677385f
2+
1.87446, 2.06253f
3+
3.33843, 5.47434f
4+
7.88282, 3.52778f
5+
9.77052, 9.16828f

‎common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
person
2+
bicycle
3+
car
4+
motorcycle
5+
airplane
6+
bus
7+
train
8+
truck
9+
boat
10+
traffic light
11+
fire hydrant
12+
stop sign
13+
parking meter
14+
bench
15+
bird
16+
cat
17+
dog
18+
horse
19+
sheep
20+
cow
21+
elephant
22+
bear
23+
zebra
24+
giraffe
25+
backpack
26+
umbrella
27+
handbag
28+
tie
29+
suitcase
30+
frisbee
31+
skis
32+
snowboard
33+
sports ball
34+
kite
35+
baseball bat
36+
baseball glove
37+
skateboard
38+
surfboard
39+
tennis racket
40+
bottle
41+
wine glass
42+
cup
43+
fork
44+
knife
45+
spoon
46+
bowl
47+
banana
48+
apple
49+
sandwich
50+
orange
51+
broccoli
52+
carrot
53+
hot dog
54+
pizza
55+
donut
56+
cake
57+
chair
58+
couch
59+
potted plant
60+
bed
61+
dining table
62+
toilet
63+
tv
64+
laptop
65+
mouse
66+
remote
67+
keyboard
68+
cell phone
69+
microwave
70+
oven
71+
toaster
72+
sink
73+
refrigerator
74+
book
75+
clock
76+
vase
77+
scissors
78+
teddy bear
79+
hair drier
80+
toothbrush

‎common/tvm_utility/design/tvm-utility-design.md

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# TVM Utility {#tvm-utility-design}
2+
3+
This is the design document for the `tvm_utility` package. For instructions on how to build the tests for YOLOv2 Tiny,
4+
see the @subpage tvm-utility-yolo-v2-tiny-tests. For information about where to store test artifacts see the @subpage tvm-utility-artifacts-readme.
5+
6+
## Purpose / Use cases
7+
8+
A set of c++ utilities to help build a TVM based machine learning inference pipeline. The library contains a pipeline
9+
class which helps building the pipeline and a number of utility functions that are common in machine learning.
10+
11+
## Design
12+
13+
The Pipeline Class is a standardized way to write an inference pipeline. The pipeline class contains 3 different stages:
14+
the pre-processor, the inference engine and the post-processor. The TVM implementation of an inference engine stage is
15+
provided.
16+
17+
### Inputs / Outputs / API
18+
19+
The pre-processor and post-processor need to be implemented by the user before instantiating the pipeline. You can see example
20+
usage in this [example_pipeline](../test/yolo_v2_tiny).
21+
22+
Each stage in the pipeline has a `schedule` function which takes input data as a parameter and return the output data.
23+
Once the pipeline object is created, `pipeline.schedule` is called to run the pipeline.
24+
25+
```{cpp}
26+
int main() {
27+
create_subscription<sensor_msgs::msg::PointCloud2>("points_raw",
28+
rclcpp::QoS{1}, [this](const sensor_msgs::msg::PointCloud2::SharedPtr msg)
29+
{pipeline.schedule(msg);});
30+
}
31+
```
32+
33+
#### Outputs
34+
35+
- `autoware_check_neural_network` cmake macro to check if a specific network and backend combination exists
36+
37+
#### Backend
38+
39+
Dependent packages are expected to include `model_zoo.hpp` in order to get the TVM configuration structure of the targeted model/backend combination.
40+
The backend used to do the inference can be specified by setting `NETWORKS_BACKEND` as a compile definition.
41+
It defaults to `llvm`.
42+
43+
### Error detection and handling
44+
45+
`std::runtime_error` should be thrown whenever an error is encountered. It should be populated with an appropriate text
46+
error description.
47+
48+
## Security considerations
49+
50+
Both the input and output are controlled by the same actor, so the following security concerns are out-of-scope:
51+
52+
- Spoofing
53+
- Tampering
54+
55+
Leaking data to another actor would require a flaw in TVM or the host operating system that allows arbitrary memory to
56+
be read, a significant security flaw in itself. This is also true for an external actor operating the pipeline early:
57+
only the object that initiated the pipeline can run the methods to receive its output.
58+
59+
A Denial-of-Service attack could make the target hardware unusable for other pipelines but would require being able to
60+
run code on the CPU, which would already allow a more severe Denial-of-Service attack.
61+
62+
No elevation of privilege is required for this package.
63+
64+
## Future extensions / Unimplemented parts
65+
66+
Future packages will use tvm_utility as part of the perception stack to run machine learning workloads.
67+
68+
## Related issues
69+
70+
<https://github.com/autowarefoundation/autoware/discussions/226>

‎common/tvm_utility/design/tvm-utility-yolo-v2-tiny-tests.md

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# YOLOv2 Tiny Example Pipeline {#tvm-utility-yolo-v2-tiny-tests}
2+
3+
This is an example implementation of an inference pipeline using the pipeline
4+
framework. This example pipeline executes the
5+
[YOLO V2 Tiny](https://pjreddie.com/darknet/yolov2/) model and decodes its
6+
output.
7+
8+
## Compiling the Example
9+
10+
1. Download an example image to be used as test input. this image needs to be
11+
saved in the `artifacts/yolo_v2_tiny/` folder
12+
13+
```sh
14+
curl https://raw.githubusercontent.com/pjreddie/darknet/master/data/dog.jpg \
15+
> artifacts/yolo_v2_tiny/test_image_0.jpg
16+
```
17+
18+
1. Build and test with the `DOWNLOAD_ARTIFACTS` flag set.
19+
20+
```sh
21+
colcon build --packages-up-to tvm_utility --cmake-args -DDOWNLOAD_ARTIFACTS=ON
22+
colcon test --packages-select tvm_utility
23+
```
24+
25+
## GPU backend
26+
27+
Vulkan is supported by default by the tvm_vendor package.
28+
It can be selected by setting the `tvm_utility_BACKEND` variable:
29+
30+
```sh
31+
colcon build --packages-up-to tvm_utility --cmake-args -DDOWNLOAD_ARTIFACTS=ON -Dtvm_utility_BACKEND=vulkan
32+
```

0 commit comments

Comments
 (0)