Skip to content

Commit cf6f473

Browse files
Xiangxu-0103Xiangxu-0103
authored
[Feature] File I/O migration and reconstruction (open-mmlab#2319)
* refactor leio * update mmengine and mmcv version * update * update docs * update version
1 parent b2e5ad6 commit cf6f473

File tree

95 files changed

+1386
-803
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

95 files changed

+1386
-803
lines changed

‎configs/3dssd/3dssd_4xb4_kitti-3d-car.py

+20-10
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
class_names = ['Car']
1010
point_cloud_range = [0, -40, -5, 70, 40, 3]
1111
input_modality = dict(use_lidar=True, use_camera=False)
12+
backend_args = None
13+
1214
db_sampler = dict(
1315
data_root=data_root,
1416
info_path=data_root + 'kitti_dbinfos_train.pkl',
@@ -17,17 +19,20 @@
1719
classes=class_names,
1820
sample_groups=dict(Car=15),
1921
points_loader=dict(
20-
type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4))
21-
22-
file_client_args = dict(backend='disk')
23-
# Uncomment the following if use ceph or other file clients.
24-
# See https://mmcv.readthedocs.io/en/latest/api.html#mmcv.fileio.FileClient
25-
# for more details.
26-
# file_client_args = dict(
27-
# backend='petrel', path_mapping=dict(data='s3://kitti_data/'))
22+
type='LoadPointsFromFile',
23+
coord_type='LIDAR',
24+
load_dim=4,
25+
use_dim=4,
26+
backend_args=backend_args),
27+
backend_args=backend_args)
2828

2929
train_pipeline = [
30-
dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
30+
dict(
31+
type='LoadPointsFromFile',
32+
coord_type='LIDAR',
33+
load_dim=4,
34+
use_dim=4,
35+
backend_args=backend_args),
3136
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
3237
dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range),
3338
dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
@@ -52,7 +57,12 @@
5257
]
5358

5459
test_pipeline = [
55-
dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
60+
dict(
61+
type='LoadPointsFromFile',
62+
coord_type='LIDAR',
63+
load_dim=4,
64+
use_dim=4,
65+
backend_args=backend_args),
5666
dict(
5767
type='MultiScaleFlipAug3D',
5868
img_scale=(1333, 800),

‎configs/_base_/datasets/kitti-3d-3class.py

+43-8
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,21 @@
66
input_modality = dict(use_lidar=True, use_camera=False)
77
metainfo = dict(classes=class_names)
88

9+
# Example to use different file client
10+
# Method 1: simply set the data root and let the file I/O module
11+
# automatically infer from prefix (not support LMDB and Memcache yet)
12+
13+
# data_root = 's3://openmmlab/datasets/detection3d/kitti/'
14+
15+
# Method 2: Use backend_args, file_client_args in versions before 1.1.0rc4
16+
# backend_args = dict(
17+
# backend='petrel',
18+
# path_mapping=dict({
19+
# './data/': 's3://openmmlab/datasets/detection3d/',
20+
# 'data/': 's3://openmmlab/datasets/detection3d/'
21+
# }))
22+
backend_args = None
23+
924
db_sampler = dict(
1025
data_root=data_root,
1126
info_path=data_root + 'kitti_dbinfos_train.pkl',
@@ -16,14 +31,20 @@
1631
classes=class_names,
1732
sample_groups=dict(Car=12, Pedestrian=6, Cyclist=6),
1833
points_loader=dict(
19-
type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4))
34+
type='LoadPointsFromFile',
35+
coord_type='LIDAR',
36+
load_dim=4,
37+
use_dim=4,
38+
backend_args=backend_args),
39+
backend_args=backend_args)
2040

2141
train_pipeline = [
2242
dict(
2343
type='LoadPointsFromFile',
2444
coord_type='LIDAR',
2545
load_dim=4, # x, y, z, intensity
26-
use_dim=4),
46+
use_dim=4,
47+
backend_args=backend_args),
2748
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
2849
dict(type='ObjectSample', db_sampler=db_sampler),
2950
dict(
@@ -45,7 +66,12 @@
4566
keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
4667
]
4768
test_pipeline = [
48-
dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
69+
dict(
70+
type='LoadPointsFromFile',
71+
coord_type='LIDAR',
72+
load_dim=4,
73+
use_dim=4,
74+
backend_args=backend_args),
4975
dict(
5076
type='MultiScaleFlipAug3D',
5177
img_scale=(1333, 800),
@@ -66,7 +92,12 @@
6692
# construct a pipeline for data and gt loading in show function
6793
# please keep its loading function consistent with test_pipeline (e.g. client)
6894
eval_pipeline = [
69-
dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
95+
dict(
96+
type='LoadPointsFromFile',
97+
coord_type='LIDAR',
98+
load_dim=4,
99+
use_dim=4,
100+
backend_args=backend_args),
70101
dict(type='Pack3DDetInputs', keys=['points'])
71102
]
72103
train_dataloader = dict(
@@ -88,7 +119,8 @@
88119
metainfo=metainfo,
89120
# we use box_type_3d='LiDAR' in kitti and nuscenes dataset
90121
# and box_type_3d='Depth' in sunrgbd and scannet dataset.
91-
box_type_3d='LiDAR')))
122+
box_type_3d='LiDAR',
123+
backend_args=backend_args)))
92124
val_dataloader = dict(
93125
batch_size=1,
94126
num_workers=1,
@@ -104,7 +136,8 @@
104136
modality=input_modality,
105137
test_mode=True,
106138
metainfo=metainfo,
107-
box_type_3d='LiDAR'))
139+
box_type_3d='LiDAR',
140+
backend_args=backend_args))
108141
test_dataloader = dict(
109142
batch_size=1,
110143
num_workers=1,
@@ -120,11 +153,13 @@
120153
modality=input_modality,
121154
test_mode=True,
122155
metainfo=metainfo,
123-
box_type_3d='LiDAR'))
156+
box_type_3d='LiDAR',
157+
backend_args=backend_args))
124158
val_evaluator = dict(
125159
type='KittiMetric',
126160
ann_file=data_root + 'kitti_infos_val.pkl',
127-
metric='bbox')
161+
metric='bbox',
162+
backend_args=backend_args)
128163
test_evaluator = val_evaluator
129164

130165
vis_backends = [dict(type='LocalVisBackend')]

‎configs/_base_/datasets/kitti-3d-car.py

+43-8
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,21 @@
66
input_modality = dict(use_lidar=True, use_camera=False)
77
metainfo = dict(classes=class_names)
88

9+
# Example to use different file client
10+
# Method 1: simply set the data root and let the file I/O module
11+
# automatically infer from prefix (not support LMDB and Memcache yet)
12+
13+
# data_root = 's3://openmmlab/datasets/detection3d/kitti/'
14+
15+
# Method 2: Use backend_args, file_client_args in versions before 1.1.0rc4
16+
# backend_args = dict(
17+
# backend='petrel',
18+
# path_mapping=dict({
19+
# './data/': 's3://openmmlab/datasets/detection3d/',
20+
# 'data/': 's3://openmmlab/datasets/detection3d/'
21+
# }))
22+
backend_args = None
23+
924
db_sampler = dict(
1025
data_root=data_root,
1126
info_path=data_root + 'kitti_dbinfos_train.pkl',
@@ -14,14 +29,20 @@
1429
classes=class_names,
1530
sample_groups=dict(Car=15),
1631
points_loader=dict(
17-
type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4))
32+
type='LoadPointsFromFile',
33+
coord_type='LIDAR',
34+
load_dim=4,
35+
use_dim=4,
36+
backend_args=backend_args),
37+
backend_args=backend_args)
1838

1939
train_pipeline = [
2040
dict(
2141
type='LoadPointsFromFile',
2242
coord_type='LIDAR',
2343
load_dim=4, # x, y, z, intensity
24-
use_dim=4),
44+
use_dim=4,
45+
backend_args=backend_args),
2546
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
2647
dict(type='ObjectSample', db_sampler=db_sampler),
2748
dict(
@@ -43,7 +64,12 @@
4364
keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
4465
]
4566
test_pipeline = [
46-
dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
67+
dict(
68+
type='LoadPointsFromFile',
69+
coord_type='LIDAR',
70+
load_dim=4,
71+
use_dim=4,
72+
backend_args=backend_args),
4773
dict(
4874
type='MultiScaleFlipAug3D',
4975
img_scale=(1333, 800),
@@ -64,7 +90,12 @@
6490
# construct a pipeline for data and gt loading in show function
6591
# please keep its loading function consistent with test_pipeline (e.g. client)
6692
eval_pipeline = [
67-
dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
93+
dict(
94+
type='LoadPointsFromFile',
95+
coord_type='LIDAR',
96+
load_dim=4,
97+
use_dim=4,
98+
backend_args=backend_args),
6899
dict(type='Pack3DDetInputs', keys=['points'])
69100
]
70101
train_dataloader = dict(
@@ -86,7 +117,8 @@
86117
metainfo=metainfo,
87118
# we use box_type_3d='LiDAR' in kitti and nuscenes dataset
88119
# and box_type_3d='Depth' in sunrgbd and scannet dataset.
89-
box_type_3d='LiDAR')))
120+
box_type_3d='LiDAR',
121+
backend_args=backend_args)))
90122
val_dataloader = dict(
91123
batch_size=1,
92124
num_workers=1,
@@ -102,7 +134,8 @@
102134
modality=input_modality,
103135
test_mode=True,
104136
metainfo=metainfo,
105-
box_type_3d='LiDAR'))
137+
box_type_3d='LiDAR',
138+
backend_args=backend_args))
106139
test_dataloader = dict(
107140
batch_size=1,
108141
num_workers=1,
@@ -118,11 +151,13 @@
118151
modality=input_modality,
119152
test_mode=True,
120153
metainfo=metainfo,
121-
box_type_3d='LiDAR'))
154+
box_type_3d='LiDAR',
155+
backend_args=backend_args))
122156
val_evaluator = dict(
123157
type='KittiMetric',
124158
ann_file=data_root + 'kitti_infos_val.pkl',
125-
metric='bbox')
159+
metric='bbox',
160+
backend_args=backend_args)
126161
test_evaluator = val_evaluator
127162

128163
vis_backends = [dict(type='LocalVisBackend')]

‎configs/_base_/datasets/kitti-mono3d.py

+23-12
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,23 @@
44
input_modality = dict(use_lidar=False, use_camera=True)
55
metainfo = dict(classes=class_names)
66

7-
file_client_args = dict(backend='disk')
8-
# Uncomment the following if use ceph or other file clients.
9-
# See https://mmcv.readthedocs.io/en/latest/api.html#mmcv.fileio.FileClient
10-
# for more details.
11-
# file_client_args = dict(
12-
# backend='petrel', path_mapping=dict(data='s3://kitti_data/'))
7+
# Example to use different file client
8+
# Method 1: simply set the data root and let the file I/O module
9+
# automatically infer from prefix (not support LMDB and Memcache yet)
10+
11+
# data_root = 's3://openmmlab/datasets/detection3d/kitti/'
12+
13+
# Method 2: Use backend_args, file_client_args in versions before 1.1.0rc4
14+
# backend_args = dict(
15+
# backend='petrel',
16+
# path_mapping=dict({
17+
# './data/': 's3://openmmlab/datasets/detection3d/',
18+
# 'data/': 's3://openmmlab/datasets/detection3d/'
19+
# }))
20+
backend_args = None
1321

1422
train_pipeline = [
15-
dict(type='LoadImageFromFileMono3D'),
23+
dict(type='LoadImageFromFileMono3D', backend_args=backend_args),
1624
dict(
1725
type='LoadAnnotations3D',
1826
with_bbox=True,
@@ -31,12 +39,12 @@
3139
]),
3240
]
3341
test_pipeline = [
34-
dict(type='LoadImageFromFileMono3D'),
42+
dict(type='LoadImageFromFileMono3D', backend_args=backend_args),
3543
dict(type='Resize', scale=(1242, 375), keep_ratio=True),
3644
dict(type='Pack3DDetInputs', keys=['img'])
3745
]
3846
eval_pipeline = [
39-
dict(type='LoadImageFromFileMono3D'),
47+
dict(type='LoadImageFromFileMono3D', backend_args=backend_args),
4048
dict(type='Pack3DDetInputs', keys=['img'])
4149
]
4250

@@ -57,7 +65,8 @@
5765
metainfo=metainfo,
5866
# we use box_type_3d='Camera' in monocular 3d
5967
# detection task
60-
box_type_3d='Camera'))
68+
box_type_3d='Camera',
69+
backend_args=backend_args))
6170
val_dataloader = dict(
6271
batch_size=1,
6372
num_workers=2,
@@ -74,13 +83,15 @@
7483
load_type='fov_image_based',
7584
metainfo=metainfo,
7685
test_mode=True,
77-
box_type_3d='Camera'))
86+
box_type_3d='Camera',
87+
backend_args=backend_args))
7888
test_dataloader = val_dataloader
7989

8090
val_evaluator = dict(
8191
type='KittiMetric',
8292
ann_file=data_root + 'kitti_infos_val.pkl',
83-
metric='bbox')
93+
metric='bbox',
94+
backend_args=backend_args)
8495

8596
test_evaluator = val_evaluator
8697

0 commit comments

Comments
 (0)