Skip to content

Commit 98ef10e

Browse files
JingweiZhang12JingweiZhang12
authored
[Fix] Fix benchmark (open-mmlab#2402)
* init commit * set random seed for groupfree and imvotenet * replace lyft by nus * add fcos3d resnet34 * Revert "add fcos3d resnet34" This reverts commit f3da3d9. * replace waymo by kitti for pointpillars
1 parent b481efc commit 98ef10e

17 files changed

+67
-60
lines changed

‎.dev_scripts/benchmark_full_models.txt

+2-3
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,11 @@ configs/pgd/pgd_r101-caffe_fpn_head-gn_4xb3-4x_kitti-mono3d.py
1616
configs/point_rcnn/point-rcnn_8xb2_kitti-3d-3class.py
1717
configs/pointnet2/pointnet2_msg_2xb16-cosine-80e_s3dis-seg.py
1818
configs/pointnet2/pointnet2_msg_2xb16-cosine-250e_scannet-seg.py
19-
configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py
20-
configs/pointpillars/pointpillars_hv_secfpn_sbn-all_16xb2-2x_waymoD5-3d-3class.py
19+
configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py
2120
configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py
2221
configs/regnet/pointpillars_hv_regnet-1.6gf_fpn_sbn-all_8xb4-2x_nus-3d.py
2322
configs/second/second_hv_secfpn_8xb6-80e_kitti-3d-3class.py
2423
configs/second/second_hv_secfpn_8xb6-amp-80e_kitti-3d-3class.py
2524
configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py
26-
configs/ssn/ssn_hv_secfpn_sbn-all_16xb2-2x_lyft-3d.py
25+
configs/ssn/ssn_hv_secfpn_sbn-all_16xb2-2x_nus-3d.py
2726
configs/votenet/votenet_8xb8_scannet-3d.py

‎.dev_scripts/benchmark_train_models.txt

+1-2
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ configs/mvxnet/mvxnet_fpn_dv_second_secfpn_8xb2-80e_kitti-3d-3class.py
66
configs/point_rcnn/point-rcnn_8xb2_kitti-3d-3class.py
77
configs/pointnet2/pointnet2_msg_2xb16-cosine-80e_s3dis-seg.py
88
configs/pointnet2/pointnet2_msg_2xb16-cosine-250e_scannet-seg.py
9-
configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py
10-
configs/pointpillars/pointpillars_hv_secfpn_sbn-all_16xb2-2x_waymoD5-3d-3class.py
9+
configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py
1110
configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py
1211
configs/second/second_hv_secfpn_8xb6-80e_kitti-3d-3class.py
1312
configs/second/second_hv_secfpn_8xb6-amp-80e_kitti-3d-3class.py

‎configs/_base_/datasets/lyft-3d-range100.py

+39-24
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
]
99
dataset_type = 'LyftDataset'
1010
data_root = 'data/lyft/'
11+
data_prefix = dict(pts='v1.01-train/lidar', img='', sweeps='v1.01-train/lidar')
1112
# Input modality for Lyft dataset, this is consistent with the submission
1213
# format which requires the information in input_modality.
1314
input_modality = dict(
@@ -101,35 +102,49 @@
101102
dict(type='Pack3DDetInputs', keys=['points'])
102103
]
103104

104-
data = dict(
105-
samples_per_gpu=2,
106-
workers_per_gpu=2,
107-
train=dict(
105+
train_dataloader = dict(
106+
batch_size=2,
107+
num_workers=2,
108+
persistent_workers=True,
109+
sampler=dict(type='DefaultSampler', shuffle=True),
110+
dataset=dict(
108111
type=dataset_type,
109112
data_root=data_root,
110-
ann_file=data_root + 'lyft_infos_train.pkl',
113+
ann_file='lyft_infos_train.pkl',
111114
pipeline=train_pipeline,
112-
classes=class_names,
115+
metainfo=dict(classes=class_names),
113116
modality=input_modality,
114-
test_mode=False),
115-
val=dict(
117+
data_prefix=data_prefix,
118+
test_mode=False,
119+
box_type_3d='LiDAR',
120+
backend_args=backend_args))
121+
val_dataloader = dict(
122+
batch_size=1,
123+
num_workers=1,
124+
persistent_workers=True,
125+
drop_last=False,
126+
sampler=dict(type='DefaultSampler', shuffle=False),
127+
dataset=dict(
116128
type=dataset_type,
117129
data_root=data_root,
118-
ann_file=data_root + 'lyft_infos_val.pkl',
130+
ann_file='lyft_infos_val.pkl',
119131
pipeline=test_pipeline,
120-
classes=class_names,
132+
metainfo=dict(classes=class_names),
121133
modality=input_modality,
122-
test_mode=True),
123-
test=dict(
124-
type=dataset_type,
125-
data_root=data_root,
126-
ann_file=data_root + 'lyft_infos_test.pkl',
127-
pipeline=test_pipeline,
128-
classes=class_names,
129-
modality=input_modality,
130-
test_mode=True))
131-
# For Lyft dataset, we usually evaluate the model at the end of training.
132-
# Since the models are trained by 24 epochs by default, we set evaluation
133-
# interval to be 24. Please change the interval accordingly if you do not
134-
# use a default schedule.
135-
evaluation = dict(interval=24, pipeline=eval_pipeline)
134+
test_mode=True,
135+
data_prefix=data_prefix,
136+
box_type_3d='LiDAR',
137+
backend_args=backend_args))
138+
test_dataloader = val_dataloader
139+
140+
val_evaluator = dict(
141+
type='LyftMetric',
142+
data_root=data_root,
143+
ann_file='lyft_infos_val.pkl',
144+
metric='bbox',
145+
backend_args=backend_args)
146+
test_evaluator = val_evaluator
147+
148+
vis_backends = [dict(type='LocalVisBackend')]
149+
visualizer = dict(
150+
type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

‎configs/_base_/datasets/lyft-3d.py

+4-7
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# Input modality for Lyft dataset, this is consistent with the submission
1212
# format which requires the information in input_modality.
1313
input_modality = dict(use_lidar=True, use_camera=False)
14-
data_prefix = dict(pts='samples/LIDAR_TOP', img='', sweeps='sweeps/LIDAR_TOP')
14+
data_prefix = dict(pts='v1.01-train/lidar', img='', sweeps='v1.01-train/lidar')
1515

1616
# Example to use different file client
1717
# Method 1: simply set the data root and let the file I/O module
@@ -149,14 +149,11 @@
149149

150150
val_evaluator = dict(
151151
type='LyftMetric',
152-
ann_file=data_root + 'lyft_infos_val.pkl',
153-
metric='bbox',
154-
backend_args=backend_args)
155-
test_evaluator = dict(
156-
type='LyftMetric',
157-
ann_file=data_root + 'lyft_infos_val.pkl',
152+
data_root=data_root,
153+
ann_file='lyft_infos_val.pkl',
158154
metric='bbox',
159155
backend_args=backend_args)
156+
test_evaluator = val_evaluator
160157

161158
vis_backends = [dict(type='LocalVisBackend')]
162159
visualizer = dict(

‎configs/groupfree3d/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ We implement Group-Free-3D and provide the result and checkpoints on ScanNet dat
2222

2323
| Method | Backbone | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 | AP@0.5 | Download |
2424
| :--------------------------------------------------------------: | :-----------: | :-----: | :------: | :------------: | :-------------: | :-------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
25-
| [L6, O256](./groupfree3d_head-L6-O256_4xb8_scannet-seg.py) | PointNet++ | 3x | 6.7 | | 66.32 (65.67\*) | 47.82 (47.74\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347.log.json) |
25+
| [L6, O256](./groupfree3d_head-L6-O256_4xb8_scannet-seg.py) | PointNet++ | 3x | 6.7 | | 66.17 (65.67\*) | 48.47 (47.74\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347.log.json) |
2626
| [L12, O256](./groupfree3d_head-L12-O256_4xb8_scannet-seg.py) | PointNet++ | 3x | 9.4 | | 66.57 (66.22\*) | 48.21 (48.95\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907-1c5551ad.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907.log.json) |
2727
| [L12, O256](./groupfree3d_w2x-head-L12-O256_4xb8_scannet-seg.py) | PointNet++w2x | 3x | 13.3 | | 68.20 (67.30\*) | 51.02 (50.44\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301-944f0ac0.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301.log.json) |
2828
| [L12, O512](./groupfree3d_w2x-head-L12-O512_4xb8_scannet-seg.py) | PointNet++w2x | 3x | 18.8 | | 68.22 (68.20\*) | 52.61 (51.31\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204-187b71c7.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204.log.json) |

‎configs/groupfree3d/groupfree3d_head-L6-O256_4xb8_scannet-seg.py

+1
Original file line numberDiff line numberDiff line change
@@ -224,3 +224,4 @@
224224

225225
default_hooks = dict(
226226
checkpoint=dict(type='CheckpointHook', interval=1, max_keep_ckpts=10))
227+
randomness = dict(seed=4)

‎configs/groupfree3d/metafile.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ Models:
2525
- Task: 3D Object Detection
2626
Dataset: ScanNet
2727
Metrics:
28-
AP@0.25: 66.32
29-
AP@0.5: 47.82
28+
AP@0.25: 66.17
29+
AP@0.5: 48.47
3030
Weights: https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth
3131

3232
- Name: groupfree3d_head-L12-O256_4xb8_scannet-seg.py

‎configs/h3dnet/README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ We implement H3DNet and provide the result and checkpoints on ScanNet datasets.
2020

2121
### ScanNet
2222

23-
| Backbone | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 | AP@0.5 | Download |
24-
| :-------------------------------------------: | :-----: | :------: | :------------: | :-----: | :----: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
25-
| [MultiBackbone](./h3dnet_8xb3_scannet-seg.py) | 3x | 7.9 | | 66.07 | 47.68 | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149.log.json) |
23+
| Backbone | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 | AP@0.5 | Download |
24+
| :-------------------------------------------: | :-----: | :------: | :------------: | :-----: | :----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
25+
| [MultiBackbone](./h3dnet_8xb3_scannet-seg.py) | 3x | 7.9 | | 66.07 | 47.68 | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_3x8_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149.log.json) |
2626

2727
**Notice**: If your current mmdetection3d version >= 0.6.0, and you are using the checkpoints downloaded from the above links or using checkpoints trained with mmdetection3d version \< 0.6.0, the checkpoints have to be first converted via [tools/model_converters/convert_h3dnet_checkpoints.py](../../tools/model_converters/convert_h3dnet_checkpoints.py):
2828

‎configs/h3dnet/metafile.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,4 @@ Models:
2626
Metrics:
2727
AP@0.25: 66.07
2828
AP@0.5: 47.68
29-
Weights: https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth
29+
Weights: https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_3x8_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth

‎configs/imvotenet/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ We implement ImVoteNet and provide the result and checkpoints on SUNRGBD.
2828

2929
| Backbone | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 | AP@0.5 | Download |
3030
| :--------------------------------------------------: | :-----: | :------: | :------------: | :-----: | :----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
31-
| [PointNet++](./imvotenet_stage2_8xb16_sunrgbd-3d.py) | 3x | 9.4 | | 64.55 | | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851-1bcd1b97.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851.log.json) |
31+
| [PointNet++](./imvotenet_stage2_8xb16_sunrgbd-3d.py) | 3x | 9.4 | | 64.48 | | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851-1bcd1b97.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851.log.json) |
3232

3333
## Citation
3434

‎configs/imvotenet/imvotenet_faster-rcnn-r50_fpn_4xb2_sunrgbd-3d.py

+1-7
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,6 @@
2525

2626
test_pipeline = [
2727
dict(type='LoadImageFromFile', backend_args=backend_args),
28-
# online evaluation
29-
dict(
30-
type='LoadAnnotations3D',
31-
with_bbox=True,
32-
with_label=True,
33-
with_bbox_3d=False,
34-
with_label_3d=False),
3528
dict(type='Resize', scale=(1333, 600), keep_ratio=True),
3629
dict(
3730
type='Pack3DDetInputs',
@@ -66,6 +59,7 @@
6659
gamma=0.1)
6760
]
6861
val_evaluator = dict(type='Indoor2DMetric')
62+
test_evaluator = val_evaluator
6963

7064
# optimizer
7165
optim_wrapper = dict(

‎configs/imvotenet/imvotenet_stage2_8xb16_sunrgbd-3d.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,10 @@
219219
test_dataloader = dict(dataset=dict(pipeline=test_pipeline))
220220

221221
# may also use your own pre-trained image branch
222-
load_from = 'https://download.openmmlab.com/mmdetection3d/v0.1.0_models/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class_20210323_173222-cad62aeb.pth' # noqa
222+
load_from = 'https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class_20210819_225618-62eba6ce.pth' # noqa
223223
# Default setting for scaling LR automatically
224224
# - `enable` means enable scaling LR automatically
225225
# or not by default.
226226
# - `base_batch_size` = (8 GPUs) x (16 samples per GPU).
227227
auto_scale_lr = dict(enable=False, base_batch_size=128)
228+
randomness = dict(seed=8)

‎configs/imvotenet/metafile.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,5 @@ Models:
3939
- Task: 3D Object Detection
4040
Dataset: SUNRGBD-3D
4141
Metrics:
42-
AP@0.25: 64.55
42+
AP@0.25: 64.48
4343
Weights: https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851-1bcd1b97.pth

‎configs/pv_rcnn/metafile.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,4 @@ Models:
2626
Dataset: KITTI
2727
Metrics:
2828
mAP: 72.28
29-
Weights: <https://download.openmmlab.com/mmdetection3d/v1.1.0_models/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class/pv_rcnn_8xb2-80e_kitti-3d-3class_20221117_234428-b384d22f.pth
29+
Weights: https://download.openmmlab.com/mmdetection3d/v1.1.0_models/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class/pv_rcnn_8xb2-80e_kitti-3d-3class_20221117_234428-b384d22f.pth

‎mmdet3d/evaluation/metrics/lyft_metric.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ def compute_metrics(self, results: List[dict]) -> Dict[str, float]:
122122

123123
# load annotations
124124
self.data_infos = load(
125-
self.ann_file, backend_args=self.backend_args)['data_list']
125+
osp.join(self.data_root, self.ann_file),
126+
backend_args=self.backend_args)['data_list']
126127
result_dict, tmp_dir = self.format_results(results, classes,
127128
self.jsonfile_prefix,
128129
self.csv_savepath)
@@ -360,7 +361,7 @@ def output_to_lyft_box(detection: dict) -> List[LyftBox]:
360361
Returns:
361362
List[:obj:`LyftBox`]: List of standard LyftBoxes.
362363
"""
363-
bbox3d = detection['bbox_3d']
364+
bbox3d = detection['bboxes_3d']
364365
scores = detection['scores_3d'].numpy()
365366
labels = detection['labels_3d'].numpy()
366367

‎tools/dataset_converters/lyft_converter.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ def create_lyft_infos(root_path,
4040
available_vers = ['v1.01-train', 'v1.01-test']
4141
assert version in available_vers
4242
if version == 'v1.01-train':
43-
train_scenes = mmcv.list_from_file('data/lyft/train.txt')
44-
val_scenes = mmcv.list_from_file('data/lyft/val.txt')
43+
train_scenes = mmengine.list_from_file('data/lyft/train.txt')
44+
val_scenes = mmengine.list_from_file('data/lyft/val.txt')
4545
elif version == 'v1.01-test':
46-
train_scenes = mmcv.list_from_file('data/lyft/test.txt')
46+
train_scenes = mmengine.list_from_file('data/lyft/test.txt')
4747
val_scenes = []
4848
else:
4949
raise ValueError('unknown')

‎tools/dataset_converters/waymo_converter.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
try:
77
from waymo_open_dataset import dataset_pb2
88
except ImportError:
9-
raise ImportError('Please run "pip install waymo-open-dataset-tf-2-5-0" '
9+
raise ImportError('Please run "pip install waymo-open-dataset-tf-2-6-0" '
1010
'>1.4.5 to install the official devkit first.')
1111

1212
import os

0 commit comments

Comments
 (0)