From 9987ff913cfd67cfb0f8ab15cc708a2fa4174cfe Mon Sep 17 00:00:00 2001 From: Hengkai Guo Date: Wed, 5 Feb 2025 22:03:04 +0800 Subject: [PATCH] Support GLOMAP as a better SfM backend (#33) * fix compilation of optimization on point trajectory * support glomap * update README * fix table * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md --- README.md | 28 ++++++++++++++++++++- point_trajectory/optimize/CMakeLists.txt | 1 + run_particlesfm.py | 16 ++++++++---- sfm/__init__.py | 2 +- sfm/main_sfm.py | 32 ++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 24652c6..744c2ca 100755 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-Code release for our ECCV 2022 paper "ParticleSfM: Exploiting Dense Point Trajectories for Localizing Moving Cameras in the Wild." by [Wang Zhao](https://github.com/thuzhaowang), [Shaohui Liu](http://www.b1ueber2y.me/), [Hengkai Guo](https://github.com/guohengkai), [Wenping Wang](https://engineering.tamu.edu/cse/profiles/Wang-Wenping.html) and [Yong-Jin Liu](https://cg.cs.tsinghua.edu.cn/people/~Yongjin/Yongjin.htm). +Code release for our ECCV 2022 paper "ParticleSfM: Exploiting Dense Point Trajectories for Localizing Moving Cameras in the Wild." by [Wang Zhao](https://thuzhaowang.github.io/), [Shaohui Liu](http://www.b1ueber2y.me/), [Hengkai Guo](https://guohengkai.github.io/), [Wenping Wang](https://engineering.tamu.edu/cse/profiles/Wang-Wenping.html) and [Yong-Jin Liu](https://cg.cs.tsinghua.edu.cn/people/~Yongjin/Yongjin.htm). **[Introduction]** ParticleSfM is an offline structure-from-motion system for videos (image sequences). Inspired by [Particle video](http://rvsn.csail.mit.edu/pv/), our method connects pairwise optical flows and optimizes dense point trajectories as long-range video correpondences, which are used in a customized global structure-from-motion framework with similarity averaging and global bundle adjustment. In particular, for dynamic scenes, the acquired dense point trajectories can be fed into a specially designed trajectory-based motion segmentation module to select static point tracks, enabling the system to produce reliable camera trajectories on in-the-wild sequences with complex foreground motion. @@ -14,12 +14,33 @@ Contact Wang Zhao (thuzhaowang@163.com), Shaohui Liu (b1ueber2y@gmail.com) and H If you are interested in potential collaboration or internship at ByteDance, please feel free to contact Hengkai Guo (guohengkai@bytedance.com). +## Update by 2025.02.05 +We support [GLOMAP](https://github.com/colmap/glomap) in our pipeline, which achieves more accurate results on 13 sequences of the [Sintel dataset](http://sintel.is.tue.mpg.de/): + +| Method | ATE (m) | RPE trans (m) | RPE rot (deg) | SfM runtime (min) | #Frames | +|:-:|:-:|:-:|:-:|:-:|:-:| +| Global SfM - Ours w/ gcolmap(Theia) | 0.104 | 0.054 | 0.414 | **3.35** | 45.6 | +| Global SfM - Ours w/ GLOMAP | **0.057** | **0.031** | **0.201** | 6.97 | 45.6 | + +Test it by simply changing the `sfm_type` to `global_glomap`: +``` +python run_particlesfm.py --image_dir /path/to/the/image/folder/ \ + --output_dir /path/to/output/workspace/ \ + --sfm_type global_glomap # "global_theia" for the paper version +``` + ## Installation 1. Install dependencies: + +For using gcolmap (Theia) as in the original ParticleSfM paper: * Ceres 2.0.0 [[Guide](./misc/doc/ceres.md)] * COLMAP <= 3.8 [[Guide](./misc/doc/colmap.md)] * Theia SfM (customized version) [[Guide](./misc/doc/theia.md)] +Alternatively, if you want to use our latest [GLOMAP](https://github.com/colmap/glomap) support: +* Ceres with lastest version +* GLOMAP + 2. Set up Python environment with Conda: ``` conda env create -f particlesfm_env.yaml @@ -148,6 +169,11 @@ python train_seq.py ./configs/your-config-file cd .. ``` +## Applications +* Motion Control for Video Generation: [MotionCtrl](https://wzhouxiff.github.io/projects/MotionCtrl/), [CamCo](https://ir1d.github.io/CamCo/) +* Motion Evaluation for Video Generation: [AnimateAnything](https://yu-shaonian.github.io/Animate_Anything/), [AC3D](https://snap-research.github.io/ac3d/) +* Kinematic Control Annotation: [EgoVid-5M](https://egovid.github.io/) + ## Citation ``` @inproceedings{zhao2022particlesfm, diff --git a/point_trajectory/optimize/CMakeLists.txt b/point_trajectory/optimize/CMakeLists.txt index f9c817d..ac476d5 100755 --- a/point_trajectory/optimize/CMakeLists.txt +++ b/point_trajectory/optimize/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.4) project(traj_ceres) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +set(CMAKE_CXX_STANDARD 14) find_package(Ceres REQUIRED) include_directories(${CERES_INCLUDE_DIRS}) diff --git a/run_particlesfm.py b/run_particlesfm.py index 7b04084..0ffc98d 100755 --- a/run_particlesfm.py +++ b/run_particlesfm.py @@ -75,13 +75,19 @@ def sfm_reconstruction(args, image_dir, output_dir, traj_dir, skip_exists=False, sfm_dir = os.path.join(output_dir, "sfm") # sfm reconstruction - from sfm import main_global_sfm, main_incremental_sfm, write_depth_pose_from_colmap_format - if not args.incremental_sfm: - print("[ParticleSfM] Running global structure-from-motion........") + from sfm import main_global_sfm, main_incremental_sfm, main_global_sfm_glomap, write_depth_pose_from_colmap_format + if args.sfm_type == 'global_theia': + print("[ParticleSfM] Running global structure-from-motion with Theia........") main_global_sfm(sfm_dir, image_dir, traj_dir, remove_dynamic=(not args.assume_static), skip_exists=skip_exists) - else: + elif args.sfm_type == 'incremental_colmap': print("[ParticleSfM] Running incremental structure-from-motion with COLMAP........") main_incremental_sfm(sfm_dir, image_dir, traj_dir, remove_dynamic=(not args.assume_static), skip_exists=skip_exists) + elif args.sfm_type == 'global_glomap': + print("[ParticleSfM] Running global structure-from-motion with GLOMAP........") + main_global_sfm_glomap(sfm_dir, image_dir, traj_dir, remove_dynamic=(not args.assume_static), skip_exists=skip_exists) + else: + print('error sfm type: ', args.sfm_type) + assert False # # write depth and pose files from COLMAP format write_depth_pose_from_colmap_format(sfm_dir, os.path.join(output_dir, "colmap_outputs_converted")) @@ -122,7 +128,7 @@ def parse_args(): parser.add_argument("--window_size", type=int, default=10, help='the window size for trajectory motion segmentation') parser.add_argument("--traj_max_num", type=int, default=100000, help='the maximum number of trajs inside a window') # sfm - parser.add_argument("--incremental_sfm", action='store_true', help='whether to use incremental sfm or not') + parser.add_argument("--sfm_type", default='global_theia', choices=['incremental_colmap', 'global_theia', 'global_glomap'], help='sfm type') # pipeline control parser.add_argument("--skip_path_consistency", action='store_true', help='whether to skip the path consistency optimization or not') parser.add_argument("--assume_static", action='store_true', help='whether to skip the motion segmentation or not') diff --git a/sfm/__init__.py b/sfm/__init__.py index 03716d0..6ddf28c 100755 --- a/sfm/__init__.py +++ b/sfm/__init__.py @@ -1,2 +1,2 @@ -from .main_sfm import main_global_sfm, main_incremental_sfm +from .main_sfm import main_global_sfm, main_incremental_sfm, main_global_sfm_glomap from .convert import write_depth_pose_from_colmap_format diff --git a/sfm/main_sfm.py b/sfm/main_sfm.py index 3a0befc..06e96d1 100755 --- a/sfm/main_sfm.py +++ b/sfm/main_sfm.py @@ -21,6 +21,7 @@ from pathlib import Path import shutil import pprint +import time import os, sys sys.path.append(os.path.dirname(os.path.abspath(__file__))) @@ -113,7 +114,10 @@ def main_incremental_sfm(sfm_dir, image_dir, traj_dir, colmap_path="colmap", sin '--Mapper.ba_refine_principal_point', str(0), '--Mapper.ba_refine_extra_params', str(0)] print(' '.join(cmd)) + start = time.time() subprocess.run(cmd, check=True) + end = time.time() + print('incremental sfm: {}s'.format(end - start)) # analyze model stats = compute_model_stats(model_path) @@ -144,7 +148,35 @@ def main_global_sfm(sfm_dir, image_dir, traj_dir, gcolmap_path=None, colmap_path cmd += ['--GlobalMapper.ba_refine_principal_point', str(0), '--GlobalMapper.ba_refine_extra_params', str(0)] print(' '.join(cmd)) + start = time.time() subprocess.run(cmd, check=True) + end = time.time() + print('global sfm: {}s'.format(end - start)) + + # analyze model + stats = compute_model_stats(model_path) + if stats is not None: + print(stats) + +def main_global_sfm_glomap(sfm_dir, image_dir, traj_dir, gcolmap_path=None, glomap_path="glomap", colmap_path="colmap", single_camera=True, remove_dynamic=True, skip_geometric_verification=False, min_num_matches=None, skip_exists=False): + """ + Global structure-from-motion for videos using GLOMAP + """ + database_path, pair_txt_path = build_database(sfm_dir, image_dir, traj_dir, colmap_path=colmap_path, single_camera=single_camera, remove_dynamic=remove_dynamic, skip_geometric_verification=skip_geometric_verification, skip_exists=skip_exists) + model_path = Path(sfm_dir) / 'model' + model_path.mkdir(exist_ok=True, parents=True) + + # run global structure-from-motion + cmd = [ + str(glomap_path), 'mapper', + '--database_path', str(database_path), + '--image_path', str(image_dir), + '--output_path', str(model_path)] + print(' '.join(cmd)) + start = time.time() + subprocess.run(cmd, check=True) + end = time.time() + print('global sfm with GLOMAP: {}s'.format(end - start)) # analyze model stats = compute_model_stats(model_path)