From e6fa1ef757bdce91ee5b396f0405b3c94e332871 Mon Sep 17 00:00:00 2001 From: danellecline Date: Thu, 20 Feb 2025 08:36:32 -0800 Subject: [PATCH] chore: updated just recipes --- README.md | 100 +++++++++++++++++++++++++++++------------------------- justfile | 88 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 120 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 2a8649c..0f31282 100644 --- a/README.md +++ b/README.md @@ -70,53 +70,59 @@ Recipes are available to run the pipelines. To see the available recipes, run t just list ``` # Available Recipes - -| **Recipe** | **Description** | -|-------------------------------------|----------------------------------------------------------------------------------------------------------| -| `list` | List recipes. | -| `install` | Setup the environment. | -| `cp-env` | Copy the default `.env` file to the project. | -| `update_trackers` | Update the environment. Run this command after checking out any code changes. | -| `update-env` | Update environment. | -| `cp-core` | Copy core dev code to the project on Doris. | -| `cp-dev-uav` | Copy UAV dev code to the project on Doris. | -| `cp-dev-bio` | Copy bio dev code to the project on Doris. | -| `cp-dev-i2map` | Copy i2map dev code to the project on Doris. | -| `plot-tsne-vss project='uav'` | Generate a t-SNE plot of the VSS database. | -| `optimize-vss project='uav'` | Optimize the VSS database. | -| `calc-acc-vss project='uav'` | Calculate the accuracy of the VSS database; run after download, then optimize. | -| `reset-vss-all` | Reset the VSS database, removing all data. Proceed with caution! | -| `reset-vss project='uav'` | Reset the VSS database for the specified project (e.g., `uav`). | -| `remove-vss project='uav'` | Remove an entry from the VSS database (e.g., `remove-vss i2map --doc 'doc:marine organism:*'`). | -| `init-vss project='uav'` | Initialize the VSS database for a project. | -| `load-vss project='uav'` | Load precomputed exemplars into the VSS database. | -| `load-cfe-isiis-videos` | Load CFE ISII mission videos listed in `missions-to-process.txt`. | -| `cluster-uav` | Cluster missions listed in `missions-to-process.txt`. | -| `detect-uav` | Detect missions listed in `missions-to-process.txt`. | -| `detect-uav-test` | Detect mission data for testing purposes. | -| `load-uav-images` | Load UAV mission images listed in `missions-to-process.txt`. | -| `load-uav type="cluster"` | Load UAV detections or clusters from `missions-to-process.txt`. | -| `fix-uav-metadata` | Fix UAV metadata for latitude, longitude, and altitude. | -| `compute-saliency project='uav'` | Compute saliency for downloaded VOC data and update the Tator database. | -| `crop project='uav'` | Crop detections from VOC-formatted downloads. | -| `download-crop project='uav'` | Download and crop images. | -| `download project='uav'` | Download only. | -| `predict-vss project='uav'` | Predict images using the VSS database. | -| `run-ctenoA-test` | Run strided inference on a single video. | -| `run-ctenoA-prod` | Run strided inference on a collection of videos listed in a TSV file. | -| `run-mega-inference` | Run mega strided inference on a single video. | -| `run-mega-track-bio-video` | Run mega strided tracking for a single video in the bio project. | -| `run-mega-track-bio-dive` | Run mega strided tracking for an entire dive in the bio project. | -| `run-mega-track-i2map-video` | Run mega strided tracking for a single video in the i2map project. | -| `run-mega-track-test-1min` | Run mega strided tracking on a single video to test the pipeline. | -| `run-mega-track-test-fastapiyv5` | Run mega strided tracking on a single video with FastAPI. | -| `cluster-i2mapbulk` | Run inference and clustering on i2map bulk data. | -| `load-i2mapbulk` | Load i2map bulk data using a `cluster_detections.csv` file. | -| `gen-cfe-data` | Generate training data for the CFE project. | -| `gen-i2map-data` | Generate training data for the i2map project. | -| `gen-i2mapbulk-data` | Generate training data for i2map from bulk server data. | -| `gen-uav-data` | Generate training data for the UAV project. | -| `gen-stats-csv project='UAV'` | Generate training data statistics. | +| Recipe | Description | +|---------------------------------|-------------| +| list | List recipes | +| install | Setup the environment | +| cp-env | Copy the default .env file to the project | +| update_trackers | Update the environment. Run after checking out code changes | +| update-env | Update environment | +| cp-core | Copy core dev code to the project on doris | +| cp-dev-ptvr | Copy planktivore dev code to the project on doris | +| cp-dev-uav | Copy UAV dev code to the project on doris | +| cp-dev-bio | Copy bio dev code to the project on doris | +| cp-dev-i2map | Copy i2map dev code to the project on doris | +| plot-tsne-vss project='uav' | Generate a t-SNE plot of the VSS database | +| optimize-vss project='uav' | Optimize the VSS database | +| calc-acc-vss project='uav' | Calculate the accuracy of the VSS database | +| reset-vss-all | Reset the VSS database, removing all data (caution!) | +| reset-vss project='uav' | Reset the VSS database before initializing or creating it | +| remove-vss project='uav' | Remove an entry from the VSS database | +| init-vss project='uav' | Initialize the VSS database for a project | +| load-vss project='uav' | Load computed exemplars into the VSS database | +| load-cfe-isiis-videos | Load CFE ISIIS mission videos | +| load-ptvr-images images='tmp/roi' | Load planktivore ROI images | +| cluster-ptvr-images images='tmp/roi' | Cluster planktivore ROI images | +| load-ptvr-clusters clusters='tmp/roi/cluster.csv' | Load planktivore ROI clusters | +| cluster-uav | Cluster UAV mission data | +| detect-uav | Detect UAV mission data | +| detect-uav-test | Detect UAV mission data (test mode) | +| load-uav-images | Load UAV mission images | +| load-uav type="cluster" | Load UAV detections/clusters | +| fix-uav-metadata | Fix UAV metadata (lat/lon/alt) | +| compute-saliency project='uav' | Compute saliency for VOC data and update Trackers in Tator | +| crop project='uav' | Crop detections from VOC formatted downloads | +| download-crop project='uav' | Download and crop with defaults for a project | +| download project='uav' | Download only | +| predict-vss project='uav' | Predict images using the VSS database | +| run-ctenoA-test | Run strided inference on a single video | +| run-ctenoA-prod | Run strided inference on a collection of videos in a TSV file | +| run-mega-inference | Run mega strided inference on a single video | +| run-mega-track-bio-video | Run mega tracking on a single bio project video | +| run-m | Run mega tracking on a single bio project video | +| run-mega-track-bio-dive | Run mega tracking on an entire dive for the bio project | +| run-mega-track-i2map-video | Run mega tracking on an i2MAP video | +| run-mega-track-test-1min | Test mega tracking on a single video (1 min) | +| run-mega-track-test-fastapiyv5 | Test mega tracking with FastAPI | +| cluster-i2mapbulk | Run inference and cluster on i2MAP bulk data | +| init-i2map-vss | Initialize VSS with combined i2MAP training data | +| load-i2mapbulk data='data' | Load i2MAP bulk data | +| download-i2mapbulk-unlabeled | Download unlabeled i2MAP bulk data | +| gen-cfe-data | Generate training data for the CFE project | +| gen-i2map-data | Generate training data for the i2MAP project | +| gen-i2mapbulk-data | Generate training data for the i2MAP bulk server | +| gen-uav-data | Generate training data for the UAV project | +| gen-stats-csv project='UAV' | Generate training data stats | -- diff --git a/justfile b/justfile index 5e9d80e..ad8afcf 100755 --- a/justfile +++ b/justfile @@ -29,9 +29,13 @@ update-env: conda env update --file environment.yml --prune # Copy core dev code to the project on doris cp-core: + cp justfile /Volumes/dcline/code/aipipeline/justfile cp ./aipipeline/prediction/*.py /Volumes/dcline/code/aipipeline/aipipeline/prediction/ cp ./aipipeline/metrics/*.py /Volumes/dcline/code/aipipeline/aipipeline/metrics/ - cp justfile /Volumes/dcline/code/aipipeline/justfile + +# Copy planktivore dev code to the project on doris +cp-dev-ptvr: + cp ./aipipeline/projects/planktivore/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/planktivore/config/ # Copy uav dev code to the project on doris cp-dev-uav: @@ -41,15 +45,20 @@ cp-dev-uav: # Copy bio dev code to the project on doris cp-dev-bio: cp ./aipipeline/projects/bio/*.py /Volumes/dcline/code/aipipeline/aipipeline/projects/bio/ - cp ./aipipeline/projects/bio/config/ /Volumes/dcline/code/aipipeline/aipipeline/projects/bio/config/ + cp ./aipipeline/projects/bio/core/*.py /Volumes/dcline/code/aipipeline/aipipeline/projects/bio/core + cp ./aipipeline/projects/bio/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/bio/config/ cp ./aipipeline/projects/bio/model/*.py /Volumes/dcline/code/aipipeline/aipipeline/projects/bio/model/ # Copy i2map dev code to the project on doris cp-dev-i2map: - cp ./aipipeline/projects/i2map/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2map/config/ + cp ./aipipeline/projects/i2map/*.py /Volumes/dcline/code/aipipeline/aipipeline/projects/i2map/ cp ./aipipeline/projects/i2mapbulk/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2mapbulk/config/ cp ./aipipeline/projects/i2mapbulk/*.py /Volumes/dcline/code/aipipeline/aipipeline/projects/i2mapbulk/ -# cp ./aipipeline/projects/i2mapbulk/data/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2mapbulk/data/ + cp ./aipipeline/projects/i2map/data/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2map/data/ + cp ./aipipeline/projects/i2map/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2map/config/ + cp ./aipipeline/projects/i2mapbulk/data/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2mapbulk/data/ + cp ./aipipeline/projects/i2mapbulk/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2mapbulk/config/ + cp ./aipipeline/projects/i2map/config/* /Volumes/dcline/code/aipipeline/aipipeline/projects/i2map/config/ # Generate a tsne plot of the VSS database plot-tsne-vss project='uav': @@ -120,6 +129,21 @@ load-cfe-isiis-videos *more_args="": --missions $PROJECT_DIR/data/missions-to-process.txt \ --config $PROJECT_DIR/config/config.yml \ {{more_args}} +# Load planktivore ROI images +load-ptvr-images images='tmp/roi' *more_args="": + #!/usr/bin/env bash + export PROJECT_DIR=./aipipeline/projects/planktivore + time aidata load images --config $PROJECT_DIR/config/config.yml --input {{images}} --token $TATOR_TOKEN --dry-run {{more_args}} +# Cluster planktivore ROI images +cluster-ptvr-images images='tmp/roi' *more_args="": + #!/usr/bin/env bash + export PROJECT_DIR=./aipipeline/projects/planktivore + time sdcat cluster roi --config-ini $PROJECT_DIR/config/sdcat.ini --roi-dir {{images}} {{more_args}} +# Load planktivore ROI clusters +load-ptvr-clusters clusters='tmp/roi/cluster.csv' *more_args="": + #!/usr/bin/env bash + export PROJECT_DIR=./aipipeline/projects/planktivore + time aidata load clusters --config $PROJECT_DIR/config/config.yml --input {{clusters}} --token $TATOR_TOKEN {{more_args}} # Cluster mission in aipipeline/projects/uav/data/missions-to-process.txt cluster-uav *more_args="": #!/usr/bin/env bash @@ -222,7 +246,7 @@ predict-vss project='uav' image_dir='/tmp/download' *more_args="": run-ctenoA-test: #!/usr/bin/env bash export PYTHONPATH=. - time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/cluster_pipeline.py \ + time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/vss_init.py \ --config ./aipipeline/projects/bio/config/config.yml \ --class_name "Ctenophora sp. A" \ --endpoint-url "http://localhost:8001/predict" \ @@ -232,7 +256,7 @@ run-ctenoA-test: run-ctenoA-prod: #!/usr/bin/env bash export PYTHONPATH=. - time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/cluster_pipeline.py \ + time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/vss_init.py \ --config ./aipipeline/projects/bio/config/config.yml \ --class_name "Ctenophora sp. A" \ --endpoint-url "http://fastap-fasta-0riu3xafmhua-337062127.us-west-2.elb.amazonaws.com/predict" \ @@ -242,7 +266,7 @@ run-ctenoA-prod: run-mega-inference: #!/usr/bin/env bash export PYTHONPATH=. - time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/cluster_pipeline.py \ + time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/vss_init.py \ --config ./aipipeline/projects/bio/config/config.yml \ --class_name "animal" \ --class-remap "{\"animal\":\"marine organism\"}" \ @@ -263,10 +287,22 @@ run-mega-track-bio-video video='/mnt/M3/mezzanine/Ventana/2022/09/4432/V4432_202 time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/process.py \ --config ./aipipeline/projects/bio/config/config.yml \ --max-frames-tracked 200 --min-score-det 0.1 --min-score-track 0.1 --batch-size 15 --min-frames 10 --version delme \ - --vits-model /mnt/DeepSea-AI/models/m3midwater-vit-b-16 \ - --det-model /mnt/DeepSea-AI/models/megadet \ - --stride-fps 1 --video {{video}} --max-seconds 10 --flush --gpu-id {{gpu_id}} + --vits-model /Volumes/DeepSea-AI/models/i2MAP/mbari-i2map-vits-b-16ntnr-20250130/ \ + --det-model /Volumes/DeepSea-AI/models/megadet --skip-load \ + --stride 1 --video {{video}} --max-seconds 10 --flush --gpu-id {{gpu_id}} +# Run the mega strided tracking pipeline on a single video for the bio project for 30 seconds +run-m video='/mnt/M3/mezzanine/Ventana/2022/09/4432/V4432_20220914T210637Z_h264.mp4' gpu_id='0': + #!/usr/bin/env bash + export PYTHONPATH=deps:deps/biotrack:. + time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/bio/process.py \ + --config ./aipipeline/projects/bio/config/config.yml \ + --max-frames-tracked 200 --min-score-det 0.1 --min-score-track 0.1 --batch-size 15 --min-frames 20 --version delme \ + --vits-model /mnt/DeepSea-AI/models/i2MAP/mbari-i2map-m3s-vits-b-8nt-20250216 \ + --det-model /mnt/DeepSea-AI/models/megadetrt --skip-load --create-video --imshow \ + --stride 4 --video {{video}} --max-seconds 5 --flush --gpu-id {{gpu_id}} + +#./models/mbari_452k_yolov10 # Run the mega strided tracking pipeline on an entire dive for the bio project run-mega-track-bio-dive dive='/mnt/M3/mezzanine/Ventana/2022/09/4432' gpu_id='0': #!/usr/bin/env bash @@ -294,15 +330,16 @@ run-mega-track-bio-dive dive='/mnt/M3/mezzanine/Ventana/2022/09/4432' gpu_id='0' find "{{dive}}" -name '*.m*' ! -name "._*.m*" -type f | xargs -P 1 -n 1 -I {} bash -c 'process_file "{}"' # Run the mega strided tracking pipeline on a single video for the i2map project -run-mega-track-i2map-video video='/mnt/M3/master/i2MAP/2019/02/20190204/i2MAP_20190205T102700Z_200m_F031_17.mov' gpu_id='0': +#run-mega-track-i2map-video video='/mnt/M3/master/i2MAP/2019/02/20190204/i2MAP_20190205T102700Z_200m_F031_17.mov' gpu_id='0': +run-mega-track-i2map-video video='/mnt/M3/master/i2MAP/2024/11/20241119/i2MAP_20241119T165914Z_200m_F031_4.mov' gpu_id='0': #!/usr/bin/env bash export PYTHONPATH=.:deps/biotrack:. time python3 aipipeline/projects/bio/process.py \ --config ./aipipeline/projects/i2map/config/config.yml \ - --det-model /mnt/DeepSea-AI/models/megadet \ - --vits-model /mnt/DeepSea-AI/models/i2MAP-vit-b-16 \ - --max-frames-tracked 200 --min-score-det 0.0002 --min-score-track 0.5 --min-frames 5 --version megadet-vits-track \ - --stride-fps 15 --max-seconds 60 --imshow --skip-load \ + --det-model /mnt/DeepSea-AI/models/FathomNet/megalodon/ \ + --vits-model /mnt/DeepSea-AI/models/i2MAP/mbari-i2map-vits-b-8-20250216\ + --max-frames-tracked 200 --min-score-det 0.02 --min-score-track 0.1 --min-frames 8 --version metadetrt-vits-track-ft \ + --stride 1 --skip-load --batch-size 16 --create-video \ --video {{video}} --gpu-id {{gpu_id}} # Run the mega strided tracking pipeline on a single video to test the pipeline @@ -314,7 +351,7 @@ run-mega-track-test-1min: --det-model /Volumes/DeepSea-AI/models/megadet \ --vits-model /Volumes/DeepSea-AI/models/m3midwater-vit-b-16 \ --max-frames-tracked 200 --min-score-det 0.0002 --min-score-track 0.5 --min-frames 5 --version mega-vits-track-gcam \ - --stride-fps 15 --max-seconds 60 --imshow --skip-load \ + --stride 8 --max-seconds 60 --imshow --skip-load \ --video aipipeline/projects/bio/data/V4361_20211006T163256Z_h265_1min.mp4 # Run the mega strided tracking pipeline on a single video for the bio project with FastAPI @@ -325,7 +362,7 @@ run-mega-track-test-fastapiyv5: --config ./aipipeline/projects/bio/config/config.yml \ --vits-model /Volumes/DeepSea-AI/models/m3midwater-vit-b-16 \ --max-frames-tracked 200 --min-score-det 0.0002 --min-score-track 0.5 --min-frames 5 --version mega-vits-track-gcam \ - --stride-fps 15 --max-seconds 60 --imshow --skip-load \ + --stride 15 --max-seconds 60 --imshow --skip-load \ --endpoint-url http://FastAP-FastA-0RIu3xAfMhUa-337062127.us-west-2.elb.amazonaws.com/predict \ --video aipipeline/projects/bio/data/V4361_20211006T163256Z_h265_1min.mp4 @@ -334,10 +371,19 @@ cluster-i2mapbulk: #!/usr/bin/env bash export PYTHONPATH=. export MPLCONFIGDIR=/tmp - time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/i2mapbulk/cluster_pipeline.py \ + time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/i2mapbulk/vss_init.py \ --config ./aipipeline/projects/i2mapbulk/config/config_unknown.yml \ --data aipipeline/projects/i2mapbulk/data/bydepth.txt +# Init VSS with combined i2MAP training data. Run with just init-i2map-vss +init-i2map-vss: + #!/usr/bin/env bash + export PYTHONPATH=. + export MPLCONFIGDIR=/tmp + time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/i2map/vss_init.py \ + --config ./aipipeline/projects/i2map/config/config.yml \ + --processed-path /mnt/ML_SCRATCH/i2map/Combined/ + # Load i2MAP bulk data run with ENV_FILE=.env.i2map just load-i2mapbulk load-i2mapbulk data='data': #!/usr/bin/env bash @@ -345,8 +391,8 @@ load-i2mapbulk data='data': time conda run -n aipipeline --no-capture-output python3 aipipeline/projects/i2mapbulk/load_pipeline.py \ --config ./aipipeline/projects/i2mapbulk/config/config_unknown.yml --data {{data}} -# Download i2mpabulk data run with ENV_FILE=.env.i2map just download-i2mapbulk -download-i2mapbulk: +# Download i2mpabulk unlabeled data run with ENV_FILE=.env.i2map just download-i2mapbulk-unlabeled +download-i2mapbulk-unlabeled: just --justfile {{justfile()}} download-crop i2mapbulk --config ./aipipeline/projects/i2mapbulk/config/config_unknown.yml # Generate training data for the CFE project @@ -355,7 +401,7 @@ gen-cfe-data: # Generate training data for the i2map project gen-i2map-data: - just --justfile {{justfile()}} download-crop i2map --skip-clean True T --use-cleanvision True + just --justfile {{justfile()}} download-crop i2map --skip-clean True --use-cleanvision True # Generate training data for the i2map project from the bulk server, run with ENV_FILE=.env.i2map just gen-i2mapbulk-data gen-i2mapbulk-data: