Merge pull request #53 from valentingol/reorganize_repo

🆙 Update to 2.2.0

.github/workflows/pydocstyle.yaml

@@ -23,4 +23,4 @@ jobs:
       run: |
         n_errors=$(wc -l < pydocstyle.txt)
         echo "Number of errors in docstrings: $n_errors"
-        python utils/github_actions/pydocstyle_manager.py --n_errors=$n_errors
+        python github_actions/pydocstyle_manager.py --n_errors=$n_errors

.github/workflows/pylint.yaml

@@ -23,7 +23,7 @@ jobs:
       run: |
         score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' .pylint.txt)
         echo "Pylint score was $score"
-        color=$(python utils/github_actions/pylint_manager.py --score=$score --score_min=8.0)
+        color=$(python github_actions/pylint_manager.py --score=$score --score_min=8.0)
         echo "PYLINT_COLOR=$color"
         echo "PYLINT_COLOR=$color" >> $GITHUB_ENV
         echo "PYLINT_SCORE=$score/10.00"

README.md

@@ -1,31 +1,54 @@
 # Facies modeling with GANs
 
-Underground facies (= kind of rocks) modeling with GANs.
+Underground facies (= kind of sediment) modeling with GANs.
+
+**Disclaimer**: This repository is a work in progress. This code aims to be the
+official implementation of a not already published research paper. It provides
+a complete code to train and evaluate some GANs for facies modeling as well as
+unit tests and a small ready-to-use dataset (see below).
 
 ![PythonVersion](https://img.shields.io/badge/python-3.7%20%7E%203.10-informational)
 ![PytorchVersion](https://img.shields.io/badge/Pytorch-1.8%20%7E%201.12-blue)
-[![License](https://img.shields.io/badge/license-MIT-white)](https://stringfixer.com/fr/MIT_license)
+[![License](https://img.shields.io/badge/license-MIT-white)](
+    https://stringfixer.com/fr/MIT_license)
 ![WandB](https://img.shields.io/badge/WandB-supported-brightgreen)
 ![ClearML](https://img.shields.io/badge/ClearML-supported-brightgreen)
 
-[![Flake8](https://github.com/valentingol/gan-facies-modeling/actions/workflows/flake.yaml/badge.svg)](https://github.com/valentingol/gan-facies-modeling/actions/workflows/flake.yaml)
-[![Pydocstyle](https://github.com/valentingol/gan-facies-modeling/actions/workflows/pydocstyle.yaml/badge.svg)](https://github.com/valentingol/gan-facies-modeling/actions/workflows/pydocstyle.yaml)
-[![MyPy](https://github.com/valentingol/gan-facies-modeling/actions/workflows/mypy.yaml/badge.svg)](https://github.com/valentingol/gan-facies-modeling/actions/workflows/mypy.yaml)
-[![Isort](https://github.com/valentingol/gan-facies-modeling/actions/workflows/isort.yaml/badge.svg)](https://github.com/valentingol/gan-facies-modeling/actions/workflows/isort.yaml)
-[![PyLint](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/valentingol/106c646ac67294657bccf02bbe22208f/raw/gan_facies_modeling_pylint.json)](https://github.com/valentingol/gan-facies-modeling/actions/workflows/pylint.yaml)
+[![Flake8](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/flake.yaml/badge.svg)](
+        https://github.com/valentingol/gan-facies-modeling/actions/workflows/flake.yaml)
+[![Pydocstyle](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/pydocstyle.yaml/badge.svg)](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/pydocstyle.yaml)
+[![MyPy](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/mypy.yaml/badge.svg)](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/mypy.yaml)
+[![Isort](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/isort.yaml/badge.svg)](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/isort.yaml)
+[![PyLint](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/valentingol/106c646ac67294657bccf02bbe22208f/raw/gan_facies_modeling_pylint.json)](
+    https://github.com/valentingol/gan-facies-modeling/actions/workflows/pylint.yaml)
 
 ---
 
 Date: 2022-07-20
 
 Author: [github@Valentingol](https://github.com/valentingol)
 
+[![GitHub User followers](
+    https://img.shields.io/github/followers/valentingol?label=Owner%20followers&style=social)](
+    https://github.com/valentingol)
+[![GitHub User's User stars](
+    https://img.shields.io/github/stars/valentingol?label=Owner%20Stars&style=social)](
+    https://github.com/valentingol)
+
 Work during my 5 months internship at [IFPEN](https://www.ifpenergiesnouvelles.com/),
 supervised by [UVSQ](https://www.uvsq.fr/english)
 and financed by [DATAIA Paris Saclay](https://dataia.eu/en/dataia-paris-saclay-institute).
 
-[![GitHub User followers](https://img.shields.io/github/followers/valentingol?label=Owner%20followers&style=social)](https://github.com/valentingol)
-[![GitHub User's User stars](https://img.shields.io/github/stars/valentingol?label=Owner%20Stars&style=social)](https://github.com/valentingol)
+Copyright © 2022 Goldite Valentin
+
+MIT License ([see here](LICENSE.md))
 
 ---
 
@@ -35,40 +58,22 @@ and financed by [DATAIA Paris Saclay](https://dataia.eu/en/dataia-paris-saclay-i
 
 - `sagan` - Unconditional SAGAN (based on
 [Self-Attention Generative Adversarial Networks](https://arxiv.org/abs/1805.08318)
-and [Modeling of subsurface sedimentary facies using SAGANs](https://www.sciencedirect.com/science/article/abs/pii/S0920410522003540))
+and [Modeling of subsurface sedimentary facies using SAGANs](
+    https://www.sciencedirect.com/science/article/abs/pii/S0920410522003540))
 
 - `cond_sagan` - Conditional SAGAN (based on
 papers above
 for SAGAN part and
-[GANSim: Conditional Facies Simulation Using an Improved Progressive Growing of GANS](https://ideas.repec.org/p/osf/eartha/fm24b.html)
-for conditional part)
+[GANSim: Conditional Facies Simulation Using an Improved Progressive Growing of GANS](
+    https://ideas.repec.org/p/osf/eartha/fm24b.html)
+for conditional part) that also reconstructs input pixel maps
+
+Note that you can disable self-attention in the configurations (DCGAN architecture).
 
 **3D Models:**
 
 - soon 🚧
 
-## Examples
-
-### GANSim dataset
-
-| <img src="./assets/images/gansim_real.png" width="512">
-|:--:|
-| **Real Images** (64 $\times$ 64)|
-
-| <img src="./assets/images/gansim_generated.png" width="512">
-|:--:|
-| **Generated Images** (128 $\times$ 128)|
-
-### Stanford-VI dataset (first part)
-
-| <img src="./assets/images/stanfordp1_real.png" width="512">
-|:--:|
-| **Real Images**|
-
-| <img src="./assets/images/stanfordp1_generated.png" width="512">
-|:--:|
-| **Generated Images**|
-
 ## Quick start
 
 ### Installation
@@ -78,78 +83,95 @@ Install the module and dependencies in a virtual environment with Python 3.7-3.1
 ```bash
 pip install -e .
 pip install -r requirements.txt
-# for dev only:
+# for developers only:
 pip install -r requirements-dev.txt
 ```
 
-### Train on a dataset
+### Train on the default dataset
 
 A small dataset is available by default in this repository. It contains 2000
 synthesized images representing some channels and 3 kind of facies and was
 generated in the [GANSim project](https://github.com/SuihongSong/GeoModeling_GANSim-2D_Condition_to_Well_Facies_and_Global_Features)
 (under [MIT license](./assets/third_party_licenses/GANSim%20MIT%20LICENSE)).
 More synthesized data are available
-[here](https://zenodo.org/record/3993791#.X1FQuMhKhaR).
+[here](https://zenodo.org/record/3993791#.X1FQuMhKhaR). **If you use this dataset
+in your work, please cite the original authors.**
 
-Of course, you can put your own dataset in the `datasets` folder. The dataset
-should be a Numpy (`.npy`) file containing a 3D ndarray with format
-(z/depth/n_samples, y, x) of type `uint8` with a different number for each
-facies, starting from 0. The number of facies is then `dataset.max() + 1`.
-
-Now you can simply run a train on the default dataset with unconditional SAGAN
-model using the following command:
+You can simply run a train on the default dataset with unconditional SAGAN
+model using the following command in `gan_facies` folder:
 
 ```bash
-python apps/train.py
+python gan_facies/apps/train.py
 ```
 
 You can see the progress of the training in the terminal and the resulted
 images and trained networks in the `res` folder.
 
-This repository contains a lot of configurations to customize the training and
-will be explained in the next section.
+## Use your own dataset
+
+Of course, you can use your own dataset. Simply drop it in the `datasets` folder.
+The dataset should be a Numpy file (`.npy`) containing a 3D ndarray with format
+(z/depth/n_samples, y, x) of type `uint8` with a different number for each
+facies, starting from 0. The number of facies is then `dataset.max() + 1`.
+Now tou can run the training adding the `--dataset_path=<mypath>` argument.
+You can also change the dataset path via configuration files. The next section
+explains how to do that.
 
 ## Configurations
 
-Of course it is always interesting to customize the training with flexibility.
-Thus this repository use the smart configuration manager
-[YAECS](https://github.com/valentingol/yaecs).
+It is always interesting to customize the training with your own configurations.
+This repository contains a lot of configuration organized in multiple sub-configurations.
+The management of the configurations is simply done thanks to the smart configuration
+manager [YAECS](https://github.com/valentingol/yaecs).
 
 The default sub-configurations (for models, training, ...) are organized in
-different sub-folders in `configs/default`. You can launch an experiment by
-writing an other configuration `.yaml` file that will be merged with the
-default one. Some examples are available in `configs/exp`. For example,
-this will override the default value of the name of the run and discriminator
+different json files in `configs/default`. You can launch your own experiment by
+writing a new `.yaml` file that will be merged with the default configuration.
+Some examples are available in `configs/exp`. For example, the following file
+will override the default value of the name of the run as well as discriminator
 learning rate to 0.001:
 
 ```yaml
-# configs/exp/my_experiment.yaml
+# >> file 'configs/exp/my_config.yaml'
 run_name: my_experiment
 training.d_lr: 0.001
 ```
 
 Then you can run the experiment by adding the configuration in command line.
 
 ```bash
-python apps/train.py --config configs/exp/my_config.yaml
+python gan_facies/apps/train.py --config gan_facies/configs/exp/my_config.yaml
 ```
 
 *Note: The space between the `--config` and the configuration file is important.*
 
-You can also put instead a **list** of configuration paths to merge together
-several experiment files (from the begin of the list to the end).
-
 Moreover you can override parameters also by adding them in the **command line**.
 For example this will override the default configuration with your experiment
 configuration, then set the generator learning rate to 0.001 and the generator
 random input dimension to 64:
 
 ```bash
-python apps/train.py --config configs/exp/my_config.yaml --training.g_lr=0.001 --model.z_dim=64
+python gan_facies/apps/train.py --config gan_facies/configs/exp/my_config.yaml--training.g_lr=0.001\
+ --model.z_dim=64
 ```
 
 *Note: The `=` between the `--param` and the value is important.*
 
+To use conditional model you can check the `configs/exp/conditional.yaml` file
+and adapt it to your needs. An other way is to use the merging of configurations
+in cascade provided by yaecs. In fact, if you can put **list** of configuration
+file for `--configs`, they will be merge together (from the begin of the list
+to the end). Example:
+
+```batch
+python gan_facies/apps/train.py --config [gan_facies/configs/exp/models/cond_sagan.yaml,gan_facies/configs/exp/my_config.yaml]
+```
+
+First `configs/exp/models/cond_sagan.yaml` will be merged (changing model configuration)
+then `configs/exp/my_config` (overwriting model configuration if needed).
+You can create your own specific configurations (for data, models, metric, ...)
+and merge as many of them as you want.
+
 Finally, the configurations will be automatically saved (by default in `res/configs`)
 to ensure that you can always recover the exact configuration used for the runs.
 The "hierarchy of merging" is also saved to understand quickly how the configuration
@@ -165,11 +187,11 @@ more interesting parameters and many utilities to explore parameters space
 (collaboratively or not), etc.
 
 This repository allows to use WandB and ClearML very simply. You can check the default
-configuration implying WandB in `configs/default/wandb.yaml` and ClearML in
-`configs/default/clearml.yaml`. To use Wandb or ClearML you need to install
-one of them, create an account if you don't have one and set the configuration
-`wandb.use_wandb: True` or `clearml.use_clearml: True` in addition to the parameters
-for initialize the WandB run or ClearML task.
+configuration implying WandB and ClearML in `configs/default/experiment_tracking.yaml`.
+To use Wandb or ClearML you first need to install them, create an account if
+you don't have one and set the configuration `wandb.use_wandb: True` or
+`clearml.use_clearml: True` in addition to the parameters for initialize
+the WandB run or ClearML task.
 
 Plus, you can explore the parameters space using `wandb.sweep`. To do so, you
 simply need to create a sweep config such as in `configs/sweep/ex_sweep.yaml`
@@ -181,18 +203,42 @@ in `configs/exp`.
 Note:
 
 - It is currently not possible to use both ClearML and WandB at the same time
-- It is currently not possible to use hyperparameter search with ClearML in
-this repository (🚧)
+- It is currently not possible to use hyperparameter search with ClearML
+  (only with wandb sweep). But we welcome any contribution to add this feature
+  (see [`CONTRIBUTE.md`](CONTRIBUTE.md))
+
+## Examples of generated images
+
+### GANSim dataset
+
+| <img src="./assets/images/gansim_real.png" width="512">
+|:--:|
+| **Real Images** (64 $\times$ 64)|
+
+| <img src="./assets/images/gansim_generated.png" width="512">
+|:--:|
+| **Generated Images** (128 $\times$ 128)|
+
+### Stanford-VI dataset (first part)
+
+| <img src="./assets/images/stanfordp1_real.png" width="512">
+|:--:|
+| **Real Images**|
+
+| <img src="./assets/images/stanfordp1_generated.png" width="512">
+|:--:|
+| **Generated Images**|
 
 ## TODO list
 
 - [x] Add test for generator in `apps/train.py`
 - [x] Add generated results on GANSim dataset and
 [Stanford VI dataset](https://github.com/SCRFpublic/Stanford-VI-E/tree/master/Facies)
 - [x] Add conditional SAGAN
+- [ ] Add images generated by conditional model and metrics in README
 - [ ] Add 3D models
-- [ ] Explore other architectures
 
 ## How to contribute
 
-Please have a look on [CONTRIBUTE.md](./CONTRIBUTE.md). Thank you very much! 🙏
+We welcome any contribution to improve this repository. Please have a look on
+[CONTRIBUTE.md](./CONTRIBUTE.md). Thank you very much! 🙏

gan_facies/__init__.py

@@ -0,0 +1,25 @@
+"""GAN facies modeling repository..
+
+MIT License
+
+Copyright (c) 2022 Goldite Valentin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files
+(the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+"""

apps/eval.py → gan_facies/apps/eval.py

@@ -9,14 +9,15 @@
 from PIL import Image
 from thop import profile
 
-from utils.auxiliaries import set_global_seed
-from utils.conditioning import colorize_pixel_map, generate_pixel_maps
-from utils.configs import ConfigType, GlobalConfig
-from utils.data.data_loader import DatasetCond2D, DistributedDataLoader
-from utils.data.process import to_img_grid
-from utils.gan.cond_sagan.modules import CondSAGenerator
-from utils.gan.uncond_sagan.modules import UncondSAGenerator
-from utils.metrics import compute_save_indicators, evaluate, print_metrics
+from gan_facies.data.data_loader import DatasetCond2D, DistributedDataLoader
+from gan_facies.data.process import to_img_grid
+from gan_facies.gan.cond_sagan.modules import CondSAGenerator
+from gan_facies.gan.uncond_sagan.modules import UncondSAGenerator
+from gan_facies.metrics import compute_save_indicators, evaluate, print_metrics
+from gan_facies.utils.auxiliaries import set_global_seed
+from gan_facies.utils.conditioning import (colorize_pixel_map,
+                                           generate_pixel_maps)
+from gan_facies.utils.configs import ConfigType, GlobalConfig
 
 
 def test(config: ConfigType) -> None:
@@ -87,7 +88,7 @@ def test(config: ConfigType) -> None:
         with torch.no_grad():
             images, attn_list = generator.generate(z_input, with_attn=True)
             # pylint: disable=unbalanced-tuple-unpacking
-            macs, _ = profile(generator, inputs=(z_input, pixel_maps))
+            macs, _ = profile(generator, inputs=(z_input,))
 
     # Save and show sample images in a grid
     img_out_dir = osp.join(config.output_dir, config.run_name, 'samples')
@@ -133,6 +134,6 @@ def save_and_show(image: Optional[np.ndarray], path: str) -> None:
 
 if __name__ == '__main__':
     global_config = GlobalConfig.build_from_argv(
-        fallback='configs/exp/base.yaml')
+        fallback='gan_facies/configs/exp/base.yaml')
     # NOTE: The config is not saved when testing only
     test(global_config)

apps/tools/compute_ref_metrics.py → gan_facies/apps/tools/compute_ref_metrics.py

@@ -2,13 +2,13 @@
 
 import numpy as np
 
-from utils.configs import GlobalConfig
-from utils.metrics import print_metrics
-from utils.metrics.metric import wasserstein_distances
+from gan_facies.metrics import print_metrics
+from gan_facies.metrics.metric import wasserstein_distances
+from gan_facies.utils.configs import GlobalConfig
 
 if __name__ == '__main__':
     global_config = GlobalConfig.build_from_argv(
-        fallback='configs/exp/base.yaml')
+        fallback='gan_facies/configs/exp/base.yaml')
     data = np.load(global_config.dataset_path)
     data1 = data[:len(data) // 2][:5000]
     data2 = data[len(data) // 2:][:5000]