#396: Make JSON_data_files_validator_loader.py callable from the command line (#400)

tlamonthezie · web-flow · commit 7bee1741beb7 · 2023-06-12T13:17:04.000+02:00
* #396: fix conflicts and make 2 modules callable * #396: fix logger module import * #396: reorganize imports in 2 modules * #396: remove unused file * #396: fix some invalid import * #396: fix bad import * #396: enable to run upgrade_configuration to only reformat a configuration file * #396: remove trailing whitespace
diff --git a/docs/docs_config.py b/docs/docs_config.py
@@ -40,7 +40,7 @@
 import lbaf.Utils.csv_2_json_data_converter as Csv2JsonConverter
 import lbaf.Utils.data_stat_files_updater as DataStatFilesUpdater
 import lbaf.Utils.exception_handler as exc_handler
-import lbaf.Utils.logging as logging
+import lbaf.Utils.logger as logger
 import lbaf.Utils.vt_data_extractor as VTDataExtractor
 
 
@@ -97,7 +97,7 @@
     Csv2JsonConverter,
     DataStatFilesUpdater,
     exc_handler,
-    logging,
+    logger,
     VTDataExtractor,
     configurationUpgrader]
 
diff --git a/scripts/upgrade_configuration.py b/scripts/upgrade_configuration.py
@@ -2,7 +2,7 @@
 import argparse
 
 from lbaf import PROJECT_PATH
-from lbaf.Utils.logging import get_logger
+from lbaf.Utils.logger import get_logger
 from lbaf.IO.lbsConfigurationUpgrader import ConfigurationUpgrader, UpgradeAction
 
 
@@ -27,8 +27,15 @@
                     )
 args = parser.parse_args()
 
-if not args.add and not args.remove:
-    raise ValueError("Missing either add (-a or --add xxx) or remove arg (-r or --remove xxx)")
+ACTION = UpgradeAction.FORMAT_ONLY
+KEY = None
+if args.add:
+    ACTION = UpgradeAction.ADD_KEY
+    KEY = args.add
+elif args.remove:
+    ACTION = UpgradeAction.REMOVE_KEY
+    KEY = args.remove
+
 if args.add and args.remove:
     raise ValueError("Cannot set both add and remove args")
 if args.add and not args.value:
@@ -45,9 +52,9 @@
 upgrader = ConfigurationUpgrader(logger=upg_logger)
 upgrader.upgrade_all(
     pattern = args.pattern,
-    relative_to = PROJECT_PATH,
-    action=(UpgradeAction.ADD_KEY if args.add is not None else UpgradeAction.REMOVE_KEY),
-    key=(args.add if args.add is not None else args.remove),
+    relative_to=PROJECT_PATH,
+    action=ACTION,
+    key=KEY,
     value=args.value,
     value_type=args.type
 )
diff --git a/somefile b/somefile
diff --git a/src/lbaf/Applications/JSON_data_files_validator_loader.py b/src/lbaf/Applications/JSON_data_files_validator_loader.py
@@ -5,13 +5,15 @@
 from urllib.request import urlretrieve
 from urllib.error import HTTPError, URLError
 
-from .. import PROJECT_PATH
-from ..Utils.logging import get_logger
-from ..Utils.exception_handler import exc_handler
+from lbaf import PROJECT_PATH
+from lbaf.Utils.logger import get_logger
+from lbaf.Utils.exception_handler import exc_handler
 
 TARGET_DIR = os.path.join(PROJECT_PATH, "src", "lbaf", "imported")
 TARGET_SCRIPT_NAME = "JSON_data_files_validator.py"
 SOURCE_SCRIPT_URL=f"https://raw.githubusercontent.com/DARMA-tasking/vt/develop/scripts/{TARGET_SCRIPT_NAME}"
+IMPORT_DIR = os.path.join(PROJECT_PATH, "src", "lbaf", "imported")
+
 
 def _save_schema_validator_and_init_file():
     """Downloads the JSON data files validator module to the lbaf/imported directory.
@@ -80,5 +82,7 @@ def is_loaded():
     :return: True if the script exists in the target location otherwise False
     :rtype: bool
     """
-    import_dir = os.path.join(PROJECT_PATH, "src", "lbaf", "imported")
-    return os.path.isfile(os.path.join(import_dir, TARGET_SCRIPT_NAME))
+    return os.path.isfile(os.path.join(IMPORT_DIR, TARGET_SCRIPT_NAME))
+
+if __name__ == "__main__":
+    load()
diff --git a/src/lbaf/Applications/LBAF_app.py b/src/lbaf/Applications/LBAF_app.py
@@ -13,7 +13,7 @@
 from lbaf.Applications import JSON_data_files_validator_loader
 from lbaf.Utils.exception_handler import exc_handler
 from lbaf.Utils.path import abspath
-from lbaf.Utils.logging import get_logger, Logger
+from lbaf.Utils.logger import get_logger, Logger
 import lbaf.IO.lbsStatistics as lbstats
 from lbaf.IO.lbsConfigurationValidator import ConfigurationValidator
 from lbaf.IO.lbsVTDataReader import LoadReader
diff --git a/src/lbaf/Applications/MoveCountsViewer.py b/src/lbaf/Applications/MoveCountsViewer.py
@@ -7,7 +7,7 @@
 
 from lbaf import PROJECT_PATH
 from lbaf.Utils.exception_handler import exc_handler
-from lbaf.Utils.logging import get_logger, Logger
+from lbaf.Utils.logger import get_logger, Logger
 
 
 class MoveCountsViewerParameters:
diff --git a/src/lbaf/Applications/rank_object_enumerator.py b/src/lbaf/Applications/rank_object_enumerator.py
@@ -8,7 +8,7 @@
 from lbaf import PROJECT_PATH
 from lbaf.IO.lbsVTDataReader import LoadReader
 from lbaf.Utils.exception_handler import exc_handler
-from lbaf.Utils.logging import get_logger
+from lbaf.Utils.logger import get_logger
 from lbaf.Utils.path import abspath
 from lbaf.Model.lbsPhase import Phase
 from lbaf.IO.lbsStatistics import (
diff --git a/src/lbaf/Execution/lbsAlgorithmBase.py b/src/lbaf/Execution/lbsAlgorithmBase.py
@@ -6,7 +6,7 @@
 from ..Model.lbsPhase import Phase
 from ..Model.lbsWorkModelBase import WorkModelBase
 from ..Utils.exception_handler import exc_handler
-from ..Utils.logging import get_logger, Logger
+from ..Utils.logger import get_logger, Logger
 
 
 class AlgorithmBase:
diff --git a/src/lbaf/Execution/lbsCriterionBase.py b/src/lbaf/Execution/lbsCriterionBase.py
@@ -6,7 +6,6 @@
 from ..Model.lbsWorkModelBase import WorkModelBase
 from ..Model.lbsPhase import Phase
 from ..Utils.exception_handler import exc_handler
-from ..Utils.logging import get_logger
 
 
 class CriterionBase:
diff --git a/src/lbaf/IO/lbsConfigurationUpgrader.py b/src/lbaf/IO/lbsConfigurationUpgrader.py
@@ -31,28 +31,31 @@ def bool_representer(dumper, value):
         text = "False"
     return dumper.represent_scalar('tag:yaml.org,2002:bool', text)
 
+
 class UpgradeAction(Enum):
     """Upgrade action"""
 
     ADD_KEY = "add"
     REMOVE_KEY = "remove"
+    FORMAT_ONLY = "format_only"
 
 class ConfigurationDumper(yaml.Dumper):
     """Custom dumper to add indent before list items hyphens."""
 
     def increase_indent(self, flow=False, indentless=False):
         return super(ConfigurationDumper, self).increase_indent(flow, False)
 
+
+ConfigurationDumper.add_representer(bool, bool_representer)
+
+
 class ConfigurationUpgrader:
     """This class enables to bulk upgrade configuration files by adding or removing keys."""
 
-    __dumper: ConfigurationDumper
     __logger: Logger
     __sections: dict
 
     def __init__(self, logger: Logger):
-        self.__dumper = ConfigurationDumper
-        self.__dumper.add_representer(bool, bool_representer)
         self.__logger = logger
         self.__sections = cast(dict, ConfigurationValidator.allowed_keys(group=True))
 
@@ -68,42 +71,52 @@ def write_node(self, k: str, value: Any, yaml_file: IO, indent_size: int = 2):
                 indent=indent_size,
                 line_break='\n',
                 sort_keys=False,
-                Dumper=self.__dumper
+                Dumper=ConfigurationDumper
             ).replace(
                 '\n',
                 '\n' + indent_str
             )
             if yaml_node.endswith('\n' + indent_str):
                 yaml_node = yaml_node[:-(indent_size + 1)]
         else:
-            yaml_node = " " + yaml.representer.BaseRepresenter().represent_data(value).value
+            yaml_node = yaml.dump(
+                value,
+                indent=0,
+                Dumper=ConfigurationDumper,
+                explicit_start=None,
+                explicit_end=None
+            )
+            if yaml_node.endswith('...\n'):
+                yaml_node = yaml_node[:-4]
+            yaml_node = ' ' + yaml_node.strip()
         yaml_file.write(yaml_node)
         yaml_file.write('\n')
 
     def upgrade(
         self,
         file_path: Path,
         action: UpgradeAction,
-        key: str,
+        key: str = None,
         value: str = None,
         value_type: str = "str"
     ) -> None:
         """Apply an upgrade to the given configuration file."""
         self.__logger.debug("Upgrading file %s ...", file_path)
         key_path = None
-        if action == UpgradeAction.ADD_KEY:
-            self.__logger.debug("Add key `%s` with value `%s`", key, value)
-        elif action == UpgradeAction.REMOVE_KEY:
-            print(f"Remove key `{key}`")
-        key_path = key.split('.')
-        parsed_value = value
-        if value_type is not None:
-            type_v = locate(value_type)
-            if callable(type_v):
-                parsed_value = type_v(value)
-
-        if key_path is None:
-            raise ValueError("The `key` must be a valid string")
+        if action != UpgradeAction.FORMAT_ONLY:
+            if action == UpgradeAction.ADD_KEY:
+                self.__logger.debug("Add key `%s` with value `%s`", key, value)
+            elif action == UpgradeAction.REMOVE_KEY:
+                print(f"Remove key `{key}`")
+            key_path = key.split('.')
+            parsed_value = value
+            if value_type is not None:
+                type_v = locate(value_type)
+                if callable(type_v):
+                    parsed_value = type_v(value)
+
+            if key_path is None and action != UpgradeAction.FORMAT_ONLY:
+                raise ValueError("The `key` must be a valid string")
 
         conf = None
         file_comment = None
@@ -126,36 +139,36 @@ def upgrade(
                 file_comment = file_comment.strip() # removes potential eol at the end
 
             yaml_file.seek(0)
-
             yaml_content = yaml_file.read()
             conf = yaml.safe_load(yaml_content)
             node = conf
-            for i, key in enumerate(key_path):
-                is_leaf = i == len(key_path)-1
-                # if node does not exist create it
-                if not key in node.keys():
-                    if action == UpgradeAction.ADD_KEY:
-                        if is_leaf:
+            if action != UpgradeAction.FORMAT_ONLY:
+                for i, key in enumerate(key_path):
+                    is_leaf = i == len(key_path)-1
+                    # if node does not exist create it
+                    if not key in node.keys():
+                        if action == UpgradeAction.ADD_KEY:
+                            if is_leaf:
+                                node[key] = parsed_value
+                            else:
+                                new_branch = {}
+                                new_node = new_branch
+                                new_key_path = key_path[i+1:]
+                                for j, new_key in enumerate(new_key_path):
+                                    is_leaf = j == len(new_key_path) - 1
+                                    new_node[new_key] = parsed_value if is_leaf else {}
+                                    new_node = new_node[new_key]
+                                node[key] = new_branch
+                    elif is_leaf:
+                        if action == UpgradeAction.REMOVE_KEY:
+                            del node[key]
+                        elif action == UpgradeAction.ADD_KEY:
                             node[key] = parsed_value
-                        else:
-                            new_branch = {}
-                            new_node = new_branch
-                            new_key_path = key_path[i+1:]
-                            for j, new_key in enumerate(new_key_path):
-                                is_leaf = j == len(new_key_path) - 1
-                                new_node[new_key] = parsed_value if is_leaf else {}
-                                new_node = new_node[new_key]
-                            node[key] = new_branch
-                elif is_leaf:
-                    if action == UpgradeAction.REMOVE_KEY:
-                        del node[key]
-                    else:
-                        node[key] = parsed_value
-                # go next node in child tree
-                if is_leaf:
-                    break
+                    # go next node in child tree
+                    if is_leaf:
+                        break
 
-                node=node[key]
+                    node=node[key]
 
         with open(file_path, 'w', encoding="utf-8") as yaml_file:
             if file_comment is not None:
diff --git a/src/lbaf/IO/lbsConfigurationValidator.py b/src/lbaf/IO/lbsConfigurationValidator.py
@@ -186,9 +186,9 @@ def allowed_keys(group: bool =  False) -> Union[List[str], Dict[str, List[str]]]
             "work model": ["work_model"],
             "algorithm": ["brute_force_optimization", "algorithm"],
             "output": [
-                "logging_level", "log_to_file", "overwrite_validator", "check_schema", "terminal_background",
+                "logging_level", "log_to_file", "overwrite_validator", "terminal_background",
                 "generate_multimedia", "output_dir", "output_file_stem",
-                "LBAF_Viz"
+                "LBAF_Viz", "write_JSON"
             ]
         }
 
diff --git a/src/lbaf/IO/lbsGridStreamer.py b/src/lbaf/IO/lbsGridStreamer.py
@@ -2,7 +2,7 @@
 
 import vtk
 
-from ..Utils.logging import get_logger, Logger
+from ..Utils.logger import get_logger, Logger
 
 
 class GridStreamer:
diff --git a/src/lbaf/Model/lbsPhase.py b/src/lbaf/Model/lbsPhase.py
@@ -11,7 +11,7 @@
 from ..IO.lbsStatistics import print_subset_statistics, print_function_statistics, sampler
 from ..IO.lbsVTDataReader import LoadReader
 from ..Utils.exception_handler import exc_handler
-from ..Utils.logging import get_logger
+from ..Utils.logger import get_logger
 
 
 class Phase:
diff --git a/src/lbaf/Model/lbsWorkModelBase.py b/src/lbaf/Model/lbsWorkModelBase.py
@@ -3,7 +3,7 @@
 import sys
 
 from ..Utils.exception_handler import exc_handler
-from ..Utils.logging import get_logger
+from ..Utils.logger import get_logger
 
 class WorkModelBase:
     """An abstract base class of per-rank work model."""
diff --git a/src/lbaf/Utils/logger.py b/src/lbaf/Utils/logger.py
diff --git a/src/lbaf/Utils/vt_data_extractor.py b/src/lbaf/Utils/vt_data_extractor.py
diff --git a/tests/test_configuration_validator.py b/tests/test_configuration_validator.py

Original file line number	Diff line number	Diff line change
`@@ -186,9 +186,9 @@ def allowed_keys(group: bool = False) -> Union[List[str], Dict[str, List[str]]]`
`186`	`186`	`"work model": ["work_model"],`
`187`	`187`	`"algorithm": ["brute_force_optimization", "algorithm"],`
`188`	`188`	`"output": [`
`189`		`- "logging_level", "log_to_file", "overwrite_validator", "check_schema", "terminal_background",`
	`189`	`+ "logging_level", "log_to_file", "overwrite_validator", "terminal_background",`
`190`	`190`	`"generate_multimedia", "output_dir", "output_file_stem",`
`191`		`- "LBAF_Viz"`
	`191`	`+ "LBAF_Viz", "write_JSON"`
`192`	`192`	`]`
`193`	`193`	`}`
`194`	`194`