Update general logger

.github/workflows/prevent_merge_when_branch_behind.yml

@@ -1,36 +1,42 @@
-name: Require Branch to Be Up-to-Date with Production
+name: Require Branch to Be Up-to-Date
 
 # Trigger this workflow on pull request events targeting a specific branch.
 on:
   pull_request:
     branches:
       - production
       - development
-      - test-protect-main-merge # for testing
+      - fix_prevent_merge_when_branch_behind
   workflow_dispatch: # enables manual triggering
 
 jobs:
   check-branch:
     runs-on: ubuntu-latest
     steps:
+      - name: Determine Comparison Branch
+        id: comparison
+        run: |
+          if [[ "${{ github.event.pull_request.base.ref }}" == "production" && "${{ github.event.pull_request.head.ref }}" == "development" ]]; then
+            echo "branch=production" >> $GITHUB_ENV
+          else
+            echo "branch=development" >> $GITHUB_ENV
+          fi
+
       - name: Checkout pull request branch
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.ref }}
 
-      - name: Fetch production branch
+      - name: Fetch comparison branch
         run: |
           git fetch --unshallow
-          git fetch origin production
+          git fetch origin ${{ env.branch }}
 
-      - name: Compare branch with production
+      - name: Compare branch with ${{ env.branch }}
         run: |
-          if git merge-base --is-ancestor origin/production HEAD; then
-            echo "::notice ::Branch is up-to-date with production."
+          if git merge-base --is-ancestor origin/${{ env.branch }} HEAD; then
+            echo "::notice ::Branch is up-to-date with ${{ env.branch }}."
           else
-            echo "::error ::Merge Blocked: Your branch is behind the latest commits on production. Please update your branch with the latest changes from production before attempting to merge."
-            echo "Merge base: $(git merge-base HEAD origin/production)"
+            echo "::error ::Merge Blocked: Your branch is behind the latest commits on ${{ env.branch }}. Please update your branch before attempting to merge."
             exit 1
           fi
-
-

common_configs/config_log.yaml

@@ -0,0 +1,62 @@
+version: 1
+disable_existing_loggers: False
+
+formatters:
+  detailed:
+    format: '%(asctime)s %(pathname)s [%(filename)s:%(lineno)d] [%(process)d] [%(threadName)s] - %(levelname)s - %(message)s'
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: detailed
+    stream: ext://sys.stdout
+
+  info_file_handler:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: INFO
+    formatter: detailed
+    filename: "{LOG_PATH}/views_pipeline_INFO.log"
+    when: "midnight"
+    backupCount: 30
+    encoding: "utf8"
+
+  debug_file_handler:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: DEBUG
+    formatter: detailed
+    filename: "{LOG_PATH}/views_pipeline_DEBUG.log"
+    when: "midnight"
+    backupCount: 10
+    encoding: "utf8"
+
+  warning_file_handler:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: WARNING
+    formatter: detailed
+    filename: "{LOG_PATH}/views_pipeline_WARNING.log"
+    when: "midnight"
+    backupCount: 20
+    encoding: "utf8"
+
+  error_file_handler:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: ERROR
+    formatter: detailed
+    filename: "{LOG_PATH}/views_pipeline_ERROR.log"
+    when: "midnight"
+    backupCount: 60
+    encoding: "utf8"
+
+  critical_file_handler:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: CRITICAL
+    formatter: detailed
+    filename: "{LOG_PATH}/views_pipeline_CRITICAL.log"
+    when: "midnight"
+    backupCount: 90
+    encoding: "utf8"
+
+root:
+  level: DEBUG
+  handlers: [console, info_file_handler, debug_file_handler, warning_file_handler, error_file_handler, critical_file_handler]

common_utils/global_cache.py

@@ -113,12 +113,12 @@ def ensure_cache_file_exists(self):
         Ensures that the cache file exists. If it does not exist, creates a new cache file.
         """
         if not self.filepath.exists():
-            logging.info(
+            logging.warning(
                 f"Cache file: {self.filepath} does not exist. Creating new cache file..."
             )
             with open(self.filepath, "wb") as f:
                 pickle.dump({}, f)
-            logging.info(f"Created new cache file: {self.filepath}")
+            logging.debug(f"Created new cache file: {self.filepath}")
 
     def set(self, key, value):
         """
@@ -167,7 +167,7 @@ def save_cache(self):
         """
         with open(self.filepath, "wb") as f:
             pickle.dump(self.cache, f)
-        logging.info(f"Cache saved to file: {self.filepath}")
+        logging.debug(f"Cache saved to file: {self.filepath}")
 
     def load_cache(self):
         """
@@ -179,7 +179,7 @@ def load_cache(self):
                     loaded_cache = pickle.loads(f.read())
                     if isinstance(loaded_cache, dict):
                         self.cache = loaded_cache
-                        logging.info(f"Cache loaded from file: {self.filepath}")
+                        logging.debug(f"Cache loaded from file: {self.filepath}")
                     else:
                         logging.error(
                             f"Loaded cache is not a dictionary. Initializing empty cache."
@@ -192,7 +192,7 @@ def load_cache(self):
                 self.cache = {}
         else:
             self.cache = {}
-            logging.info(f"Cache file does not exist. Initialized empty cache.")
+            logging.debug(f"Cache file does not exist. Initialized empty cache.")
 
 
 def cleanup_cache_file():

common_utils/model_path.py

@@ -61,45 +61,45 @@ class ModelPath:
         _ignore_attributes (list): A list of paths to ignore.
     """
 
-    __slots__ = (
-        "_validate",
-        "target",
-        "use_global_cache",
-        "_force_cache_overwrite",
-        "root",
-        "models",
-        "common_utils",
-        "common_configs",
-        "_ignore_attributes",
-        "model_name",
-        "_instance_hash",
-        "_queryset",
-        "model_dir",
-        "architectures",
-        "artifacts",
-        "configs",
-        "data",
-        "data_generated",
-        "data_processed",
-        "data_raw",
-        "dataloaders",
-        "forecasting",
-        "management",
-        "notebooks",
-        "offline_evaluation",
-        "online_evaluation",
-        "reports",
-        "src",
-        "_templates",
-        "training",
-        "utils",
-        "visualization",
-        "_sys_paths",
-        "common_querysets",
-        "queryset_path",
-        "scripts",
-        "meta_tools",
-    )
+    # __slots__ = (
+    #     "_validate",
+    #     "target",
+    #     "use_global_cache",
+    #     "_force_cache_overwrite",
+    #     "root",
+    #     "models",
+    #     "common_utils",
+    #     "common_configs",
+    #     "_ignore_attributes",
+    #     "model_name",
+    #     "_instance_hash",
+    #     "_queryset",
+    #     "model_dir",
+    #     "architectures",
+    #     "artifacts",
+    #     "configs",
+    #     "data",
+    #     "data_generated",
+    #     "data_processed",
+    #     "data_raw",
+    #     "dataloaders",
+    #     "forecasting",
+    #     "management",
+    #     "notebooks",
+    #     "offline_evaluation",
+    #     "online_evaluation",
+    #     "reports",
+    #     "src",
+    #     "_templates",
+    #     "training",
+    #     "utils",
+    #     "visualization",
+    #     "_sys_paths",
+    #     "common_querysets",
+    #     "queryset_path",
+    #     "scripts",
+    #     "meta_tools",
+    # )
 
     _target = "model"
     _use_global_cache = True
@@ -120,6 +120,7 @@ def _initialize_class_paths(cls):
         cls._common_configs = cls._root / "common_configs"
         cls._common_querysets = cls._root / "common_querysets"
         cls._meta_tools = cls._root / "meta_tools"
+        cls._common_logs = cls._root / "common_logs"
 
     @classmethod
     def get_root(cls) -> Path:
@@ -163,6 +164,13 @@ def get_meta_tools(cls) -> Path:
             cls._initialize_class_paths()
         return cls._meta_tools
 
+    @classmethod
+    def get_common_logs(cls) -> Path:
+        """Get the common logs path."""
+        if cls._common_logs is None:
+            cls._initialize_class_paths()
+        return cls._common_logs
+
     @classmethod
     def check_if_model_dir_exists(cls, model_name: str) -> bool:
         """
@@ -575,8 +583,8 @@ def add_paths_to_sys(self) -> List[str]:
                     )
         if self._sys_paths is None:
             self._sys_paths = []
-        for attr in self.__slots__:
-            value = getattr(self, attr)
+        for attr, value in self.__dict__.items():
+            # value = getattr(self, attr)
             if str(attr) not in self._ignore_attributes:
                 if (
                     isinstance(value, Path)
@@ -641,8 +649,8 @@ def view_directories(self) -> None:
         """
         print("\n{:<20}\t{:<50}".format("Name", "Path"))
         print("=" * 72)
-        for attr in self.__slots__:
-            value = getattr(self, attr)
+        for attr, value in self.__dict__.items():
+            # value = getattr(self, attr)
             if attr not in self._ignore_attributes and isinstance(value, Path):
                 print("{:<20}\t{:<50}".format(str(attr), str(value)))
 
@@ -687,8 +695,8 @@ def get_directories(self) -> Dict[str, Optional[str]]:
         # ]
         directories = {}
         relative = False
-        for attr in self.__slots__:
-            value = getattr(self, attr)
+        for attr, value in self.__dict__.items():
+            # value = getattr(self, attr)
             if str(attr) not in [
                 "model_name",
                 "root",

common_utils/tests/test_utils_evaluation_metrics.py

@@ -3,6 +3,7 @@
 import properscoring as ps
 from sklearn.metrics import mean_squared_error, mean_absolute_error
 # WARNING: mean_squared_error is deprecated!
+import wandb
 import sys
 from pathlib import Path
 PATH = Path(__file__)
@@ -43,11 +44,11 @@ def mock_config():
         Config: A mock configuration object with attributes 'steps' and 'depvar'.
     """
 
-    class Config:
-        steps = [1, 2]
-        depvar = "depvar"
+    config = wandb.Config()
+    config.steps = [1, 2]
+    config.depvar = "depvar"
 
-    return Config()
+    return config
 
 
 def test_evaluation_metrics_default_values():

common_utils/tests/test_utils_model_outputs.py

@@ -3,7 +3,7 @@
 from utils_model_outputs import ModelOutputs, generate_output_dict
 import sys
 from pathlib import Path
-
+import wandb
 PATH = Path(__file__)
 if 'views_pipeline' in PATH.parts:
     PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1])
@@ -53,11 +53,11 @@ def mock_config():
         Config: A mock configuration object with predefined attributes.
     """
 
-    class Config:
-        steps = [1, 2]
-        depvar = "depvar"
+    config = wandb.Config()
+    config.steps = [1, 2]
+    config.depvar = "depvar"
 
-    return Config()
+    return config
 
 
 def test_model_outputs_default_values():