From bfd93b45aeb8f740583a66cf08706223b11d82e0 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:11 +0100 Subject: [PATCH 01/94] general ADR on logging --- .../ADRs/015_log_files_general_strategy.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 documentation/ADRs/015_log_files_general_strategy.md diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md new file mode 100644 index 00000000..1d0490d5 --- /dev/null +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -0,0 +1,84 @@ +# log files general strategy + +| ADR Info | Details | +|---------------------|------------------------------| +| Subject | log files general strategy | +| ADR Number | 015 | +| Status | Accepted | +| Author | Simon | +| Date | 28.10.2024 | + + +## Context + +Effective logging is essential for maintaining data integrity, monitoring model behavior, and troubleshooting issues within the model pipeline. A cohesive, centralized logging strategy ensures that logs are structured and accessible, enhancing transparency, auditability, and reliability across the model deployment lifecycle. The main goals of this logging strategy are to: + +1. **Enable Reproducibility and Traceability**: Log details such as timestamps, script paths, and process IDs are standardized to help trace model behavior and system states effectively across different environments. +2. **Support Monitoring and Real-Time Alerts**: Logs will provide data for monitoring tools, enabling real-time alerting on critical errors and pipeline health checks. +3. **Align with MLOps Best Practices**: This strategy follows MLOps standards for consistent error handling, observability, scalability, and storage management, preparing the pipeline for scalable deployment and future monitoring enhancements. + +For additional information, see also: +- [009_log_file_for_generated_data.md](009_log_file_for_generated_data.md) +- [016_log_files_for_input_data.md](016_log_files_for_input_data.md) +- [017_log_files_for_offline_evaluation.md](017_log_files_for_offline_evaluation.md) +- [018_log_files_for_online_evaluation.md](018_log_files_for_online_evaluation.md) +- [019_log_files_for_model_training.md](019_log_files_for_model_training.md) +- [020_log_files_realtime_alerts.md](020_log_files_realtime_alerts.md) + +## Decision + +To implement a robust and unified logging strategy, we have decided on the following practices: + +### Overview + +1. **Standardized Log Configuration**: All logs will follow a centralized structure defined in the configurable `common_config/config_log.yaml` file. This configuration file controls logging levels, file rotation schedules, log output formats, and target log destinations. By centralizing log settings, all models within the pipeline will have a consistent logging structure, making the setup easier to maintain and adapt across environments. + +2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. + +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. + +4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. + +5. **Error Handling and Alerts**: Real-time alerting will be implemented for critical errors and unmet conditions. Integration with alerting tools (such as Slack or email) will provide immediate notifications of key pipeline issues. Alerts will include relevant metadata like timestamps, log level, and error specifics to support rapid troubleshooting. + +6. **Dual Logging to Local Storage and Weights & Biases (W&B)**: + - **Local Storage**: Logs will be stored locally on a rotating basis for easy access and immediate troubleshooting. + - **Weights & Biases (W&B) Integration**: Model training and evaluation logs will also be sent to W&B, which allows for centralized logging of metrics, model performance tracking, and experiment comparison. The W&B integration supports MLOps best practices by making logs easily searchable, taggable (e.g., by model or pipeline stage), and accessible for experiment analysis and auditing. + +7. **Access Control and Data Sensitivity**: Logs will avoid capturing sensitive data (such as configuration secrets or personally identifiable information) to align with data governance standards. While access controls for log files are not implemented at this stage, we may restrict log access in the future as the project scales, ensuring that sensitive log data is adequately protected. + +8. **Testing and Validation**: Automated tests will validate that logs are created accurately and that rotation and level-specific separation operate as expected. These tests will cover: + - Log creation and rotation validation. + - Level-specific log file checks to confirm appropriate separation (e.g., that `INFO` logs do not include `DEBUG` messages). + - Functional testing of real-time alerts to verify that notifications trigger as configured. + +## Consequences + +**Positive Effects:** +- Provides a consistent and structured logging framework, improving troubleshooting, auditability, and compliance. +- Supports MLOps best practices by establishing robust monitoring, traceability, and data governance standards. +- Facilitates scalability and onboarding by providing a standardized, centralized approach to logging across all pipeline models. + +**Negative Effects:** +- Additional storage resources are required for log retention and rotation, and periodic monitoring of storage usage is needed. +- Initial setup and adjustment period may add complexity as team members adapt to the standardized logging and alerting practices. +- Some refactoring of the current codebase will be needed as this ADR is accepted. + +## Rationale + +The unified logging strategy aligns with MLOps best practices by combining flexibility, scalability, and robustness. This approach ensures that logging configurations are adaptable, reproducible, and traceable across the model pipeline. By establishing standardized configuration files and integrating alerting, this logging strategy proactively supports system monitoring and provides a foundation for future observability and security enhancements. + +## Considerations + +1. **Future Alerting Integrations**: Additional alerting tools, such as W&B alerts, Slack, and email notifications, will be incorporated as the project matures to ensure real-time visibility into pipeline states and failures. + +2. **Centralized Logging Platform**: In future updates, the logging system may transition to a centralized platform (e.g., ELK Stack, Grafana) to improve scalability, visualization, and monitoring. This would require adjusting the current setup to work seamlessly with a logging infrastructure, which could involve additional configurations or external services. + +3. **Access Control Expansion**: As the project scales, access control measures will be considered to ensure data protection. Log files should avoid sensitive information to comply with best practices in data governance and avoid potential data exposure risks. + +4. **Testing Resource Allocation**: Implementing automated tests for logging mechanisms may require resources such as mock environments or testing frameworks to ensure the system functions as expected under different scenarios and that alert conditions trigger correctly. + +## Additional Notes + +Future updates may involve enhancing logging with a centralized platform, providing a more scalable and observable solution for monitoring and auditability. Access control measures and security protocols will also be revisited as the project scales to protect data integrity and confidentiality. Team members are encouraged to provide feedback on specific logging configuration details or suggest improvements to the alerting and monitoring system. + From a3447fb08f684b9b1f099669298cea51853acd9d Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:30 +0100 Subject: [PATCH 02/94] New ADRs todo --- documentation/ADRs/016_log_files_for_input_data.md | 0 documentation/ADRs/017_log_files_for_offline_evaluation.md | 0 documentation/ADRs/018_log_files_for_online_evaluation.md | 1 + documentation/ADRs/019_log_files_for_model_training.md | 1 + documentation/ADRs/020_log_files_realtime_alerts.md | 1 + 5 files changed, 3 insertions(+) create mode 100644 documentation/ADRs/016_log_files_for_input_data.md create mode 100644 documentation/ADRs/017_log_files_for_offline_evaluation.md create mode 100644 documentation/ADRs/018_log_files_for_online_evaluation.md create mode 100644 documentation/ADRs/019_log_files_for_model_training.md create mode 100644 documentation/ADRs/020_log_files_realtime_alerts.md diff --git a/documentation/ADRs/016_log_files_for_input_data.md b/documentation/ADRs/016_log_files_for_input_data.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/017_log_files_for_offline_evaluation.md b/documentation/ADRs/017_log_files_for_offline_evaluation.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/018_log_files_for_online_evaluation.md b/documentation/ADRs/018_log_files_for_online_evaluation.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/018_log_files_for_online_evaluation.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/019_log_files_for_model_training.md b/documentation/ADRs/019_log_files_for_model_training.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/019_log_files_for_model_training.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/020_log_files_realtime_alerts.md b/documentation/ADRs/020_log_files_realtime_alerts.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/020_log_files_realtime_alerts.md @@ -0,0 +1 @@ +TODO \ No newline at end of file From 36c01b89fe9c3e7260b64b8415e4bf3c99ffd22c Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:45 +0100 Subject: [PATCH 03/94] the new yaml - not yet used... --- common_configs/config_log.yaml | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 common_configs/config_log.yaml diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml new file mode 100644 index 00000000..857043e2 --- /dev/null +++ b/common_configs/config_log.yaml @@ -0,0 +1,44 @@ +version: 1 +disable_existing_loggers: False + +formatters: + detailed: + format: '%(asctime)s %(pathname)s [%(filename)s:%(lineno)d] [%(process)d] [%(threadName)s] - %(levelname)s - %(message)s' + +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: detailed + stream: ext://sys.stdout + + info_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: INFO + formatter: detailed + filename: 'logs/app_INFO_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + debug_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: DEBUG + formatter: detailed + filename: 'logs/app_DEBUG_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + error_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: ERROR + formatter: detailed + filename: 'logs/app_ERROR_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + +root: + level: DEBUG + handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From cc3824012fcc71eb5e0481edc6955e157d73500e Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:57:57 +0100 Subject: [PATCH 04/94] the config_log_yaml --- common_configs/config_log.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 857043e2..a87cfc6a 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -16,29 +16,29 @@ handlers: class: logging.handlers.TimedRotatingFileHandler level: INFO formatter: detailed - filename: 'logs/app_INFO_%Y-%m-%d.log' - when: 'midnight' + filename: "{LOG_PATH}/views_pipeline_INFO.log" + when: "midnight" backupCount: 30 - encoding: 'utf8' + encoding: "utf8" debug_file_handler: class: logging.handlers.TimedRotatingFileHandler level: DEBUG formatter: detailed - filename: 'logs/app_DEBUG_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_DEBUG.log" + when: "midnight" + backupCount: 10 + encoding: "utf8" error_file_handler: class: logging.handlers.TimedRotatingFileHandler level: ERROR formatter: detailed - filename: 'logs/app_ERROR_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_ERROR.log" + when: "midnight" + backupCount: 60 + encoding: "utf8" root: - level: DEBUG + level: INFO handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 6510796b6971aae650eb6260ad167e7748a3b51d Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:30 +0100 Subject: [PATCH 05/94] new common_logs dir --- common_logs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 common_logs/.gitkeep diff --git a/common_logs/.gitkeep b/common_logs/.gitkeep new file mode 100644 index 00000000..e69de29b From 0f64d2989ba95c50aa631d68a6b5e8de7753c6fd Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:48 +0100 Subject: [PATCH 06/94] changed the central logger --- common_utils/utils_logger.py | 166 ++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 20 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index 84094b75..a3a8dd04 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -1,33 +1,159 @@ import logging +import logging.config +import yaml +import os +from pathlib import Path -def setup_logging(log_file: str, log_level=logging.INFO): +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ +def get_config_log_path() -> Path: """ - Sets up logging to both a specified file and the terminal (console). + Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - Args: - log_file (str): The file where logs should be written. - log_level (int): The logging level. Default is logging.INFO. + This function identifies the 'views_pipeline' directory within the path of the current file, + constructs a new path up to and including this directory, and then appends the relative path + to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file + is not found, it raises a ValueError. + + Returns: + pathlib.Path: The path to the 'config_log.yaml' file. + + Raises: + ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' + if not PATH_CONFIG_LOG.exists(): + raise ValueError("The 'config_log.yaml' file was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_CONFIG_LOG +# -------------------------------------------------------------------------------------------------------------- + + +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +def get_common_logs_path() -> Path: + """ + Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, then constructs + a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, + it raises a ValueError. + + Returns: + pathlib.Path: Absolute path to the 'common_logs' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' + if not PATH_COMMON_LOGS.exists(): + raise ValueError("The 'common_logs' directory was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_COMMON_LOGS +# ------------------------------------------------------------------------------------------------------------ + + +def ensure_log_directory(log_path: str) -> None: """ + Ensure the log directory exists for file-based logging handlers. + + Parameters: + log_path (str): The full path to the log file for which the directory should be verified. + """ + log_dir = os.path.dirname(log_path) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + +def setup_logging( + default_level: int = logging.INFO, env_key: str = 'LOG_CONFIG') -> logging.Logger: + + """ + Setup the logging configuration from a YAML file and return the root logger. + + Parameters: + default_level (int): The default logging level if the configuration file is not found + or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging + configuration file. Default is 'LOG_CONFIG'. + + Returns: + logging.Logger: The root logger configured based on the loaded configuration. + + Example Usage: + >>> logger = setup_logging() + >>> logger.info("Logging setup complete.") + """ + + CONFIG_LOGS_PATH = get_config_log_path() + COMMON_LOGS_PATH = get_common_logs_path() + + # Load YAML configuration + path = os.getenv(env_key, CONFIG_LOGS_PATH) - basic_logger = logging.getLogger() - basic_logger.setLevel(log_level) + if os.path.exists(path): + try: + with open(path, 'rt') as f: + config = yaml.safe_load(f.read()) + + # Replace placeholder with actual log directory path + for handler in config.get("handlers", {}).values(): + if "filename" in handler and "{LOG_PATH}" in handler["filename"]: + handler["filename"] = handler["filename"].replace("{LOG_PATH}", str(COMMON_LOGS_PATH)) + ensure_log_directory(handler["filename"]) + + # Apply logging configuration + logging.config.dictConfig(config) - file_handler = logging.FileHandler(log_file) - console_handler = logging.StreamHandler() + except Exception as e: + logging.basicConfig(level=default_level) + logging.error(f"Failed to load logging configuration from {path}. Using basic configuration. Error: {e}") + else: + logging.basicConfig(level=default_level) + logging.warning(f"Logging configuration file not found at {path}. Using basic configuration.") + + return logging.getLogger() - file_handler.setLevel(log_level) - console_handler.setLevel(log_level) - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - # Clear previous handlers if they exist - if basic_logger.hasHandlers(): - basic_logger.handlers.clear() - basic_logger.addHandler(file_handler) - basic_logger.addHandler(console_handler) - return basic_logger +## Old version +#def setup_logging(log_file: str, log_level=logging.INFO): +# """ +# Sets up logging to both a specified file and the terminal (console). +# +# Args: +# log_file (str): The file where logs should be written. +# log_level (int): The logging level. Default is logging.INFO. +# """ +# +# basic_logger = logging.getLogger() +# basic_logger.setLevel(log_level) +# +# file_handler = logging.FileHandler(log_file) +# console_handler = logging.StreamHandler() +# +# file_handler.setLevel(log_level) +# console_handler.setLevel(log_level) +# +# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +# file_handler.setFormatter(formatter) +# console_handler.setFormatter(formatter) +# +# # Clear previous handlers if they exist +# if basic_logger.hasHandlers(): +# basic_logger.handlers.clear() +# +# basic_logger.addHandler(file_handler) +# basic_logger.addHandler(console_handler) +# +# return basic_logger +# \ No newline at end of file From b1c704a1bcd05e0c1bbcdd0b7e22f7b6a245a6d0 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 06:02:41 +0100 Subject: [PATCH 07/94] detail --- documentation/ADRs/015_log_files_general_strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md index 1d0490d5..aaf3d8fc 100644 --- a/documentation/ADRs/015_log_files_general_strategy.md +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -35,7 +35,7 @@ To implement a robust and unified logging strategy, we have decided on the follo 2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. -3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files and stored under `views_pipeline/common_logs`. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. 4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. From 69e0207252863aa2f62a4f4316458ed4a48cdb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borb=C3=A1la=20Farkas?= Date: Tue, 29 Oct 2024 19:04:47 +0100 Subject: [PATCH 08/94] catalog ADR initial draft --- documentation/ADRs/022_model_catalogs.md | 104 ++++++++++++++++++ .../ADRs/{adr_templete.md => adr_template.md} | 0 2 files changed, 104 insertions(+) create mode 100644 documentation/ADRs/022_model_catalogs.md rename documentation/ADRs/{adr_templete.md => adr_template.md} (100%) diff --git a/documentation/ADRs/022_model_catalogs.md b/documentation/ADRs/022_model_catalogs.md new file mode 100644 index 00000000..726ba451 --- /dev/null +++ b/documentation/ADRs/022_model_catalogs.md @@ -0,0 +1,104 @@ + + +## Title +*Create Model Catalogs* + +| ADR Info | Details | +|---------------------|-------------------| +| Subject | Create Model Catalog | +| ADR Number | 022 | +| Status | proposed | +| Author | Borbála | +| Date | 29.10.2024. | + +## Context +*We wanted to have a catalog about all of the models in the pipeline. We needed to do that both for the old and the new pipeline because the structure of the two pipelines and the way how the querysets are organised are different. We also had to be sure that the catalogs update whenever a model is modified or added.* + +*Describe the issue that necessitated the decision, including any factors considered during the decision-making process. This should provide a clear understanding of the challenges or opportunities addressed by the ADR.* + +## Decision +### New pipeline +*In the new pipeline there are two spearate catalogs for 'country level' and 'priogrid level' models with the following structure:* +| Model Name | Algorithm | Target | Input Features | Non-default Hyperparameters | Forecasting Type | Implementation Status | Implementation Date | Author | +| ---------- | --------- | ------ | -------------- | --------------------------- | ---------------- | --------------------- | ------------------- | ------ | +| electric_relaxation | RandomForestClassifier | ged_sb_dep | - [escwa001_cflong](https://github.com/prio-data/views_pipeline/blob/main/common_querysets/queryset_electric_relaxation.py) | - [hyperparameters electric_relaxation](https://github.com/prio-data/views_pipeline/blob/main/models/electric_relaxation/configs/config_hyperparameters.py) | None | shadow | NA | Sara | + +Configs used to create the catalog: +- `views_pipeline/models/*/configs/config_meta.py` +- `views_pipeline/models/*/configs/config_deployment.py` +- `views_pipeline/models/*/configs/config_hyperparameters.py` +- `views_pipeline/common_querysets/*` + +Columns: +- **Model Name**: name of the model, always in a form of `adjective_noun` +- **Algorithm**: "algorithm" from `config_meta.py` +- **Target**: "depvar" from `config_meta.py` +- **Input Features**: "queryset" from `config_meta.py` that points to the queryset in `common_querysets`folder +- **Non-default Hyperparameters**: "hyperparameters model_name" that points to `config_hyperparameters.py` +- **Forecasting Type**: TBD +- **Implementation Status**: "deployment_status" from `config_deployment.py` (e.g. shadow, deployed, baseline, or deprecated) +- **Implementation Date**: TBD +- **Author**: "creator" from `config_meta.py` + +### Old Pipeline +*In the old pipeline there is only one catalog that contains both 'country level' and 'priogrid level' models with the following structure:* +| Model Name | Algorithm | Target | Input Features | Non-default Hyperparameters | Forecasting Type | Implementation Status | Implementation Date | Author | +| ---------- | --------- | ------ | -------------- | --------------------------- | ---------------- | --------------------- | ------------------- | ------ | +| fatalities002_baseline_rf | XGBRFRegressor | ln_ged_sb_dep | - [fatalities002_baseline](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py#L16) | n_estimators=300, n_jobs=nj | Direct multi-step | no | NA | NA | + +Configs used to create the catalog: +- [ModelDefinitions.py](https://github.com/prio-data/viewsforecasting/blob/main/SystemUpdates/ModelDefinitions.py) +- [cm_querysets.py](https://github.com/prio-data/viewsforecasting/blob/main/Tools/cm_querysets.py) +- [pgm_querysets.py](https://github.com/prio-data/viewsforecasting/blob/main/Tools/pgm_querysets.py) + +Columns: +- **Model Name**: "modelname" from `ModelDefinitions.py` +- **Algorithm**: "algorithm" from `ModelDefinitions.py` +- **Target**: "depvar" from `ModelDefinitions.py` +- **Input Features**: "queryset" from `ModelDefinitions.py` pointing to the corresponding line in `cm_querysets.py` or `pgm_querysets.py` +- **Non-default Hyperparameters**: the argument of "algorithm" from `ModelDefinitions.py` +- **Forecasting Type**: Direct multi-step +- **Implementation Status**: no (none of these models are in production) +- **Implementation Date**: TBD +- **Author**: TBD + +### GitHub actions +The catalogs are updated via GitHub actions. Action for the new pipeline: [update_views_pipeline_cm_catalog.yml](https://github.com/prio-data/viewsforecasting/blob/github_workflows/.github/workflows/update_views_pipeline_cm_catalog.yml), action for the old pipeline: [check_if_new_model_added.yml](https://github.com/prio-data/views_pipeline/blob/production/.github/workflows/check_if_new_model_added.yml). They trigger when the config files are modified on the `production` and `development` branch. These GitHub actions can also be triggered manually for testing reason. + +*Detail the decision that was made, including any alternatives that were considered and the reasons for choosing the implemented solution. Provide enough technical specifics to justify the approach.* + +### Overview +*Creating catalogs for 'country level' and 'priogrid level' that update automatically when a model is modified. Separate implementation for the old and the new pipeline.* + +*Overview of the decision in a clear and concise manner.* + +## Consequences +*Clear overview about our existing models in the `views_pipeline/documentation/catalogs/` directory.* + +*Discuss the positive and negative effects of the decision. Include both immediate outcomes and long-term implications for the project's architecture. Highlight how the decision aligns with the challenges outlined in the context.* + +**Positive Effects:** +- Our models become trackable and presentable. +- Model features are easily accessible via links. + +**Negative Effects:** +- The github actions and generator scripts require maintenance. +- If the catalogs fail to update, it might remain unnoticed for a while. + +## Rationale +*Every information about the models are found at one place* + +*Explain the reasoning behind the decision, including any specific advantages that influenced the choice. This section should reflect the factors mentioned in the context.* + +### Considerations +*We decided to separate 'country level' and 'priogrid level' models into different catalogs. GitHub actions * + +*List any considerations that were part of the decision-making process, such as potential risks, dependency issues, or impacts on existing systems.* + +## Additional Notes +Involving GitHub actions led to the separation of `production` and `development`branch, since they cannot push to a protected branch (`production`). More detailed information is found in ADR #023. + +## Feedback and Suggestions +*Feedbacks are awaited.* + +--- diff --git a/documentation/ADRs/adr_templete.md b/documentation/ADRs/adr_template.md similarity index 100% rename from documentation/ADRs/adr_templete.md rename to documentation/ADRs/adr_template.md From daa102a5b3f446f1cffc4b36257cfea2290e2840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borb=C3=A1la=20Farkas?= Date: Tue, 29 Oct 2024 19:07:59 +0100 Subject: [PATCH 09/94] catalog ADR improve draft --- documentation/ADRs/022_model_catalogs.md | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/documentation/ADRs/022_model_catalogs.md b/documentation/ADRs/022_model_catalogs.md index 726ba451..902c7aea 100644 --- a/documentation/ADRs/022_model_catalogs.md +++ b/documentation/ADRs/022_model_catalogs.md @@ -14,8 +14,6 @@ ## Context *We wanted to have a catalog about all of the models in the pipeline. We needed to do that both for the old and the new pipeline because the structure of the two pipelines and the way how the querysets are organised are different. We also had to be sure that the catalogs update whenever a model is modified or added.* -*Describe the issue that necessitated the decision, including any factors considered during the decision-making process. This should provide a clear understanding of the challenges or opportunities addressed by the ADR.* - ## Decision ### New pipeline *In the new pipeline there are two spearate catalogs for 'country level' and 'priogrid level' models with the following structure:* @@ -63,20 +61,16 @@ Columns: - **Author**: TBD ### GitHub actions -The catalogs are updated via GitHub actions. Action for the new pipeline: [update_views_pipeline_cm_catalog.yml](https://github.com/prio-data/viewsforecasting/blob/github_workflows/.github/workflows/update_views_pipeline_cm_catalog.yml), action for the old pipeline: [check_if_new_model_added.yml](https://github.com/prio-data/views_pipeline/blob/production/.github/workflows/check_if_new_model_added.yml). They trigger when the config files are modified on the `production` and `development` branch. These GitHub actions can also be triggered manually for testing reason. +The catalogs are updated via GitHub actions. Action for the new pipeline: [update_views_pipeline_cm_catalog.yml](https://github.com/prio-data/viewsforecasting/blob/github_workflows/.github/workflows/update_views_pipeline_cm_catalog.yml), action for the old pipeline: [check_if_new_model_added.yml](https://github.com/prio-data/views_pipeline/blob/production/.github/workflows/check_if_new_model_added.yml). They trigger when the config files are modified on the `production` and `development` branch. These GitHub actions can also be triggered manually for testing reason. The GitHub actions can only push to non-protected branches. -*Detail the decision that was made, including any alternatives that were considered and the reasons for choosing the implemented solution. Provide enough technical specifics to justify the approach.* ### Overview *Creating catalogs for 'country level' and 'priogrid level' that update automatically when a model is modified. Separate implementation for the old and the new pipeline.* -*Overview of the decision in a clear and concise manner.* ## Consequences *Clear overview about our existing models in the `views_pipeline/documentation/catalogs/` directory.* -*Discuss the positive and negative effects of the decision. Include both immediate outcomes and long-term implications for the project's architecture. Highlight how the decision aligns with the challenges outlined in the context.* - **Positive Effects:** - Our models become trackable and presentable. - Model features are easily accessible via links. @@ -86,14 +80,15 @@ The catalogs are updated via GitHub actions. Action for the new pipeline: [updat - If the catalogs fail to update, it might remain unnoticed for a while. ## Rationale -*Every information about the models are found at one place* +*Every information about the models are found at one place. Models can be tracked and presented, even for people not involved in the development. It is easier to involve new people to the model development. GitHub actions provide a convenient way to keep the catalogs up-to-date.* -*Explain the reasoning behind the decision, including any specific advantages that influenced the choice. This section should reflect the factors mentioned in the context.* ### Considerations -*We decided to separate 'country level' and 'priogrid level' models into different catalogs. GitHub actions * +- We decided to separate 'country level' and 'priogrid level' models into different catalogs. +- We needed a separate catalog for the old pipeline as well, which will be depreciated. +- GitHub actions push to `development` branch, but they cannot push to `production` branch, since it is protected. + -*List any considerations that were part of the decision-making process, such as potential risks, dependency issues, or impacts on existing systems.* ## Additional Notes Involving GitHub actions led to the separation of `production` and `development`branch, since they cannot push to a protected branch (`production`). More detailed information is found in ADR #023. From c9a65ec11acfc4e1e8a7cb53c0520016a6221127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borb=C3=A1la=20Farkas?= Date: Tue, 29 Oct 2024 19:11:36 +0100 Subject: [PATCH 10/94] catalog ADR change title --- documentation/ADRs/022_model_catalogs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/ADRs/022_model_catalogs.md b/documentation/ADRs/022_model_catalogs.md index 902c7aea..b4dca391 100644 --- a/documentation/ADRs/022_model_catalogs.md +++ b/documentation/ADRs/022_model_catalogs.md @@ -1,7 +1,7 @@ -## Title -*Create Model Catalogs* +## Create Model Catalogs + | ADR Info | Details | |---------------------|-------------------| From e003b96e8eab7efef05c4ab123e607e55705a9a3 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:08 +0100 Subject: [PATCH 11/94] Use this to see how the logs look now --- meta_tools/asses_logging_setup.py | 74 +++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 meta_tools/asses_logging_setup.py diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py new file mode 100644 index 00000000..452bb449 --- /dev/null +++ b/meta_tools/asses_logging_setup.py @@ -0,0 +1,74 @@ +import logging +from pathlib import Path +import sys + + +PATH = Path(__file__) + +def get_path_common_utils(): + + if 'views_pipeline' in PATH.parts: + + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + + if not PATH_COMMON_UTILS.exists(): + + raise ValueError("The 'common_utils' directory was not found in the provided path.") + + else: + + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + + return PATH_COMMON_UTILS + + +# Import your logging setup function from wherever it is defined +# from your_logging_module import setup_logging, get_common_logs_path + +def test_logging_setup(): + # Step 1: Set up the logging configuration + try: + log_directory = get_common_logs_path() # Fetch centralized log directory + logger = setup_logging() # Initialize logging setup + + except Exception as e: + print(f"Failed to initialize logging setup: {e}") + return + + # Step 2: Generate test log messages + logger.debug("This is a DEBUG log message for testing.") + logger.info("This is an INFO log message for testing.") + logger.error("This is an ERROR log message for testing.") + + # Step 3: Define expected log files + expected_files = [ + log_directory / "views_pipeline_INFO.log", + log_directory / "views_pipeline_DEBUG.log", + log_directory / "views_pipeline_ERROR.log" + ] + + # Step 4: Check if log files exist and are not empty + for file_path in expected_files: + if file_path.exists(): + print(f"Log file '{file_path}' exists.") + if file_path.stat().st_size > 0: + print(f"Log file '{file_path}' contains data.") + else: + print(f"Warning: Log file '{file_path}' is empty.") + else: + print(f"Error: Log file '{file_path}' was not created as expected.") + + print("Logging setup test completed.") + +# Run the test + +if __name__ == "__main__": + + PATH_COMMON_UTILS = get_path_common_utils() + + sys.path.append(str(PATH_COMMON_UTILS)) + + from utils_logger import setup_logging, get_common_logs_path + + test_logging_setup() From f941f465d822cbdac0050a5ffb9ad4690b43c77f Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:17 +0100 Subject: [PATCH 12/94] Updated root level --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index a87cfc6a..21c7f24e 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: INFO + level: DEBUG handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From bff09182193bf89b891f31f8a3c319a9440310c1 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:29 +0100 Subject: [PATCH 13/94] whitspace --- common_utils/utils_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index a3a8dd04..b9f5dd82 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -81,6 +81,7 @@ def setup_logging( Parameters: default_level (int): The default logging level if the configuration file is not found or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging configuration file. Default is 'LOG_CONFIG'. From 4c5069feaf11aac653878bb50a16715ae154d88f Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:08:04 +0100 Subject: [PATCH 14/94] fix wandb config object in tests --- common_utils/tests/test_utils_evaluation_metrics.py | 9 +++++---- common_utils/tests/test_utils_model_outputs.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/common_utils/tests/test_utils_evaluation_metrics.py b/common_utils/tests/test_utils_evaluation_metrics.py index c9b7870b..1989b821 100644 --- a/common_utils/tests/test_utils_evaluation_metrics.py +++ b/common_utils/tests/test_utils_evaluation_metrics.py @@ -3,6 +3,7 @@ import properscoring as ps from sklearn.metrics import mean_squared_error, mean_absolute_error # WARNING: mean_squared_error is deprecated! +import wandb import sys from pathlib import Path PATH = Path(__file__) @@ -43,11 +44,11 @@ def mock_config(): Config: A mock configuration object with attributes 'steps' and 'depvar'. """ - class Config: - steps = [1, 2] - depvar = "depvar" + config = wandb.Config() + config.steps = [1, 2] + config.depvar = "depvar" - return Config() + return config def test_evaluation_metrics_default_values(): diff --git a/common_utils/tests/test_utils_model_outputs.py b/common_utils/tests/test_utils_model_outputs.py index 9709054a..0860da43 100644 --- a/common_utils/tests/test_utils_model_outputs.py +++ b/common_utils/tests/test_utils_model_outputs.py @@ -3,7 +3,7 @@ from utils_model_outputs import ModelOutputs, generate_output_dict import sys from pathlib import Path - +import wandb PATH = Path(__file__) if 'views_pipeline' in PATH.parts: PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) @@ -53,11 +53,11 @@ def mock_config(): Config: A mock configuration object with predefined attributes. """ - class Config: - steps = [1, 2] - depvar = "depvar" + config = wandb.Config() + config.steps = [1, 2] + config.depvar = "depvar" - return Config() + return config def test_model_outputs_default_values(): From 004f40e2e596a66d054249c3c7ebc4e920cfd5f9 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:08:44 +0100 Subject: [PATCH 15/94] this is better... --- meta_tools/asses_logging_setup.py | 33 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index 452bb449..e374585c 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -5,22 +5,41 @@ PATH = Path(__file__) -def get_path_common_utils(): - if 'views_pipeline' in PATH.parts: +def set_path_common_utils(): + """ + Retrieve the path to the 'common_utils' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, + then constructs a new path to the 'common_utils' directory. If 'views_pipeline' + or 'common_utils' directories are not found, it raises a ValueError. + + If the 'common_utils' path is not already in sys.path, it appends it. + Returns: + pathlib.Path: Absolute path to the 'common_utils' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. + """ + PATH = Path(__file__) + + # Locate 'views_pipeline' in the current file's path parts + if 'views_pipeline' in PATH.parts: PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + # Check if 'common_utils' directory exists if not PATH_COMMON_UTILS.exists(): - raise ValueError("The 'common_utils' directory was not found in the provided path.") - else: + # Add 'common_utils' to sys.path if it's not already present + if str(PATH_COMMON_UTILS) not in sys.path: + sys.path.append(str(PATH_COMMON_UTILS)) + else: raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_UTILS # Import your logging setup function from wherever it is defined @@ -65,9 +84,7 @@ def test_logging_setup(): if __name__ == "__main__": - PATH_COMMON_UTILS = get_path_common_utils() - - sys.path.append(str(PATH_COMMON_UTILS)) + set_path_common_utils() from utils_logger import setup_logging, get_common_logs_path From 856a7227c7f39da27c88b8682ca499f3ca598e2d Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:09:09 +0100 Subject: [PATCH 16/94] corrected doc string --- meta_tools/asses_logging_setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index e374585c..c95604ac 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -16,9 +16,6 @@ def set_path_common_utils(): If the 'common_utils' path is not already in sys.path, it appends it. - Returns: - pathlib.Path: Absolute path to the 'common_utils' directory. - Raises: ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. """ From b389b613ff8d03ecfed40f90879b8f76e043f91b Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:08:04 +0100 Subject: [PATCH 17/94] fix wandb config object in tests --- common_utils/tests/test_utils_evaluation_metrics.py | 9 +++++---- common_utils/tests/test_utils_model_outputs.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/common_utils/tests/test_utils_evaluation_metrics.py b/common_utils/tests/test_utils_evaluation_metrics.py index c9b7870b..1989b821 100644 --- a/common_utils/tests/test_utils_evaluation_metrics.py +++ b/common_utils/tests/test_utils_evaluation_metrics.py @@ -3,6 +3,7 @@ import properscoring as ps from sklearn.metrics import mean_squared_error, mean_absolute_error # WARNING: mean_squared_error is deprecated! +import wandb import sys from pathlib import Path PATH = Path(__file__) @@ -43,11 +44,11 @@ def mock_config(): Config: A mock configuration object with attributes 'steps' and 'depvar'. """ - class Config: - steps = [1, 2] - depvar = "depvar" + config = wandb.Config() + config.steps = [1, 2] + config.depvar = "depvar" - return Config() + return config def test_evaluation_metrics_default_values(): diff --git a/common_utils/tests/test_utils_model_outputs.py b/common_utils/tests/test_utils_model_outputs.py index 9709054a..0860da43 100644 --- a/common_utils/tests/test_utils_model_outputs.py +++ b/common_utils/tests/test_utils_model_outputs.py @@ -3,7 +3,7 @@ from utils_model_outputs import ModelOutputs, generate_output_dict import sys from pathlib import Path - +import wandb PATH = Path(__file__) if 'views_pipeline' in PATH.parts: PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) @@ -53,11 +53,11 @@ def mock_config(): Config: A mock configuration object with predefined attributes. """ - class Config: - steps = [1, 2] - depvar = "depvar" + config = wandb.Config() + config.steps = [1, 2] + config.depvar = "depvar" - return Config() + return config def test_model_outputs_default_values(): From 7613b4cb4488f7496a69e74a67889553760179d7 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:11 +0100 Subject: [PATCH 18/94] general ADR on logging --- .../ADRs/015_log_files_general_strategy.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 documentation/ADRs/015_log_files_general_strategy.md diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md new file mode 100644 index 00000000..1d0490d5 --- /dev/null +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -0,0 +1,84 @@ +# log files general strategy + +| ADR Info | Details | +|---------------------|------------------------------| +| Subject | log files general strategy | +| ADR Number | 015 | +| Status | Accepted | +| Author | Simon | +| Date | 28.10.2024 | + + +## Context + +Effective logging is essential for maintaining data integrity, monitoring model behavior, and troubleshooting issues within the model pipeline. A cohesive, centralized logging strategy ensures that logs are structured and accessible, enhancing transparency, auditability, and reliability across the model deployment lifecycle. The main goals of this logging strategy are to: + +1. **Enable Reproducibility and Traceability**: Log details such as timestamps, script paths, and process IDs are standardized to help trace model behavior and system states effectively across different environments. +2. **Support Monitoring and Real-Time Alerts**: Logs will provide data for monitoring tools, enabling real-time alerting on critical errors and pipeline health checks. +3. **Align with MLOps Best Practices**: This strategy follows MLOps standards for consistent error handling, observability, scalability, and storage management, preparing the pipeline for scalable deployment and future monitoring enhancements. + +For additional information, see also: +- [009_log_file_for_generated_data.md](009_log_file_for_generated_data.md) +- [016_log_files_for_input_data.md](016_log_files_for_input_data.md) +- [017_log_files_for_offline_evaluation.md](017_log_files_for_offline_evaluation.md) +- [018_log_files_for_online_evaluation.md](018_log_files_for_online_evaluation.md) +- [019_log_files_for_model_training.md](019_log_files_for_model_training.md) +- [020_log_files_realtime_alerts.md](020_log_files_realtime_alerts.md) + +## Decision + +To implement a robust and unified logging strategy, we have decided on the following practices: + +### Overview + +1. **Standardized Log Configuration**: All logs will follow a centralized structure defined in the configurable `common_config/config_log.yaml` file. This configuration file controls logging levels, file rotation schedules, log output formats, and target log destinations. By centralizing log settings, all models within the pipeline will have a consistent logging structure, making the setup easier to maintain and adapt across environments. + +2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. + +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. + +4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. + +5. **Error Handling and Alerts**: Real-time alerting will be implemented for critical errors and unmet conditions. Integration with alerting tools (such as Slack or email) will provide immediate notifications of key pipeline issues. Alerts will include relevant metadata like timestamps, log level, and error specifics to support rapid troubleshooting. + +6. **Dual Logging to Local Storage and Weights & Biases (W&B)**: + - **Local Storage**: Logs will be stored locally on a rotating basis for easy access and immediate troubleshooting. + - **Weights & Biases (W&B) Integration**: Model training and evaluation logs will also be sent to W&B, which allows for centralized logging of metrics, model performance tracking, and experiment comparison. The W&B integration supports MLOps best practices by making logs easily searchable, taggable (e.g., by model or pipeline stage), and accessible for experiment analysis and auditing. + +7. **Access Control and Data Sensitivity**: Logs will avoid capturing sensitive data (such as configuration secrets or personally identifiable information) to align with data governance standards. While access controls for log files are not implemented at this stage, we may restrict log access in the future as the project scales, ensuring that sensitive log data is adequately protected. + +8. **Testing and Validation**: Automated tests will validate that logs are created accurately and that rotation and level-specific separation operate as expected. These tests will cover: + - Log creation and rotation validation. + - Level-specific log file checks to confirm appropriate separation (e.g., that `INFO` logs do not include `DEBUG` messages). + - Functional testing of real-time alerts to verify that notifications trigger as configured. + +## Consequences + +**Positive Effects:** +- Provides a consistent and structured logging framework, improving troubleshooting, auditability, and compliance. +- Supports MLOps best practices by establishing robust monitoring, traceability, and data governance standards. +- Facilitates scalability and onboarding by providing a standardized, centralized approach to logging across all pipeline models. + +**Negative Effects:** +- Additional storage resources are required for log retention and rotation, and periodic monitoring of storage usage is needed. +- Initial setup and adjustment period may add complexity as team members adapt to the standardized logging and alerting practices. +- Some refactoring of the current codebase will be needed as this ADR is accepted. + +## Rationale + +The unified logging strategy aligns with MLOps best practices by combining flexibility, scalability, and robustness. This approach ensures that logging configurations are adaptable, reproducible, and traceable across the model pipeline. By establishing standardized configuration files and integrating alerting, this logging strategy proactively supports system monitoring and provides a foundation for future observability and security enhancements. + +## Considerations + +1. **Future Alerting Integrations**: Additional alerting tools, such as W&B alerts, Slack, and email notifications, will be incorporated as the project matures to ensure real-time visibility into pipeline states and failures. + +2. **Centralized Logging Platform**: In future updates, the logging system may transition to a centralized platform (e.g., ELK Stack, Grafana) to improve scalability, visualization, and monitoring. This would require adjusting the current setup to work seamlessly with a logging infrastructure, which could involve additional configurations or external services. + +3. **Access Control Expansion**: As the project scales, access control measures will be considered to ensure data protection. Log files should avoid sensitive information to comply with best practices in data governance and avoid potential data exposure risks. + +4. **Testing Resource Allocation**: Implementing automated tests for logging mechanisms may require resources such as mock environments or testing frameworks to ensure the system functions as expected under different scenarios and that alert conditions trigger correctly. + +## Additional Notes + +Future updates may involve enhancing logging with a centralized platform, providing a more scalable and observable solution for monitoring and auditability. Access control measures and security protocols will also be revisited as the project scales to protect data integrity and confidentiality. Team members are encouraged to provide feedback on specific logging configuration details or suggest improvements to the alerting and monitoring system. + From 41bfd516ce5210c3f0bc0a2c06015dbd6b55aab1 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:30 +0100 Subject: [PATCH 19/94] New ADRs todo --- documentation/ADRs/016_log_files_for_input_data.md | 0 documentation/ADRs/017_log_files_for_offline_evaluation.md | 0 documentation/ADRs/018_log_files_for_online_evaluation.md | 1 + documentation/ADRs/019_log_files_for_model_training.md | 1 + documentation/ADRs/020_log_files_realtime_alerts.md | 1 + 5 files changed, 3 insertions(+) create mode 100644 documentation/ADRs/016_log_files_for_input_data.md create mode 100644 documentation/ADRs/017_log_files_for_offline_evaluation.md create mode 100644 documentation/ADRs/018_log_files_for_online_evaluation.md create mode 100644 documentation/ADRs/019_log_files_for_model_training.md create mode 100644 documentation/ADRs/020_log_files_realtime_alerts.md diff --git a/documentation/ADRs/016_log_files_for_input_data.md b/documentation/ADRs/016_log_files_for_input_data.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/017_log_files_for_offline_evaluation.md b/documentation/ADRs/017_log_files_for_offline_evaluation.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/018_log_files_for_online_evaluation.md b/documentation/ADRs/018_log_files_for_online_evaluation.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/018_log_files_for_online_evaluation.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/019_log_files_for_model_training.md b/documentation/ADRs/019_log_files_for_model_training.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/019_log_files_for_model_training.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/020_log_files_realtime_alerts.md b/documentation/ADRs/020_log_files_realtime_alerts.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/020_log_files_realtime_alerts.md @@ -0,0 +1 @@ +TODO \ No newline at end of file From bd77bb8068668cd58bb88e689744a2976f702642 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:45 +0100 Subject: [PATCH 20/94] the new yaml - not yet used... --- common_configs/config_log.yaml | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 common_configs/config_log.yaml diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml new file mode 100644 index 00000000..857043e2 --- /dev/null +++ b/common_configs/config_log.yaml @@ -0,0 +1,44 @@ +version: 1 +disable_existing_loggers: False + +formatters: + detailed: + format: '%(asctime)s %(pathname)s [%(filename)s:%(lineno)d] [%(process)d] [%(threadName)s] - %(levelname)s - %(message)s' + +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: detailed + stream: ext://sys.stdout + + info_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: INFO + formatter: detailed + filename: 'logs/app_INFO_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + debug_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: DEBUG + formatter: detailed + filename: 'logs/app_DEBUG_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + error_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: ERROR + formatter: detailed + filename: 'logs/app_ERROR_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + +root: + level: DEBUG + handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From c9fe3d4d0517657d752a805f69684f2b5a790cb6 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:57:57 +0100 Subject: [PATCH 21/94] the config_log_yaml --- common_configs/config_log.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 857043e2..a87cfc6a 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -16,29 +16,29 @@ handlers: class: logging.handlers.TimedRotatingFileHandler level: INFO formatter: detailed - filename: 'logs/app_INFO_%Y-%m-%d.log' - when: 'midnight' + filename: "{LOG_PATH}/views_pipeline_INFO.log" + when: "midnight" backupCount: 30 - encoding: 'utf8' + encoding: "utf8" debug_file_handler: class: logging.handlers.TimedRotatingFileHandler level: DEBUG formatter: detailed - filename: 'logs/app_DEBUG_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_DEBUG.log" + when: "midnight" + backupCount: 10 + encoding: "utf8" error_file_handler: class: logging.handlers.TimedRotatingFileHandler level: ERROR formatter: detailed - filename: 'logs/app_ERROR_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_ERROR.log" + when: "midnight" + backupCount: 60 + encoding: "utf8" root: - level: DEBUG + level: INFO handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 1ec2ce3e1853681a9ad256caf5eff3c7d2f938e2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:30 +0100 Subject: [PATCH 22/94] new common_logs dir --- common_logs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 common_logs/.gitkeep diff --git a/common_logs/.gitkeep b/common_logs/.gitkeep new file mode 100644 index 00000000..e69de29b From 2e8277af59359c963838480fb6dcac88aa68e733 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:48 +0100 Subject: [PATCH 23/94] changed the central logger --- common_utils/utils_logger.py | 166 ++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 20 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index 9cf03a18..a3a8dd04 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -1,33 +1,159 @@ import logging +import logging.config +import yaml +import os +from pathlib import Path -def setup_logging(log_file: str, log_level=logging.INFO) -> logging.Logger: +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ +def get_config_log_path() -> Path: """ - Sets up logging to both a specified file and the terminal (console). + Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - Args: - log_file (str): The file where logs should be written. - log_level (int): The logging level. Default is logging.INFO. + This function identifies the 'views_pipeline' directory within the path of the current file, + constructs a new path up to and including this directory, and then appends the relative path + to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file + is not found, it raises a ValueError. + + Returns: + pathlib.Path: The path to the 'config_log.yaml' file. + + Raises: + ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' + if not PATH_CONFIG_LOG.exists(): + raise ValueError("The 'config_log.yaml' file was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_CONFIG_LOG +# -------------------------------------------------------------------------------------------------------------- + + +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +def get_common_logs_path() -> Path: + """ + Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, then constructs + a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, + it raises a ValueError. + + Returns: + pathlib.Path: Absolute path to the 'common_logs' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' + if not PATH_COMMON_LOGS.exists(): + raise ValueError("The 'common_logs' directory was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_COMMON_LOGS +# ------------------------------------------------------------------------------------------------------------ + + +def ensure_log_directory(log_path: str) -> None: """ + Ensure the log directory exists for file-based logging handlers. + + Parameters: + log_path (str): The full path to the log file for which the directory should be verified. + """ + log_dir = os.path.dirname(log_path) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + +def setup_logging( + default_level: int = logging.INFO, env_key: str = 'LOG_CONFIG') -> logging.Logger: + + """ + Setup the logging configuration from a YAML file and return the root logger. + + Parameters: + default_level (int): The default logging level if the configuration file is not found + or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging + configuration file. Default is 'LOG_CONFIG'. + + Returns: + logging.Logger: The root logger configured based on the loaded configuration. + + Example Usage: + >>> logger = setup_logging() + >>> logger.info("Logging setup complete.") + """ + + CONFIG_LOGS_PATH = get_config_log_path() + COMMON_LOGS_PATH = get_common_logs_path() + + # Load YAML configuration + path = os.getenv(env_key, CONFIG_LOGS_PATH) - basic_logger = logging.getLogger() - basic_logger.setLevel(log_level) + if os.path.exists(path): + try: + with open(path, 'rt') as f: + config = yaml.safe_load(f.read()) + + # Replace placeholder with actual log directory path + for handler in config.get("handlers", {}).values(): + if "filename" in handler and "{LOG_PATH}" in handler["filename"]: + handler["filename"] = handler["filename"].replace("{LOG_PATH}", str(COMMON_LOGS_PATH)) + ensure_log_directory(handler["filename"]) + + # Apply logging configuration + logging.config.dictConfig(config) - file_handler = logging.FileHandler(log_file) - console_handler = logging.StreamHandler() + except Exception as e: + logging.basicConfig(level=default_level) + logging.error(f"Failed to load logging configuration from {path}. Using basic configuration. Error: {e}") + else: + logging.basicConfig(level=default_level) + logging.warning(f"Logging configuration file not found at {path}. Using basic configuration.") + + return logging.getLogger() - file_handler.setLevel(log_level) - console_handler.setLevel(log_level) - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - # Clear previous handlers if they exist - if basic_logger.hasHandlers(): - basic_logger.handlers.clear() - basic_logger.addHandler(file_handler) - basic_logger.addHandler(console_handler) - return basic_logger +## Old version +#def setup_logging(log_file: str, log_level=logging.INFO): +# """ +# Sets up logging to both a specified file and the terminal (console). +# +# Args: +# log_file (str): The file where logs should be written. +# log_level (int): The logging level. Default is logging.INFO. +# """ +# +# basic_logger = logging.getLogger() +# basic_logger.setLevel(log_level) +# +# file_handler = logging.FileHandler(log_file) +# console_handler = logging.StreamHandler() +# +# file_handler.setLevel(log_level) +# console_handler.setLevel(log_level) +# +# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +# file_handler.setFormatter(formatter) +# console_handler.setFormatter(formatter) +# +# # Clear previous handlers if they exist +# if basic_logger.hasHandlers(): +# basic_logger.handlers.clear() +# +# basic_logger.addHandler(file_handler) +# basic_logger.addHandler(console_handler) +# +# return basic_logger +# \ No newline at end of file From 3d2511a39de00b86ad0942f5553886bcb5584a87 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 06:02:41 +0100 Subject: [PATCH 24/94] detail --- documentation/ADRs/015_log_files_general_strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md index 1d0490d5..aaf3d8fc 100644 --- a/documentation/ADRs/015_log_files_general_strategy.md +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -35,7 +35,7 @@ To implement a robust and unified logging strategy, we have decided on the follo 2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. -3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files and stored under `views_pipeline/common_logs`. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. 4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. From 221d54ba391727cf7f630f8c4bfb314d0b4516a2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:08 +0100 Subject: [PATCH 25/94] Use this to see how the logs look now --- meta_tools/asses_logging_setup.py | 74 +++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 meta_tools/asses_logging_setup.py diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py new file mode 100644 index 00000000..452bb449 --- /dev/null +++ b/meta_tools/asses_logging_setup.py @@ -0,0 +1,74 @@ +import logging +from pathlib import Path +import sys + + +PATH = Path(__file__) + +def get_path_common_utils(): + + if 'views_pipeline' in PATH.parts: + + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + + if not PATH_COMMON_UTILS.exists(): + + raise ValueError("The 'common_utils' directory was not found in the provided path.") + + else: + + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + + return PATH_COMMON_UTILS + + +# Import your logging setup function from wherever it is defined +# from your_logging_module import setup_logging, get_common_logs_path + +def test_logging_setup(): + # Step 1: Set up the logging configuration + try: + log_directory = get_common_logs_path() # Fetch centralized log directory + logger = setup_logging() # Initialize logging setup + + except Exception as e: + print(f"Failed to initialize logging setup: {e}") + return + + # Step 2: Generate test log messages + logger.debug("This is a DEBUG log message for testing.") + logger.info("This is an INFO log message for testing.") + logger.error("This is an ERROR log message for testing.") + + # Step 3: Define expected log files + expected_files = [ + log_directory / "views_pipeline_INFO.log", + log_directory / "views_pipeline_DEBUG.log", + log_directory / "views_pipeline_ERROR.log" + ] + + # Step 4: Check if log files exist and are not empty + for file_path in expected_files: + if file_path.exists(): + print(f"Log file '{file_path}' exists.") + if file_path.stat().st_size > 0: + print(f"Log file '{file_path}' contains data.") + else: + print(f"Warning: Log file '{file_path}' is empty.") + else: + print(f"Error: Log file '{file_path}' was not created as expected.") + + print("Logging setup test completed.") + +# Run the test + +if __name__ == "__main__": + + PATH_COMMON_UTILS = get_path_common_utils() + + sys.path.append(str(PATH_COMMON_UTILS)) + + from utils_logger import setup_logging, get_common_logs_path + + test_logging_setup() From ef52f58c81fb2e82967e921f22c2d9d24089ee95 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:17 +0100 Subject: [PATCH 26/94] Updated root level --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index a87cfc6a..21c7f24e 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: INFO + level: DEBUG handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 0fc98d243de30ffe630fc9223c48279e69a55c6e Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:29 +0100 Subject: [PATCH 27/94] whitspace --- common_utils/utils_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index a3a8dd04..b9f5dd82 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -81,6 +81,7 @@ def setup_logging( Parameters: default_level (int): The default logging level if the configuration file is not found or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging configuration file. Default is 'LOG_CONFIG'. From 689fb4de8583fdf9433711a2ee5ea7e64f369011 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:08:44 +0100 Subject: [PATCH 28/94] this is better... --- meta_tools/asses_logging_setup.py | 33 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index 452bb449..e374585c 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -5,22 +5,41 @@ PATH = Path(__file__) -def get_path_common_utils(): - if 'views_pipeline' in PATH.parts: +def set_path_common_utils(): + """ + Retrieve the path to the 'common_utils' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, + then constructs a new path to the 'common_utils' directory. If 'views_pipeline' + or 'common_utils' directories are not found, it raises a ValueError. + + If the 'common_utils' path is not already in sys.path, it appends it. + Returns: + pathlib.Path: Absolute path to the 'common_utils' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. + """ + PATH = Path(__file__) + + # Locate 'views_pipeline' in the current file's path parts + if 'views_pipeline' in PATH.parts: PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + # Check if 'common_utils' directory exists if not PATH_COMMON_UTILS.exists(): - raise ValueError("The 'common_utils' directory was not found in the provided path.") - else: + # Add 'common_utils' to sys.path if it's not already present + if str(PATH_COMMON_UTILS) not in sys.path: + sys.path.append(str(PATH_COMMON_UTILS)) + else: raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_UTILS # Import your logging setup function from wherever it is defined @@ -65,9 +84,7 @@ def test_logging_setup(): if __name__ == "__main__": - PATH_COMMON_UTILS = get_path_common_utils() - - sys.path.append(str(PATH_COMMON_UTILS)) + set_path_common_utils() from utils_logger import setup_logging, get_common_logs_path From a930a95a8389226baa3437b14d918cc4145c06ac Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:09:09 +0100 Subject: [PATCH 29/94] corrected doc string --- meta_tools/asses_logging_setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index e374585c..c95604ac 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -16,9 +16,6 @@ def set_path_common_utils(): If the 'common_utils' path is not already in sys.path, it appends it. - Returns: - pathlib.Path: Absolute path to the 'common_utils' directory. - Raises: ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. """ From 7ca3a10a041bc1f4777da4b339a26946bc02dabd Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:13:22 +0100 Subject: [PATCH 30/94] add modelpath and split by model support --- common_utils/global_cache.py | 10 +-- common_utils/model_path.py | 98 ++++++++++++++++------------- common_utils/utils_logger.py | 118 ++++++++++++++++++----------------- models/lavender_haze/main.py | 6 +- 4 files changed, 124 insertions(+), 108 deletions(-) diff --git a/common_utils/global_cache.py b/common_utils/global_cache.py index cecc515d..e519c363 100644 --- a/common_utils/global_cache.py +++ b/common_utils/global_cache.py @@ -113,12 +113,12 @@ def ensure_cache_file_exists(self): Ensures that the cache file exists. If it does not exist, creates a new cache file. """ if not self.filepath.exists(): - logging.info( + logging.warning( f"Cache file: {self.filepath} does not exist. Creating new cache file..." ) with open(self.filepath, "wb") as f: pickle.dump({}, f) - logging.info(f"Created new cache file: {self.filepath}") + logging.debug(f"Created new cache file: {self.filepath}") def set(self, key, value): """ @@ -167,7 +167,7 @@ def save_cache(self): """ with open(self.filepath, "wb") as f: pickle.dump(self.cache, f) - logging.info(f"Cache saved to file: {self.filepath}") + logging.debug(f"Cache saved to file: {self.filepath}") def load_cache(self): """ @@ -179,7 +179,7 @@ def load_cache(self): loaded_cache = pickle.loads(f.read()) if isinstance(loaded_cache, dict): self.cache = loaded_cache - logging.info(f"Cache loaded from file: {self.filepath}") + logging.debug(f"Cache loaded from file: {self.filepath}") else: logging.error( f"Loaded cache is not a dictionary. Initializing empty cache." @@ -192,7 +192,7 @@ def load_cache(self): self.cache = {} else: self.cache = {} - logging.info(f"Cache file does not exist. Initialized empty cache.") + logging.debug(f"Cache file does not exist. Initialized empty cache.") def cleanup_cache_file(): diff --git a/common_utils/model_path.py b/common_utils/model_path.py index b06642cd..ac884b3b 100644 --- a/common_utils/model_path.py +++ b/common_utils/model_path.py @@ -61,45 +61,45 @@ class ModelPath: _ignore_attributes (list): A list of paths to ignore. """ - __slots__ = ( - "_validate", - "target", - "use_global_cache", - "_force_cache_overwrite", - "root", - "models", - "common_utils", - "common_configs", - "_ignore_attributes", - "model_name", - "_instance_hash", - "_queryset", - "model_dir", - "architectures", - "artifacts", - "configs", - "data", - "data_generated", - "data_processed", - "data_raw", - "dataloaders", - "forecasting", - "management", - "notebooks", - "offline_evaluation", - "online_evaluation", - "reports", - "src", - "_templates", - "training", - "utils", - "visualization", - "_sys_paths", - "common_querysets", - "queryset_path", - "scripts", - "meta_tools", - ) + # __slots__ = ( + # "_validate", + # "target", + # "use_global_cache", + # "_force_cache_overwrite", + # "root", + # "models", + # "common_utils", + # "common_configs", + # "_ignore_attributes", + # "model_name", + # "_instance_hash", + # "_queryset", + # "model_dir", + # "architectures", + # "artifacts", + # "configs", + # "data", + # "data_generated", + # "data_processed", + # "data_raw", + # "dataloaders", + # "forecasting", + # "management", + # "notebooks", + # "offline_evaluation", + # "online_evaluation", + # "reports", + # "src", + # "_templates", + # "training", + # "utils", + # "visualization", + # "_sys_paths", + # "common_querysets", + # "queryset_path", + # "scripts", + # "meta_tools", + # ) _target = "model" _use_global_cache = True @@ -120,6 +120,7 @@ def _initialize_class_paths(cls): cls._common_configs = cls._root / "common_configs" cls._common_querysets = cls._root / "common_querysets" cls._meta_tools = cls._root / "meta_tools" + cls._common_logs = cls._root / "common_logs" @classmethod def get_root(cls) -> Path: @@ -163,6 +164,13 @@ def get_meta_tools(cls) -> Path: cls._initialize_class_paths() return cls._meta_tools + @classmethod + def get_common_logs(cls) -> Path: + """Get the common logs path.""" + if cls._common_logs is None: + cls._initialize_class_paths() + return cls._common_logs + @classmethod def check_if_model_dir_exists(cls, model_name: str) -> bool: """ @@ -575,8 +583,8 @@ def add_paths_to_sys(self) -> List[str]: ) if self._sys_paths is None: self._sys_paths = [] - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if str(attr) not in self._ignore_attributes: if ( isinstance(value, Path) @@ -641,8 +649,8 @@ def view_directories(self) -> None: """ print("\n{:<20}\t{:<50}".format("Name", "Path")) print("=" * 72) - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if attr not in self._ignore_attributes and isinstance(value, Path): print("{:<20}\t{:<50}".format(str(attr), str(value))) @@ -687,8 +695,8 @@ def get_directories(self) -> Dict[str, Optional[str]]: # ] directories = {} relative = False - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if str(attr) not in [ "model_name", "root", diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index b9f5dd82..b78ee761 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -3,62 +3,63 @@ import yaml import os from pathlib import Path - - +from model_path import ModelPath +from global_cache import GlobalCache # SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ -def get_config_log_path() -> Path: - """ - Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - - This function identifies the 'views_pipeline' directory within the path of the current file, - constructs a new path up to and including this directory, and then appends the relative path - to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file - is not found, it raises a ValueError. - - Returns: - pathlib.Path: The path to the 'config_log.yaml' file. - - Raises: - ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. - """ - PATH = Path(__file__) - if 'views_pipeline' in PATH.parts: - PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) - PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' - if not PATH_CONFIG_LOG.exists(): - raise ValueError("The 'config_log.yaml' file was not found in the provided path.") - else: - raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_CONFIG_LOG -# -------------------------------------------------------------------------------------------------------------- - - -# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- -def get_common_logs_path() -> Path: - """ - Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. - - This function locates the 'views_pipeline' directory in the current file's path, then constructs - a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, - it raises a ValueError. - - Returns: - pathlib.Path: Absolute path to the 'common_logs' directory. - - Raises: - ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. - """ - PATH = Path(__file__) - if 'views_pipeline' in PATH.parts: - PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) - PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' - if not PATH_COMMON_LOGS.exists(): - raise ValueError("The 'common_logs' directory was not found in the provided path.") - else: - raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_LOGS -# ------------------------------------------------------------------------------------------------------------ - +# def get_config_log_path() -> Path: +# """ +# Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. + +# This function identifies the 'views_pipeline' directory within the path of the current file, +# constructs a new path up to and including this directory, and then appends the relative path +# to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file +# is not found, it raises a ValueError. + +# Returns: +# pathlib.Path: The path to the 'config_log.yaml' file. + +# Raises: +# ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. +# """ +# PATH = Path(__file__) +# if 'views_pipeline' in PATH.parts: +# PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) +# PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' +# if not PATH_CONFIG_LOG.exists(): +# raise ValueError("The 'config_log.yaml' file was not found in the provided path.") +# else: +# raise ValueError("The 'views_pipeline' directory was not found in the provided path.") +# return PATH_CONFIG_LOG +# # -------------------------------------------------------------------------------------------------------------- + + +# # SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +# def get_common_logs_path() -> Path: +# """ +# Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + +# This function locates the 'views_pipeline' directory in the current file's path, then constructs +# a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, +# it raises a ValueError. + +# Returns: +# pathlib.Path: Absolute path to the 'common_logs' directory. + +# Raises: +# ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. +# """ +# PATH = Path(__file__) +# if 'views_pipeline' in PATH.parts: +# PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) +# PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' +# if not PATH_COMMON_LOGS.exists(): +# raise ValueError("The 'common_logs' directory was not found in the provided path.") +# else: +# raise ValueError("The 'views_pipeline' directory was not found in the provided path.") +# return PATH_COMMON_LOGS +# # ------------------------------------------------------------------------------------------------------------ + +_split_by_model = True # Only works for lavender_haze def ensure_log_directory(log_path: str) -> None: """ @@ -93,8 +94,11 @@ def setup_logging( >>> logger.info("Logging setup complete.") """ - CONFIG_LOGS_PATH = get_config_log_path() - COMMON_LOGS_PATH = get_common_logs_path() + CONFIG_LOGS_PATH = ModelPath.get_common_configs() / 'config_log.yaml' + if _split_by_model: + COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + else: + COMMON_LOGS_PATH = ModelPath.get_common_logs() # Load YAML configuration path = os.getenv(env_key, CONFIG_LOGS_PATH) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 3176cba9..b44b8ca4 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,14 +13,18 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +from common_utils.model_path import ModelPath +from common_utils.global_cache import GlobalCache + warnings.filterwarnings("ignore") +GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) logger = setup_logging("run.log") if __name__ == "__main__": wandb.login() - + args = parse_args() validate_arguments(args) From f44180b116fd5d6ac884edddc60d23f53231c139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borb=C3=A1la=20Farkas?= Date: Wed, 30 Oct 2024 12:17:31 +0100 Subject: [PATCH 31/94] initial ADR about production development branches --- .../ADRs/023_production_development.md | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 documentation/ADRs/023_production_development.md diff --git a/documentation/ADRs/023_production_development.md b/documentation/ADRs/023_production_development.md new file mode 100644 index 00000000..2ab2bbca --- /dev/null +++ b/documentation/ADRs/023_production_development.md @@ -0,0 +1,43 @@ +## Production and Development Branches +*Using production and development branches instead of main* + +| ADR Info | Details | +|---------------------|-------------------| +| Subject | Production and Development Branches | +| ADR Number | 023 | +| Status | proposed | +| Author | Borbála | +| Date | 29.10.2024. | + +## Context +Historically, the `main` branch was used for production and development work occurred directly via feature branched pulled from `main`. This required increased testing and restrictions on `main` such as making it protected or creating a GitHub action that prevents merging if branch is behind. These additional restrictions made the development cumbersome and increased the risk of merging unstable features. + +Therefore, we decided to establish a dedicated `production`(renamed from `main`) branch and a `development` branch, from where the feature branches are pulled instead of the former `main`. The `development` branch is synchronized with the `production` branch. + + +## Decision + +### Overview + +We renamed the `main` branch to `production` and established a new `development` branch as the main source for feature branches. Development work will now occur in feature branches off `development`, which will be synced periodically with `production`. The exact synchronization procedure is To Be Discussed... The `production` branch remains protected with the condition that two reviewers should accept the Pull Request in order to be merged. There GitHub action that prevents merging if branch is behind still applies to `production` branch (although this workflow ensures that the `development` branch is never behind `production` branch) and a new action is created for the same reason targeting `development` branch. + +## Consequences + +**Positive Effects:** +- Clear separation of production-ready code and active development work. +- Only well-tested and stable features can reach production. +- Future GitHub actions can push directly to `development` branch, but `production` is still protected. +- Improved alignment with CI/CD workflows. + +**Negative Effects:** +- Existing Pull Requests to `production` cause `development` to be behind `production`. +- `production` branch requires constant synchronization and additional testing to prevent the risk of unstable features reaching production. + +## Rationale + +A separate `development` branch allows for a stable, tested `production` branch (production) that is not impacted by experimental changes. This decision supports the integration of CI/CD workflows. + +### Considerations + +- GitHub action (e.g. updating the model catalogs) could not push to the `main` branch +- Every small feature required to be approved by two reviewers, which made the development process slower. The `development` branch aims to accelerate the development of small features. From 7c4799bddae54c0b963e1dadc1fb3759e625c38f Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:43:28 +0100 Subject: [PATCH 32/94] add globalcache failsafe --- common_utils/utils_logger.py | 6 +++++- models/lavender_haze/main.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index b78ee761..cf50a339 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -96,7 +96,11 @@ def setup_logging( CONFIG_LOGS_PATH = ModelPath.get_common_configs() / 'config_log.yaml' if _split_by_model: - COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + try: + COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + except: + # Pretection in case model name is not available or GlobalCache fails. + COMMON_LOGS_PATH = ModelPath.get_common_logs() else: COMMON_LOGS_PATH = ModelPath.get_common_logs() diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index b44b8ca4..956f953c 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -17,8 +17,10 @@ from common_utils.global_cache import GlobalCache warnings.filterwarnings("ignore") - -GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +try: + GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +except Exception: + pass logger = setup_logging("run.log") From 316f2832bd4e80072f0ac5e6bd7917746652cd2a Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:44:43 +0100 Subject: [PATCH 33/94] cleanup --- models/lavender_haze/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 956f953c..360c51d1 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,11 +13,10 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run -from common_utils.model_path import ModelPath -from common_utils.global_cache import GlobalCache - warnings.filterwarnings("ignore") try: + from common_utils.model_path import ModelPath + from common_utils.global_cache import GlobalCache GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) except Exception: pass From 84fa584648a2659091f9006d131904664dcf87de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borb=C3=A1la=20Farkas?= Date: Wed, 30 Oct 2024 14:12:50 +0100 Subject: [PATCH 34/94] ADR catalogs improvement --- documentation/ADRs/022_model_catalogs.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/documentation/ADRs/022_model_catalogs.md b/documentation/ADRs/022_model_catalogs.md index b4dca391..25502ded 100644 --- a/documentation/ADRs/022_model_catalogs.md +++ b/documentation/ADRs/022_model_catalogs.md @@ -12,7 +12,7 @@ | Date | 29.10.2024. | ## Context -*We wanted to have a catalog about all of the models in the pipeline. We needed to do that both for the old and the new pipeline because the structure of the two pipelines and the way how the querysets are organised are different. We also had to be sure that the catalogs update whenever a model is modified or added.* +We wanted to have a catalog about all of the models in the pipeline. We needed to do that both for the old and the new pipeline because the structure of the two pipelines and the way how the querysets are organised are different. We also had to be sure that the catalogs update whenever a model is modified or added. ## Decision ### New pipeline @@ -65,11 +65,11 @@ The catalogs are updated via GitHub actions. Action for the new pipeline: [updat ### Overview -*Creating catalogs for 'country level' and 'priogrid level' that update automatically when a model is modified. Separate implementation for the old and the new pipeline.* +Creating catalogs for 'country level' and 'priogrid level' that update automatically when a model is modified. Separate implementation for the old and the new pipeline. ## Consequences -*Clear overview about our existing models in the `views_pipeline/documentation/catalogs/` directory.* +Clear overview about our existing models in the `views_pipeline/documentation/catalogs/` directory. **Positive Effects:** - Our models become trackable and presentable. @@ -80,7 +80,7 @@ The catalogs are updated via GitHub actions. Action for the new pipeline: [updat - If the catalogs fail to update, it might remain unnoticed for a while. ## Rationale -*Every information about the models are found at one place. Models can be tracked and presented, even for people not involved in the development. It is easier to involve new people to the model development. GitHub actions provide a convenient way to keep the catalogs up-to-date.* +Every information about the models are found at one place. Models can be tracked and presented, even for people not involved in the development. It is easier to involve new people to the model development. GitHub actions provide a convenient way to keep the catalogs up-to-date. ### Considerations @@ -91,7 +91,9 @@ The catalogs are updated via GitHub actions. Action for the new pipeline: [updat ## Additional Notes -Involving GitHub actions led to the separation of `production` and `development`branch, since they cannot push to a protected branch (`production`). More detailed information is found in ADR #023. +- Involving GitHub actions led to the separation of `production` and `development`branch, since they cannot push to a protected branch (`production`). More detailed information is found in **ADR #023**. + +- Implementation of an alerting system, if GitHub actions fail. ## Feedback and Suggestions *Feedbacks are awaited.* From 5cdca50a7a4a697c32ba855077c95be1671a69b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borb=C3=A1la=20Farkas?= Date: Wed, 30 Oct 2024 14:36:04 +0100 Subject: [PATCH 35/94] adjusting github acgtion to production-development workflow --- .../prevent_merge_when_branch_behind.yml | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/prevent_merge_when_branch_behind.yml b/.github/workflows/prevent_merge_when_branch_behind.yml index 40337a77..d9d32201 100644 --- a/.github/workflows/prevent_merge_when_branch_behind.yml +++ b/.github/workflows/prevent_merge_when_branch_behind.yml @@ -1,4 +1,4 @@ -name: Require Branch to Be Up-to-Date with Production +name: Require Branch to Be Up-to-Date # Trigger this workflow on pull request events targeting a specific branch. on: @@ -6,31 +6,37 @@ on: branches: - production - development - - test-protect-main-merge # for testing + - fix_prevent_merge_when_branch_behind workflow_dispatch: # enables manual triggering jobs: check-branch: runs-on: ubuntu-latest steps: + - name: Determine Comparison Branch + id: comparison + run: | + if [[ "${{ github.event.pull_request.base.ref }}" == "production" && "${{ github.event.pull_request.head.ref }}" == "development" ]]; then + echo "branch=production" >> $GITHUB_ENV + else + echo "branch=development" >> $GITHUB_ENV + fi + - name: Checkout pull request branch uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.ref }} - - name: Fetch production branch + - name: Fetch comparison branch run: | git fetch --unshallow - git fetch origin production + git fetch origin ${{ env.branch }} - - name: Compare branch with production + - name: Compare branch with ${{ env.branch }} run: | - if git merge-base --is-ancestor origin/production HEAD; then - echo "::notice ::Branch is up-to-date with production." + if git merge-base --is-ancestor origin/${{ env.branch }} HEAD; then + echo "::notice ::Branch is up-to-date with ${{ env.branch }}." else - echo "::error ::Merge Blocked: Your branch is behind the latest commits on production. Please update your branch with the latest changes from production before attempting to merge." - echo "Merge base: $(git merge-base HEAD origin/production)" + echo "::error ::Merge Blocked: Your branch is behind the latest commits on ${{ env.branch }}. Please update your branch before attempting to merge." exit 1 fi - - \ No newline at end of file From d304d21128e9397affe8cc4fad2ff223ad6bec1d Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:46:00 +0100 Subject: [PATCH 36/94] add new templates for darts models --- common_querysets/queryset_meow_meow.py | 41 ------- meta_tools/model_scaffold_builder.py | 39 ++++++- .../model/template_evaluate_model.py | 77 +++++++++++++ .../model/template_evaluate_sweep.py | 50 +++++++++ .../model/template_execute_model_runs.py | 60 ++++++++++ .../model/template_execute_model_tasks.py | 92 ++++++++++++++++ .../model/template_generate_forecast.py | 68 ++++++++++++ .../templates/model/template_get_data.py | 36 ++++++ meta_tools/templates/model/template_main.py | 44 ++++---- .../templates/model/template_train_model.py | 53 +++++++++ .../templates/model/template_utils_run.py | 104 ++++++++++++++++++ 11 files changed, 595 insertions(+), 69 deletions(-) delete mode 100644 common_querysets/queryset_meow_meow.py create mode 100644 meta_tools/templates/model/template_evaluate_model.py create mode 100644 meta_tools/templates/model/template_evaluate_sweep.py create mode 100644 meta_tools/templates/model/template_execute_model_runs.py create mode 100644 meta_tools/templates/model/template_execute_model_tasks.py create mode 100644 meta_tools/templates/model/template_generate_forecast.py create mode 100644 meta_tools/templates/model/template_get_data.py create mode 100644 meta_tools/templates/model/template_train_model.py create mode 100644 meta_tools/templates/model/template_utils_run.py diff --git a/common_querysets/queryset_meow_meow.py b/common_querysets/queryset_meow_meow.py deleted file mode 100644 index 02b00ecd..00000000 --- a/common_querysets/queryset_meow_meow.py +++ /dev/null @@ -1,41 +0,0 @@ -from viewser import Queryset, Column - -def generate(): - """ - Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. - This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. - There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. - - Returns: - - queryset_base (Queryset): A queryset containing the base data for the model training. - """ - - # VIEWSER 6, Example configuration. Modify as needed. - - queryset_base = (Queryset("meow_meow", "priogrid_month") - # Create a new column 'ln_sb_best' using data from 'priogrid_month' and 'ged_sb_best_count_nokgi' column - # Apply logarithmic transformation, handle missing values by replacing them with NA - .with_column(Column("ln_sb_best", from_loa="priogrid_month", from_column="ged_sb_best_count_nokgi") - .transform.ops.ln().transform.missing.replace_na()) - - # Create a new column 'ln_ns_best' using data from 'priogrid_month' and 'ged_ns_best_count_nokgi' column - # Apply logarithmic transformation, handle missing values by replacing them with NA - .with_column(Column("ln_ns_best", from_loa="priogrid_month", from_column="ged_ns_best_count_nokgi") - .transform.ops.ln().transform.missing.replace_na()) - - # Create a new column 'ln_os_best' using data from 'priogrid_month' and 'ged_os_best_count_nokgi' column - # Apply logarithmic transformation, handle missing values by replacing them with NA - .with_column(Column("ln_os_best", from_loa="priogrid_month", from_column="ged_os_best_count_nokgi") - .transform.ops.ln().transform.missing.replace_na()) - - # Create columns for month and year_id - .with_column(Column("month", from_loa="month", from_column="month")) - .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) - - # Create columns for country_id, col, and row - .with_column(Column("c_id", from_loa="country_year", from_column="country_id")) - .with_column(Column("col", from_loa="priogrid", from_column="col")) - .with_column(Column("row", from_loa="priogrid", from_column="row")) - ) - - return queryset_base diff --git a/meta_tools/model_scaffold_builder.py b/meta_tools/model_scaffold_builder.py index 6fa48bb1..0d69e0f0 100644 --- a/meta_tools/model_scaffold_builder.py +++ b/meta_tools/model_scaffold_builder.py @@ -25,6 +25,14 @@ template_config_meta, template_config_sweep, template_main, + template_get_data, + template_execute_model_runs, + template_execute_model_tasks, + template_evaluate_model, + template_evaluate_sweep, + template_train_model, + template_utils_run, + template_generate_forecast ) logging.basicConfig(level=logging.INFO) @@ -152,7 +160,7 @@ def build_model_scripts(self): f"Model directory {self._model.model_dir} does not exist. Please call build_model_directory() first. Aborting script generation." ) template_config_deployment.generate( - script_dir=self._model.model_dir / "configs/config_deployment.py" + script_dir=self._model.configs / "config_deployment.py" ) self._model_algorithm = str( input( @@ -160,7 +168,7 @@ def build_model_scripts(self): ) ) template_config_hyperparameters.generate( - script_dir=self._model.model_dir / "configs/config_hyperparameters.py", + script_dir=self._model.configs / "config_hyperparameters.py", model_algorithm=self._model_algorithm, ) template_config_input_data.generate( @@ -169,15 +177,38 @@ def build_model_scripts(self): model_name=self._model.model_name, ) template_config_meta.generate( - script_dir=self._model.model_dir / "configs/config_meta.py", + script_dir=self._model.configs / "config_meta.py", model_name=self._model.model_name, model_algorithm=self._model_algorithm, ) template_config_sweep.generate( - script_dir=self._model.model_dir / "configs/config_sweep.py", + script_dir=self._model.configs / "config_sweep.py", model_algorithm=self._model_algorithm, ) template_main.generate(script_dir=self._model.model_dir / "main.py") + template_get_data.generate(script_dir=self._model.dataloaders / "get_data.py") + template_generate_forecast.generate( + script_dir=self._model.forecasting / "generate_forecast.py" + ) + template_execute_model_runs.generate( + script_dir=self._model.management / "execute_model_runs.py") + template_execute_model_tasks.generate( + script_dir=self._model.management / "execute_model_tasks.py" + ) + template_evaluate_model.generate( + script_dir=self._model.offline_evaluation / "evaluate_model.py" + ) + template_evaluate_sweep.generate( + script_dir=self._model.offline_evaluation / "evaluate_sweep.py" + ) + template_train_model.generate( + script_dir=self._model.training / "train_model.py" + ) + template_utils_run.generate( + script_dir=self._model.utils / "utils_run.py" + ) + # INFO: utils_outputs.py was not templated because it will probably be moved to common_utils in the future. + def assess_model_directory(self) -> dict: """ diff --git a/meta_tools/templates/model/template_evaluate_model.py b/meta_tools/templates/model/template_evaluate_model.py new file mode 100644 index 00000000..bdfdd47f --- /dev/null +++ b/meta_tools/templates/model/template_evaluate_model.py @@ -0,0 +1,77 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to evaluate a model artifact. It handles loading the model, + making predictions, standardizing the data, generating evaluation metrics, and saving the outputs. + It also logs relevant information using Weights & Biases (wandb). + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +from datetime import datetime +import pandas as pd +import logging +from model_path import ModelPath +from utils_log_files import create_log_file, read_log_file +from utils_outputs import save_model_outputs, save_predictions +from utils_run import get_standardized_df +from utils_artifacts import get_latest_model_artifact +from utils_evaluation_metrics import generate_metric_dict +from utils_model_outputs import generate_output_dict +from utils_wandb import log_wandb_log_dict +from views_forecasts.extensions import * + +logger = logging.getLogger(__name__) + +def evaluate_model_artifact(config, artifact_name): + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + path_generated = model_path.data_generated + path_artifacts = model_path.artifacts + run_type = config["run_type"] + + # if an artifact name is provided through the CLI, use it. + # Otherwise, get the latest model artifact based on the run type + if artifact_name: + logger.info(f"Using (non-default) artifact: {{artifact_name}}") + + if not artifact_name.endswith(".pkl"): + artifact_name += ".pkl" + PATH_ARTIFACT = path_artifacts / artifact_name + else: + # use the latest model artifact based on the run type + logger.info(f"Using latest (default) run type ({{run_type}}) specific artifact") + PATH_ARTIFACT = get_latest_model_artifact(path_artifacts, run_type) + + config["timestamp"] = PATH_ARTIFACT.stem[-15:] + df_viewser = pd.read_pickle(path_raw / f"{{run_type}}_viewser_df.pkl") + + try: + stepshift_model = pd.read_pickle(PATH_ARTIFACT) + except FileNotFoundError: + logger.exception(f"Model artifact not found at {{PATH_ARTIFACT}}") + + df = stepshift_model.predict(run_type, df_viewser) + df = get_standardized_df(df, config) + data_generation_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + date_fetch_timestamp = read_log_file(path_raw / f"{{run_type}}_data_fetch_log.txt").get("Data Fetch Timestamp", None) + + _, df_output = generate_output_dict(df, config) + evaluation, df_evaluation = generate_metric_dict(df, config) + log_wandb_log_dict(config, evaluation) + + save_model_outputs(df_evaluation, df_output, path_generated, config) + save_predictions(df, path_generated, config) + create_log_file(path_generated, config, config["timestamp"], data_generation_timestamp, date_fetch_timestamp) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_evaluate_sweep.py b/meta_tools/templates/model/template_evaluate_sweep.py new file mode 100644 index 00000000..d4c254c3 --- /dev/null +++ b/meta_tools/templates/model/template_evaluate_sweep.py @@ -0,0 +1,50 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to evaluate a sweep of model runs. It handles loading the model, + making predictions, standardizing the data, calculating the mean squared error (MSE), generating evaluation metrics, + and logging the results using Weights & Biases (wandb). + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import pandas as pd +import wandb +from sklearn.metrics import mean_squared_error +from model_path import ModelPath +from utils_run import get_standardized_df +from utils_wandb import log_wandb_log_dict +from utils_evaluation_metrics import generate_metric_dict + + +def evaluate_sweep(config, stepshift_model): + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + run_type = config["run_type"] + steps = config["steps"] + + df_viewser = pd.read_pickle(path_raw / f"{{{{run_type}}}}_viewser_df.pkl") + df = stepshift_model.predict(run_type, df_viewser) + df = get_standardized_df(df, config) + + # Temporarily keep this because the metric to minimize is MSE + pred_cols = [f"step_pred_{{{{str(i)}}}}" for i in steps] + df["mse"] = df.apply(lambda row: mean_squared_error([row[config["depvar"]]] * 36, + [row[col] for col in pred_cols]), axis=1) + + wandb.log({{"MSE": df["mse"].mean()}}) + + evaluation, _ = generate_metric_dict(df, config) + log_wandb_log_dict(config, evaluation) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_execute_model_runs.py b/meta_tools/templates/model/template_execute_model_runs.py new file mode 100644 index 00000000..95a142cd --- /dev/null +++ b/meta_tools/templates/model/template_execute_model_runs.py @@ -0,0 +1,60 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes functions to execute model runs, either as a sweep or a single run. + It uses configurations for deployment, hyperparameters, meta, and sweep, and integrates with Weights & Biases (wandb) for experiment tracking. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import wandb +from config_deployment import get_deployment_config +from config_hyperparameters import get_hp_config +from config_meta import get_meta_config +from config_sweep import get_sweep_config +from execute_model_tasks import execute_model_tasks +from get_data import get_data +from utils_run import update_config, update_sweep_config + + +def execute_sweep_run(args): + sweep_config = get_sweep_config() + meta_config = get_meta_config() + update_sweep_config(sweep_config, args, meta_config) + + get_data(args, sweep_config["name"]) + + project = f"{{sweep_config['name']}}_sweep" # we can name the sweep in the config file + sweep_id = wandb.sweep(sweep_config, project=project, entity="views_pipeline") + wandb.agent(sweep_id, execute_model_tasks, entity="views_pipeline") + + +def execute_single_run(args): + hp_config = get_hp_config() + meta_config = get_meta_config() + dp_config = get_deployment_config() + config = update_config(hp_config, meta_config, dp_config, args) + + get_data(args, config["name"]) + + project = f"{{config['name']}}_{{args.run_type}}" + + if args.run_type == "calibration" or args.run_type == "testing": + execute_model_tasks(config=config, project=project, train=args.train, eval=args.evaluate, + forecast=False, artifact_name=args.artifact_name) + + elif args.run_type == "forecasting": + execute_model_tasks(config=config, project=project, train=args.train, eval=False, + forecast=args.forecast, artifact_name=args.artifact_name) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_execute_model_tasks.py b/meta_tools/templates/model/template_execute_model_tasks.py new file mode 100644 index 00000000..a62cf057 --- /dev/null +++ b/meta_tools/templates/model/template_execute_model_tasks.py @@ -0,0 +1,92 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to execute various model-related tasks such as training, + evaluation, and forecasting. It integrates with Weights & Biases (wandb) for experiment tracking + and logging. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import wandb +import logging +import time +from evaluate_model import evaluate_model_artifact +from evaluate_sweep import evaluate_sweep +from generate_forecast import forecast_model_artifact +from train_model import train_model_artifact +from utils_run import split_hurdle_parameters +from utils_wandb import add_wandb_monthly_metrics + +logger = logging.getLogger(__name__) + +def execute_model_tasks(config=None, project=None, train=None, eval=None, forecast=None, artifact_name=None): + \""" + Executes various model-related tasks including training, evaluation, and forecasting. + + This function manages the execution of different tasks such as training the model, + evaluating an existing model, or performing forecasting. + It also initializes the WandB project. + + Args: + config: Configuration object containing parameters and settings. + project: The WandB project name. + train: Flag to indicate if the model should be trained. + eval: Flag to indicate if the model should be evaluated. + forecast: Flag to indicate if forecasting should be performed. + artifact_name (optional): Specific name of the model artifact to load for evaluation or forecasting. + \""" + + start_t = time.time() + + # Initialize WandB + with wandb.init(project=project, entity="views_pipeline", + config=config): # project and config ignored when running a sweep + + # add the monthly metrics to WandB + add_wandb_monthly_metrics() + + # Update config from WandB initialization above + config = wandb.config + + # W&B does not directly support nested dictionaries for hyperparameters + # This will make the sweep config super ugly, but we don't have to distinguish between sweep and single runs + if config["sweep"] and config["algorithm"] == "HurdleRegression": + config["parameters"] = {{}} + config["parameters"]["clf"], config["parameters"]["reg"] = split_hurdle_parameters(config) + + if config["sweep"]: + logger.info(f"Sweeping model {{config['name']}}...") + stepshift_model = train_model_artifact(config) + logger.info(f"Evaluating model {{config['name']}}...") + evaluate_sweep(config, stepshift_model) + + # Handle the single model runs: train and save the model as an artifact + if train: + logger.info(f"Training model {{config['name']}}...") + train_model_artifact(config) + + # Handle the single model runs: evaluate a trained model (artifact) + if eval: + logger.info(f"Evaluating model {{config['name']}}...") + evaluate_model_artifact(config, artifact_name) + + if forecast: + logger.info(f"Forecasting model {{config['name']}}...") + forecast_model_artifact(config, artifact_name) + + end_t = time.time() + minutes = (end_t - start_t) / 60 + logger.info(f"Done. Runtime: {{minutes:.3f}} minutes.\\n") +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_generate_forecast.py b/meta_tools/templates/model/template_generate_forecast.py new file mode 100644 index 00000000..cf513bc0 --- /dev/null +++ b/meta_tools/templates/model/template_generate_forecast.py @@ -0,0 +1,68 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to forecast using a model artifact. It handles loading the model, + making predictions, standardizing the data, and saving the predictions and log files. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import pandas as pd +from datetime import datetime +import logging +from model_path import ModelPath +from utils_log_files import create_log_file, read_log_file +from utils_run import get_standardized_df +from utils_outputs import save_predictions +from utils_artifacts import get_latest_model_artifact + +logger = logging.getLogger(__name__) + + +def forecast_model_artifact(config, artifact_name): + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + path_generated = model_path.data_generated + path_artifacts = model_path.artifacts + run_type = config["run_type"] + + # if an artifact name is provided through the CLI, use it. + # Otherwise, get the latest model artifact based on the run type + if artifact_name: + logger.info(f"Using (non-default) artifact: {{{{artifact_name}}}}") + + if not artifact_name.endswith(".pkl"): + artifact_name += ".pkl" + path_artifact = path_artifacts / artifact_name + else: + # use the latest model artifact based on the run type + logger.info(f"Using latest (default) run type ({{{{run_type}}}}) specific artifact") + path_artifact = get_latest_model_artifact(path_artifacts, run_type) + + config["timestamp"] = path_artifact.stem[-15:] + df_viewser = pd.read_pickle(path_raw / f"{{{{run_type}}}}_viewser_df.pkl") + + try: + stepshift_model = pd.read_pickle(path_artifact) + except FileNotFoundError: + logger.exception(f"Model artifact not found at {{{{path_artifact}}}}") + + df_predictions = stepshift_model.predict(run_type, df_viewser) + df_predictions = get_standardized_df(df_predictions, config) + data_generation_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + date_fetch_timestamp = read_log_file(path_raw / f"{{{{run_type}}}}_data_fetch_log.txt").get("Data Fetch Timestamp", None) + + save_predictions(df_predictions, path_generated, config) + create_log_file(path_generated, config, config["timestamp"], data_generation_timestamp, date_fetch_timestamp) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_get_data.py b/meta_tools/templates/model/template_get_data.py new file mode 100644 index 00000000..d2cf4bc8 --- /dev/null +++ b/meta_tools/templates/model/template_get_data.py @@ -0,0 +1,36 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import logging +from model_path import ModelPath +from utils_dataloaders import fetch_or_load_views_df + +logger = logging.getLogger(__name__) + +def get_data(args, model_name): + model_path = ModelPath(model_name) + path_raw = model_path.data_raw + + data, alerts = fetch_or_load_views_df(model_name, args.run_type, path_raw, use_saved=args.saved) + logger.debug(f"DataFrame shape: {{data.shape if data is not None else 'None'}}") + + for ialert, alert in enumerate(str(alerts).strip('[').strip(']').split('Input')): + if 'offender' in alert: + logger.warning({{f"{{args.run_type}} data alert {{ialert}}": str(alert)}}) + + return data +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_main.py b/meta_tools/templates/model/template_main.py index c41e5eed..4706a03a 100644 --- a/meta_tools/templates/model/template_main.py +++ b/meta_tools/templates/model/template_main.py @@ -33,44 +33,40 @@ def generate(script_dir: Path) -> bool: specified script directory. - The generated script is designed to be executed as a standalone Python script. """ - code = """import time -import wandb + code = """import wandb import sys -import logging -logging.basicConfig(filename='run.log', encoding='utf-8', level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) +import warnings + from pathlib import Path -# Set up the path to include common_utils module PATH = Path(__file__) sys.path.insert(0, str(Path( *[i for i in PATH.parts[:PATH.parts.index("views_pipeline") + 1]]) / "common_utils")) # PATH_COMMON_UTILS -# Import necessary functions for project setup and model execution from set_path import setup_project_paths setup_project_paths(PATH) + from utils_cli_parser import parse_args, validate_arguments +from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +warnings.filterwarnings("ignore") +try: + from common_utils.model_path import ModelPath + from common_utils.global_cache import GlobalCache + GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +except Exception: + pass +logger = setup_logging("run.log") + + if __name__ == "__main__": - # Parse command-line arguments - args = parse_args() + wandb.login() - # Validate the arguments to ensure they are correct + args = parse_args() validate_arguments(args) - # Log in to Weights & Biases (wandb) - wandb.login() - # Record the start time - start_t = time.time() - # Execute the model run based on the sweep flag + if args.sweep: - execute_sweep_run(args) # Execute sweep run + execute_sweep_run(args) else: - execute_single_run(args) # Execute single run - # Record the end time - end_t = time.time() - - # Calculate and print the runtime in minutes - minutes = (end_t - start_t) / 60 - logger.info(f'Done. Runtime: {minutes:.3f} minutes') + execute_single_run(args) """ return utils_script_gen.save_script(script_dir, code) diff --git a/meta_tools/templates/model/template_train_model.py b/meta_tools/templates/model/template_train_model.py new file mode 100644 index 00000000..2be82037 --- /dev/null +++ b/meta_tools/templates/model/template_train_model.py @@ -0,0 +1,53 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes functions to train a model artifact. It handles loading the data, + training the model, saving the model artifact, and creating log files. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f"""from datetime import datetime +import pandas as pd +from model_path import ModelPath +from utils_log_files import create_log_file, read_log_file +from utils_run import get_model +from set_partition import get_partitioner_dict +from views_forecasts.extensions import * + + +def train_model_artifact(config): + # print(config) + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + path_generated = model_path.data_generated + path_artifacts = model_path.artifacts + run_type = config["run_type"] + df_viewser = pd.read_pickle(path_raw / f"{{{{run_type}}}}_viewser_df.pkl") + + stepshift_model = stepshift_training(config, run_type, df_viewser) + if not config["sweep"]: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + model_filename = f"{{{{run_type}}}}_model_{{{{timestamp}}}}.pkl" + stepshift_model.save(path_artifacts / model_filename) + date_fetch_timestamp = read_log_file(path_raw / f"{{{{run_type}}}}_data_fetch_log.txt").get("Data Fetch Timestamp", None) + create_log_file(path_generated, config, timestamp, None, date_fetch_timestamp) + return stepshift_model + + +def stepshift_training(config, partition_name, dataset): + partitioner_dict = get_partitioner_dict(partition_name) + stepshift_model = get_model(config, partitioner_dict) + stepshift_model.fit(dataset) + return stepshift_model +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_utils_run.py b/meta_tools/templates/model/template_utils_run.py new file mode 100644 index 00000000..65c8d12e --- /dev/null +++ b/meta_tools/templates/model/template_utils_run.py @@ -0,0 +1,104 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes functions to get a model based on the configuration, standardize a DataFrame, + split hurdle parameters, and update configurations for both single runs and sweeps. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import numpy as np +from views_stepshifter_darts.stepshifter import StepshifterModel +from views_stepshifter_darts.hurdle_model import HurdleModel +from views_forecasts.extensions import * + + +def get_model(config, partitioner_dict): + \""" + Get the model based on the algorithm specified in the config + \""" + + if config["algorithm"] == "HurdleRegression": + model = HurdleModel(config, partitioner_dict) + else: + config["model_reg"] = config["algorithm"] + model = StepshifterModel(config, partitioner_dict) + + return model + + +def get_standardized_df(df, config): + \""" + Standardize the DataFrame based on the run type + \""" + + run_type = config["run_type"] + steps = config["steps"] + depvar = config["depvar"] + + # choose the columns to keep based on the run type and replace negative values with 0 + if run_type in ["calibration", "testing"]: + cols = [depvar] + df.forecasts.prediction_columns + elif run_type == "forecasting": + cols = [f"step_pred_{{{{i}}}}" for i in steps] + df = df.replace([np.inf, -np.inf], 0)[cols] + df = df.mask(df < 0, 0) + return df + + +def split_hurdle_parameters(parameters_dict): + \""" + Split the parameters dictionary into two separate dictionaries, one for the + classification model and one for the regression model. + \""" + + cls_dict = {{}} + reg_dict = {{}} + + for key, value in parameters_dict.items(): + if key.startswith("cls_"): + cls_key = key.replace("cls_", "") + cls_dict[cls_key] = value + elif key.startswith("reg_"): + reg_key = key.replace("reg_", "") + reg_dict[reg_key] = value + + return cls_dict, reg_dict + + +def update_config(hp_config, meta_config, dp_config, args): + config = hp_config.copy() + config["run_type"] = args.run_type + config["sweep"] = False + config["name"] = meta_config["name"] + config["depvar"] = meta_config["depvar"] + config["algorithm"] = meta_config["algorithm"] + if meta_config["algorithm"] == "HurdleRegression": + config["model_clf"] = meta_config["model_clf"] + config["model_reg"] = meta_config["model_reg"] + config["deployment_status"] = dp_config["deployment_status"] + + return config + + +def update_sweep_config(sweep_config, args, meta_config): + sweep_config["parameters"]["run_type"] = {{"value": args.run_type}} + sweep_config["parameters"]["sweep"] = {{"value": True}} + sweep_config["parameters"]["name"] = {{"value": meta_config["name"]}} + sweep_config["parameters"]["depvar"] = {{"value": meta_config["depvar"]}} + sweep_config["parameters"]["algorithm"] = {{"value": meta_config["algorithm"]}} + if meta_config["algorithm"] == "HurdleRegression": + sweep_config["parameters"]["model_clf"] = {{"value": meta_config["model_clf"]}} + sweep_config["parameters"]["model_reg"] = {{"value": meta_config["model_reg"]}} +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file From 4c65b6982aa13addb99e307928db9437691899f6 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:51:21 +0100 Subject: [PATCH 37/94] cleanup --- meta_tools/model_scaffold_builder.py | 1 + meta_tools/templates/model/template_evaluate_model.py | 3 +-- meta_tools/templates/model/template_evaluate_sweep.py | 3 +-- meta_tools/templates/model/template_execute_model_runs.py | 3 +-- meta_tools/templates/model/template_execute_model_tasks.py | 3 +-- meta_tools/templates/model/template_generate_forecast.py | 3 +-- meta_tools/templates/model/template_get_data.py | 3 +-- meta_tools/templates/model/template_utils_run.py | 3 +-- 8 files changed, 8 insertions(+), 14 deletions(-) diff --git a/meta_tools/model_scaffold_builder.py b/meta_tools/model_scaffold_builder.py index 0d69e0f0..4cc8d7d7 100644 --- a/meta_tools/model_scaffold_builder.py +++ b/meta_tools/model_scaffold_builder.py @@ -208,6 +208,7 @@ def build_model_scripts(self): script_dir=self._model.utils / "utils_run.py" ) # INFO: utils_outputs.py was not templated because it will probably be moved to common_utils in the future. + logging.info(f"Remember to update the queryset file at {self._model.queryset_path}!") def assess_model_directory(self) -> dict: diff --git a/meta_tools/templates/model/template_evaluate_model.py b/meta_tools/templates/model/template_evaluate_model.py index bdfdd47f..e0ce4a3d 100644 --- a/meta_tools/templates/model/template_evaluate_model.py +++ b/meta_tools/templates/model/template_evaluate_model.py @@ -17,8 +17,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -from datetime import datetime + code = f"""from datetime import datetime import pandas as pd import logging from model_path import ModelPath diff --git a/meta_tools/templates/model/template_evaluate_sweep.py b/meta_tools/templates/model/template_evaluate_sweep.py index d4c254c3..805181a8 100644 --- a/meta_tools/templates/model/template_evaluate_sweep.py +++ b/meta_tools/templates/model/template_evaluate_sweep.py @@ -17,8 +17,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import pandas as pd + code = f"""import pandas as pd import wandb from sklearn.metrics import mean_squared_error from model_path import ModelPath diff --git a/meta_tools/templates/model/template_execute_model_runs.py b/meta_tools/templates/model/template_execute_model_runs.py index 95a142cd..1a59dddb 100644 --- a/meta_tools/templates/model/template_execute_model_runs.py +++ b/meta_tools/templates/model/template_execute_model_runs.py @@ -16,8 +16,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import wandb + code = f"""import wandb from config_deployment import get_deployment_config from config_hyperparameters import get_hp_config from config_meta import get_meta_config diff --git a/meta_tools/templates/model/template_execute_model_tasks.py b/meta_tools/templates/model/template_execute_model_tasks.py index a62cf057..67f9e212 100644 --- a/meta_tools/templates/model/template_execute_model_tasks.py +++ b/meta_tools/templates/model/template_execute_model_tasks.py @@ -17,8 +17,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import wandb + code = f"""import wandb import logging import time from evaluate_model import evaluate_model_artifact diff --git a/meta_tools/templates/model/template_generate_forecast.py b/meta_tools/templates/model/template_generate_forecast.py index cf513bc0..bd49882e 100644 --- a/meta_tools/templates/model/template_generate_forecast.py +++ b/meta_tools/templates/model/template_generate_forecast.py @@ -16,8 +16,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import pandas as pd + code = f"""import pandas as pd from datetime import datetime import logging from model_path import ModelPath diff --git a/meta_tools/templates/model/template_get_data.py b/meta_tools/templates/model/template_get_data.py index d2cf4bc8..fea091a5 100644 --- a/meta_tools/templates/model/template_get_data.py +++ b/meta_tools/templates/model/template_get_data.py @@ -13,8 +13,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import logging + code = f"""import logging from model_path import ModelPath from utils_dataloaders import fetch_or_load_views_df diff --git a/meta_tools/templates/model/template_utils_run.py b/meta_tools/templates/model/template_utils_run.py index 65c8d12e..2b84ed64 100644 --- a/meta_tools/templates/model/template_utils_run.py +++ b/meta_tools/templates/model/template_utils_run.py @@ -16,8 +16,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import numpy as np + code = f"""import numpy as np from views_stepshifter_darts.stepshifter import StepshifterModel from views_stepshifter_darts.hurdle_model import HurdleModel from views_forecasts.extensions import * From a9d888e8230a4906e1968c15f982065210f9fefb Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 14:53:15 +0100 Subject: [PATCH 38/94] Accepted --- documentation/ADRs/022_model_catalogs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/ADRs/022_model_catalogs.md b/documentation/ADRs/022_model_catalogs.md index 25502ded..376fe715 100644 --- a/documentation/ADRs/022_model_catalogs.md +++ b/documentation/ADRs/022_model_catalogs.md @@ -7,7 +7,7 @@ |---------------------|-------------------| | Subject | Create Model Catalog | | ADR Number | 022 | -| Status | proposed | +| Status | Accepted | | Author | Borbála | | Date | 29.10.2024. | From 3733aa73aad927f3232f495d07996de9b388e62b Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:46:00 +0100 Subject: [PATCH 39/94] add new templates for darts models --- common_querysets/queryset_meow_meow.py | 41 ------- meta_tools/model_scaffold_builder.py | 39 ++++++- .../model/template_evaluate_model.py | 77 +++++++++++++ .../model/template_evaluate_sweep.py | 50 +++++++++ .../model/template_execute_model_runs.py | 60 ++++++++++ .../model/template_execute_model_tasks.py | 92 ++++++++++++++++ .../model/template_generate_forecast.py | 68 ++++++++++++ .../templates/model/template_get_data.py | 36 ++++++ meta_tools/templates/model/template_main.py | 44 ++++---- .../templates/model/template_train_model.py | 53 +++++++++ .../templates/model/template_utils_run.py | 104 ++++++++++++++++++ 11 files changed, 595 insertions(+), 69 deletions(-) delete mode 100644 common_querysets/queryset_meow_meow.py create mode 100644 meta_tools/templates/model/template_evaluate_model.py create mode 100644 meta_tools/templates/model/template_evaluate_sweep.py create mode 100644 meta_tools/templates/model/template_execute_model_runs.py create mode 100644 meta_tools/templates/model/template_execute_model_tasks.py create mode 100644 meta_tools/templates/model/template_generate_forecast.py create mode 100644 meta_tools/templates/model/template_get_data.py create mode 100644 meta_tools/templates/model/template_train_model.py create mode 100644 meta_tools/templates/model/template_utils_run.py diff --git a/common_querysets/queryset_meow_meow.py b/common_querysets/queryset_meow_meow.py deleted file mode 100644 index 02b00ecd..00000000 --- a/common_querysets/queryset_meow_meow.py +++ /dev/null @@ -1,41 +0,0 @@ -from viewser import Queryset, Column - -def generate(): - """ - Contains the configuration for the input data in the form of a viewser queryset. That is the data from viewser that is used to train the model. - This configuration is "behavioral" so modifying it will affect the model's runtime behavior and integration into the deployment system. - There is no guarantee that the model will work if the input data configuration is changed here without changing the model settings and algorithm accordingly. - - Returns: - - queryset_base (Queryset): A queryset containing the base data for the model training. - """ - - # VIEWSER 6, Example configuration. Modify as needed. - - queryset_base = (Queryset("meow_meow", "priogrid_month") - # Create a new column 'ln_sb_best' using data from 'priogrid_month' and 'ged_sb_best_count_nokgi' column - # Apply logarithmic transformation, handle missing values by replacing them with NA - .with_column(Column("ln_sb_best", from_loa="priogrid_month", from_column="ged_sb_best_count_nokgi") - .transform.ops.ln().transform.missing.replace_na()) - - # Create a new column 'ln_ns_best' using data from 'priogrid_month' and 'ged_ns_best_count_nokgi' column - # Apply logarithmic transformation, handle missing values by replacing them with NA - .with_column(Column("ln_ns_best", from_loa="priogrid_month", from_column="ged_ns_best_count_nokgi") - .transform.ops.ln().transform.missing.replace_na()) - - # Create a new column 'ln_os_best' using data from 'priogrid_month' and 'ged_os_best_count_nokgi' column - # Apply logarithmic transformation, handle missing values by replacing them with NA - .with_column(Column("ln_os_best", from_loa="priogrid_month", from_column="ged_os_best_count_nokgi") - .transform.ops.ln().transform.missing.replace_na()) - - # Create columns for month and year_id - .with_column(Column("month", from_loa="month", from_column="month")) - .with_column(Column("year_id", from_loa="country_year", from_column="year_id")) - - # Create columns for country_id, col, and row - .with_column(Column("c_id", from_loa="country_year", from_column="country_id")) - .with_column(Column("col", from_loa="priogrid", from_column="col")) - .with_column(Column("row", from_loa="priogrid", from_column="row")) - ) - - return queryset_base diff --git a/meta_tools/model_scaffold_builder.py b/meta_tools/model_scaffold_builder.py index 6fa48bb1..0d69e0f0 100644 --- a/meta_tools/model_scaffold_builder.py +++ b/meta_tools/model_scaffold_builder.py @@ -25,6 +25,14 @@ template_config_meta, template_config_sweep, template_main, + template_get_data, + template_execute_model_runs, + template_execute_model_tasks, + template_evaluate_model, + template_evaluate_sweep, + template_train_model, + template_utils_run, + template_generate_forecast ) logging.basicConfig(level=logging.INFO) @@ -152,7 +160,7 @@ def build_model_scripts(self): f"Model directory {self._model.model_dir} does not exist. Please call build_model_directory() first. Aborting script generation." ) template_config_deployment.generate( - script_dir=self._model.model_dir / "configs/config_deployment.py" + script_dir=self._model.configs / "config_deployment.py" ) self._model_algorithm = str( input( @@ -160,7 +168,7 @@ def build_model_scripts(self): ) ) template_config_hyperparameters.generate( - script_dir=self._model.model_dir / "configs/config_hyperparameters.py", + script_dir=self._model.configs / "config_hyperparameters.py", model_algorithm=self._model_algorithm, ) template_config_input_data.generate( @@ -169,15 +177,38 @@ def build_model_scripts(self): model_name=self._model.model_name, ) template_config_meta.generate( - script_dir=self._model.model_dir / "configs/config_meta.py", + script_dir=self._model.configs / "config_meta.py", model_name=self._model.model_name, model_algorithm=self._model_algorithm, ) template_config_sweep.generate( - script_dir=self._model.model_dir / "configs/config_sweep.py", + script_dir=self._model.configs / "config_sweep.py", model_algorithm=self._model_algorithm, ) template_main.generate(script_dir=self._model.model_dir / "main.py") + template_get_data.generate(script_dir=self._model.dataloaders / "get_data.py") + template_generate_forecast.generate( + script_dir=self._model.forecasting / "generate_forecast.py" + ) + template_execute_model_runs.generate( + script_dir=self._model.management / "execute_model_runs.py") + template_execute_model_tasks.generate( + script_dir=self._model.management / "execute_model_tasks.py" + ) + template_evaluate_model.generate( + script_dir=self._model.offline_evaluation / "evaluate_model.py" + ) + template_evaluate_sweep.generate( + script_dir=self._model.offline_evaluation / "evaluate_sweep.py" + ) + template_train_model.generate( + script_dir=self._model.training / "train_model.py" + ) + template_utils_run.generate( + script_dir=self._model.utils / "utils_run.py" + ) + # INFO: utils_outputs.py was not templated because it will probably be moved to common_utils in the future. + def assess_model_directory(self) -> dict: """ diff --git a/meta_tools/templates/model/template_evaluate_model.py b/meta_tools/templates/model/template_evaluate_model.py new file mode 100644 index 00000000..bdfdd47f --- /dev/null +++ b/meta_tools/templates/model/template_evaluate_model.py @@ -0,0 +1,77 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to evaluate a model artifact. It handles loading the model, + making predictions, standardizing the data, generating evaluation metrics, and saving the outputs. + It also logs relevant information using Weights & Biases (wandb). + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +from datetime import datetime +import pandas as pd +import logging +from model_path import ModelPath +from utils_log_files import create_log_file, read_log_file +from utils_outputs import save_model_outputs, save_predictions +from utils_run import get_standardized_df +from utils_artifacts import get_latest_model_artifact +from utils_evaluation_metrics import generate_metric_dict +from utils_model_outputs import generate_output_dict +from utils_wandb import log_wandb_log_dict +from views_forecasts.extensions import * + +logger = logging.getLogger(__name__) + +def evaluate_model_artifact(config, artifact_name): + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + path_generated = model_path.data_generated + path_artifacts = model_path.artifacts + run_type = config["run_type"] + + # if an artifact name is provided through the CLI, use it. + # Otherwise, get the latest model artifact based on the run type + if artifact_name: + logger.info(f"Using (non-default) artifact: {{artifact_name}}") + + if not artifact_name.endswith(".pkl"): + artifact_name += ".pkl" + PATH_ARTIFACT = path_artifacts / artifact_name + else: + # use the latest model artifact based on the run type + logger.info(f"Using latest (default) run type ({{run_type}}) specific artifact") + PATH_ARTIFACT = get_latest_model_artifact(path_artifacts, run_type) + + config["timestamp"] = PATH_ARTIFACT.stem[-15:] + df_viewser = pd.read_pickle(path_raw / f"{{run_type}}_viewser_df.pkl") + + try: + stepshift_model = pd.read_pickle(PATH_ARTIFACT) + except FileNotFoundError: + logger.exception(f"Model artifact not found at {{PATH_ARTIFACT}}") + + df = stepshift_model.predict(run_type, df_viewser) + df = get_standardized_df(df, config) + data_generation_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + date_fetch_timestamp = read_log_file(path_raw / f"{{run_type}}_data_fetch_log.txt").get("Data Fetch Timestamp", None) + + _, df_output = generate_output_dict(df, config) + evaluation, df_evaluation = generate_metric_dict(df, config) + log_wandb_log_dict(config, evaluation) + + save_model_outputs(df_evaluation, df_output, path_generated, config) + save_predictions(df, path_generated, config) + create_log_file(path_generated, config, config["timestamp"], data_generation_timestamp, date_fetch_timestamp) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_evaluate_sweep.py b/meta_tools/templates/model/template_evaluate_sweep.py new file mode 100644 index 00000000..d4c254c3 --- /dev/null +++ b/meta_tools/templates/model/template_evaluate_sweep.py @@ -0,0 +1,50 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to evaluate a sweep of model runs. It handles loading the model, + making predictions, standardizing the data, calculating the mean squared error (MSE), generating evaluation metrics, + and logging the results using Weights & Biases (wandb). + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import pandas as pd +import wandb +from sklearn.metrics import mean_squared_error +from model_path import ModelPath +from utils_run import get_standardized_df +from utils_wandb import log_wandb_log_dict +from utils_evaluation_metrics import generate_metric_dict + + +def evaluate_sweep(config, stepshift_model): + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + run_type = config["run_type"] + steps = config["steps"] + + df_viewser = pd.read_pickle(path_raw / f"{{{{run_type}}}}_viewser_df.pkl") + df = stepshift_model.predict(run_type, df_viewser) + df = get_standardized_df(df, config) + + # Temporarily keep this because the metric to minimize is MSE + pred_cols = [f"step_pred_{{{{str(i)}}}}" for i in steps] + df["mse"] = df.apply(lambda row: mean_squared_error([row[config["depvar"]]] * 36, + [row[col] for col in pred_cols]), axis=1) + + wandb.log({{"MSE": df["mse"].mean()}}) + + evaluation, _ = generate_metric_dict(df, config) + log_wandb_log_dict(config, evaluation) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_execute_model_runs.py b/meta_tools/templates/model/template_execute_model_runs.py new file mode 100644 index 00000000..95a142cd --- /dev/null +++ b/meta_tools/templates/model/template_execute_model_runs.py @@ -0,0 +1,60 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes functions to execute model runs, either as a sweep or a single run. + It uses configurations for deployment, hyperparameters, meta, and sweep, and integrates with Weights & Biases (wandb) for experiment tracking. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import wandb +from config_deployment import get_deployment_config +from config_hyperparameters import get_hp_config +from config_meta import get_meta_config +from config_sweep import get_sweep_config +from execute_model_tasks import execute_model_tasks +from get_data import get_data +from utils_run import update_config, update_sweep_config + + +def execute_sweep_run(args): + sweep_config = get_sweep_config() + meta_config = get_meta_config() + update_sweep_config(sweep_config, args, meta_config) + + get_data(args, sweep_config["name"]) + + project = f"{{sweep_config['name']}}_sweep" # we can name the sweep in the config file + sweep_id = wandb.sweep(sweep_config, project=project, entity="views_pipeline") + wandb.agent(sweep_id, execute_model_tasks, entity="views_pipeline") + + +def execute_single_run(args): + hp_config = get_hp_config() + meta_config = get_meta_config() + dp_config = get_deployment_config() + config = update_config(hp_config, meta_config, dp_config, args) + + get_data(args, config["name"]) + + project = f"{{config['name']}}_{{args.run_type}}" + + if args.run_type == "calibration" or args.run_type == "testing": + execute_model_tasks(config=config, project=project, train=args.train, eval=args.evaluate, + forecast=False, artifact_name=args.artifact_name) + + elif args.run_type == "forecasting": + execute_model_tasks(config=config, project=project, train=args.train, eval=False, + forecast=args.forecast, artifact_name=args.artifact_name) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_execute_model_tasks.py b/meta_tools/templates/model/template_execute_model_tasks.py new file mode 100644 index 00000000..a62cf057 --- /dev/null +++ b/meta_tools/templates/model/template_execute_model_tasks.py @@ -0,0 +1,92 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to execute various model-related tasks such as training, + evaluation, and forecasting. It integrates with Weights & Biases (wandb) for experiment tracking + and logging. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import wandb +import logging +import time +from evaluate_model import evaluate_model_artifact +from evaluate_sweep import evaluate_sweep +from generate_forecast import forecast_model_artifact +from train_model import train_model_artifact +from utils_run import split_hurdle_parameters +from utils_wandb import add_wandb_monthly_metrics + +logger = logging.getLogger(__name__) + +def execute_model_tasks(config=None, project=None, train=None, eval=None, forecast=None, artifact_name=None): + \""" + Executes various model-related tasks including training, evaluation, and forecasting. + + This function manages the execution of different tasks such as training the model, + evaluating an existing model, or performing forecasting. + It also initializes the WandB project. + + Args: + config: Configuration object containing parameters and settings. + project: The WandB project name. + train: Flag to indicate if the model should be trained. + eval: Flag to indicate if the model should be evaluated. + forecast: Flag to indicate if forecasting should be performed. + artifact_name (optional): Specific name of the model artifact to load for evaluation or forecasting. + \""" + + start_t = time.time() + + # Initialize WandB + with wandb.init(project=project, entity="views_pipeline", + config=config): # project and config ignored when running a sweep + + # add the monthly metrics to WandB + add_wandb_monthly_metrics() + + # Update config from WandB initialization above + config = wandb.config + + # W&B does not directly support nested dictionaries for hyperparameters + # This will make the sweep config super ugly, but we don't have to distinguish between sweep and single runs + if config["sweep"] and config["algorithm"] == "HurdleRegression": + config["parameters"] = {{}} + config["parameters"]["clf"], config["parameters"]["reg"] = split_hurdle_parameters(config) + + if config["sweep"]: + logger.info(f"Sweeping model {{config['name']}}...") + stepshift_model = train_model_artifact(config) + logger.info(f"Evaluating model {{config['name']}}...") + evaluate_sweep(config, stepshift_model) + + # Handle the single model runs: train and save the model as an artifact + if train: + logger.info(f"Training model {{config['name']}}...") + train_model_artifact(config) + + # Handle the single model runs: evaluate a trained model (artifact) + if eval: + logger.info(f"Evaluating model {{config['name']}}...") + evaluate_model_artifact(config, artifact_name) + + if forecast: + logger.info(f"Forecasting model {{config['name']}}...") + forecast_model_artifact(config, artifact_name) + + end_t = time.time() + minutes = (end_t - start_t) / 60 + logger.info(f"Done. Runtime: {{minutes:.3f}} minutes.\\n") +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_generate_forecast.py b/meta_tools/templates/model/template_generate_forecast.py new file mode 100644 index 00000000..cf513bc0 --- /dev/null +++ b/meta_tools/templates/model/template_generate_forecast.py @@ -0,0 +1,68 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes a function to forecast using a model artifact. It handles loading the model, + making predictions, standardizing the data, and saving the predictions and log files. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import pandas as pd +from datetime import datetime +import logging +from model_path import ModelPath +from utils_log_files import create_log_file, read_log_file +from utils_run import get_standardized_df +from utils_outputs import save_predictions +from utils_artifacts import get_latest_model_artifact + +logger = logging.getLogger(__name__) + + +def forecast_model_artifact(config, artifact_name): + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + path_generated = model_path.data_generated + path_artifacts = model_path.artifacts + run_type = config["run_type"] + + # if an artifact name is provided through the CLI, use it. + # Otherwise, get the latest model artifact based on the run type + if artifact_name: + logger.info(f"Using (non-default) artifact: {{{{artifact_name}}}}") + + if not artifact_name.endswith(".pkl"): + artifact_name += ".pkl" + path_artifact = path_artifacts / artifact_name + else: + # use the latest model artifact based on the run type + logger.info(f"Using latest (default) run type ({{{{run_type}}}}) specific artifact") + path_artifact = get_latest_model_artifact(path_artifacts, run_type) + + config["timestamp"] = path_artifact.stem[-15:] + df_viewser = pd.read_pickle(path_raw / f"{{{{run_type}}}}_viewser_df.pkl") + + try: + stepshift_model = pd.read_pickle(path_artifact) + except FileNotFoundError: + logger.exception(f"Model artifact not found at {{{{path_artifact}}}}") + + df_predictions = stepshift_model.predict(run_type, df_viewser) + df_predictions = get_standardized_df(df_predictions, config) + data_generation_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + date_fetch_timestamp = read_log_file(path_raw / f"{{{{run_type}}}}_data_fetch_log.txt").get("Data Fetch Timestamp", None) + + save_predictions(df_predictions, path_generated, config) + create_log_file(path_generated, config, config["timestamp"], data_generation_timestamp, date_fetch_timestamp) +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_get_data.py b/meta_tools/templates/model/template_get_data.py new file mode 100644 index 00000000..d2cf4bc8 --- /dev/null +++ b/meta_tools/templates/model/template_get_data.py @@ -0,0 +1,36 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import logging +from model_path import ModelPath +from utils_dataloaders import fetch_or_load_views_df + +logger = logging.getLogger(__name__) + +def get_data(args, model_name): + model_path = ModelPath(model_name) + path_raw = model_path.data_raw + + data, alerts = fetch_or_load_views_df(model_name, args.run_type, path_raw, use_saved=args.saved) + logger.debug(f"DataFrame shape: {{data.shape if data is not None else 'None'}}") + + for ialert, alert in enumerate(str(alerts).strip('[').strip(']').split('Input')): + if 'offender' in alert: + logger.warning({{f"{{args.run_type}} data alert {{ialert}}": str(alert)}}) + + return data +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_main.py b/meta_tools/templates/model/template_main.py index c41e5eed..4706a03a 100644 --- a/meta_tools/templates/model/template_main.py +++ b/meta_tools/templates/model/template_main.py @@ -33,44 +33,40 @@ def generate(script_dir: Path) -> bool: specified script directory. - The generated script is designed to be executed as a standalone Python script. """ - code = """import time -import wandb + code = """import wandb import sys -import logging -logging.basicConfig(filename='run.log', encoding='utf-8', level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) +import warnings + from pathlib import Path -# Set up the path to include common_utils module PATH = Path(__file__) sys.path.insert(0, str(Path( *[i for i in PATH.parts[:PATH.parts.index("views_pipeline") + 1]]) / "common_utils")) # PATH_COMMON_UTILS -# Import necessary functions for project setup and model execution from set_path import setup_project_paths setup_project_paths(PATH) + from utils_cli_parser import parse_args, validate_arguments +from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +warnings.filterwarnings("ignore") +try: + from common_utils.model_path import ModelPath + from common_utils.global_cache import GlobalCache + GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +except Exception: + pass +logger = setup_logging("run.log") + + if __name__ == "__main__": - # Parse command-line arguments - args = parse_args() + wandb.login() - # Validate the arguments to ensure they are correct + args = parse_args() validate_arguments(args) - # Log in to Weights & Biases (wandb) - wandb.login() - # Record the start time - start_t = time.time() - # Execute the model run based on the sweep flag + if args.sweep: - execute_sweep_run(args) # Execute sweep run + execute_sweep_run(args) else: - execute_single_run(args) # Execute single run - # Record the end time - end_t = time.time() - - # Calculate and print the runtime in minutes - minutes = (end_t - start_t) / 60 - logger.info(f'Done. Runtime: {minutes:.3f} minutes') + execute_single_run(args) """ return utils_script_gen.save_script(script_dir, code) diff --git a/meta_tools/templates/model/template_train_model.py b/meta_tools/templates/model/template_train_model.py new file mode 100644 index 00000000..2be82037 --- /dev/null +++ b/meta_tools/templates/model/template_train_model.py @@ -0,0 +1,53 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes functions to train a model artifact. It handles loading the data, + training the model, saving the model artifact, and creating log files. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f"""from datetime import datetime +import pandas as pd +from model_path import ModelPath +from utils_log_files import create_log_file, read_log_file +from utils_run import get_model +from set_partition import get_partitioner_dict +from views_forecasts.extensions import * + + +def train_model_artifact(config): + # print(config) + model_path = ModelPath(config["name"]) + path_raw = model_path.data_raw + path_generated = model_path.data_generated + path_artifacts = model_path.artifacts + run_type = config["run_type"] + df_viewser = pd.read_pickle(path_raw / f"{{{{run_type}}}}_viewser_df.pkl") + + stepshift_model = stepshift_training(config, run_type, df_viewser) + if not config["sweep"]: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + model_filename = f"{{{{run_type}}}}_model_{{{{timestamp}}}}.pkl" + stepshift_model.save(path_artifacts / model_filename) + date_fetch_timestamp = read_log_file(path_raw / f"{{{{run_type}}}}_data_fetch_log.txt").get("Data Fetch Timestamp", None) + create_log_file(path_generated, config, timestamp, None, date_fetch_timestamp) + return stepshift_model + + +def stepshift_training(config, partition_name, dataset): + partitioner_dict = get_partitioner_dict(partition_name) + stepshift_model = get_model(config, partitioner_dict) + stepshift_model.fit(dataset) + return stepshift_model +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file diff --git a/meta_tools/templates/model/template_utils_run.py b/meta_tools/templates/model/template_utils_run.py new file mode 100644 index 00000000..65c8d12e --- /dev/null +++ b/meta_tools/templates/model/template_utils_run.py @@ -0,0 +1,104 @@ +from utils import utils_script_gen +from pathlib import Path + + +def generate(script_dir: Path) -> bool: + """ + Generates a Python script with a predefined template and saves it to the specified directory. + + The generated script includes functions to get a model based on the configuration, standardize a DataFrame, + split hurdle parameters, and update configurations for both single runs and sweeps. + + Args: + script_dir (Path): The directory where the generated script will be saved. + + Returns: + bool: True if the script was successfully saved, False otherwise. + """ + + code = f""" +import numpy as np +from views_stepshifter_darts.stepshifter import StepshifterModel +from views_stepshifter_darts.hurdle_model import HurdleModel +from views_forecasts.extensions import * + + +def get_model(config, partitioner_dict): + \""" + Get the model based on the algorithm specified in the config + \""" + + if config["algorithm"] == "HurdleRegression": + model = HurdleModel(config, partitioner_dict) + else: + config["model_reg"] = config["algorithm"] + model = StepshifterModel(config, partitioner_dict) + + return model + + +def get_standardized_df(df, config): + \""" + Standardize the DataFrame based on the run type + \""" + + run_type = config["run_type"] + steps = config["steps"] + depvar = config["depvar"] + + # choose the columns to keep based on the run type and replace negative values with 0 + if run_type in ["calibration", "testing"]: + cols = [depvar] + df.forecasts.prediction_columns + elif run_type == "forecasting": + cols = [f"step_pred_{{{{i}}}}" for i in steps] + df = df.replace([np.inf, -np.inf], 0)[cols] + df = df.mask(df < 0, 0) + return df + + +def split_hurdle_parameters(parameters_dict): + \""" + Split the parameters dictionary into two separate dictionaries, one for the + classification model and one for the regression model. + \""" + + cls_dict = {{}} + reg_dict = {{}} + + for key, value in parameters_dict.items(): + if key.startswith("cls_"): + cls_key = key.replace("cls_", "") + cls_dict[cls_key] = value + elif key.startswith("reg_"): + reg_key = key.replace("reg_", "") + reg_dict[reg_key] = value + + return cls_dict, reg_dict + + +def update_config(hp_config, meta_config, dp_config, args): + config = hp_config.copy() + config["run_type"] = args.run_type + config["sweep"] = False + config["name"] = meta_config["name"] + config["depvar"] = meta_config["depvar"] + config["algorithm"] = meta_config["algorithm"] + if meta_config["algorithm"] == "HurdleRegression": + config["model_clf"] = meta_config["model_clf"] + config["model_reg"] = meta_config["model_reg"] + config["deployment_status"] = dp_config["deployment_status"] + + return config + + +def update_sweep_config(sweep_config, args, meta_config): + sweep_config["parameters"]["run_type"] = {{"value": args.run_type}} + sweep_config["parameters"]["sweep"] = {{"value": True}} + sweep_config["parameters"]["name"] = {{"value": meta_config["name"]}} + sweep_config["parameters"]["depvar"] = {{"value": meta_config["depvar"]}} + sweep_config["parameters"]["algorithm"] = {{"value": meta_config["algorithm"]}} + if meta_config["algorithm"] == "HurdleRegression": + sweep_config["parameters"]["model_clf"] = {{"value": meta_config["model_clf"]}} + sweep_config["parameters"]["model_reg"] = {{"value": meta_config["model_reg"]}} +""" + return utils_script_gen.save_script(script_dir, code) \ No newline at end of file From 73b1c094ad77cebb973ddf0def3f98651cb0171f Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:51:21 +0100 Subject: [PATCH 40/94] cleanup --- meta_tools/model_scaffold_builder.py | 1 + meta_tools/templates/model/template_evaluate_model.py | 3 +-- meta_tools/templates/model/template_evaluate_sweep.py | 3 +-- meta_tools/templates/model/template_execute_model_runs.py | 3 +-- meta_tools/templates/model/template_execute_model_tasks.py | 3 +-- meta_tools/templates/model/template_generate_forecast.py | 3 +-- meta_tools/templates/model/template_get_data.py | 3 +-- meta_tools/templates/model/template_utils_run.py | 3 +-- 8 files changed, 8 insertions(+), 14 deletions(-) diff --git a/meta_tools/model_scaffold_builder.py b/meta_tools/model_scaffold_builder.py index 0d69e0f0..4cc8d7d7 100644 --- a/meta_tools/model_scaffold_builder.py +++ b/meta_tools/model_scaffold_builder.py @@ -208,6 +208,7 @@ def build_model_scripts(self): script_dir=self._model.utils / "utils_run.py" ) # INFO: utils_outputs.py was not templated because it will probably be moved to common_utils in the future. + logging.info(f"Remember to update the queryset file at {self._model.queryset_path}!") def assess_model_directory(self) -> dict: diff --git a/meta_tools/templates/model/template_evaluate_model.py b/meta_tools/templates/model/template_evaluate_model.py index bdfdd47f..e0ce4a3d 100644 --- a/meta_tools/templates/model/template_evaluate_model.py +++ b/meta_tools/templates/model/template_evaluate_model.py @@ -17,8 +17,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -from datetime import datetime + code = f"""from datetime import datetime import pandas as pd import logging from model_path import ModelPath diff --git a/meta_tools/templates/model/template_evaluate_sweep.py b/meta_tools/templates/model/template_evaluate_sweep.py index d4c254c3..805181a8 100644 --- a/meta_tools/templates/model/template_evaluate_sweep.py +++ b/meta_tools/templates/model/template_evaluate_sweep.py @@ -17,8 +17,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import pandas as pd + code = f"""import pandas as pd import wandb from sklearn.metrics import mean_squared_error from model_path import ModelPath diff --git a/meta_tools/templates/model/template_execute_model_runs.py b/meta_tools/templates/model/template_execute_model_runs.py index 95a142cd..1a59dddb 100644 --- a/meta_tools/templates/model/template_execute_model_runs.py +++ b/meta_tools/templates/model/template_execute_model_runs.py @@ -16,8 +16,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import wandb + code = f"""import wandb from config_deployment import get_deployment_config from config_hyperparameters import get_hp_config from config_meta import get_meta_config diff --git a/meta_tools/templates/model/template_execute_model_tasks.py b/meta_tools/templates/model/template_execute_model_tasks.py index a62cf057..67f9e212 100644 --- a/meta_tools/templates/model/template_execute_model_tasks.py +++ b/meta_tools/templates/model/template_execute_model_tasks.py @@ -17,8 +17,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import wandb + code = f"""import wandb import logging import time from evaluate_model import evaluate_model_artifact diff --git a/meta_tools/templates/model/template_generate_forecast.py b/meta_tools/templates/model/template_generate_forecast.py index cf513bc0..bd49882e 100644 --- a/meta_tools/templates/model/template_generate_forecast.py +++ b/meta_tools/templates/model/template_generate_forecast.py @@ -16,8 +16,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import pandas as pd + code = f"""import pandas as pd from datetime import datetime import logging from model_path import ModelPath diff --git a/meta_tools/templates/model/template_get_data.py b/meta_tools/templates/model/template_get_data.py index d2cf4bc8..fea091a5 100644 --- a/meta_tools/templates/model/template_get_data.py +++ b/meta_tools/templates/model/template_get_data.py @@ -13,8 +13,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import logging + code = f"""import logging from model_path import ModelPath from utils_dataloaders import fetch_or_load_views_df diff --git a/meta_tools/templates/model/template_utils_run.py b/meta_tools/templates/model/template_utils_run.py index 65c8d12e..2b84ed64 100644 --- a/meta_tools/templates/model/template_utils_run.py +++ b/meta_tools/templates/model/template_utils_run.py @@ -16,8 +16,7 @@ def generate(script_dir: Path) -> bool: bool: True if the script was successfully saved, False otherwise. """ - code = f""" -import numpy as np + code = f"""import numpy as np from views_stepshifter_darts.stepshifter import StepshifterModel from views_stepshifter_darts.hurdle_model import HurdleModel from views_forecasts.extensions import * From 7067ebd2e8e6dd3423a8b3f814e639dc01760fab Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 15:06:11 +0100 Subject: [PATCH 41/94] In anticipation --- documentation/ADRs/023_production_development.md | 2 +- documentation/ADRs/024_development_and_production_sync.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 documentation/ADRs/024_development_and_production_sync.md diff --git a/documentation/ADRs/023_production_development.md b/documentation/ADRs/023_production_development.md index 2ab2bbca..6e0897b8 100644 --- a/documentation/ADRs/023_production_development.md +++ b/documentation/ADRs/023_production_development.md @@ -19,7 +19,7 @@ Therefore, we decided to establish a dedicated `production`(renamed from `main`) ### Overview -We renamed the `main` branch to `production` and established a new `development` branch as the main source for feature branches. Development work will now occur in feature branches off `development`, which will be synced periodically with `production`. The exact synchronization procedure is To Be Discussed... The `production` branch remains protected with the condition that two reviewers should accept the Pull Request in order to be merged. There GitHub action that prevents merging if branch is behind still applies to `production` branch (although this workflow ensures that the `development` branch is never behind `production` branch) and a new action is created for the same reason targeting `development` branch. +We renamed the `main` branch to `production` and established a new `development` branch as the main source for feature branches. Development work will now occur in feature branches off `development`, which will be synced periodically with `production`. The exact synchronization procedure is to be discussed, decided, and documented in [024_development_and_production_sync.md](/documentation/ADRs/024_development_and_production_sync.md). The `production` branch remains protected with the condition that two reviewers should accept the Pull Request in order to be merged. There GitHub action that prevents merging if branch is behind still applies to `production` branch (although this workflow ensures that the `development` branch is never behind `production` branch) and a new action is created for the same reason targeting `development` branch. ## Consequences diff --git a/documentation/ADRs/024_development_and_production_sync.md b/documentation/ADRs/024_development_and_production_sync.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/024_development_and_production_sync.md @@ -0,0 +1 @@ +TODO \ No newline at end of file From bdfb3e5155352ff5a42b45a7588370b4192d6ad2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:11 +0100 Subject: [PATCH 42/94] general ADR on logging --- .../ADRs/015_log_files_general_strategy.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 documentation/ADRs/015_log_files_general_strategy.md diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md new file mode 100644 index 00000000..1d0490d5 --- /dev/null +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -0,0 +1,84 @@ +# log files general strategy + +| ADR Info | Details | +|---------------------|------------------------------| +| Subject | log files general strategy | +| ADR Number | 015 | +| Status | Accepted | +| Author | Simon | +| Date | 28.10.2024 | + + +## Context + +Effective logging is essential for maintaining data integrity, monitoring model behavior, and troubleshooting issues within the model pipeline. A cohesive, centralized logging strategy ensures that logs are structured and accessible, enhancing transparency, auditability, and reliability across the model deployment lifecycle. The main goals of this logging strategy are to: + +1. **Enable Reproducibility and Traceability**: Log details such as timestamps, script paths, and process IDs are standardized to help trace model behavior and system states effectively across different environments. +2. **Support Monitoring and Real-Time Alerts**: Logs will provide data for monitoring tools, enabling real-time alerting on critical errors and pipeline health checks. +3. **Align with MLOps Best Practices**: This strategy follows MLOps standards for consistent error handling, observability, scalability, and storage management, preparing the pipeline for scalable deployment and future monitoring enhancements. + +For additional information, see also: +- [009_log_file_for_generated_data.md](009_log_file_for_generated_data.md) +- [016_log_files_for_input_data.md](016_log_files_for_input_data.md) +- [017_log_files_for_offline_evaluation.md](017_log_files_for_offline_evaluation.md) +- [018_log_files_for_online_evaluation.md](018_log_files_for_online_evaluation.md) +- [019_log_files_for_model_training.md](019_log_files_for_model_training.md) +- [020_log_files_realtime_alerts.md](020_log_files_realtime_alerts.md) + +## Decision + +To implement a robust and unified logging strategy, we have decided on the following practices: + +### Overview + +1. **Standardized Log Configuration**: All logs will follow a centralized structure defined in the configurable `common_config/config_log.yaml` file. This configuration file controls logging levels, file rotation schedules, log output formats, and target log destinations. By centralizing log settings, all models within the pipeline will have a consistent logging structure, making the setup easier to maintain and adapt across environments. + +2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. + +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. + +4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. + +5. **Error Handling and Alerts**: Real-time alerting will be implemented for critical errors and unmet conditions. Integration with alerting tools (such as Slack or email) will provide immediate notifications of key pipeline issues. Alerts will include relevant metadata like timestamps, log level, and error specifics to support rapid troubleshooting. + +6. **Dual Logging to Local Storage and Weights & Biases (W&B)**: + - **Local Storage**: Logs will be stored locally on a rotating basis for easy access and immediate troubleshooting. + - **Weights & Biases (W&B) Integration**: Model training and evaluation logs will also be sent to W&B, which allows for centralized logging of metrics, model performance tracking, and experiment comparison. The W&B integration supports MLOps best practices by making logs easily searchable, taggable (e.g., by model or pipeline stage), and accessible for experiment analysis and auditing. + +7. **Access Control and Data Sensitivity**: Logs will avoid capturing sensitive data (such as configuration secrets or personally identifiable information) to align with data governance standards. While access controls for log files are not implemented at this stage, we may restrict log access in the future as the project scales, ensuring that sensitive log data is adequately protected. + +8. **Testing and Validation**: Automated tests will validate that logs are created accurately and that rotation and level-specific separation operate as expected. These tests will cover: + - Log creation and rotation validation. + - Level-specific log file checks to confirm appropriate separation (e.g., that `INFO` logs do not include `DEBUG` messages). + - Functional testing of real-time alerts to verify that notifications trigger as configured. + +## Consequences + +**Positive Effects:** +- Provides a consistent and structured logging framework, improving troubleshooting, auditability, and compliance. +- Supports MLOps best practices by establishing robust monitoring, traceability, and data governance standards. +- Facilitates scalability and onboarding by providing a standardized, centralized approach to logging across all pipeline models. + +**Negative Effects:** +- Additional storage resources are required for log retention and rotation, and periodic monitoring of storage usage is needed. +- Initial setup and adjustment period may add complexity as team members adapt to the standardized logging and alerting practices. +- Some refactoring of the current codebase will be needed as this ADR is accepted. + +## Rationale + +The unified logging strategy aligns with MLOps best practices by combining flexibility, scalability, and robustness. This approach ensures that logging configurations are adaptable, reproducible, and traceable across the model pipeline. By establishing standardized configuration files and integrating alerting, this logging strategy proactively supports system monitoring and provides a foundation for future observability and security enhancements. + +## Considerations + +1. **Future Alerting Integrations**: Additional alerting tools, such as W&B alerts, Slack, and email notifications, will be incorporated as the project matures to ensure real-time visibility into pipeline states and failures. + +2. **Centralized Logging Platform**: In future updates, the logging system may transition to a centralized platform (e.g., ELK Stack, Grafana) to improve scalability, visualization, and monitoring. This would require adjusting the current setup to work seamlessly with a logging infrastructure, which could involve additional configurations or external services. + +3. **Access Control Expansion**: As the project scales, access control measures will be considered to ensure data protection. Log files should avoid sensitive information to comply with best practices in data governance and avoid potential data exposure risks. + +4. **Testing Resource Allocation**: Implementing automated tests for logging mechanisms may require resources such as mock environments or testing frameworks to ensure the system functions as expected under different scenarios and that alert conditions trigger correctly. + +## Additional Notes + +Future updates may involve enhancing logging with a centralized platform, providing a more scalable and observable solution for monitoring and auditability. Access control measures and security protocols will also be revisited as the project scales to protect data integrity and confidentiality. Team members are encouraged to provide feedback on specific logging configuration details or suggest improvements to the alerting and monitoring system. + From 522edc20a535d442366b1c46a48721e2003f34b9 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:30 +0100 Subject: [PATCH 43/94] New ADRs todo --- documentation/ADRs/016_log_files_for_input_data.md | 0 documentation/ADRs/017_log_files_for_offline_evaluation.md | 0 documentation/ADRs/018_log_files_for_online_evaluation.md | 1 + documentation/ADRs/019_log_files_for_model_training.md | 1 + documentation/ADRs/020_log_files_realtime_alerts.md | 1 + 5 files changed, 3 insertions(+) create mode 100644 documentation/ADRs/016_log_files_for_input_data.md create mode 100644 documentation/ADRs/017_log_files_for_offline_evaluation.md create mode 100644 documentation/ADRs/018_log_files_for_online_evaluation.md create mode 100644 documentation/ADRs/019_log_files_for_model_training.md create mode 100644 documentation/ADRs/020_log_files_realtime_alerts.md diff --git a/documentation/ADRs/016_log_files_for_input_data.md b/documentation/ADRs/016_log_files_for_input_data.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/017_log_files_for_offline_evaluation.md b/documentation/ADRs/017_log_files_for_offline_evaluation.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/018_log_files_for_online_evaluation.md b/documentation/ADRs/018_log_files_for_online_evaluation.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/018_log_files_for_online_evaluation.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/019_log_files_for_model_training.md b/documentation/ADRs/019_log_files_for_model_training.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/019_log_files_for_model_training.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/020_log_files_realtime_alerts.md b/documentation/ADRs/020_log_files_realtime_alerts.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/020_log_files_realtime_alerts.md @@ -0,0 +1 @@ +TODO \ No newline at end of file From d9feb354052b848e7fc2748c3fd8c26eec1e3ee1 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:45 +0100 Subject: [PATCH 44/94] the new yaml - not yet used... --- common_configs/config_log.yaml | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 common_configs/config_log.yaml diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml new file mode 100644 index 00000000..857043e2 --- /dev/null +++ b/common_configs/config_log.yaml @@ -0,0 +1,44 @@ +version: 1 +disable_existing_loggers: False + +formatters: + detailed: + format: '%(asctime)s %(pathname)s [%(filename)s:%(lineno)d] [%(process)d] [%(threadName)s] - %(levelname)s - %(message)s' + +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: detailed + stream: ext://sys.stdout + + info_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: INFO + formatter: detailed + filename: 'logs/app_INFO_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + debug_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: DEBUG + formatter: detailed + filename: 'logs/app_DEBUG_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + error_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: ERROR + formatter: detailed + filename: 'logs/app_ERROR_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + +root: + level: DEBUG + handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 7885a44626bbe4993695cd0897b43e04d0bc7b10 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:57:57 +0100 Subject: [PATCH 45/94] the config_log_yaml --- common_configs/config_log.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 857043e2..a87cfc6a 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -16,29 +16,29 @@ handlers: class: logging.handlers.TimedRotatingFileHandler level: INFO formatter: detailed - filename: 'logs/app_INFO_%Y-%m-%d.log' - when: 'midnight' + filename: "{LOG_PATH}/views_pipeline_INFO.log" + when: "midnight" backupCount: 30 - encoding: 'utf8' + encoding: "utf8" debug_file_handler: class: logging.handlers.TimedRotatingFileHandler level: DEBUG formatter: detailed - filename: 'logs/app_DEBUG_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_DEBUG.log" + when: "midnight" + backupCount: 10 + encoding: "utf8" error_file_handler: class: logging.handlers.TimedRotatingFileHandler level: ERROR formatter: detailed - filename: 'logs/app_ERROR_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_ERROR.log" + when: "midnight" + backupCount: 60 + encoding: "utf8" root: - level: DEBUG + level: INFO handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 4f677eab791322041b4899705b011fc033eefbf7 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:30 +0100 Subject: [PATCH 46/94] new common_logs dir --- common_logs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 common_logs/.gitkeep diff --git a/common_logs/.gitkeep b/common_logs/.gitkeep new file mode 100644 index 00000000..e69de29b From 18ac2a6bf288f8043fda35c508fa5a7c567261fd Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:48 +0100 Subject: [PATCH 47/94] changed the central logger --- common_utils/utils_logger.py | 166 ++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 20 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index 9cf03a18..a3a8dd04 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -1,33 +1,159 @@ import logging +import logging.config +import yaml +import os +from pathlib import Path -def setup_logging(log_file: str, log_level=logging.INFO) -> logging.Logger: +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ +def get_config_log_path() -> Path: """ - Sets up logging to both a specified file and the terminal (console). + Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - Args: - log_file (str): The file where logs should be written. - log_level (int): The logging level. Default is logging.INFO. + This function identifies the 'views_pipeline' directory within the path of the current file, + constructs a new path up to and including this directory, and then appends the relative path + to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file + is not found, it raises a ValueError. + + Returns: + pathlib.Path: The path to the 'config_log.yaml' file. + + Raises: + ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' + if not PATH_CONFIG_LOG.exists(): + raise ValueError("The 'config_log.yaml' file was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_CONFIG_LOG +# -------------------------------------------------------------------------------------------------------------- + + +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +def get_common_logs_path() -> Path: + """ + Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, then constructs + a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, + it raises a ValueError. + + Returns: + pathlib.Path: Absolute path to the 'common_logs' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' + if not PATH_COMMON_LOGS.exists(): + raise ValueError("The 'common_logs' directory was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_COMMON_LOGS +# ------------------------------------------------------------------------------------------------------------ + + +def ensure_log_directory(log_path: str) -> None: """ + Ensure the log directory exists for file-based logging handlers. + + Parameters: + log_path (str): The full path to the log file for which the directory should be verified. + """ + log_dir = os.path.dirname(log_path) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + +def setup_logging( + default_level: int = logging.INFO, env_key: str = 'LOG_CONFIG') -> logging.Logger: + + """ + Setup the logging configuration from a YAML file and return the root logger. + + Parameters: + default_level (int): The default logging level if the configuration file is not found + or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging + configuration file. Default is 'LOG_CONFIG'. + + Returns: + logging.Logger: The root logger configured based on the loaded configuration. + + Example Usage: + >>> logger = setup_logging() + >>> logger.info("Logging setup complete.") + """ + + CONFIG_LOGS_PATH = get_config_log_path() + COMMON_LOGS_PATH = get_common_logs_path() + + # Load YAML configuration + path = os.getenv(env_key, CONFIG_LOGS_PATH) - basic_logger = logging.getLogger() - basic_logger.setLevel(log_level) + if os.path.exists(path): + try: + with open(path, 'rt') as f: + config = yaml.safe_load(f.read()) + + # Replace placeholder with actual log directory path + for handler in config.get("handlers", {}).values(): + if "filename" in handler and "{LOG_PATH}" in handler["filename"]: + handler["filename"] = handler["filename"].replace("{LOG_PATH}", str(COMMON_LOGS_PATH)) + ensure_log_directory(handler["filename"]) + + # Apply logging configuration + logging.config.dictConfig(config) - file_handler = logging.FileHandler(log_file) - console_handler = logging.StreamHandler() + except Exception as e: + logging.basicConfig(level=default_level) + logging.error(f"Failed to load logging configuration from {path}. Using basic configuration. Error: {e}") + else: + logging.basicConfig(level=default_level) + logging.warning(f"Logging configuration file not found at {path}. Using basic configuration.") + + return logging.getLogger() - file_handler.setLevel(log_level) - console_handler.setLevel(log_level) - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - # Clear previous handlers if they exist - if basic_logger.hasHandlers(): - basic_logger.handlers.clear() - basic_logger.addHandler(file_handler) - basic_logger.addHandler(console_handler) - return basic_logger +## Old version +#def setup_logging(log_file: str, log_level=logging.INFO): +# """ +# Sets up logging to both a specified file and the terminal (console). +# +# Args: +# log_file (str): The file where logs should be written. +# log_level (int): The logging level. Default is logging.INFO. +# """ +# +# basic_logger = logging.getLogger() +# basic_logger.setLevel(log_level) +# +# file_handler = logging.FileHandler(log_file) +# console_handler = logging.StreamHandler() +# +# file_handler.setLevel(log_level) +# console_handler.setLevel(log_level) +# +# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +# file_handler.setFormatter(formatter) +# console_handler.setFormatter(formatter) +# +# # Clear previous handlers if they exist +# if basic_logger.hasHandlers(): +# basic_logger.handlers.clear() +# +# basic_logger.addHandler(file_handler) +# basic_logger.addHandler(console_handler) +# +# return basic_logger +# \ No newline at end of file From d81a152693663261a7423c3fe850e1773fb567e5 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 06:02:41 +0100 Subject: [PATCH 48/94] detail --- documentation/ADRs/015_log_files_general_strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md index 1d0490d5..aaf3d8fc 100644 --- a/documentation/ADRs/015_log_files_general_strategy.md +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -35,7 +35,7 @@ To implement a robust and unified logging strategy, we have decided on the follo 2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. -3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files and stored under `views_pipeline/common_logs`. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. 4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. From 8cc961f762911e06322e4a8432f171cb8d54b1fa Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:08 +0100 Subject: [PATCH 49/94] Use this to see how the logs look now --- meta_tools/asses_logging_setup.py | 74 +++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 meta_tools/asses_logging_setup.py diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py new file mode 100644 index 00000000..452bb449 --- /dev/null +++ b/meta_tools/asses_logging_setup.py @@ -0,0 +1,74 @@ +import logging +from pathlib import Path +import sys + + +PATH = Path(__file__) + +def get_path_common_utils(): + + if 'views_pipeline' in PATH.parts: + + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + + if not PATH_COMMON_UTILS.exists(): + + raise ValueError("The 'common_utils' directory was not found in the provided path.") + + else: + + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + + return PATH_COMMON_UTILS + + +# Import your logging setup function from wherever it is defined +# from your_logging_module import setup_logging, get_common_logs_path + +def test_logging_setup(): + # Step 1: Set up the logging configuration + try: + log_directory = get_common_logs_path() # Fetch centralized log directory + logger = setup_logging() # Initialize logging setup + + except Exception as e: + print(f"Failed to initialize logging setup: {e}") + return + + # Step 2: Generate test log messages + logger.debug("This is a DEBUG log message for testing.") + logger.info("This is an INFO log message for testing.") + logger.error("This is an ERROR log message for testing.") + + # Step 3: Define expected log files + expected_files = [ + log_directory / "views_pipeline_INFO.log", + log_directory / "views_pipeline_DEBUG.log", + log_directory / "views_pipeline_ERROR.log" + ] + + # Step 4: Check if log files exist and are not empty + for file_path in expected_files: + if file_path.exists(): + print(f"Log file '{file_path}' exists.") + if file_path.stat().st_size > 0: + print(f"Log file '{file_path}' contains data.") + else: + print(f"Warning: Log file '{file_path}' is empty.") + else: + print(f"Error: Log file '{file_path}' was not created as expected.") + + print("Logging setup test completed.") + +# Run the test + +if __name__ == "__main__": + + PATH_COMMON_UTILS = get_path_common_utils() + + sys.path.append(str(PATH_COMMON_UTILS)) + + from utils_logger import setup_logging, get_common_logs_path + + test_logging_setup() From a1a6673d00c1d8d40c70c6b2e545369c3717df03 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:17 +0100 Subject: [PATCH 50/94] Updated root level --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index a87cfc6a..21c7f24e 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: INFO + level: DEBUG handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 87d804b19e0196aa0697a0a021de8658776839a5 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:29 +0100 Subject: [PATCH 51/94] whitspace --- common_utils/utils_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index a3a8dd04..b9f5dd82 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -81,6 +81,7 @@ def setup_logging( Parameters: default_level (int): The default logging level if the configuration file is not found or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging configuration file. Default is 'LOG_CONFIG'. From 172f23b53de34f073953eaba26b1b470328d9478 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:08:44 +0100 Subject: [PATCH 52/94] this is better... --- meta_tools/asses_logging_setup.py | 33 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index 452bb449..e374585c 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -5,22 +5,41 @@ PATH = Path(__file__) -def get_path_common_utils(): - if 'views_pipeline' in PATH.parts: +def set_path_common_utils(): + """ + Retrieve the path to the 'common_utils' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, + then constructs a new path to the 'common_utils' directory. If 'views_pipeline' + or 'common_utils' directories are not found, it raises a ValueError. + + If the 'common_utils' path is not already in sys.path, it appends it. + Returns: + pathlib.Path: Absolute path to the 'common_utils' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. + """ + PATH = Path(__file__) + + # Locate 'views_pipeline' in the current file's path parts + if 'views_pipeline' in PATH.parts: PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + # Check if 'common_utils' directory exists if not PATH_COMMON_UTILS.exists(): - raise ValueError("The 'common_utils' directory was not found in the provided path.") - else: + # Add 'common_utils' to sys.path if it's not already present + if str(PATH_COMMON_UTILS) not in sys.path: + sys.path.append(str(PATH_COMMON_UTILS)) + else: raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_UTILS # Import your logging setup function from wherever it is defined @@ -65,9 +84,7 @@ def test_logging_setup(): if __name__ == "__main__": - PATH_COMMON_UTILS = get_path_common_utils() - - sys.path.append(str(PATH_COMMON_UTILS)) + set_path_common_utils() from utils_logger import setup_logging, get_common_logs_path From 7d752eb8515506c75ece1f0623f5fde376edb7cd Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:09:09 +0100 Subject: [PATCH 53/94] corrected doc string --- meta_tools/asses_logging_setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index e374585c..c95604ac 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -16,9 +16,6 @@ def set_path_common_utils(): If the 'common_utils' path is not already in sys.path, it appends it. - Returns: - pathlib.Path: Absolute path to the 'common_utils' directory. - Raises: ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. """ From 6ed5564ca0d1e142b353ef1569912d0d83240b7e Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:13:22 +0100 Subject: [PATCH 54/94] add modelpath and split by model support --- common_utils/global_cache.py | 10 +-- common_utils/model_path.py | 98 ++++++++++++++++------------- common_utils/utils_logger.py | 118 ++++++++++++++++++----------------- models/lavender_haze/main.py | 6 +- 4 files changed, 124 insertions(+), 108 deletions(-) diff --git a/common_utils/global_cache.py b/common_utils/global_cache.py index cecc515d..e519c363 100644 --- a/common_utils/global_cache.py +++ b/common_utils/global_cache.py @@ -113,12 +113,12 @@ def ensure_cache_file_exists(self): Ensures that the cache file exists. If it does not exist, creates a new cache file. """ if not self.filepath.exists(): - logging.info( + logging.warning( f"Cache file: {self.filepath} does not exist. Creating new cache file..." ) with open(self.filepath, "wb") as f: pickle.dump({}, f) - logging.info(f"Created new cache file: {self.filepath}") + logging.debug(f"Created new cache file: {self.filepath}") def set(self, key, value): """ @@ -167,7 +167,7 @@ def save_cache(self): """ with open(self.filepath, "wb") as f: pickle.dump(self.cache, f) - logging.info(f"Cache saved to file: {self.filepath}") + logging.debug(f"Cache saved to file: {self.filepath}") def load_cache(self): """ @@ -179,7 +179,7 @@ def load_cache(self): loaded_cache = pickle.loads(f.read()) if isinstance(loaded_cache, dict): self.cache = loaded_cache - logging.info(f"Cache loaded from file: {self.filepath}") + logging.debug(f"Cache loaded from file: {self.filepath}") else: logging.error( f"Loaded cache is not a dictionary. Initializing empty cache." @@ -192,7 +192,7 @@ def load_cache(self): self.cache = {} else: self.cache = {} - logging.info(f"Cache file does not exist. Initialized empty cache.") + logging.debug(f"Cache file does not exist. Initialized empty cache.") def cleanup_cache_file(): diff --git a/common_utils/model_path.py b/common_utils/model_path.py index b06642cd..ac884b3b 100644 --- a/common_utils/model_path.py +++ b/common_utils/model_path.py @@ -61,45 +61,45 @@ class ModelPath: _ignore_attributes (list): A list of paths to ignore. """ - __slots__ = ( - "_validate", - "target", - "use_global_cache", - "_force_cache_overwrite", - "root", - "models", - "common_utils", - "common_configs", - "_ignore_attributes", - "model_name", - "_instance_hash", - "_queryset", - "model_dir", - "architectures", - "artifacts", - "configs", - "data", - "data_generated", - "data_processed", - "data_raw", - "dataloaders", - "forecasting", - "management", - "notebooks", - "offline_evaluation", - "online_evaluation", - "reports", - "src", - "_templates", - "training", - "utils", - "visualization", - "_sys_paths", - "common_querysets", - "queryset_path", - "scripts", - "meta_tools", - ) + # __slots__ = ( + # "_validate", + # "target", + # "use_global_cache", + # "_force_cache_overwrite", + # "root", + # "models", + # "common_utils", + # "common_configs", + # "_ignore_attributes", + # "model_name", + # "_instance_hash", + # "_queryset", + # "model_dir", + # "architectures", + # "artifacts", + # "configs", + # "data", + # "data_generated", + # "data_processed", + # "data_raw", + # "dataloaders", + # "forecasting", + # "management", + # "notebooks", + # "offline_evaluation", + # "online_evaluation", + # "reports", + # "src", + # "_templates", + # "training", + # "utils", + # "visualization", + # "_sys_paths", + # "common_querysets", + # "queryset_path", + # "scripts", + # "meta_tools", + # ) _target = "model" _use_global_cache = True @@ -120,6 +120,7 @@ def _initialize_class_paths(cls): cls._common_configs = cls._root / "common_configs" cls._common_querysets = cls._root / "common_querysets" cls._meta_tools = cls._root / "meta_tools" + cls._common_logs = cls._root / "common_logs" @classmethod def get_root(cls) -> Path: @@ -163,6 +164,13 @@ def get_meta_tools(cls) -> Path: cls._initialize_class_paths() return cls._meta_tools + @classmethod + def get_common_logs(cls) -> Path: + """Get the common logs path.""" + if cls._common_logs is None: + cls._initialize_class_paths() + return cls._common_logs + @classmethod def check_if_model_dir_exists(cls, model_name: str) -> bool: """ @@ -575,8 +583,8 @@ def add_paths_to_sys(self) -> List[str]: ) if self._sys_paths is None: self._sys_paths = [] - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if str(attr) not in self._ignore_attributes: if ( isinstance(value, Path) @@ -641,8 +649,8 @@ def view_directories(self) -> None: """ print("\n{:<20}\t{:<50}".format("Name", "Path")) print("=" * 72) - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if attr not in self._ignore_attributes and isinstance(value, Path): print("{:<20}\t{:<50}".format(str(attr), str(value))) @@ -687,8 +695,8 @@ def get_directories(self) -> Dict[str, Optional[str]]: # ] directories = {} relative = False - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if str(attr) not in [ "model_name", "root", diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index b9f5dd82..b78ee761 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -3,62 +3,63 @@ import yaml import os from pathlib import Path - - +from model_path import ModelPath +from global_cache import GlobalCache # SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ -def get_config_log_path() -> Path: - """ - Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - - This function identifies the 'views_pipeline' directory within the path of the current file, - constructs a new path up to and including this directory, and then appends the relative path - to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file - is not found, it raises a ValueError. - - Returns: - pathlib.Path: The path to the 'config_log.yaml' file. - - Raises: - ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. - """ - PATH = Path(__file__) - if 'views_pipeline' in PATH.parts: - PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) - PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' - if not PATH_CONFIG_LOG.exists(): - raise ValueError("The 'config_log.yaml' file was not found in the provided path.") - else: - raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_CONFIG_LOG -# -------------------------------------------------------------------------------------------------------------- - - -# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- -def get_common_logs_path() -> Path: - """ - Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. - - This function locates the 'views_pipeline' directory in the current file's path, then constructs - a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, - it raises a ValueError. - - Returns: - pathlib.Path: Absolute path to the 'common_logs' directory. - - Raises: - ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. - """ - PATH = Path(__file__) - if 'views_pipeline' in PATH.parts: - PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) - PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' - if not PATH_COMMON_LOGS.exists(): - raise ValueError("The 'common_logs' directory was not found in the provided path.") - else: - raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_LOGS -# ------------------------------------------------------------------------------------------------------------ - +# def get_config_log_path() -> Path: +# """ +# Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. + +# This function identifies the 'views_pipeline' directory within the path of the current file, +# constructs a new path up to and including this directory, and then appends the relative path +# to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file +# is not found, it raises a ValueError. + +# Returns: +# pathlib.Path: The path to the 'config_log.yaml' file. + +# Raises: +# ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. +# """ +# PATH = Path(__file__) +# if 'views_pipeline' in PATH.parts: +# PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) +# PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' +# if not PATH_CONFIG_LOG.exists(): +# raise ValueError("The 'config_log.yaml' file was not found in the provided path.") +# else: +# raise ValueError("The 'views_pipeline' directory was not found in the provided path.") +# return PATH_CONFIG_LOG +# # -------------------------------------------------------------------------------------------------------------- + + +# # SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +# def get_common_logs_path() -> Path: +# """ +# Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + +# This function locates the 'views_pipeline' directory in the current file's path, then constructs +# a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, +# it raises a ValueError. + +# Returns: +# pathlib.Path: Absolute path to the 'common_logs' directory. + +# Raises: +# ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. +# """ +# PATH = Path(__file__) +# if 'views_pipeline' in PATH.parts: +# PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) +# PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' +# if not PATH_COMMON_LOGS.exists(): +# raise ValueError("The 'common_logs' directory was not found in the provided path.") +# else: +# raise ValueError("The 'views_pipeline' directory was not found in the provided path.") +# return PATH_COMMON_LOGS +# # ------------------------------------------------------------------------------------------------------------ + +_split_by_model = True # Only works for lavender_haze def ensure_log_directory(log_path: str) -> None: """ @@ -93,8 +94,11 @@ def setup_logging( >>> logger.info("Logging setup complete.") """ - CONFIG_LOGS_PATH = get_config_log_path() - COMMON_LOGS_PATH = get_common_logs_path() + CONFIG_LOGS_PATH = ModelPath.get_common_configs() / 'config_log.yaml' + if _split_by_model: + COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + else: + COMMON_LOGS_PATH = ModelPath.get_common_logs() # Load YAML configuration path = os.getenv(env_key, CONFIG_LOGS_PATH) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 3176cba9..b44b8ca4 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,14 +13,18 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +from common_utils.model_path import ModelPath +from common_utils.global_cache import GlobalCache + warnings.filterwarnings("ignore") +GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) logger = setup_logging("run.log") if __name__ == "__main__": wandb.login() - + args = parse_args() validate_arguments(args) From badc6196ea5fedac77bc57b8194295210ec1eae9 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:43:28 +0100 Subject: [PATCH 55/94] add globalcache failsafe --- common_utils/utils_logger.py | 6 +++++- models/lavender_haze/main.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index b78ee761..cf50a339 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -96,7 +96,11 @@ def setup_logging( CONFIG_LOGS_PATH = ModelPath.get_common_configs() / 'config_log.yaml' if _split_by_model: - COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + try: + COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + except: + # Pretection in case model name is not available or GlobalCache fails. + COMMON_LOGS_PATH = ModelPath.get_common_logs() else: COMMON_LOGS_PATH = ModelPath.get_common_logs() diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index b44b8ca4..956f953c 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -17,8 +17,10 @@ from common_utils.global_cache import GlobalCache warnings.filterwarnings("ignore") - -GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +try: + GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +except Exception: + pass logger = setup_logging("run.log") From a45b630c2d2b137876ccf17e69840669eda909d2 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:44:43 +0100 Subject: [PATCH 56/94] cleanup --- models/lavender_haze/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 956f953c..360c51d1 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,11 +13,10 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run -from common_utils.model_path import ModelPath -from common_utils.global_cache import GlobalCache - warnings.filterwarnings("ignore") try: + from common_utils.model_path import ModelPath + from common_utils.global_cache import GlobalCache GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) except Exception: pass From 3f2b6e452f1b7e70a72c682dea2451fb42734b74 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 15:42:14 +0100 Subject: [PATCH 57/94] added warning and critical --- common_configs/config_log.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 21c7f24e..f75eb956 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -30,6 +30,15 @@ handlers: backupCount: 10 encoding: "utf8" + warning_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: WARNING + formatter: detailed + filename: "{LOG_PATH}/views_pipeline_WARNING.log" + when: "midnight" + backupCount: 20 + encoding: "utf8" + error_file_handler: class: logging.handlers.TimedRotatingFileHandler level: ERROR @@ -39,6 +48,15 @@ handlers: backupCount: 60 encoding: "utf8" + critical_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: CRITICAL + formatter: detailed + filename: "{LOG_PATH}/views_pipeline_CRITICAL.log" + when: "midnight" + backupCount: 90 + encoding: "utf8" + root: level: DEBUG - handlers: [console, info_file_handler, debug_file_handler, error_file_handler] + handlers: [console, info_file_handler, debug_file_handler, warning_file_handler, error_file_handler, critical_file_handler] From b249a6a92efc01d9b9316eb5353ea24bdbaf917b Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 15:55:14 +0100 Subject: [PATCH 58/94] And ADR to justify this --- .../ADRs/025_log _level_standards.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 documentation/ADRs/025_log _level_standards.md diff --git a/documentation/ADRs/025_log _level_standards.md b/documentation/ADRs/025_log _level_standards.md new file mode 100644 index 00000000..f2ead9e1 --- /dev/null +++ b/documentation/ADRs/025_log _level_standards.md @@ -0,0 +1,77 @@ +## Title +*Log Level Standards* + +| ADR Info | Details | +|---------------------|-------------------| +| Subject | Logging Levels Configuration | +| ADR Number | 025 | +| Status | Proposed | +| Author | Simon | +| Date | 30.10.2024 | + +## Context +We aim to establish a new benchmark in MLOps for early warning systems (EWS), emphasizing robust and transparent logging practices. The conflict forecasting pipeline must have a comprehensive logging strategy to support the continuous quality assurance, real-time monitoring, and rapid model updates critical for high-stakes decision-making and early action. This ADR specifically addresses the standardized use of log levels within the pipeline, which helps the team capture relevant system states and provides clear visibility into operations, potential issues, and crucial decision points across the pipeline. + +The following log levels—DEBUG, INFO, WARNING, ERROR, and CRITICAL—are configured to ensure appropriate information is logged for various scenarios, supporting both ongoing development and long-term system monitoring and troubleshooting. + +## Decision +The following log levels are implemented as standard for the conflict forecasting pipeline: + +### Overview +The system’s log levels are structured as follows: + +1. **DEBUG**: Provides detailed diagnostic information, primarily used during development and debugging. +2. **INFO**: Captures general system information about normal operations to give an overview without verbosity. +3. **WARNING**: Logs potentially problematic situations that require attention but do not yet impact execution. +4. **ERROR**: Indicates issues where specific processes fail but the overall system remains operational. +5. **CRITICAL**: Records severe errors that require immediate attention as they could lead to system failures or data loss. + +### Examples and Use Cases + +- **DEBUG**: + - Example: Logging input data shapes, intermediate transformations, or model hyperparameters during experimentation phases. + - Use: Essential for development and debugging stages, enabling the team to trace and diagnose precise pipeline states. + +- **INFO**: + - Example: Model training and evaluation start/completion times or successful data fetching messages. + - Use: Provides a clear history of system operations without overwhelming detail, facilitating audits and general status tracking. + +- **WARNING**: + - Example: Detection of minor schema mismatches in input data or slower-than-expected execution times. + - Use: Highlights potentially impactful issues that may worsen if not addressed, useful for preventive monitoring. + +- **ERROR**: + - Example: Failure in data loading or model checkpoint saving. + - Use: Captures failures within components, supporting root cause analysis without interrupting the entire pipeline’s execution. + +- **CRITICAL**: + - Example: Data corruption detected during ingestion or a complete failure in the model orchestration module. + - Use: Alerts the team to major issues demanding immediate intervention, ensuring prompt actions to mitigate risks. + +## Consequences +Implementing this structured logging approach provides several benefits and potential drawbacks: + +**Positive Effects:** +- **Improved Observability**: Each log level offers a distinct lens on system operations, enhancing real-time monitoring and troubleshooting capabilities. +- **Data-Driven Issue Resolution**: The logging structure supports continuous quality assurance by capturing detailed information, aiding root cause analysis and enabling proactive interventions. +- **MLOps Standardization**: This approach aligns with best practices in MLOps, facilitating future integrations, scaling, and consistent team understanding. + +**Negative Effects:** +- **Increased Storage Use**: Higher log granularity, especially at DEBUG and INFO levels, may increase storage requirements. +- **Operational Overhead**: Maintenance of log file management, such as purging or archiving, may require periodic oversight, particularly with large volume logs like DEBUG. + +## Rationale +The chosen logging levels reflect best practices in MLOps, which call for clearly defined, purposeful logging to maintain high observability and diagnostic precision within production environments. By setting granular logging levels, we are able to balance immediate operational needs with long-term maintenance, ensuring that the system remains adaptable to shifting conditions and that degraded performance can be preemptively managed. + +### Considerations +Key considerations include: +- **Dependency Management**: Log retention and storage solutions must scale with the forecast pipeline, ensuring longevity without excessive operational costs. +- **Data Security**: Sensitive information should not be logged, especially at DEBUG or INFO levels, to maintain data protection standards. +- **Resource Constraints**: Frequent updates to logging configurations may impose operational overheads; thus, the current structure should be periodically reviewed but not frequently changed. + +## Additional Notes +- **Log Rotation**: Log files are set up with TimedRotatingFileHandler to rotate daily, retaining a set number of logs per level (e.g., INFO logs for 30 days), which balances traceability and storage efficiency. +- **Future Enhancements**: Potential additions may include automated alerts for CRITICAL logs or integration with monitoring dashboards for centralized logging analysis. + +## Feedback and Suggestions +Team members and stakeholders are encouraged to provide feedback on the logging configuration’s effectiveness, particularly regarding its impact on troubleshooting and operational transparency. Suggestions for improvement are welcome as we continue refining MLOps practices to support reliable, high-stakes forecasting. From c934874dadb74e95c469a8ffbfb4cdc0b15d72da Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 16:01:17 +0100 Subject: [PATCH 59/94] updated with new default level INFO --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index f75eb956..c9deee22 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -8,7 +8,7 @@ formatters: handlers: console: class: logging.StreamHandler - level: DEBUG + level: INFO formatter: detailed stream: ext://sys.stdout From 99cfe9f75f2b07c34850b45309591d19fb1edefd Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:11 +0100 Subject: [PATCH 60/94] general ADR on logging --- .../ADRs/015_log_files_general_strategy.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 documentation/ADRs/015_log_files_general_strategy.md diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md new file mode 100644 index 00000000..1d0490d5 --- /dev/null +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -0,0 +1,84 @@ +# log files general strategy + +| ADR Info | Details | +|---------------------|------------------------------| +| Subject | log files general strategy | +| ADR Number | 015 | +| Status | Accepted | +| Author | Simon | +| Date | 28.10.2024 | + + +## Context + +Effective logging is essential for maintaining data integrity, monitoring model behavior, and troubleshooting issues within the model pipeline. A cohesive, centralized logging strategy ensures that logs are structured and accessible, enhancing transparency, auditability, and reliability across the model deployment lifecycle. The main goals of this logging strategy are to: + +1. **Enable Reproducibility and Traceability**: Log details such as timestamps, script paths, and process IDs are standardized to help trace model behavior and system states effectively across different environments. +2. **Support Monitoring and Real-Time Alerts**: Logs will provide data for monitoring tools, enabling real-time alerting on critical errors and pipeline health checks. +3. **Align with MLOps Best Practices**: This strategy follows MLOps standards for consistent error handling, observability, scalability, and storage management, preparing the pipeline for scalable deployment and future monitoring enhancements. + +For additional information, see also: +- [009_log_file_for_generated_data.md](009_log_file_for_generated_data.md) +- [016_log_files_for_input_data.md](016_log_files_for_input_data.md) +- [017_log_files_for_offline_evaluation.md](017_log_files_for_offline_evaluation.md) +- [018_log_files_for_online_evaluation.md](018_log_files_for_online_evaluation.md) +- [019_log_files_for_model_training.md](019_log_files_for_model_training.md) +- [020_log_files_realtime_alerts.md](020_log_files_realtime_alerts.md) + +## Decision + +To implement a robust and unified logging strategy, we have decided on the following practices: + +### Overview + +1. **Standardized Log Configuration**: All logs will follow a centralized structure defined in the configurable `common_config/config_log.yaml` file. This configuration file controls logging levels, file rotation schedules, log output formats, and target log destinations. By centralizing log settings, all models within the pipeline will have a consistent logging structure, making the setup easier to maintain and adapt across environments. + +2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. + +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. + +4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. + +5. **Error Handling and Alerts**: Real-time alerting will be implemented for critical errors and unmet conditions. Integration with alerting tools (such as Slack or email) will provide immediate notifications of key pipeline issues. Alerts will include relevant metadata like timestamps, log level, and error specifics to support rapid troubleshooting. + +6. **Dual Logging to Local Storage and Weights & Biases (W&B)**: + - **Local Storage**: Logs will be stored locally on a rotating basis for easy access and immediate troubleshooting. + - **Weights & Biases (W&B) Integration**: Model training and evaluation logs will also be sent to W&B, which allows for centralized logging of metrics, model performance tracking, and experiment comparison. The W&B integration supports MLOps best practices by making logs easily searchable, taggable (e.g., by model or pipeline stage), and accessible for experiment analysis and auditing. + +7. **Access Control and Data Sensitivity**: Logs will avoid capturing sensitive data (such as configuration secrets or personally identifiable information) to align with data governance standards. While access controls for log files are not implemented at this stage, we may restrict log access in the future as the project scales, ensuring that sensitive log data is adequately protected. + +8. **Testing and Validation**: Automated tests will validate that logs are created accurately and that rotation and level-specific separation operate as expected. These tests will cover: + - Log creation and rotation validation. + - Level-specific log file checks to confirm appropriate separation (e.g., that `INFO` logs do not include `DEBUG` messages). + - Functional testing of real-time alerts to verify that notifications trigger as configured. + +## Consequences + +**Positive Effects:** +- Provides a consistent and structured logging framework, improving troubleshooting, auditability, and compliance. +- Supports MLOps best practices by establishing robust monitoring, traceability, and data governance standards. +- Facilitates scalability and onboarding by providing a standardized, centralized approach to logging across all pipeline models. + +**Negative Effects:** +- Additional storage resources are required for log retention and rotation, and periodic monitoring of storage usage is needed. +- Initial setup and adjustment period may add complexity as team members adapt to the standardized logging and alerting practices. +- Some refactoring of the current codebase will be needed as this ADR is accepted. + +## Rationale + +The unified logging strategy aligns with MLOps best practices by combining flexibility, scalability, and robustness. This approach ensures that logging configurations are adaptable, reproducible, and traceable across the model pipeline. By establishing standardized configuration files and integrating alerting, this logging strategy proactively supports system monitoring and provides a foundation for future observability and security enhancements. + +## Considerations + +1. **Future Alerting Integrations**: Additional alerting tools, such as W&B alerts, Slack, and email notifications, will be incorporated as the project matures to ensure real-time visibility into pipeline states and failures. + +2. **Centralized Logging Platform**: In future updates, the logging system may transition to a centralized platform (e.g., ELK Stack, Grafana) to improve scalability, visualization, and monitoring. This would require adjusting the current setup to work seamlessly with a logging infrastructure, which could involve additional configurations or external services. + +3. **Access Control Expansion**: As the project scales, access control measures will be considered to ensure data protection. Log files should avoid sensitive information to comply with best practices in data governance and avoid potential data exposure risks. + +4. **Testing Resource Allocation**: Implementing automated tests for logging mechanisms may require resources such as mock environments or testing frameworks to ensure the system functions as expected under different scenarios and that alert conditions trigger correctly. + +## Additional Notes + +Future updates may involve enhancing logging with a centralized platform, providing a more scalable and observable solution for monitoring and auditability. Access control measures and security protocols will also be revisited as the project scales to protect data integrity and confidentiality. Team members are encouraged to provide feedback on specific logging configuration details or suggest improvements to the alerting and monitoring system. + From a3e8bcaaf02293dba8e8a74c6d256c8f29902406 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:30 +0100 Subject: [PATCH 61/94] New ADRs todo --- documentation/ADRs/016_log_files_for_input_data.md | 0 documentation/ADRs/017_log_files_for_offline_evaluation.md | 0 documentation/ADRs/018_log_files_for_online_evaluation.md | 1 + documentation/ADRs/019_log_files_for_model_training.md | 1 + documentation/ADRs/020_log_files_realtime_alerts.md | 1 + 5 files changed, 3 insertions(+) create mode 100644 documentation/ADRs/016_log_files_for_input_data.md create mode 100644 documentation/ADRs/017_log_files_for_offline_evaluation.md create mode 100644 documentation/ADRs/018_log_files_for_online_evaluation.md create mode 100644 documentation/ADRs/019_log_files_for_model_training.md create mode 100644 documentation/ADRs/020_log_files_realtime_alerts.md diff --git a/documentation/ADRs/016_log_files_for_input_data.md b/documentation/ADRs/016_log_files_for_input_data.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/017_log_files_for_offline_evaluation.md b/documentation/ADRs/017_log_files_for_offline_evaluation.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/ADRs/018_log_files_for_online_evaluation.md b/documentation/ADRs/018_log_files_for_online_evaluation.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/018_log_files_for_online_evaluation.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/019_log_files_for_model_training.md b/documentation/ADRs/019_log_files_for_model_training.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/019_log_files_for_model_training.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/documentation/ADRs/020_log_files_realtime_alerts.md b/documentation/ADRs/020_log_files_realtime_alerts.md new file mode 100644 index 00000000..30404ce4 --- /dev/null +++ b/documentation/ADRs/020_log_files_realtime_alerts.md @@ -0,0 +1 @@ +TODO \ No newline at end of file From 3150e3492b200232906b6d7bc607ea9454b2ca9d Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 04:48:45 +0100 Subject: [PATCH 62/94] the new yaml - not yet used... --- common_configs/config_log.yaml | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 common_configs/config_log.yaml diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml new file mode 100644 index 00000000..857043e2 --- /dev/null +++ b/common_configs/config_log.yaml @@ -0,0 +1,44 @@ +version: 1 +disable_existing_loggers: False + +formatters: + detailed: + format: '%(asctime)s %(pathname)s [%(filename)s:%(lineno)d] [%(process)d] [%(threadName)s] - %(levelname)s - %(message)s' + +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: detailed + stream: ext://sys.stdout + + info_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: INFO + formatter: detailed + filename: 'logs/app_INFO_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + debug_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: DEBUG + formatter: detailed + filename: 'logs/app_DEBUG_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + + error_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: ERROR + formatter: detailed + filename: 'logs/app_ERROR_%Y-%m-%d.log' + when: 'midnight' + backupCount: 30 + encoding: 'utf8' + +root: + level: DEBUG + handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 4abe11c21a718a86d8414878aef2d79f0356ac10 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:57:57 +0100 Subject: [PATCH 63/94] the config_log_yaml --- common_configs/config_log.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 857043e2..a87cfc6a 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -16,29 +16,29 @@ handlers: class: logging.handlers.TimedRotatingFileHandler level: INFO formatter: detailed - filename: 'logs/app_INFO_%Y-%m-%d.log' - when: 'midnight' + filename: "{LOG_PATH}/views_pipeline_INFO.log" + when: "midnight" backupCount: 30 - encoding: 'utf8' + encoding: "utf8" debug_file_handler: class: logging.handlers.TimedRotatingFileHandler level: DEBUG formatter: detailed - filename: 'logs/app_DEBUG_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_DEBUG.log" + when: "midnight" + backupCount: 10 + encoding: "utf8" error_file_handler: class: logging.handlers.TimedRotatingFileHandler level: ERROR formatter: detailed - filename: 'logs/app_ERROR_%Y-%m-%d.log' - when: 'midnight' - backupCount: 30 - encoding: 'utf8' + filename: "{LOG_PATH}/views_pipeline_ERROR.log" + when: "midnight" + backupCount: 60 + encoding: "utf8" root: - level: DEBUG + level: INFO handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From e84fa24069cba5a30f26e08c81676d2ec2e3378b Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:30 +0100 Subject: [PATCH 64/94] new common_logs dir --- common_logs/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 common_logs/.gitkeep diff --git a/common_logs/.gitkeep b/common_logs/.gitkeep new file mode 100644 index 00000000..e69de29b From af07de598ca4bca0ff88b303eab071e65b83170b Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:48 +0100 Subject: [PATCH 65/94] changed the central logger --- common_utils/utils_logger.py | 166 ++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 20 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index 9cf03a18..a3a8dd04 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -1,33 +1,159 @@ import logging +import logging.config +import yaml +import os +from pathlib import Path -def setup_logging(log_file: str, log_level=logging.INFO) -> logging.Logger: +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ +def get_config_log_path() -> Path: """ - Sets up logging to both a specified file and the terminal (console). + Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - Args: - log_file (str): The file where logs should be written. - log_level (int): The logging level. Default is logging.INFO. + This function identifies the 'views_pipeline' directory within the path of the current file, + constructs a new path up to and including this directory, and then appends the relative path + to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file + is not found, it raises a ValueError. + + Returns: + pathlib.Path: The path to the 'config_log.yaml' file. + + Raises: + ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' + if not PATH_CONFIG_LOG.exists(): + raise ValueError("The 'config_log.yaml' file was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_CONFIG_LOG +# -------------------------------------------------------------------------------------------------------------- + + +# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +def get_common_logs_path() -> Path: + """ + Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, then constructs + a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, + it raises a ValueError. + + Returns: + pathlib.Path: Absolute path to the 'common_logs' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. + """ + PATH = Path(__file__) + if 'views_pipeline' in PATH.parts: + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' + if not PATH_COMMON_LOGS.exists(): + raise ValueError("The 'common_logs' directory was not found in the provided path.") + else: + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + return PATH_COMMON_LOGS +# ------------------------------------------------------------------------------------------------------------ + + +def ensure_log_directory(log_path: str) -> None: """ + Ensure the log directory exists for file-based logging handlers. + + Parameters: + log_path (str): The full path to the log file for which the directory should be verified. + """ + log_dir = os.path.dirname(log_path) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + +def setup_logging( + default_level: int = logging.INFO, env_key: str = 'LOG_CONFIG') -> logging.Logger: + + """ + Setup the logging configuration from a YAML file and return the root logger. + + Parameters: + default_level (int): The default logging level if the configuration file is not found + or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging + configuration file. Default is 'LOG_CONFIG'. + + Returns: + logging.Logger: The root logger configured based on the loaded configuration. + + Example Usage: + >>> logger = setup_logging() + >>> logger.info("Logging setup complete.") + """ + + CONFIG_LOGS_PATH = get_config_log_path() + COMMON_LOGS_PATH = get_common_logs_path() + + # Load YAML configuration + path = os.getenv(env_key, CONFIG_LOGS_PATH) - basic_logger = logging.getLogger() - basic_logger.setLevel(log_level) + if os.path.exists(path): + try: + with open(path, 'rt') as f: + config = yaml.safe_load(f.read()) + + # Replace placeholder with actual log directory path + for handler in config.get("handlers", {}).values(): + if "filename" in handler and "{LOG_PATH}" in handler["filename"]: + handler["filename"] = handler["filename"].replace("{LOG_PATH}", str(COMMON_LOGS_PATH)) + ensure_log_directory(handler["filename"]) + + # Apply logging configuration + logging.config.dictConfig(config) - file_handler = logging.FileHandler(log_file) - console_handler = logging.StreamHandler() + except Exception as e: + logging.basicConfig(level=default_level) + logging.error(f"Failed to load logging configuration from {path}. Using basic configuration. Error: {e}") + else: + logging.basicConfig(level=default_level) + logging.warning(f"Logging configuration file not found at {path}. Using basic configuration.") + + return logging.getLogger() - file_handler.setLevel(log_level) - console_handler.setLevel(log_level) - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - # Clear previous handlers if they exist - if basic_logger.hasHandlers(): - basic_logger.handlers.clear() - basic_logger.addHandler(file_handler) - basic_logger.addHandler(console_handler) - return basic_logger +## Old version +#def setup_logging(log_file: str, log_level=logging.INFO): +# """ +# Sets up logging to both a specified file and the terminal (console). +# +# Args: +# log_file (str): The file where logs should be written. +# log_level (int): The logging level. Default is logging.INFO. +# """ +# +# basic_logger = logging.getLogger() +# basic_logger.setLevel(log_level) +# +# file_handler = logging.FileHandler(log_file) +# console_handler = logging.StreamHandler() +# +# file_handler.setLevel(log_level) +# console_handler.setLevel(log_level) +# +# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +# file_handler.setFormatter(formatter) +# console_handler.setFormatter(formatter) +# +# # Clear previous handlers if they exist +# if basic_logger.hasHandlers(): +# basic_logger.handlers.clear() +# +# basic_logger.addHandler(file_handler) +# basic_logger.addHandler(console_handler) +# +# return basic_logger +# \ No newline at end of file From aa03a6ee4a9a731f6673234da240da98bc6381d3 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 06:02:41 +0100 Subject: [PATCH 66/94] detail --- documentation/ADRs/015_log_files_general_strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/ADRs/015_log_files_general_strategy.md b/documentation/ADRs/015_log_files_general_strategy.md index 1d0490d5..aaf3d8fc 100644 --- a/documentation/ADRs/015_log_files_general_strategy.md +++ b/documentation/ADRs/015_log_files_general_strategy.md @@ -35,7 +35,7 @@ To implement a robust and unified logging strategy, we have decided on the follo 2. **Daily Rotation and Retention Policy**: Logs will rotate daily, keeping the last 30 days of logs by default. This policy provides sufficient historical data for troubleshooting and auditing without excessive storage usage. Rotation is achieved using a `TimedRotatingFileHandler`, with daily timestamped log filenames for easy access. -3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. +3. **Log Separation by Level**: Logs are separated into `INFO`, `DEBUG`, and `ERROR` files and stored under `views_pipeline/common_logs`. This separation improves monitoring and helps maintain focus on the desired logging level when troubleshooting (e.g., reviewing only errors or detailed debugging information). Each log file will capture messages specific to its level, ensuring modularity and readability in logs. 4. **Inclusion of Path and Process Details**: Log messages include additional context such as script path (`%(pathname)s`), filename (`%(filename)s`), line number (`%(lineno)d`), process ID (`%(process)d`), and thread name (`%(threadName)s`). This information aids in tracing logs back to their source, supporting traceability and aiding debugging. From 7f0a34a709bc11f725ea88844e5eb26658c05255 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:08 +0100 Subject: [PATCH 67/94] Use this to see how the logs look now --- meta_tools/asses_logging_setup.py | 74 +++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 meta_tools/asses_logging_setup.py diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py new file mode 100644 index 00000000..452bb449 --- /dev/null +++ b/meta_tools/asses_logging_setup.py @@ -0,0 +1,74 @@ +import logging +from pathlib import Path +import sys + + +PATH = Path(__file__) + +def get_path_common_utils(): + + if 'views_pipeline' in PATH.parts: + + PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) + PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + + if not PATH_COMMON_UTILS.exists(): + + raise ValueError("The 'common_utils' directory was not found in the provided path.") + + else: + + raise ValueError("The 'views_pipeline' directory was not found in the provided path.") + + return PATH_COMMON_UTILS + + +# Import your logging setup function from wherever it is defined +# from your_logging_module import setup_logging, get_common_logs_path + +def test_logging_setup(): + # Step 1: Set up the logging configuration + try: + log_directory = get_common_logs_path() # Fetch centralized log directory + logger = setup_logging() # Initialize logging setup + + except Exception as e: + print(f"Failed to initialize logging setup: {e}") + return + + # Step 2: Generate test log messages + logger.debug("This is a DEBUG log message for testing.") + logger.info("This is an INFO log message for testing.") + logger.error("This is an ERROR log message for testing.") + + # Step 3: Define expected log files + expected_files = [ + log_directory / "views_pipeline_INFO.log", + log_directory / "views_pipeline_DEBUG.log", + log_directory / "views_pipeline_ERROR.log" + ] + + # Step 4: Check if log files exist and are not empty + for file_path in expected_files: + if file_path.exists(): + print(f"Log file '{file_path}' exists.") + if file_path.stat().st_size > 0: + print(f"Log file '{file_path}' contains data.") + else: + print(f"Warning: Log file '{file_path}' is empty.") + else: + print(f"Error: Log file '{file_path}' was not created as expected.") + + print("Logging setup test completed.") + +# Run the test + +if __name__ == "__main__": + + PATH_COMMON_UTILS = get_path_common_utils() + + sys.path.append(str(PATH_COMMON_UTILS)) + + from utils_logger import setup_logging, get_common_logs_path + + test_logging_setup() From 4bba63dbb4b54f446d89d8e7d06d49ad3e291590 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:17 +0100 Subject: [PATCH 68/94] Updated root level --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index a87cfc6a..21c7f24e 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: INFO + level: DEBUG handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 35dd16d9830f3b4e7a957b316ce2fd1aa57eaee6 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:29 +0100 Subject: [PATCH 69/94] whitspace --- common_utils/utils_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index a3a8dd04..b9f5dd82 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -81,6 +81,7 @@ def setup_logging( Parameters: default_level (int): The default logging level if the configuration file is not found or cannot be loaded. Default is logging.INFO. + env_key (str): Environment variable key to override the default path to the logging configuration file. Default is 'LOG_CONFIG'. From c27d74d4d4c24dae992c651859a0197214ba50ab Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:08:44 +0100 Subject: [PATCH 70/94] this is better... --- meta_tools/asses_logging_setup.py | 33 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index 452bb449..e374585c 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -5,22 +5,41 @@ PATH = Path(__file__) -def get_path_common_utils(): - if 'views_pipeline' in PATH.parts: +def set_path_common_utils(): + """ + Retrieve the path to the 'common_utils' directory within the 'views_pipeline' structure. + + This function locates the 'views_pipeline' directory in the current file's path, + then constructs a new path to the 'common_utils' directory. If 'views_pipeline' + or 'common_utils' directories are not found, it raises a ValueError. + + If the 'common_utils' path is not already in sys.path, it appends it. + Returns: + pathlib.Path: Absolute path to the 'common_utils' directory. + + Raises: + ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. + """ + PATH = Path(__file__) + + # Locate 'views_pipeline' in the current file's path parts + if 'views_pipeline' in PATH.parts: PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) PATH_COMMON_UTILS = PATH_ROOT / 'common_utils' + # Check if 'common_utils' directory exists if not PATH_COMMON_UTILS.exists(): - raise ValueError("The 'common_utils' directory was not found in the provided path.") - else: + # Add 'common_utils' to sys.path if it's not already present + if str(PATH_COMMON_UTILS) not in sys.path: + sys.path.append(str(PATH_COMMON_UTILS)) + else: raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_UTILS # Import your logging setup function from wherever it is defined @@ -65,9 +84,7 @@ def test_logging_setup(): if __name__ == "__main__": - PATH_COMMON_UTILS = get_path_common_utils() - - sys.path.append(str(PATH_COMMON_UTILS)) + set_path_common_utils() from utils_logger import setup_logging, get_common_logs_path From 174cac6828e6ef0c29c59eee0d2c46f72c46ad10 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:09:09 +0100 Subject: [PATCH 71/94] corrected doc string --- meta_tools/asses_logging_setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index e374585c..c95604ac 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -16,9 +16,6 @@ def set_path_common_utils(): If the 'common_utils' path is not already in sys.path, it appends it. - Returns: - pathlib.Path: Absolute path to the 'common_utils' directory. - Raises: ValueError: If the 'views_pipeline' or 'common_utils' directory is not found. """ From d911e8c12e16b4e6e050b33d50e6121e43d906c7 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:57:57 +0100 Subject: [PATCH 72/94] the config_log_yaml --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 21c7f24e..a87cfc6a 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: DEBUG + level: INFO handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 792e830fa180a1405ea6e11f59f8ff1cd686de27 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:58:48 +0100 Subject: [PATCH 73/94] changed the central logger --- common_utils/utils_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index b9f5dd82..971579d8 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -82,7 +82,7 @@ def setup_logging( default_level (int): The default logging level if the configuration file is not found or cannot be loaded. Default is logging.INFO. - env_key (str): Environment variable key to override the default path to the logging + env_key (str): Environment variablei key to override the default path to the logging configuration file. Default is 'LOG_CONFIG'. Returns: From 79ea54ffdaacfcb9301314e039ea7d819a08dd82 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:08 +0100 Subject: [PATCH 74/94] Use this to see how the logs look now --- meta_tools/asses_logging_setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index c95604ac..6e319cfa 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -81,7 +81,13 @@ def test_logging_setup(): if __name__ == "__main__": +<<<<<<< HEAD set_path_common_utils() +======= + PATH_COMMON_UTILS = get_path_common_utils() + + sys.path.append(str(PATH_COMMON_UTILS)) +>>>>>>> 8cc961f (Use this to see how the logs look now) from utils_logger import setup_logging, get_common_logs_path From 824688b103e627e9a690979813a46e577676f2d5 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:17 +0100 Subject: [PATCH 75/94] Updated root level --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index a87cfc6a..21c7f24e 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: INFO + level: DEBUG handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From e23bc088e0f76706a022f5b0f2ea98379d7f9f09 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:08:44 +0100 Subject: [PATCH 76/94] this is better... --- meta_tools/asses_logging_setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index 6e319cfa..697f8848 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -81,6 +81,7 @@ def test_logging_setup(): if __name__ == "__main__": +<<<<<<< HEAD <<<<<<< HEAD set_path_common_utils() ======= @@ -88,6 +89,9 @@ def test_logging_setup(): sys.path.append(str(PATH_COMMON_UTILS)) >>>>>>> 8cc961f (Use this to see how the logs look now) +======= + set_path_common_utils() +>>>>>>> 172f23b (this is better...) from utils_logger import setup_logging, get_common_logs_path From 6dfc99365ed57af16cb6c21f35a1a0cec4e5ac0a Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:13:22 +0100 Subject: [PATCH 77/94] add modelpath and split by model support --- common_utils/global_cache.py | 10 +-- common_utils/model_path.py | 98 ++++++++++++++++------------- common_utils/utils_logger.py | 118 ++++++++++++++++++----------------- models/lavender_haze/main.py | 6 +- 4 files changed, 124 insertions(+), 108 deletions(-) diff --git a/common_utils/global_cache.py b/common_utils/global_cache.py index cecc515d..e519c363 100644 --- a/common_utils/global_cache.py +++ b/common_utils/global_cache.py @@ -113,12 +113,12 @@ def ensure_cache_file_exists(self): Ensures that the cache file exists. If it does not exist, creates a new cache file. """ if not self.filepath.exists(): - logging.info( + logging.warning( f"Cache file: {self.filepath} does not exist. Creating new cache file..." ) with open(self.filepath, "wb") as f: pickle.dump({}, f) - logging.info(f"Created new cache file: {self.filepath}") + logging.debug(f"Created new cache file: {self.filepath}") def set(self, key, value): """ @@ -167,7 +167,7 @@ def save_cache(self): """ with open(self.filepath, "wb") as f: pickle.dump(self.cache, f) - logging.info(f"Cache saved to file: {self.filepath}") + logging.debug(f"Cache saved to file: {self.filepath}") def load_cache(self): """ @@ -179,7 +179,7 @@ def load_cache(self): loaded_cache = pickle.loads(f.read()) if isinstance(loaded_cache, dict): self.cache = loaded_cache - logging.info(f"Cache loaded from file: {self.filepath}") + logging.debug(f"Cache loaded from file: {self.filepath}") else: logging.error( f"Loaded cache is not a dictionary. Initializing empty cache." @@ -192,7 +192,7 @@ def load_cache(self): self.cache = {} else: self.cache = {} - logging.info(f"Cache file does not exist. Initialized empty cache.") + logging.debug(f"Cache file does not exist. Initialized empty cache.") def cleanup_cache_file(): diff --git a/common_utils/model_path.py b/common_utils/model_path.py index b06642cd..ac884b3b 100644 --- a/common_utils/model_path.py +++ b/common_utils/model_path.py @@ -61,45 +61,45 @@ class ModelPath: _ignore_attributes (list): A list of paths to ignore. """ - __slots__ = ( - "_validate", - "target", - "use_global_cache", - "_force_cache_overwrite", - "root", - "models", - "common_utils", - "common_configs", - "_ignore_attributes", - "model_name", - "_instance_hash", - "_queryset", - "model_dir", - "architectures", - "artifacts", - "configs", - "data", - "data_generated", - "data_processed", - "data_raw", - "dataloaders", - "forecasting", - "management", - "notebooks", - "offline_evaluation", - "online_evaluation", - "reports", - "src", - "_templates", - "training", - "utils", - "visualization", - "_sys_paths", - "common_querysets", - "queryset_path", - "scripts", - "meta_tools", - ) + # __slots__ = ( + # "_validate", + # "target", + # "use_global_cache", + # "_force_cache_overwrite", + # "root", + # "models", + # "common_utils", + # "common_configs", + # "_ignore_attributes", + # "model_name", + # "_instance_hash", + # "_queryset", + # "model_dir", + # "architectures", + # "artifacts", + # "configs", + # "data", + # "data_generated", + # "data_processed", + # "data_raw", + # "dataloaders", + # "forecasting", + # "management", + # "notebooks", + # "offline_evaluation", + # "online_evaluation", + # "reports", + # "src", + # "_templates", + # "training", + # "utils", + # "visualization", + # "_sys_paths", + # "common_querysets", + # "queryset_path", + # "scripts", + # "meta_tools", + # ) _target = "model" _use_global_cache = True @@ -120,6 +120,7 @@ def _initialize_class_paths(cls): cls._common_configs = cls._root / "common_configs" cls._common_querysets = cls._root / "common_querysets" cls._meta_tools = cls._root / "meta_tools" + cls._common_logs = cls._root / "common_logs" @classmethod def get_root(cls) -> Path: @@ -163,6 +164,13 @@ def get_meta_tools(cls) -> Path: cls._initialize_class_paths() return cls._meta_tools + @classmethod + def get_common_logs(cls) -> Path: + """Get the common logs path.""" + if cls._common_logs is None: + cls._initialize_class_paths() + return cls._common_logs + @classmethod def check_if_model_dir_exists(cls, model_name: str) -> bool: """ @@ -575,8 +583,8 @@ def add_paths_to_sys(self) -> List[str]: ) if self._sys_paths is None: self._sys_paths = [] - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if str(attr) not in self._ignore_attributes: if ( isinstance(value, Path) @@ -641,8 +649,8 @@ def view_directories(self) -> None: """ print("\n{:<20}\t{:<50}".format("Name", "Path")) print("=" * 72) - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if attr not in self._ignore_attributes and isinstance(value, Path): print("{:<20}\t{:<50}".format(str(attr), str(value))) @@ -687,8 +695,8 @@ def get_directories(self) -> Dict[str, Optional[str]]: # ] directories = {} relative = False - for attr in self.__slots__: - value = getattr(self, attr) + for attr, value in self.__dict__.items(): + # value = getattr(self, attr) if str(attr) not in [ "model_name", "root", diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index 971579d8..4603cd75 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -3,62 +3,63 @@ import yaml import os from pathlib import Path - - +from model_path import ModelPath +from global_cache import GlobalCache # SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...------------------------------ -def get_config_log_path() -> Path: - """ - Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. - - This function identifies the 'views_pipeline' directory within the path of the current file, - constructs a new path up to and including this directory, and then appends the relative path - to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file - is not found, it raises a ValueError. - - Returns: - pathlib.Path: The path to the 'config_log.yaml' file. - - Raises: - ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. - """ - PATH = Path(__file__) - if 'views_pipeline' in PATH.parts: - PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) - PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' - if not PATH_CONFIG_LOG.exists(): - raise ValueError("The 'config_log.yaml' file was not found in the provided path.") - else: - raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_CONFIG_LOG -# -------------------------------------------------------------------------------------------------------------- - - -# SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- -def get_common_logs_path() -> Path: - """ - Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. - - This function locates the 'views_pipeline' directory in the current file's path, then constructs - a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, - it raises a ValueError. - - Returns: - pathlib.Path: Absolute path to the 'common_logs' directory. - - Raises: - ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. - """ - PATH = Path(__file__) - if 'views_pipeline' in PATH.parts: - PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) - PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' - if not PATH_COMMON_LOGS.exists(): - raise ValueError("The 'common_logs' directory was not found in the provided path.") - else: - raise ValueError("The 'views_pipeline' directory was not found in the provided path.") - return PATH_COMMON_LOGS -# ------------------------------------------------------------------------------------------------------------ - +# def get_config_log_path() -> Path: +# """ +# Retrieves the path to the 'config_log.yaml' file within the 'views_pipeline' directory. + +# This function identifies the 'views_pipeline' directory within the path of the current file, +# constructs a new path up to and including this directory, and then appends the relative path +# to the 'config_log.yaml' file. If the 'views_pipeline' directory or the 'config_log.yaml' file +# is not found, it raises a ValueError. + +# Returns: +# pathlib.Path: The path to the 'config_log.yaml' file. + +# Raises: +# ValueError: If the 'views_pipeline' directory or the 'config_log.yaml' file is not found in the provided path. +# """ +# PATH = Path(__file__) +# if 'views_pipeline' in PATH.parts: +# PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) +# PATH_CONFIG_LOG = PATH_ROOT / 'common_configs/config_log.yaml' +# if not PATH_CONFIG_LOG.exists(): +# raise ValueError("The 'config_log.yaml' file was not found in the provided path.") +# else: +# raise ValueError("The 'views_pipeline' directory was not found in the provided path.") +# return PATH_CONFIG_LOG +# # -------------------------------------------------------------------------------------------------------------- + + +# # SINCE WE ARE IN COMMON_UTILS, WE CAN JUST USE THE MODEL_PATH OBJECT HERE...----------------------------------- +# def get_common_logs_path() -> Path: +# """ +# Retrieve the absolute path to the 'common_logs' directory within the 'views_pipeline' structure. + +# This function locates the 'views_pipeline' directory in the current file's path, then constructs +# a new path to the 'common_logs' directory. If 'common_logs' or 'views_pipeline' directories are not found, +# it raises a ValueError. + +# Returns: +# pathlib.Path: Absolute path to the 'common_logs' directory. + +# Raises: +# ValueError: If the 'views_pipeline' or 'common_logs' directory is not found. +# """ +# PATH = Path(__file__) +# if 'views_pipeline' in PATH.parts: +# PATH_ROOT = Path(*PATH.parts[:PATH.parts.index('views_pipeline') + 1]) +# PATH_COMMON_LOGS = PATH_ROOT / 'common_logs' +# if not PATH_COMMON_LOGS.exists(): +# raise ValueError("The 'common_logs' directory was not found in the provided path.") +# else: +# raise ValueError("The 'views_pipeline' directory was not found in the provided path.") +# return PATH_COMMON_LOGS +# # ------------------------------------------------------------------------------------------------------------ + +_split_by_model = True # Only works for lavender_haze def ensure_log_directory(log_path: str) -> None: """ @@ -93,8 +94,11 @@ def setup_logging( >>> logger.info("Logging setup complete.") """ - CONFIG_LOGS_PATH = get_config_log_path() - COMMON_LOGS_PATH = get_common_logs_path() + CONFIG_LOGS_PATH = ModelPath.get_common_configs() / 'config_log.yaml' + if _split_by_model: + COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + else: + COMMON_LOGS_PATH = ModelPath.get_common_logs() # Load YAML configuration path = os.getenv(env_key, CONFIG_LOGS_PATH) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 3176cba9..b44b8ca4 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,14 +13,18 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +from common_utils.model_path import ModelPath +from common_utils.global_cache import GlobalCache + warnings.filterwarnings("ignore") +GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) logger = setup_logging("run.log") if __name__ == "__main__": wandb.login() - + args = parse_args() validate_arguments(args) From 37a91c2bee66d08335ebf8a00fbc8eb6a2500f7a Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:43:28 +0100 Subject: [PATCH 78/94] add globalcache failsafe --- common_utils/utils_logger.py | 6 +++++- models/lavender_haze/main.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common_utils/utils_logger.py b/common_utils/utils_logger.py index 4603cd75..6b6093d5 100644 --- a/common_utils/utils_logger.py +++ b/common_utils/utils_logger.py @@ -96,7 +96,11 @@ def setup_logging( CONFIG_LOGS_PATH = ModelPath.get_common_configs() / 'config_log.yaml' if _split_by_model: - COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + try: + COMMON_LOGS_PATH = ModelPath.get_common_logs() / GlobalCache["current_model"] + except: + # Pretection in case model name is not available or GlobalCache fails. + COMMON_LOGS_PATH = ModelPath.get_common_logs() else: COMMON_LOGS_PATH = ModelPath.get_common_logs() diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index b44b8ca4..956f953c 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -17,8 +17,10 @@ from common_utils.global_cache import GlobalCache warnings.filterwarnings("ignore") - -GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +try: + GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) +except Exception: + pass logger = setup_logging("run.log") From c92e17d4f11783d8be70f63c3d8ab0ffc37cc7ad Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:44:43 +0100 Subject: [PATCH 79/94] cleanup --- models/lavender_haze/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 956f953c..360c51d1 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,11 +13,10 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run -from common_utils.model_path import ModelPath -from common_utils.global_cache import GlobalCache - warnings.filterwarnings("ignore") try: + from common_utils.model_path import ModelPath + from common_utils.global_cache import GlobalCache GlobalCache["current_model"] = ModelPath.get_model_name_from_path(Path(__file__)) except Exception: pass From 440234a53d1632d3a33386174f7cf69eb84a92f2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 28 Oct 2024 05:57:57 +0100 Subject: [PATCH 80/94] the config_log_yaml --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 21c7f24e..a87cfc6a 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: DEBUG + level: INFO handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From 8ff4b41cf29a22c170bbafaf01aa0baae441a1ea Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:08 +0100 Subject: [PATCH 81/94] Use this to see how the logs look now --- meta_tools/asses_logging_setup.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/asses_logging_setup.py index 697f8848..c95604ac 100644 --- a/meta_tools/asses_logging_setup.py +++ b/meta_tools/asses_logging_setup.py @@ -81,17 +81,7 @@ def test_logging_setup(): if __name__ == "__main__": -<<<<<<< HEAD -<<<<<<< HEAD set_path_common_utils() -======= - PATH_COMMON_UTILS = get_path_common_utils() - - sys.path.append(str(PATH_COMMON_UTILS)) ->>>>>>> 8cc961f (Use this to see how the logs look now) -======= - set_path_common_utils() ->>>>>>> 172f23b (this is better...) from utils_logger import setup_logging, get_common_logs_path From f0b98046553173ea711044d750b34d4c7d809e85 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 11:03:17 +0100 Subject: [PATCH 82/94] Updated root level --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index a87cfc6a..21c7f24e 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -40,5 +40,5 @@ handlers: encoding: "utf8" root: - level: INFO + level: DEBUG handlers: [console, info_file_handler, debug_file_handler, error_file_handler] From c77e7c2b3a99450a4027020f5982b4201b421e32 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:13:22 +0100 Subject: [PATCH 83/94] add modelpath and split by model support --- models/lavender_haze/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 360c51d1..b24fbc36 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,6 +13,9 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +from common_utils.model_path import ModelPath +from common_utils.global_cache import GlobalCache + warnings.filterwarnings("ignore") try: from common_utils.model_path import ModelPath From 44094de4c9cac0e9db48072ac9b7bfe775814b60 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:44:43 +0100 Subject: [PATCH 84/94] cleanup --- models/lavender_haze/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index b24fbc36..360c51d1 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,9 +13,6 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run -from common_utils.model_path import ModelPath -from common_utils.global_cache import GlobalCache - warnings.filterwarnings("ignore") try: from common_utils.model_path import ModelPath From 0fc243802bdea46ab9b988b0d7c59e56410a5bf2 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 15:42:14 +0100 Subject: [PATCH 85/94] added warning and critical --- common_configs/config_log.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index 21c7f24e..f75eb956 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -30,6 +30,15 @@ handlers: backupCount: 10 encoding: "utf8" + warning_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: WARNING + formatter: detailed + filename: "{LOG_PATH}/views_pipeline_WARNING.log" + when: "midnight" + backupCount: 20 + encoding: "utf8" + error_file_handler: class: logging.handlers.TimedRotatingFileHandler level: ERROR @@ -39,6 +48,15 @@ handlers: backupCount: 60 encoding: "utf8" + critical_file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: CRITICAL + formatter: detailed + filename: "{LOG_PATH}/views_pipeline_CRITICAL.log" + when: "midnight" + backupCount: 90 + encoding: "utf8" + root: level: DEBUG - handlers: [console, info_file_handler, debug_file_handler, error_file_handler] + handlers: [console, info_file_handler, debug_file_handler, warning_file_handler, error_file_handler, critical_file_handler] From 7a07d437e2b234e8539305f3620105ded2a32ca3 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 15:55:14 +0100 Subject: [PATCH 86/94] And ADR to justify this --- .../ADRs/025_log _level_standards.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 documentation/ADRs/025_log _level_standards.md diff --git a/documentation/ADRs/025_log _level_standards.md b/documentation/ADRs/025_log _level_standards.md new file mode 100644 index 00000000..f2ead9e1 --- /dev/null +++ b/documentation/ADRs/025_log _level_standards.md @@ -0,0 +1,77 @@ +## Title +*Log Level Standards* + +| ADR Info | Details | +|---------------------|-------------------| +| Subject | Logging Levels Configuration | +| ADR Number | 025 | +| Status | Proposed | +| Author | Simon | +| Date | 30.10.2024 | + +## Context +We aim to establish a new benchmark in MLOps for early warning systems (EWS), emphasizing robust and transparent logging practices. The conflict forecasting pipeline must have a comprehensive logging strategy to support the continuous quality assurance, real-time monitoring, and rapid model updates critical for high-stakes decision-making and early action. This ADR specifically addresses the standardized use of log levels within the pipeline, which helps the team capture relevant system states and provides clear visibility into operations, potential issues, and crucial decision points across the pipeline. + +The following log levels—DEBUG, INFO, WARNING, ERROR, and CRITICAL—are configured to ensure appropriate information is logged for various scenarios, supporting both ongoing development and long-term system monitoring and troubleshooting. + +## Decision +The following log levels are implemented as standard for the conflict forecasting pipeline: + +### Overview +The system’s log levels are structured as follows: + +1. **DEBUG**: Provides detailed diagnostic information, primarily used during development and debugging. +2. **INFO**: Captures general system information about normal operations to give an overview without verbosity. +3. **WARNING**: Logs potentially problematic situations that require attention but do not yet impact execution. +4. **ERROR**: Indicates issues where specific processes fail but the overall system remains operational. +5. **CRITICAL**: Records severe errors that require immediate attention as they could lead to system failures or data loss. + +### Examples and Use Cases + +- **DEBUG**: + - Example: Logging input data shapes, intermediate transformations, or model hyperparameters during experimentation phases. + - Use: Essential for development and debugging stages, enabling the team to trace and diagnose precise pipeline states. + +- **INFO**: + - Example: Model training and evaluation start/completion times or successful data fetching messages. + - Use: Provides a clear history of system operations without overwhelming detail, facilitating audits and general status tracking. + +- **WARNING**: + - Example: Detection of minor schema mismatches in input data or slower-than-expected execution times. + - Use: Highlights potentially impactful issues that may worsen if not addressed, useful for preventive monitoring. + +- **ERROR**: + - Example: Failure in data loading or model checkpoint saving. + - Use: Captures failures within components, supporting root cause analysis without interrupting the entire pipeline’s execution. + +- **CRITICAL**: + - Example: Data corruption detected during ingestion or a complete failure in the model orchestration module. + - Use: Alerts the team to major issues demanding immediate intervention, ensuring prompt actions to mitigate risks. + +## Consequences +Implementing this structured logging approach provides several benefits and potential drawbacks: + +**Positive Effects:** +- **Improved Observability**: Each log level offers a distinct lens on system operations, enhancing real-time monitoring and troubleshooting capabilities. +- **Data-Driven Issue Resolution**: The logging structure supports continuous quality assurance by capturing detailed information, aiding root cause analysis and enabling proactive interventions. +- **MLOps Standardization**: This approach aligns with best practices in MLOps, facilitating future integrations, scaling, and consistent team understanding. + +**Negative Effects:** +- **Increased Storage Use**: Higher log granularity, especially at DEBUG and INFO levels, may increase storage requirements. +- **Operational Overhead**: Maintenance of log file management, such as purging or archiving, may require periodic oversight, particularly with large volume logs like DEBUG. + +## Rationale +The chosen logging levels reflect best practices in MLOps, which call for clearly defined, purposeful logging to maintain high observability and diagnostic precision within production environments. By setting granular logging levels, we are able to balance immediate operational needs with long-term maintenance, ensuring that the system remains adaptable to shifting conditions and that degraded performance can be preemptively managed. + +### Considerations +Key considerations include: +- **Dependency Management**: Log retention and storage solutions must scale with the forecast pipeline, ensuring longevity without excessive operational costs. +- **Data Security**: Sensitive information should not be logged, especially at DEBUG or INFO levels, to maintain data protection standards. +- **Resource Constraints**: Frequent updates to logging configurations may impose operational overheads; thus, the current structure should be periodically reviewed but not frequently changed. + +## Additional Notes +- **Log Rotation**: Log files are set up with TimedRotatingFileHandler to rotate daily, retaining a set number of logs per level (e.g., INFO logs for 30 days), which balances traceability and storage efficiency. +- **Future Enhancements**: Potential additions may include automated alerts for CRITICAL logs or integration with monitoring dashboards for centralized logging analysis. + +## Feedback and Suggestions +Team members and stakeholders are encouraged to provide feedback on the logging configuration’s effectiveness, particularly regarding its impact on troubleshooting and operational transparency. Suggestions for improvement are welcome as we continue refining MLOps practices to support reliable, high-stakes forecasting. From f30f0d7a28a93c3c7e982605e285f7fc4aebb097 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Wed, 30 Oct 2024 16:01:17 +0100 Subject: [PATCH 87/94] updated with new default level INFO --- common_configs/config_log.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_configs/config_log.yaml b/common_configs/config_log.yaml index f75eb956..c9deee22 100644 --- a/common_configs/config_log.yaml +++ b/common_configs/config_log.yaml @@ -8,7 +8,7 @@ formatters: handlers: console: class: logging.StreamHandler - level: DEBUG + level: INFO formatter: detailed stream: ext://sys.stdout From 9c2006f7d8c48e92d15400349490347928e192a4 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:13:22 +0100 Subject: [PATCH 88/94] add modelpath and split by model support --- models/lavender_haze/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 360c51d1..b24fbc36 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,6 +13,9 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +from common_utils.model_path import ModelPath +from common_utils.global_cache import GlobalCache + warnings.filterwarnings("ignore") try: from common_utils.model_path import ModelPath From f060f3147caeb17b13f082f65596b80e7348ffa8 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:44:43 +0100 Subject: [PATCH 89/94] cleanup --- models/lavender_haze/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index b24fbc36..360c51d1 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,9 +13,6 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run -from common_utils.model_path import ModelPath -from common_utils.global_cache import GlobalCache - warnings.filterwarnings("ignore") try: from common_utils.model_path import ModelPath From 2bfddce3cfa4145e74c679c516123ea0f2ec256a Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:13:22 +0100 Subject: [PATCH 90/94] add modelpath and split by model support --- models/lavender_haze/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index 360c51d1..b24fbc36 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,6 +13,9 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run +from common_utils.model_path import ModelPath +from common_utils.global_cache import GlobalCache + warnings.filterwarnings("ignore") try: from common_utils.model_path import ModelPath From eccbb8c1db5dfb51fa0366d04b169e6e6b192952 Mon Sep 17 00:00:00 2001 From: Dylan <52908667+smellycloud@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:44:43 +0100 Subject: [PATCH 91/94] cleanup --- models/lavender_haze/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/lavender_haze/main.py b/models/lavender_haze/main.py index b24fbc36..360c51d1 100644 --- a/models/lavender_haze/main.py +++ b/models/lavender_haze/main.py @@ -13,9 +13,6 @@ from utils_logger import setup_logging from execute_model_runs import execute_sweep_run, execute_single_run -from common_utils.model_path import ModelPath -from common_utils.global_cache import GlobalCache - warnings.filterwarnings("ignore") try: from common_utils.model_path import ModelPath From ec15615f263b67020130b4b6eefe49d97b9f81ec Mon Sep 17 00:00:00 2001 From: Polichinl Date: Thu, 31 Oct 2024 12:26:32 +0100 Subject: [PATCH 92/94] bc Dylan is no fun --- meta_tools/{asses_logging_setup.py => assess_logging_setup.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename meta_tools/{asses_logging_setup.py => assess_logging_setup.py} (100%) diff --git a/meta_tools/asses_logging_setup.py b/meta_tools/assess_logging_setup.py similarity index 100% rename from meta_tools/asses_logging_setup.py rename to meta_tools/assess_logging_setup.py From 8a82a68daa3be71a4bf07a5597c7160f8587d67b Mon Sep 17 00:00:00 2001 From: Polichinl Date: Thu, 31 Oct 2024 12:27:54 +0100 Subject: [PATCH 93/94] slots no longer in use --- common_utils/model_path.py | 40 -------------------------------------- 1 file changed, 40 deletions(-) diff --git a/common_utils/model_path.py b/common_utils/model_path.py index ac884b3b..88f06d70 100644 --- a/common_utils/model_path.py +++ b/common_utils/model_path.py @@ -61,46 +61,6 @@ class ModelPath: _ignore_attributes (list): A list of paths to ignore. """ - # __slots__ = ( - # "_validate", - # "target", - # "use_global_cache", - # "_force_cache_overwrite", - # "root", - # "models", - # "common_utils", - # "common_configs", - # "_ignore_attributes", - # "model_name", - # "_instance_hash", - # "_queryset", - # "model_dir", - # "architectures", - # "artifacts", - # "configs", - # "data", - # "data_generated", - # "data_processed", - # "data_raw", - # "dataloaders", - # "forecasting", - # "management", - # "notebooks", - # "offline_evaluation", - # "online_evaluation", - # "reports", - # "src", - # "_templates", - # "training", - # "utils", - # "visualization", - # "_sys_paths", - # "common_querysets", - # "queryset_path", - # "scripts", - # "meta_tools", - # ) - _target = "model" _use_global_cache = True __instances__ = 0 From 49e33a6d68b5ea69f9db29bf509283f60bf76669 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Thu, 31 Oct 2024 12:32:34 +0100 Subject: [PATCH 94/94] No longer using slots anywhere --- common_utils/model_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_utils/model_path.py b/common_utils/model_path.py index 88f06d70..51ae9c9b 100644 --- a/common_utils/model_path.py +++ b/common_utils/model_path.py @@ -656,7 +656,7 @@ def get_directories(self) -> Dict[str, Optional[str]]: directories = {} relative = False for attr, value in self.__dict__.items(): - # value = getattr(self, attr) + if str(attr) not in [ "model_name", "root",