diff --git a/resources/config.sample.json b/resources/config.sample.json index 1ef930f..eebed4b 100644 --- a/resources/config.sample.json +++ b/resources/config.sample.json @@ -17,6 +17,11 @@ "username": "USERNAME", "password": "PASSWORD" }, + "graylog": { + "host" : "", + "port" : "", + "facility" : "scicat.ingestor" + }, "dataset": { "instrument_id" : "", "instrument" : "INSTRUMENT_NAME", @@ -37,6 +42,7 @@ "system_log_facility" : "mail", "log_message_prefix" : " SFI: ", "check_by_job_id" : true, - "pyscicat": null + "pyscicat": null, + "graylog" : false } } diff --git a/resources/dream.configuration.json.example b/resources/dream.configuration.json.example new file mode 100644 index 0000000..9943053 --- /dev/null +++ b/resources/dream.configuration.json.example @@ -0,0 +1,55 @@ +{ + "proposal_id" : { + "field_type" : "high_level", + "machine_name" : "proposalId", + "source" : "NXS:/entry/experiment_identifier", + "type" : "string" + }, + "principal_investigator" : { + "field_type" : "high_level", + "machine_name" : "principalInvestigator", + "source" : "UO:proposal:/proposer/firstname + UO:proposal:/proposer/lastname", + "type" : "string" + }, + "dataset_name" : { + "field_type" : "high_level", + "machine_name" : "datasetName", + "source" : "NXS:/entry/title", + "type" : "string" + }, + "techniques" : { + "field_type" : "high_level", + "machine_name" : "techniques", + "human_name" : "techniques", + "value" : "", + "type" : "object" + }, + "start_time" : { + "field_type" : "scientific_metadata", + "machine_name" : "start_time", + "human_name" : "Start Time", + "source" : "WRDN:/metadata/start_time", + "transformation" : "timestamp_to_iso8601", + "type" : "date" + } + "end_time" : { + "field_type" : "scientific_metadata", + "machine_name" : "end_time", + "human_name" : "End Time", + "source" : "WRDN:/metadata/end_time", + "transformation" : "timestamp_to_iso8601", + "type" : "date" + }, + "sample_id" : { + "field_type" : "high_level", + "machine_name" : "sampleId", + "value" : "", + "type" : "string" + }, + "instrument_id" : { + "field_type" : "high_level", + "machine_name" : "instrumentId", + "value" : "DREAM", + "type" : "string" + }, +] \ No newline at end of file diff --git a/src/scicat_configuration.py b/src/scicat_configuration.py index fe99737..d4413e3 100644 --- a/src/scicat_configuration.py +++ b/src/scicat_configuration.py @@ -8,85 +8,99 @@ def build_main_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() - group = parser.add_argument_group('Scicat Ingestor Options') + group = parser.add_argument_group("Scicat Ingestor Options") group.add_argument( - '-c', - '--cf', - '--config', - '--config-file', - default='config.20240405.json', - dest='config_file', - help='Configuration file name. Default: config.20240405.json', + "-c", + "--cf", + "--config", + "--config-file", + default="config.20240405.json", + dest="config_file", + help="Configuration file name. Default: config.20240405.json", type=str, ) group.add_argument( - '-v', - '--verbose', - dest='verbose', - help='Provide logging on stdout', - action='store_true', + "-v", + "--verbose", + dest="verbose", + help="Provide logging on stdout", + action="store_true", default=False, ) group.add_argument( - '--file-log', - dest='file_log', - help='Provide logging on file', - action='store_true', + "--file-log", + dest="file_log", + help="Provide logging on file", + action="store_true", default=False, ) group.add_argument( - '--log-filepath-prefix', - dest='log_filepath_prefix', - help='Prefix of the log file path', - default='.scicat_ingestor_log', + "--log-filepath-prefix", + dest="log_filepath_prefix", + help="Prefix of the log file path", + default=".scicat_ingestor_log", ) group.add_argument( - '--file-log-timestamp', - dest='file_log_timestamp', - help='Provide logging on the system log', - action='store_true', + "--file-log-timestamp", + dest="file_log_timestamp", + help="Provide logging on the system log", + action="store_true", default=False, ) group.add_argument( - '--system-log', - dest='system_log', - help='Provide logging on the system log', - action='store_true', + "--system-log", + dest="system_log", + help="Provide logging on the system log", + action="store_true", default=False, ) group.add_argument( - '--system-log-facility', - dest='system_log_facility', - help='Facility for system log', - default='mail', + "--system-log-facility", + dest="system_log_facility", + help="Facility for system log", + default="mail", ) group.add_argument( - '--log-message-prefix', - dest='log_message_prefix', - help='Prefix for log messages', - default=' SFI: ', + "--log-message-prefix", + dest="log_message_prefix", + help="Prefix for log messages", + default=" SFI: ", ) group.add_argument( - '--log-level', dest='log_level', help='Logging level', default='INFO', type=str + "--log-level", dest="log_level", help="Logging level", default="INFO", type=str ) group.add_argument( - '--check-by-job-id', - dest='check_by_job_id', - help='Check the status of a job by job_id', - action='store_true', + "--check-by-job-id", + dest="check_by_job_id", + help="Check the status of a job by job_id", + action="store_true", default=True, ) group.add_argument( - '--pyscicat', - dest='pyscicat', - help='Location where a specific version of pyscicat is available', + "--pyscicat", + dest="pyscicat", + help="Location where a specific version of pyscicat is available", default=None, type=str, ) + group.add_argument( + "--graylog", + dest="graylog", + help="Use graylog for additional logs", + action="store_true", + default=False, + ) return parser +@dataclass +class GraylogOptions: + host: str = "" + port: str = "" + facility: str = "scicat.ingestor" + + @dataclass class RunOptions: """RunOptions dataclass to store the configuration options. @@ -107,6 +121,7 @@ class RunOptions: check_by_job_id: bool system_log_facility: Optional[str] = None pyscicat: Optional[str] = None + graylog: bool = False @dataclass @@ -139,6 +154,8 @@ class ScicatConfig: """Merged configuration dictionary with command line arguments.""" kafka_options: kafkaOptions """Kafka configuration options read from files.""" + graylog_options: GraylogOptions + """Graylog configuration options for streaming logs.""" def to_dict(self) -> dict: """Return the configuration as a dictionary.""" @@ -150,7 +167,9 @@ def to_dict(self) -> dict: if isinstance(value, Mapping): original_dict[key] = dict(value) - copied = ScicatConfig(original_dict, self.run_options, self.kafka_options) + copied = ScicatConfig( + original_dict, self.run_options, self.kafka_options, self.graylog_options + ) return asdict(copied) @@ -171,7 +190,7 @@ def build_scicat_config(input_args: argparse.Namespace) -> ScicatConfig: config_dict = dict() # Overwrite deep-copied options with command line arguments - run_option_dict: dict = copy.deepcopy(config_dict.setdefault('options', dict())) + run_option_dict: dict = copy.deepcopy(config_dict.setdefault("options", dict())) for arg_name, arg_value in vars(input_args).items(): if arg_value is not None: run_option_dict[arg_name] = arg_value @@ -184,5 +203,6 @@ def build_scicat_config(input_args: argparse.Namespace) -> ScicatConfig: return ScicatConfig( original_dict=MappingProxyType(config_dict), run_options=RunOptions(**run_option_dict), - kafka_options=kafkaOptions(**config_dict.setdefault('kafka', dict())), + kafka_options=kafkaOptions(**config_dict.setdefault("kafka", dict())), + graylog_options=GraylogOptions(**config_dict.setdefault("graylog", dict())), ) diff --git a/src/scicat_logging.py b/src/scicat_logging.py index ab99800..2017fe0 100644 --- a/src/scicat_logging.py +++ b/src/scicat_logging.py @@ -4,6 +4,8 @@ import logging.handlers from datetime import datetime +import graypy + from scicat_configuration import ScicatConfig @@ -33,6 +35,16 @@ def build_logger(config: ScicatConfig) -> logging.Logger: if run_options.system_log: logger.addHandler(logging.handlers.SysLogHandler(address='/dev/log')) + # Add graylog handler + if run_options.graylog: + graylog_config = config.graylog_options + graylog_handler = graypy.GELFTCPHandler( + graylog_config.host, + int(graylog_config.port), + facility=graylog_config.facility, + ) + logger.addHandler(graylog_handler) + # Set the level and formatter for all handlers logger.setLevel(run_options.log_level) for handler in logger.handlers: diff --git a/tests/test_logging.py b/tests/test_logging.py index daf3f2e..b89129f 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -2,7 +2,7 @@ import pytest -from scicat_configuration import RunOptions, ScicatConfig, kafkaOptions +from scicat_configuration import GraylogOptions, RunOptions, ScicatConfig, kafkaOptions @pytest.fixture @@ -23,6 +23,7 @@ def scicat_config(tmp_path: pathlib.Path) -> ScicatConfig: pyscicat='test', ), kafka_options=kafkaOptions(), + graylog_options=GraylogOptions(), )