diff --git a/client/src/components/WebinputCSV.vue b/client/src/components/WebinputCSV.vue index 84d1c62..f5b7add 100644 --- a/client/src/components/WebinputCSV.vue +++ b/client/src/components/WebinputCSV.vue @@ -337,7 +337,11 @@ spinner-variant="primary" class="d-inline-block" > - Submit to {{ customWorkflow ? 'custom workflow' : 'standard workflow' }} + Submit to {{ customWorkflow ? 'custom workflow' : 'standard workflow' }} @@ -854,6 +858,8 @@ export default ({ mailgenTemplateValidationText: '', mailgenTemplateValidationStatus: null, mailgenTemplatePrototype: null, + validationNumErrors: null, + validatedCurrentData: false } }, computed: { @@ -867,7 +873,7 @@ export default ({ // returns only the names of assigned columns return this.tableHeader.slice(1).map(header => header.field).filter(entry => entry) }, - ...mapState(['user', 'loggedIn', 'hasAuth', 'classificationTypes', 'harmonizationFields', 'customFieldsMapping', 'requiredFields', 'mailgenAvailable', 'botsAvailable', 'mailgenAvailableTargetGroups', 'mailgenAvailableTargetGroupsStatus', 'backendVersion', 'mailgenTemplatesServer', 'mailgenTemplates', 'mailgenMultiTemplatesEnabled', 'mailgenTemplateDefaultTemplateName', 'customWorkflowDefault']), + ...mapState(['user', 'loggedIn', 'hasAuth', 'classificationTypes', 'harmonizationFields', 'customFieldsMapping', 'requiredFields', 'mailgenAvailable', 'botsAvailable', 'mailgenAvailableTargetGroups', 'mailgenAvailableTargetGroupsStatus', 'backendVersion', 'mailgenTemplatesServer', 'mailgenTemplates', 'mailgenMultiTemplatesEnabled', 'mailgenTemplateDefaultTemplateName', 'customWorkflowDefault', 'allowValidationOverride']), }, mounted() { this.$store.dispatch("fetchBackendVersion"); @@ -969,18 +975,21 @@ export default ({ me.inProgress = false; } - const num_errors = Object.keys(data.errors).length; - me.transferred = (submit ? "Submitted " : "Validated ") + (data.input_lines) + " lines" + (submit ? (this.customWorkflow ? " to IntelMQ database" : " to IntelMQ processing queue") : "") + ". Of these, " + (data.input_lines - data.input_lines_invalid) + " were valid. This resulted in " + num_errors + " validation errors and in total " + data.input_lines_invalid + " lines were invalid" + (submit ? ", these were not submitted" : "") + "."; + this.validatedCurrentData = true; // independent of the result, the data validation was run + + this.validationNumErrors = Object.keys(data.errors).length; + me.transferred = (submit ? "Submitted " : "Validated ") + (data.input_lines) + " lines" + (submit ? (this.customWorkflow ? " to IntelMQ database" : " to IntelMQ processing queue") : "") + ". Of these, " + (data.input_lines - data.input_lines_invalid) + " were valid. This resulted in " + this.validationNumErrors + " validation errors and in total " + data.input_lines_invalid + " lines were invalid" + (submit ? ", these were not submitted" : "") + "."; if (this.customWorkflow) { me.transferred = me.transferred + " After bot validation the input data resulted in " + data.output_lines + " events and " + data.output_lines_invalid + " errors occured (invalid events)."; } me.dataErrors = data.errors; - if (num_errors) { + if (this.validationNumErrors) { me.transferStatus = "text-danger"; } else { me.transferStatus = "text-black"; } me.inProgress = false; + console.log("Debug:", this.validatedCurrentData, this.validationNumErrors > 0, this.allowValidationOverride, this.allowValidationOverride == false, this.validationNumErrors > 0 && this.allowValidationOverride == false, (!this.validatedCurrentData || this.validationNumErrors > 0) && this.allowValidationOverride == false) }) }, (response) => { // error if (response.status == 401) { @@ -1191,6 +1200,8 @@ export default ({ * Parse the csv data and apply user options. */ parseCSV() { + this.validatedCurrentData = false; + if (this.csvText === "") { this.overlay = false; } diff --git a/client/src/store/index.js b/client/src/store/index.js index f627cfb..290b8af 100644 --- a/client/src/store/index.js +++ b/client/src/store/index.js @@ -27,6 +27,7 @@ export default new Vuex.Store({ mailgenMultiTemplatesEnabled: false, mailgenTemplateDefaultTemplateName: null, customWorkflowDefault: false, + allowValidationOverride: true }, mutations: { SET_USER (state, user) { @@ -87,6 +88,7 @@ export default new Vuex.Store({ }, SET_SETTNGS(state, data) { state.customWorkflowDefault = data['custom_workflow_default']; + state.allowValidationOverride = data['allow_validation_override']; } }, actions: { diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 1f975e9..01c1d7a 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -90,6 +90,8 @@ Usual configuration parameters The check does **not** apply to constant fields and custom input fields! - ``custom_workflow_default``: If true (default: false), then the switch "Use custom workflow" in the "Data validation and submission" section is enabled by default. +- ``allow_validation_override``: If true (default), the the user is allowed to submit data if + the previous validation run did not succeed (not all lines/events were valid). Mailgen configuration parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/intelmq_webinput_csv/data.py b/intelmq_webinput_csv/data.py index 88778a8..28b0cc1 100644 --- a/intelmq_webinput_csv/data.py +++ b/intelmq_webinput_csv/data.py @@ -1,5 +1,5 @@ """ -SPDX-FileCopyrightText: 2016, 2017, 2022-2023 Bundesamt für Sicherheit in der Informationstechnik +SPDX-FileCopyrightText: 2016, 2017, 2022-2024 Bundesamt für Sicherheit in der Informationstechnik SPDX-License-Identifier: AGPL-3.0-or-later Software engineering by Intevation GmbH """ diff --git a/intelmq_webinput_csv/serve.py b/intelmq_webinput_csv/serve.py index fe04a65..4a50524 100644 --- a/intelmq_webinput_csv/serve.py +++ b/intelmq_webinput_csv/serve.py @@ -9,7 +9,7 @@ Several configuration methods are shown within the code. -SPDX-FileCopyrightText: 2016, 2017, 2022-2023 Bundesamt für Sicherheit in der Informationstechnik +SPDX-FileCopyrightText: 2016, 2017, 2022-2024 Bundesamt für Sicherheit in der Informationstechnik SPDX-License-Identifier: AGPL-3.0-or-later Software engineering by Intevation GmbH @@ -210,12 +210,13 @@ def row_to_event(item: dict, body: dict, except IntelMQException as exc: lineerrors[key].append(f"Failed to add data {value!r} as field {key!r}: {exc!s}") line_valid = False - for key in body['custom']: + for key, value in body['custom'].items(): if key.startswith('custom_') and key[7:] not in event: + key = key[7:] try: - event.add(key[7:], body['custom'][key]) + event.add(key, value) except InvalidValue as exc: - lineerrors[-1].append(f"Failed to add data {body['custom'][key]!r} as field {key!r}: {exc!s}") + lineerrors[-1].append(f"Failed to add data {value!r} as field {key!r}: {exc!s}") line_valid = False for key in CONSTANTS: if key not in event: @@ -443,7 +444,8 @@ def settings(): Returns some configuration options """ return { - 'custom_workflow_default': CONFIG.get('custom_workflow_default', False) + 'custom_workflow_default': CONFIG.get('custom_workflow_default', False), + 'allow_validation_override': CONFIG.get('allow_validation_override', True) }