ui: option to require successfull validation

before submission

fixes #9

client/src/components/WebinputCSV.vue

@@ -337,7 +337,11 @@
                             spinner-variant="primary"
                             class="d-inline-block"
                           >
-                            <b-button @click="onSendData" variant="primary">Submit to {{ customWorkflow ? 'custom workflow' : 'standard workflow' }}</b-button>
+                            <b-button @click="onSendData" variant="primary"
+                              :disabled="(!validatedCurrentData || validationNumErrors > 0) && allowValidationOverride == false"
+                              v-b-tooltip.hover
+                              :title="(!validatedCurrentData && allowValidationOverride == false) ? 'Data validation required before' : ((validationNumErrors > 0 && allowValidationOverride == false) ? 'Data validation failed' : '')"
+                              >Submit to {{ customWorkflow ? 'custom workflow' : 'standard workflow' }}</b-button>
                           </b-overlay>
                         </b-col>
                         <b-col>
@@ -854,6 +858,8 @@ export default ({
       mailgenTemplateValidationText: '',
       mailgenTemplateValidationStatus: null,
       mailgenTemplatePrototype: null,
+      validationNumErrors: null,
+      validatedCurrentData: false
     }
   },
   computed: {
@@ -867,7 +873,7 @@ export default ({
       // returns only the names of assigned columns
       return this.tableHeader.slice(1).map(header => header.field).filter(entry => entry)
     },
-    ...mapState(['user', 'loggedIn', 'hasAuth', 'classificationTypes', 'harmonizationFields', 'customFieldsMapping', 'requiredFields', 'mailgenAvailable', 'botsAvailable', 'mailgenAvailableTargetGroups', 'mailgenAvailableTargetGroupsStatus', 'backendVersion', 'mailgenTemplatesServer', 'mailgenTemplates', 'mailgenMultiTemplatesEnabled', 'mailgenTemplateDefaultTemplateName', 'customWorkflowDefault']),
+    ...mapState(['user', 'loggedIn', 'hasAuth', 'classificationTypes', 'harmonizationFields', 'customFieldsMapping', 'requiredFields', 'mailgenAvailable', 'botsAvailable', 'mailgenAvailableTargetGroups', 'mailgenAvailableTargetGroupsStatus', 'backendVersion', 'mailgenTemplatesServer', 'mailgenTemplates', 'mailgenMultiTemplatesEnabled', 'mailgenTemplateDefaultTemplateName', 'customWorkflowDefault', 'allowValidationOverride']),
   },
   mounted() {
     this.$store.dispatch("fetchBackendVersion");
@@ -969,18 +975,21 @@ export default ({
               me.inProgress = false;
             }
 
-            const num_errors = Object.keys(data.errors).length;
-            me.transferred = (submit ? "Submitted " : "Validated ") + (data.input_lines) + " lines" + (submit ? (this.customWorkflow ? " to IntelMQ database" : " to IntelMQ processing queue") : "") + ". Of these, " + (data.input_lines - data.input_lines_invalid) + " were valid. This resulted in " + num_errors + " validation errors and in total " + data.input_lines_invalid + " lines were invalid" + (submit ? ", these were not submitted" : "") + ".";
+            this.validatedCurrentData = true;  // independent of the result, the data validation was run
+
+            this.validationNumErrors = Object.keys(data.errors).length;
+            me.transferred = (submit ? "Submitted " : "Validated ") + (data.input_lines) + " lines" + (submit ? (this.customWorkflow ? " to IntelMQ database" : " to IntelMQ processing queue") : "") + ". Of these, " + (data.input_lines - data.input_lines_invalid) + " were valid. This resulted in " + this.validationNumErrors + " validation errors and in total " + data.input_lines_invalid + " lines were invalid" + (submit ? ", these were not submitted" : "") + ".";
             if (this.customWorkflow) {
               me.transferred = me.transferred + " After bot validation the input data resulted in " + data.output_lines + " events and " + data.output_lines_invalid + " errors occured (invalid events).";
             }
             me.dataErrors = data.errors;
-            if (num_errors) {
+            if (this.validationNumErrors) {
               me.transferStatus = "text-danger";
             } else {
               me.transferStatus = "text-black";
             }
             me.inProgress = false;
+            console.log("Debug:", this.validatedCurrentData, this.validationNumErrors > 0, this.allowValidationOverride, this.allowValidationOverride == false, this.validationNumErrors > 0 && this.allowValidationOverride == false, (!this.validatedCurrentData || this.validationNumErrors > 0) && this.allowValidationOverride == false)
           })
         }, (response) => { // error
             if (response.status == 401) {
@@ -1191,6 +1200,8 @@ export default ({
      * Parse the csv data and apply user options.
      */
     parseCSV() {
+      this.validatedCurrentData = false;
+
       if (this.csvText === "") {
         this.overlay = false;
       }

client/src/store/index.js

@@ -27,6 +27,7 @@ export default new Vuex.Store({
     mailgenMultiTemplatesEnabled: false,
     mailgenTemplateDefaultTemplateName: null,
     customWorkflowDefault: false,
+    allowValidationOverride: true
   },
   mutations: {
     SET_USER (state, user) {
@@ -87,6 +88,7 @@ export default new Vuex.Store({
     },
     SET_SETTNGS(state, data) {
       state.customWorkflowDefault = data['custom_workflow_default'];
+      state.allowValidationOverride = data['allow_validation_override'];
     }
   },
   actions: {

docs/user-guide.rst

@@ -90,6 +90,8 @@ Usual configuration parameters
   The check does **not** apply to constant fields and custom input fields!
 - ``custom_workflow_default``: If true (default: false), then the switch "Use custom workflow"
   in the "Data validation and submission" section is enabled by default.
+- ``allow_validation_override``: If true (default), the the user is allowed to submit data if
+  the previous validation run did not succeed (not all lines/events were valid).
 
 Mailgen configuration parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

intelmq_webinput_csv/data.py

@@ -1,5 +1,5 @@
 """
-SPDX-FileCopyrightText: 2016, 2017, 2022-2023 Bundesamt für Sicherheit in der Informationstechnik
+SPDX-FileCopyrightText: 2016, 2017, 2022-2024 Bundesamt für Sicherheit in der Informationstechnik
 SPDX-License-Identifier: AGPL-3.0-or-later
 Software engineering by Intevation GmbH <https://intevation.de>
 """

intelmq_webinput_csv/serve.py

@@ -9,7 +9,7 @@
 
 Several configuration methods are shown within the code.
 
-SPDX-FileCopyrightText: 2016, 2017, 2022-2023 Bundesamt für Sicherheit in der Informationstechnik
+SPDX-FileCopyrightText: 2016, 2017, 2022-2024 Bundesamt für Sicherheit in der Informationstechnik
 SPDX-License-Identifier: AGPL-3.0-or-later
 
 Software engineering by Intevation GmbH <https://intevation.de>
@@ -210,12 +210,13 @@ def row_to_event(item: dict, body: dict,
         except IntelMQException as exc:
             lineerrors[key].append(f"Failed to add data {value!r} as field {key!r}: {exc!s}")
             line_valid = False
-    for key in body['custom']:
+    for key, value in body['custom'].items():
         if key.startswith('custom_') and key[7:] not in event:
+            key = key[7:]
             try:
-                event.add(key[7:], body['custom'][key])
+                event.add(key, value)
             except InvalidValue as exc:
-                lineerrors[-1].append(f"Failed to add data {body['custom'][key]!r} as field {key!r}: {exc!s}")
+                lineerrors[-1].append(f"Failed to add data {value!r} as field {key!r}: {exc!s}")
                 line_valid = False
     for key in CONSTANTS:
         if key not in event:
@@ -443,7 +444,8 @@ def settings():
     Returns some configuration options
     """
     return {
-        'custom_workflow_default': CONFIG.get('custom_workflow_default', False)
+        'custom_workflow_default': CONFIG.get('custom_workflow_default', False),
+        'allow_validation_override': CONFIG.get('allow_validation_override', True)
     }