[ADD] base_import_extended: New module

TT52224

base_import_extended/README.rst

@@ -0,0 +1,83 @@
+====================
+Base import extended
+====================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:bc2594e42c1110b31090b0eb7d8f55e2899f79c43fa10a04451f2b65be0f25a2
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Beta
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--ux-lightgray.png?logo=github
+    :target: https://github.com/OCA/server-ux/tree/17.0/base_import_extended
+    :alt: OCA/server-ux
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/server-ux-17-0/server-ux-17-0-base_import_extended
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/server-ux&target_branch=17.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+This module allows to create templates for import data files
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Usage
+=====
+
+
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/server-ux/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/server-ux/issues/new?body=module:%20base_import_extended%0Aversion:%2017.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+-------
+
+* Tecnativa
+
+Contributors
+------------
+
+- `Tecnativa <https://www.tecnativa.com>`__:
+
+  - Carlos Dauden
+
+Maintainers
+-----------
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+This module is part of the `OCA/server-ux <https://github.com/OCA/server-ux/tree/17.0/base_import_extended>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

base_import_extended/__init__.py

@@ -0,0 +1 @@
+from . import models

base_import_extended/__manifest__.py

@@ -0,0 +1,22 @@
+# Copyright 2025 Tecnativa - Carlos Dauden
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
+
+{
+    "name": "Base import extended",
+    "summary": "Base import extended to manage templates and extra conversion methods",
+    "version": "17.0.1.0.0",
+    "category": "Tools",
+    "website": "https://github.com/OCA/server-ux",
+    "author": "Tecnativa, Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "installable": True,
+    "depends": [
+        "base_import",
+    ],
+    "data": [
+        "security/ir.model.access.csv",
+        "views/base_import_mapping_template_views.xml",
+        "views/base_import_mapping_value_map_views.xml",
+        # "views/invoice_import_xls_view.xml",
+    ],
+}

base_import_extended/models/__init__.py

@@ -0,0 +1,4 @@
+from . import base_import_mapping
+from . import base_import_mapping_template
+from . import base_import_mapping_value_map
+from . import ir_fields

base_import_extended/models/base_import_mapping.py

@@ -0,0 +1,70 @@
+# Copyright 2025 Tecnativa - Carlos Dauden
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo import api, fields, models
+from odoo.osv import expression
+
+
+class ImportMapping(models.Model):
+    _inherit = "base_import.mapping"
+    _rec_name = "column_name"
+
+    mapping_template_id = fields.Many2one(
+        comodel_name="base_import.mapping.template",
+        ondelete="cascade",
+    )
+    pre_process_method = fields.Selection(
+        [
+            ("str_abs_value", "Absolute value (string)"),
+            ("prefix_c", "Prefix C"),
+            ("prefix_p", "Prefix P"),
+        ]
+    )
+    mapped_value_ids = fields.Many2many(
+        comodel_name="base_import.mapping.value.map",
+        relation="base_import_mapping_value_map_rel",
+        column1="mapping_id",
+        column2="value_map_id",
+    )
+    python_code = fields.Char()
+    # Convert to compute to manage from template if is set
+    res_model = fields.Char(compute="_compute_res_model", store=True, readonly=False)
+
+    @api.depends("mapping_template_id")
+    def _compute_res_model(self):
+        for line in self.filtered("mapping_template_id"):
+            line.res_model = line.mapping_template_id.res_model
+
+    @api.model
+    def search(self, domain, offset=0, limit=None, order=None):
+        domain = expression.AND(
+            [
+                domain,
+                [
+                    (
+                        "mapping_template_id",
+                        "=",
+                        self.env.context.get("use_mapping_template_id", False),
+                    )
+                ],
+            ]
+        )
+        return super().search(domain, offset=offset, limit=limit, order=order)
+
+    @api.model_create_multi
+    def create(self, vals_list):
+        mapping_template_id = self.env.context.get("use_mapping_template_id", False)
+        if mapping_template_id:
+            for vals in vals_list:
+                if "mapping_template_id" not in vals:
+                    vals["mapping_template_id"] = mapping_template_id
+        return super().create(vals_list)
+
+    def pre_process_method_str_abs_value(self, value):
+        return value.replace("-", "")
+
+    def pre_process_method_prefix_c(self, value):
+        return f"C-{value}"
+
+    def pre_process_method_prefix_p(self, value):
+        return f"P-{value}"

base_import_extended/models/base_import_mapping_template.py

@@ -0,0 +1,177 @@
+# Copyright 2025 Tecnativa - Carlos Dauden
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+import base64
+import math
+
+from odoo import _, api, fields, models
+from odoo.exceptions import UserError
+from odoo.tools.safe_eval import safe_eval
+
+
+class ImportMappingTemplate(models.Model):
+    _inherit = ["base_import.import"]
+    _name = "base_import.mapping.template"
+    _description = "Template to group import mapping data"
+    _transient = False
+    _transient_max_hours = 0
+    _transient_max_count = 0
+
+    name = fields.Char()
+    action_id = fields.Many2one(comodel_name="ir.actions.act_window")
+    mapping_ids = fields.One2many(
+        comodel_name="base_import.mapping",
+        inverse_name="mapping_template_id",
+    )
+    forced_context = fields.Char(string="Context Value", default={}, required=True)
+    forced_options = fields.Char(default={}, required=True)
+
+    def execute_import_template(self):
+        options = {
+            "import_skip_records": [],
+            "import_set_empty_fields": [],
+            "fallback_values": {},
+            "name_create_enabled_fields": {},
+            "encoding": "",
+            "separator": "",
+            "quoting": '"',
+            "date_format": "%Y-%m-%d",
+            "datetime_format": "",
+            "float_thousand_separator": ",",
+            "float_decimal_separator": ".",
+            "advanced": True,
+            "has_headers": True,
+            "keep_matches": False,
+            "limit": 2000,
+            "sheets": [],
+            "sheet": "",
+            "skip": 0,
+            "tracking_disable": True,
+        }
+        eval_ctx = dict(self.env.context)
+        ctx = {}
+        if self.action_id:
+            ctx.update(**safe_eval(self.action_id.context, eval_ctx))
+        if self.forced_context != "{}":
+            ctx.update(**safe_eval(self.forced_context, eval_ctx))
+        self = self.with_context(**ctx)
+        self.file = base64.b64decode(self.file)
+        preview = self.parse_preview(options=options)
+        if self.forced_options != "{}":
+            options.update(**safe_eval(self.forced_options, eval_ctx))
+        columns = [col.lower() for col in preview.get("headers", [])]
+        fields = []
+        for column in columns:
+            line = self.mapping_ids.filtered(lambda x, col=column: x.column_name == col)
+            fields.append(line.field_name)
+        limit = options["limit"]
+        steps_number = math.ceil((preview.get("file_length", 1) - 1) / limit)
+        all_ids = []
+        for step in range(steps_number):
+            options["skip"] = step * limit
+            options["limit"] = limit
+            res = self.with_context(
+                use_mapping_template_id=self.id, use_cached_db_id_for=True
+            ).execute_import(fields, columns, options, dryrun=False)
+            messages = res.get("messages", [])
+            if messages:
+                text_message = "\n".join(m.get("message", "") for m in messages)
+                if step > 0:
+                    text_message = (
+                        f"Already imported {step * limit} records, but \n{text_message}"
+                    )
+                raise UserError(text_message)
+            res_ids = res.get("ids", [])
+            if not res_ids:
+                raise UserError(_("No records were imported"))
+            all_ids.extend(res_ids)
+            # self.env.registry.clear_cache()
+        self.file = False
+        return self.action_view_imported_records(all_ids, ctx)
+
+    @api.model
+    def _convert_import_data(self, fields, options):
+        data, import_fields = super()._convert_import_data(fields, options)
+        if not self.env.context.get("use_mapping_template_id"):
+            return data, import_fields
+        multilevel = "id" in import_fields and any(
+            "_ids/" in f for f in import_fields if f
+        )
+        if multilevel:
+            id_index = import_fields.index("id")
+        index_line_dict, index_column_dict = self.get_index_dictionaries(import_fields)
+        # for index, field_name in enumerate(import_fields):
+        # line = self.mapping_ids.filtered(
+        #     lambda x, f_name=field_name: x.field_name == f_name
+        # )
+        for index, line in index_line_dict.items():
+            field_name = line.field_name
+            if line.pre_process_method:
+                process_fnc = getattr(
+                    line, f"pre_process_method_{line.pre_process_method}"
+                )
+                for row in data:
+                    row[index] = process_fnc(row[index])
+            elif line.python_code:
+                self.update_data_with_python_code(
+                    data, index, line.python_code, index_column_dict
+                )
+            elif field_name == "id":
+                prefix = self.res_model.replace(".", "_")
+                for row in data:
+                    row[index] = f"{prefix}_{row[index]}_{self.id}"
+            if line.mapped_value_ids:
+                mapped_dict = {
+                    map_line.value: map_line.new_value_ref
+                    and str(map_line.new_value_ref.id)
+                    or map_line.new_value
+                    for map_line in line.mapped_value_ids
+                }
+                for row in data:
+                    row[index] = mapped_dict.get(row[index], row[index])
+            # Empty repeat values for principal record fields
+            if multilevel and "_ids/" not in field_name:
+                last_value = ""
+                for row in data:
+                    if row[id_index] in ("", last_value):
+                        row[index] = ""
+                    else:
+                        last_value = row[id_index]
+        return data, import_fields
+
+    def get_index_dictionaries(self, import_fields):
+        index_line_dict = {}
+        index_column_dict = {}
+        for index, field_name in enumerate(import_fields):
+            line = self.mapping_ids.filtered(
+                lambda x, f_name=field_name: x.field_name == f_name
+            )
+            index_line_dict[index] = line
+            index_column_dict[index] = line.column_name
+        return index_line_dict, index_column_dict
+
+    def update_data_with_python_code(self, data, index, python_code, index_column_dict):
+        for row in data:
+            col_vals = {}
+            if "col_vals" in python_code:
+                for idx, col in index_column_dict.items():
+                    col_vals[col] = row[idx]
+            row[index] = safe_eval(
+                python_code,
+                {"value": row[index], "col_vals": col_vals},
+            )
+
+    def action_view_imported_records(self, res_ids, context=None):
+        if self.action_id:
+            action = self.env["ir.actions.actions"]._for_xml_id(self.action_id.xml_id)
+        else:
+            action = {
+                "type": "ir.actions.act_window",
+                "res_model": self.res_model,
+                "name": _("Imported Records"),
+                "views": [[False, "tree"], [False, "kanban"], [False, "form"]],
+            }
+        action["domain"] = [("id", "in", res_ids)]
+        if context:
+            action["context"] = context
+        return action

base_import_extended/models/base_import_mapping_value_map.py

@@ -0,0 +1,25 @@
+# Copyright 2025 Tecnativa - Carlos Dauden
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo import fields, models
+
+
+class ImportMappingValueMap(models.Model):
+    _name = "base_import.mapping.value.map"
+    _description = "Map import value with odoo value"
+    _rec_name = "value"
+
+    mapping_ids = fields.Many2many(
+        comodel_name="base_import.mapping",
+        relation="base_import_mapping_value_map_rel",
+        column1="value_map_id",
+        column2="mapping_id",
+    )
+    value = fields.Char()
+    new_value = fields.Char()
+    new_value_ref = fields.Reference(
+        lambda self: [
+            (m.model, m.name) for m in self.env["ir.model"].sudo().search([])
+        ],
+        string="Object",
+    )