diff --git a/base_import_extended/README.rst b/base_import_extended/README.rst new file mode 100644 index 0000000000..dd0ebf4cdb --- /dev/null +++ b/base_import_extended/README.rst @@ -0,0 +1,83 @@ +==================== +Base import extended +==================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:bc2594e42c1110b31090b0eb7d8f55e2899f79c43fa10a04451f2b65be0f25a2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--ux-lightgray.png?logo=github + :target: https://github.com/OCA/server-ux/tree/17.0/base_import_extended + :alt: OCA/server-ux +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-ux-17-0/server-ux-17-0-base_import_extended + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-ux&target_branch=17.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module allows to create templates for import data files + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + + + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Tecnativa + +Contributors +------------ + +- `Tecnativa `__: + + - Carlos Dauden + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-ux `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_import_extended/__init__.py b/base_import_extended/__init__.py new file mode 100644 index 0000000000..0650744f6b --- /dev/null +++ b/base_import_extended/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/base_import_extended/__manifest__.py b/base_import_extended/__manifest__.py new file mode 100644 index 0000000000..e8c316467c --- /dev/null +++ b/base_import_extended/__manifest__.py @@ -0,0 +1,22 @@ +# Copyright 2025 Tecnativa - Carlos Dauden +# License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +{ + "name": "Base import extended", + "summary": "Base import extended to manage templates and extra conversion methods", + "version": "17.0.1.0.0", + "category": "Tools", + "website": "https://github.com/OCA/server-ux", + "author": "Tecnativa, Odoo Community Association (OCA)", + "license": "AGPL-3", + "installable": True, + "depends": [ + "base_import", + ], + "data": [ + "security/ir.model.access.csv", + "views/base_import_mapping_template_views.xml", + "views/base_import_mapping_value_map_views.xml", + # "views/invoice_import_xls_view.xml", + ], +} diff --git a/base_import_extended/models/__init__.py b/base_import_extended/models/__init__.py new file mode 100644 index 0000000000..720292defd --- /dev/null +++ b/base_import_extended/models/__init__.py @@ -0,0 +1,4 @@ +from . import base_import_mapping +from . import base_import_mapping_template +from . import base_import_mapping_value_map +from . import ir_fields diff --git a/base_import_extended/models/base_import_mapping.py b/base_import_extended/models/base_import_mapping.py new file mode 100644 index 0000000000..15fe615b5c --- /dev/null +++ b/base_import_extended/models/base_import_mapping.py @@ -0,0 +1,70 @@ +# Copyright 2025 Tecnativa - Carlos Dauden +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models +from odoo.osv import expression + + +class ImportMapping(models.Model): + _inherit = "base_import.mapping" + _rec_name = "column_name" + + mapping_template_id = fields.Many2one( + comodel_name="base_import.mapping.template", + ondelete="cascade", + ) + pre_process_method = fields.Selection( + [ + ("str_abs_value", "Absolute value (string)"), + ("prefix_c", "Prefix C"), + ("prefix_p", "Prefix P"), + ] + ) + mapped_value_ids = fields.Many2many( + comodel_name="base_import.mapping.value.map", + relation="base_import_mapping_value_map_rel", + column1="mapping_id", + column2="value_map_id", + ) + python_code = fields.Char() + # Convert to compute to manage from template if is set + res_model = fields.Char(compute="_compute_res_model", store=True, readonly=False) + + @api.depends("mapping_template_id") + def _compute_res_model(self): + for line in self.filtered("mapping_template_id"): + line.res_model = line.mapping_template_id.res_model + + @api.model + def search(self, domain, offset=0, limit=None, order=None): + domain = expression.AND( + [ + domain, + [ + ( + "mapping_template_id", + "=", + self.env.context.get("use_mapping_template_id", False), + ) + ], + ] + ) + return super().search(domain, offset=offset, limit=limit, order=order) + + @api.model_create_multi + def create(self, vals_list): + mapping_template_id = self.env.context.get("use_mapping_template_id", False) + if mapping_template_id: + for vals in vals_list: + if "mapping_template_id" not in vals: + vals["mapping_template_id"] = mapping_template_id + return super().create(vals_list) + + def pre_process_method_str_abs_value(self, value): + return value.replace("-", "") + + def pre_process_method_prefix_c(self, value): + return f"C-{value}" + + def pre_process_method_prefix_p(self, value): + return f"P-{value}" diff --git a/base_import_extended/models/base_import_mapping_template.py b/base_import_extended/models/base_import_mapping_template.py new file mode 100644 index 0000000000..ef79176613 --- /dev/null +++ b/base_import_extended/models/base_import_mapping_template.py @@ -0,0 +1,177 @@ +# Copyright 2025 Tecnativa - Carlos Dauden +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import base64 +import math + +from odoo import _, api, fields, models +from odoo.exceptions import UserError +from odoo.tools.safe_eval import safe_eval + + +class ImportMappingTemplate(models.Model): + _inherit = ["base_import.import"] + _name = "base_import.mapping.template" + _description = "Template to group import mapping data" + _transient = False + _transient_max_hours = 0 + _transient_max_count = 0 + + name = fields.Char() + action_id = fields.Many2one(comodel_name="ir.actions.act_window") + mapping_ids = fields.One2many( + comodel_name="base_import.mapping", + inverse_name="mapping_template_id", + ) + forced_context = fields.Char(string="Context Value", default={}, required=True) + forced_options = fields.Char(default={}, required=True) + + def execute_import_template(self): + options = { + "import_skip_records": [], + "import_set_empty_fields": [], + "fallback_values": {}, + "name_create_enabled_fields": {}, + "encoding": "", + "separator": "", + "quoting": '"', + "date_format": "%Y-%m-%d", + "datetime_format": "", + "float_thousand_separator": ",", + "float_decimal_separator": ".", + "advanced": True, + "has_headers": True, + "keep_matches": False, + "limit": 2000, + "sheets": [], + "sheet": "", + "skip": 0, + "tracking_disable": True, + } + eval_ctx = dict(self.env.context) + ctx = {} + if self.action_id: + ctx.update(**safe_eval(self.action_id.context, eval_ctx)) + if self.forced_context != "{}": + ctx.update(**safe_eval(self.forced_context, eval_ctx)) + self = self.with_context(**ctx) + self.file = base64.b64decode(self.file) + preview = self.parse_preview(options=options) + if self.forced_options != "{}": + options.update(**safe_eval(self.forced_options, eval_ctx)) + columns = [col.lower() for col in preview.get("headers", [])] + fields = [] + for column in columns: + line = self.mapping_ids.filtered(lambda x, col=column: x.column_name == col) + fields.append(line.field_name) + limit = options["limit"] + steps_number = math.ceil((preview.get("file_length", 1) - 1) / limit) + all_ids = [] + for step in range(steps_number): + options["skip"] = step * limit + options["limit"] = limit + res = self.with_context( + use_mapping_template_id=self.id, use_cached_db_id_for=True + ).execute_import(fields, columns, options, dryrun=False) + messages = res.get("messages", []) + if messages: + text_message = "\n".join(m.get("message", "") for m in messages) + if step > 0: + text_message = ( + f"Already imported {step * limit} records, but \n{text_message}" + ) + raise UserError(text_message) + res_ids = res.get("ids", []) + if not res_ids: + raise UserError(_("No records were imported")) + all_ids.extend(res_ids) + # self.env.registry.clear_cache() + self.file = False + return self.action_view_imported_records(all_ids, ctx) + + @api.model + def _convert_import_data(self, fields, options): + data, import_fields = super()._convert_import_data(fields, options) + if not self.env.context.get("use_mapping_template_id"): + return data, import_fields + multilevel = "id" in import_fields and any( + "_ids/" in f for f in import_fields if f + ) + if multilevel: + id_index = import_fields.index("id") + index_line_dict, index_column_dict = self.get_index_dictionaries(import_fields) + # for index, field_name in enumerate(import_fields): + # line = self.mapping_ids.filtered( + # lambda x, f_name=field_name: x.field_name == f_name + # ) + for index, line in index_line_dict.items(): + field_name = line.field_name + if line.pre_process_method: + process_fnc = getattr( + line, f"pre_process_method_{line.pre_process_method}" + ) + for row in data: + row[index] = process_fnc(row[index]) + elif line.python_code: + self.update_data_with_python_code( + data, index, line.python_code, index_column_dict + ) + elif field_name == "id": + prefix = self.res_model.replace(".", "_") + for row in data: + row[index] = f"{prefix}_{row[index]}_{self.id}" + if line.mapped_value_ids: + mapped_dict = { + map_line.value: map_line.new_value_ref + and str(map_line.new_value_ref.id) + or map_line.new_value + for map_line in line.mapped_value_ids + } + for row in data: + row[index] = mapped_dict.get(row[index], row[index]) + # Empty repeat values for principal record fields + if multilevel and "_ids/" not in field_name: + last_value = "" + for row in data: + if row[id_index] in ("", last_value): + row[index] = "" + else: + last_value = row[id_index] + return data, import_fields + + def get_index_dictionaries(self, import_fields): + index_line_dict = {} + index_column_dict = {} + for index, field_name in enumerate(import_fields): + line = self.mapping_ids.filtered( + lambda x, f_name=field_name: x.field_name == f_name + ) + index_line_dict[index] = line + index_column_dict[index] = line.column_name + return index_line_dict, index_column_dict + + def update_data_with_python_code(self, data, index, python_code, index_column_dict): + for row in data: + col_vals = {} + if "col_vals" in python_code: + for idx, col in index_column_dict.items(): + col_vals[col] = row[idx] + row[index] = safe_eval( + python_code, + {"value": row[index], "col_vals": col_vals}, + ) + + def action_view_imported_records(self, res_ids, context=None): + if self.action_id: + action = self.env["ir.actions.actions"]._for_xml_id(self.action_id.xml_id) + else: + action = { + "type": "ir.actions.act_window", + "res_model": self.res_model, + "name": _("Imported Records"), + "views": [[False, "tree"], [False, "kanban"], [False, "form"]], + } + action["domain"] = [("id", "in", res_ids)] + if context: + action["context"] = context + return action diff --git a/base_import_extended/models/base_import_mapping_value_map.py b/base_import_extended/models/base_import_mapping_value_map.py new file mode 100644 index 0000000000..f6537d2773 --- /dev/null +++ b/base_import_extended/models/base_import_mapping_value_map.py @@ -0,0 +1,25 @@ +# Copyright 2025 Tecnativa - Carlos Dauden +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + + +class ImportMappingValueMap(models.Model): + _name = "base_import.mapping.value.map" + _description = "Map import value with odoo value" + _rec_name = "value" + + mapping_ids = fields.Many2many( + comodel_name="base_import.mapping", + relation="base_import_mapping_value_map_rel", + column1="value_map_id", + column2="mapping_id", + ) + value = fields.Char() + new_value = fields.Char() + new_value_ref = fields.Reference( + lambda self: [ + (m.model, m.name) for m in self.env["ir.model"].sudo().search([]) + ], + string="Object", + ) diff --git a/base_import_extended/models/ir_fields.py b/base_import_extended/models/ir_fields.py new file mode 100644 index 0000000000..d370407598 --- /dev/null +++ b/base_import_extended/models/ir_fields.py @@ -0,0 +1,23 @@ +# Copyright 2025 Tecnativa - Carlos Dauden +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, models +from odoo.tools import ormcache + + +class IrFieldsConverter(models.AbstractModel): + _inherit = "ir.fields.converter" + + @api.model + @ormcache("model", "field", "subfield", "value") + def cached_db_id_for(self, model, field, subfield, value): + res_id, field_type, warnings = self.with_context( + use_cached_db_id_for=False + ).db_id_for(model, field, subfield, value) + return res_id, field_type, warnings + + @api.model + def db_id_for(self, model, field, subfield, value): + if self.env.context.get("use_cached_db_id_for", False): + return self.cached_db_id_for(model, field, subfield, value) + return super().db_id_for(model, field, subfield, value) diff --git a/base_import_extended/pyproject.toml b/base_import_extended/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/base_import_extended/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/base_import_extended/readme/CONTRIBUTORS.md b/base_import_extended/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..37d0fcf44b --- /dev/null +++ b/base_import_extended/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- [Tecnativa](https://www.tecnativa.com): + - Carlos Dauden diff --git a/base_import_extended/readme/DESCRIPTION.md b/base_import_extended/readme/DESCRIPTION.md new file mode 100644 index 0000000000..983b4a53ba --- /dev/null +++ b/base_import_extended/readme/DESCRIPTION.md @@ -0,0 +1 @@ +This module allows to create templates for import data files diff --git a/base_import_extended/readme/USAGE.md b/base_import_extended/readme/USAGE.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/base_import_extended/security/ir.model.access.csv b/base_import_extended/security/ir.model.access.csv new file mode 100644 index 0000000000..e475f01f6e --- /dev/null +++ b/base_import_extended/security/ir.model.access.csv @@ -0,0 +1,3 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +base_import_extended.access_base_import_mapping_template,access_base_import_mapping_template,base_import_extended.model_base_import_mapping_template,base.group_user,1,1,1,1 +base_import_extended.access_base_import_mapping_value_map,access_base_import_mapping_value_map,base_import_extended.model_base_import_mapping_value_map,base.group_user,1,1,1,1 diff --git a/base_import_extended/static/description/icon.png b/base_import_extended/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/base_import_extended/static/description/icon.png differ diff --git a/base_import_extended/static/description/index.html b/base_import_extended/static/description/index.html new file mode 100644 index 0000000000..933f9ced12 --- /dev/null +++ b/base_import_extended/static/description/index.html @@ -0,0 +1,430 @@ + + + + + +Base import extended + + + +
+

Base import extended

+ + +

Beta License: AGPL-3 OCA/server-ux Translate me on Weblate Try me on Runboat

+

This module allows to create templates for import data files

+

Table of contents

+ +
+

Usage

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Tecnativa
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/server-ux project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/base_import_extended/views/base_import_mapping_template_views.xml b/base_import_extended/views/base_import_mapping_template_views.xml new file mode 100644 index 0000000000..251244c9e0 --- /dev/null +++ b/base_import_extended/views/base_import_mapping_template_views.xml @@ -0,0 +1,84 @@ + + + + + base.import.mapping.template.form + base_import.mapping.template + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +