diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst new file mode 100644 index 000000000..fb78af9ed --- /dev/null +++ b/edi_storage_oca/README.rst @@ -0,0 +1,116 @@ +=========================== +EDI Storage backend support +=========================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3806d6eaf0c3ce46d7d24b5c5e002f0c49a42390385f23e4480a33dd94e85451 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi--framework-lightgray.png?logo=github + :target: https://github.com/OCA/edi-framework/tree/16.0/edi_storage_oca + :alt: OCA/edi-framework +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/edi-framework-16-0/edi-framework-16-0-edi_storage_oca + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=16.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. + +Known issues / Roadmap +====================== + +* clean deprecated methods in the storage + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* ACSONE + +Contributors +~~~~~~~~~~~~ + +* Simone Orsi +* Foram Shah +* Lois Rilo +* Duong (Tran Quoc) + +Other credits +~~~~~~~~~~~~~ + +The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp. + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/edi-framework `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__init__.py b/edi_storage_oca/__init__.py new file mode 100644 index 000000000..f24d3e242 --- /dev/null +++ b/edi_storage_oca/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py new file mode 100644 index 000000000..7a88dcef9 --- /dev/null +++ b/edi_storage_oca/__manifest__.py @@ -0,0 +1,24 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +{ + "name": "EDI Storage backend support", + "summary": """ + Base module to allow exchanging files via storage backend (eg: SFTP). + """, + "version": "16.0.1.0.0", + "development_status": "Beta", + "license": "LGPL-3", + "website": "https://github.com/OCA/edi-framework", + "author": "ACSONE,Odoo Community Association (OCA)", + "depends": ["edi_oca", "fs_storage", "component"], + "data": [ + "data/cron.xml", + "data/job_channel_data.xml", + "data/queue_job_function_data.xml", + "security/ir_model_access.xml", + "views/edi_backend_views.xml", + ], + "demo": ["demo/edi_backend_demo.xml"], +} diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py new file mode 100644 index 000000000..7bff9ddfc --- /dev/null +++ b/edi_storage_oca/components/__init__.py @@ -0,0 +1,5 @@ +from . import base +from . import check +from . import send +from . import receive +from . import listener diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py new file mode 100644 index 000000000..3081f77cc --- /dev/null +++ b/edi_storage_oca/components/base.py @@ -0,0 +1,86 @@ +# Copyright 2020 ACSONE +# Copyright 2022 Camptocamp +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import logging +from pathlib import PurePath + +from odoo.addons.component.core import AbstractComponent + +_logger = logging.getLogger(__file__) + + +class EDIStorageComponentMixin(AbstractComponent): + + _name = "edi.storage.component.mixin" + _inherit = "edi.component.mixin" + # Components having `_storage_type` will have precedence. + # If the value is not set, generic components will be used. + _storage_type = None + + @classmethod + def _component_match(cls, work, usage=None, model_name=None, **kw): + res = super()._component_match(work, usage=usage, model_name=model_name, **kw) + storage_type = kw.get("storage_type") + if storage_type and cls._storage_type: + return cls._storage_type == storage_type + return res + + @property + def storage(self): + return self.backend.storage_id + + def _dir_by_state(self, direction, state): + """Return remote directory path by direction and state. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :return: PurePath object + """ + assert direction in ("input", "output") + assert state in ("pending", "done", "error") + return PurePath( + (self.backend[direction + "_dir_" + state] or "").strip().rstrip("/") + ) + + def _get_remote_file_path(self, state, filename=None): + """Retrieve remote path for current exchange record.""" + filename = filename or self.exchange_record.exchange_filename + direction = self.exchange_record.direction + directory = self._dir_by_state(direction, state).as_posix() + path = self.exchange_record.type_id._storage_fullpath( + directory=directory, filename=filename + ) + return path + + def _get_remote_file(self, state, filename=None, binary=False): + """Get file for current exchange_record in the given destination state. + + :param state: string ("pending", "done", "error") + :param filename: custom file name, exchange_record filename used by default + :return: remote file content as string + """ + path = self._get_remote_file_path(state, filename=filename) + try: + # TODO: support match via pattern (eg: filename-prefix-*) + # otherwise is impossible to retrieve input files and acks + # (the date will never match) + # TODO: clean this up, .get is deprecated in fs_storage + return self.storage.get(path.as_posix(), binary=binary) + except FileNotFoundError: + _logger.info( + "Ignored FileNotFoundError when trying " + "to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None + except OSError: + _logger.info( + "Ignored OSError when trying to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py new file mode 100644 index 000000000..4643f8c9c --- /dev/null +++ b/edi_storage_oca/components/check.py @@ -0,0 +1,71 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import logging + +from odoo.tools import pycompat + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + + +class EDIStorageCheckComponentMixin(Component): + + _name = "edi.storage.component.check" + _inherit = [ + "edi.component.check.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.check" + + def check(self): + return self._exchange_output_check() + + def _exchange_output_check(self): + """Check status output exchange and update record. + + 1. check if the file has been processed already (done) + 2. if yes, post message and exit + 3. if not, check for errors + 4. if no errors, return + + :return: boolean + * False if there's nothing else to be done + * True if file still need action + """ + if self._get_remote_file("done"): + _logger.info( + "%s done", + self.exchange_record.identifier, + ) + if ( + not self.exchange_record.edi_exchange_state + == "output_sent_and_processed" + ): + self.exchange_record.edi_exchange_state = "output_sent_and_processed" + self.exchange_record._notify_done() + return False + + error = self._get_remote_file("error") + if error: + _logger.info( + "%s error", + self.exchange_record.identifier, + ) + # Assume a text file will be placed there w/ the same name and error suffix + err_filename = self.exchange_record.exchange_filename + ".error" + error_report = ( + self._get_remote_file("error", filename=err_filename) or "no-report" + ) + if self.exchange_record.edi_exchange_state == "output_sent": + self.exchange_record.update( + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": pycompat.to_text(error_report), + } + ) + self.exchange_record._notify_error("process_ko") + return False + return True diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py new file mode 100644 index 000000000..5b3f49300 --- /dev/null +++ b/edi_storage_oca/components/listener.py @@ -0,0 +1,74 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import functools +import os +from pathlib import PurePath + +from odoo.addons.component.core import Component + + +class EdiStorageListener(Component): + _name = "edi.storage.component.listener" + _inherit = "base.event.listener" + + def _move_file(self, storage, from_dir_str, to_dir_str, filename): + from_dir = PurePath(from_dir_str) + to_dir = PurePath(to_dir_str) + # - storage.list_files now includes path in fs_storage, breaking change + # - we remove path + # TODO: clean this up, .list_files is deprecated in fs_storage + files = storage.list_files(from_dir.as_posix()) + files = [os.path.basename(f) for f in files] + if filename not in files: + return False + # TODO: clean this up, .move_files is deprecated in fs_storage + self._add_post_commit_hook( + storage.move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() + ) + return True + + def _add_post_commit_hook(self, move_func, sftp_filepath, sftp_destination_path): + """Add hook after commit to move the file when transaction is over.""" + self.env.cr.postcommit.add( + functools.partial(move_func, sftp_filepath, sftp_destination_path) + ) + + def on_edi_exchange_done(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + done_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_done + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() + if not done_dir: + return res + res = self._move_file(storage, pending_dir, done_dir, file) + if not res: + # If a file previously failed it should have been previously + # moved to the error dir, therefore it is not present in the + # pending dir and we need to retry from error dir. + res = self._move_file(storage, error_dir, done_dir, file) + return res + + def on_edi_exchange_error(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() + if error_dir: + res = self._move_file(storage, pending_dir, error_dir, file) + return res diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py new file mode 100644 index 000000000..c10a6e629 --- /dev/null +++ b/edi_storage_oca/components/receive.py @@ -0,0 +1,20 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageReceiveComponent(Component): + + _name = "edi.storage.component.receive" + _inherit = [ + "edi.component.receive.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.receive" + + def receive(self): + path = self._get_remote_file_path("pending") + # TODO: clean this up, .get is deprecated in fs_storage + filedata = self.storage.get(path.as_posix()) + return filedata diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py new file mode 100644 index 000000000..fe1437ca2 --- /dev/null +++ b/edi_storage_oca/components/send.py @@ -0,0 +1,37 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageSendComponent(Component): + + _name = "edi.storage.component.send" + _inherit = [ + "edi.component.send.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.send" + + def send(self): + # If the file has been sent already, refresh its state + # TODO: double check if this is useless + # since the backend checks the state already + checker = self.component(usage="storage.check") + result = checker.check() + if not result: + # all good here + return True + filedata = self.exchange_record.exchange_file + path = self._get_remote_file_path("pending") + # TODO: clean this up, .add is deprecated in fs_storage + self.storage.add(path.as_posix(), filedata, binary=False) + # TODO: delegate this to generic storage backend + # except paramiko.ssh_exception.AuthenticationException: + # # TODO this exc handling should be moved to sftp backend IMO + # error = _("Authentication error") + # state = "error_on_send" + # TODO: catch other specific exceptions + # this will swallow all the exceptions! + return True diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml new file mode 100644 index 000000000..df21892a0 --- /dev/null +++ b/edi_storage_oca/data/cron.xml @@ -0,0 +1,17 @@ + + + + EDI backend storage check pending input + + + 1 + hours + -1 + + + code + model.search([('storage_id', '!=', False)])._storage_cron_check_pending_input() + + diff --git a/edi_storage_oca/data/job_channel_data.xml b/edi_storage_oca/data/job_channel_data.xml new file mode 100644 index 000000000..b3b3b770d --- /dev/null +++ b/edi_storage_oca/data/job_channel_data.xml @@ -0,0 +1,6 @@ + + + edi_storage + + + diff --git a/edi_storage_oca/data/queue_job_function_data.xml b/edi_storage_oca/data/queue_job_function_data.xml new file mode 100644 index 000000000..196528419 --- /dev/null +++ b/edi_storage_oca/data/queue_job_function_data.xml @@ -0,0 +1,7 @@ + + + + _storage_create_record_if_missing + + + diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml new file mode 100644 index 000000000..83e26cc2a --- /dev/null +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -0,0 +1,14 @@ + + + + Storage Demo EDI backend + + + demo_in/pending + demo_in/done + demo_in/error + demo_out/pending + demo_out/done + demo_out/error + + diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py new file mode 100644 index 000000000..f34a72163 --- /dev/null +++ b/edi_storage_oca/models/__init__.py @@ -0,0 +1,2 @@ +from . import edi_backend +from . import edi_exchange_type diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py new file mode 100644 index 000000000..6751247a3 --- /dev/null +++ b/edi_storage_oca/models/edi_backend.py @@ -0,0 +1,169 @@ +# Copyright 2020 ACSONE SA +# @author Simone Orsi +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import logging +import os + +from odoo import fields, models + +_logger = logging.getLogger(__name__) + + +class EDIBackend(models.Model): + + _inherit = "edi.backend" + + storage_id = fields.Many2one( + string="FS Storage", + comodel_name="fs.storage", + help="Storage for in-out files", + ondelete="restrict", + ) + """ + We assume the exchanges happen it 2 ways (input, output) + and we have a hierarchy of directory like: + + from_A_to_B + |- pending + |- done + |- error + from_B_to_A + |- pending + |- done + |- error + + where A and B are the partners exchanging data and they are in turn + sender and receiver and vice versa. + """ + # TODO: these paths should probably be by type instead + # Here we can maybe set a common root folder for this exchange. + input_dir_pending = fields.Char( + "Input pending directory", help="Path to folder for pending operations" + ) + input_dir_done = fields.Char( + "Input done directory", help="Path to folder for doneful operations" + ) + input_dir_error = fields.Char( + "Input error directory", help="Path to folder for error operations" + ) + output_dir_pending = fields.Char( + "Output pending directory", help="Path to folder for pending operations" + ) + output_dir_done = fields.Char( + "Output done directory", help="Path to folder for doneful operations" + ) + output_dir_error = fields.Char( + "Output error directory", help="Path to folder for error operations" + ) + + _storage_actions = ("check", "send", "receive") + + def _get_component_usage_candidates(self, exchange_record, key): + candidates = super()._get_component_usage_candidates(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return candidates + return ["storage.{}".format(key)] + candidates + + def _component_match_attrs(self, exchange_record, key): + # Override to inject storage_type + res = super()._component_match_attrs(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return res + res["storage_type"] = self.storage_id.protocol + return res + + def _component_sort_key(self, component_class): + res = super()._component_sort_key(component_class) + # Override to give precedence by storage_type when needed. + if not self.storage_id: + return res + return (1 if getattr(component_class, "_storage_type", False) else 0,) + res + + def _storage_cron_check_pending_input(self, **kw): + for backend in self: + backend._storage_check_pending_input(**kw) + + def _storage_check_pending_input(self, **kw): + """Create new exchange records if new files found. + + Collect input exchange types and for each of them, + check by pattern if the a new exchange record is required. + """ + self.ensure_one() + if not self.storage_id or not self.input_dir_pending: + _logger.info( + "%s ignored: no storage and/or input directory specified.", self.name + ) + return False + + exchange_types = self.env["edi.exchange.type"].search( + self._storage_exchange_type_pending_input_domain() + ) + for exchange_type in exchange_types: + # NOTE: this call might keep hanging the cron + # if the remote storage is slow (eg: too many files) + # We should probably run this code in a separate job per exchange type. + file_names = self._storage_get_input_filenames(exchange_type) + _logger.info( + "Processing exchange type '%s': found %s files to process", + exchange_type.display_name, + len(file_names), + ) + for file_name in file_names: + self.with_delay()._storage_create_record_if_missing( + exchange_type, file_name + ) + return True + + def _storage_exchange_type_pending_input_domain(self): + """Domain for retrieving input exchange types.""" + return [ + ("backend_type_id", "=", self.backend_type_id.id), + ("direction", "=", "input"), + "|", + ("backend_id", "=", False), + ("backend_id", "=", self.id), + ] + + def _storage_create_record_if_missing(self, exchange_type, remote_file_name): + """Create a new exchange record for given type and file name if missing.""" + file_name = os.path.basename(remote_file_name) + extra_domain = [("exchange_filename", "=", file_name)] + existing = self._find_existing_exchange_records( + exchange_type, extra_domain=extra_domain, count_only=True + ) + if existing: + return + record = self.create_record( + exchange_type.code, self._storage_new_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return record.identifier + + def _storage_get_input_filenames(self, exchange_type): + full_input_dir_pending = exchange_type._storage_fullpath( + self.input_dir_pending + ).as_posix() + if not exchange_type.exchange_filename_pattern: + # If there is not pattern, return everything + # TODO: clean this up, .list_files is deprecated in fs_storage + filenames = [ + x + for x in self.storage_id.list_files(full_input_dir_pending) + if x.strip("/") + ] + return filenames + + bits = [exchange_type.exchange_filename_pattern] + if exchange_type.exchange_file_ext: + bits.append(r"\." + exchange_type.exchange_file_ext) + pattern = "".join(bits) + # TODO: clean this up, .find_files is deprecated in fs_storage + full_paths = self.storage_id.find_files(pattern, full_input_dir_pending) + pending_path_len = len(full_input_dir_pending) + return [p[pending_path_len:].strip("/") for p in full_paths] + + def _storage_new_exchange_record_vals(self, file_name): + return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py new file mode 100644 index 000000000..258576edc --- /dev/null +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -0,0 +1,62 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from pathlib import PurePath + +from odoo import fields, models + + +class EDIExchangeType(models.Model): + _inherit = "edi.exchange.type" + + # Extend help to explain new usage. + exchange_filename_pattern = fields.Char( + help="For output exchange types this should be a formatting string " + "with the following variables available (to be used between " + "brackets, `{}`): `exchange_record`, `record_name`, `type` and " + "`dt`. For instance, a valid string would be " + "{record_name}-{type.code}-{dt}\n" + "For input exchange types related to storage backends " + "it should be a regex expression to filter " + "the files to be fetched from the pending directory in the related " + "storage. E.g: `.*my-type-[0-9]*.\\.csv`" + ) + + def _storage_path(self): + """Retrieve specific path for current exchange type. + + In your exchange type you can pass this config: + + storage: + # simple string + path: path/to/file + + Or + + storage: + # name of the param containing the path + path_config_param: path/to/file + + Thanks to the param you could even configure it by env. + """ + self.ensure_one() + storage_settings = self.advanced_settings.get("storage", {}) + path = storage_settings.get("path") + if path: + return PurePath(path) + path_config_param = storage_settings.get("path_config_param") + if path_config_param: + icp = self.env["ir.config_parameter"].sudo() + path = icp.get_param(path_config_param) + if path: + return PurePath(path) + + def _storage_fullpath(self, directory=None, filename=None): + self.ensure_one() + path_prefix = self._storage_path() + path = PurePath((directory or "").strip().rstrip("/")) + if path_prefix: + path = path_prefix / path + if filename: + path = path / filename.strip("/") + return path diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000..fdb4b1c8a --- /dev/null +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -0,0 +1,4 @@ +* Simone Orsi +* Foram Shah +* Lois Rilo +* Duong (Tran Quoc) diff --git a/edi_storage_oca/readme/CREDITS.rst b/edi_storage_oca/readme/CREDITS.rst new file mode 100644 index 000000000..5511b681b --- /dev/null +++ b/edi_storage_oca/readme/CREDITS.rst @@ -0,0 +1 @@ +The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp. diff --git a/edi_storage_oca/readme/DESCRIPTION.rst b/edi_storage_oca/readme/DESCRIPTION.rst new file mode 100644 index 000000000..bdef14a8a --- /dev/null +++ b/edi_storage_oca/readme/DESCRIPTION.rst @@ -0,0 +1,23 @@ +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. diff --git a/edi_storage_oca/readme/ROADMAP.rst b/edi_storage_oca/readme/ROADMAP.rst new file mode 100644 index 000000000..fd353688e --- /dev/null +++ b/edi_storage_oca/readme/ROADMAP.rst @@ -0,0 +1 @@ +* clean deprecated methods in the storage diff --git a/edi_storage_oca/readme/USAGE.rst b/edi_storage_oca/readme/USAGE.rst new file mode 100644 index 000000000..a47c64c84 --- /dev/null +++ b/edi_storage_oca/readme/USAGE.rst @@ -0,0 +1 @@ +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. diff --git a/edi_storage_oca/security/ir_model_access.xml b/edi_storage_oca/security/ir_model_access.xml new file mode 100644 index 000000000..d50ace4c4 --- /dev/null +++ b/edi_storage_oca/security/ir_model_access.xml @@ -0,0 +1,12 @@ + + + + access_fs_storage EDI manager + + + + + + + + diff --git a/edi_storage_oca/static/description/icon.png b/edi_storage_oca/static/description/icon.png new file mode 100644 index 000000000..3a0328b51 Binary files /dev/null and b/edi_storage_oca/static/description/icon.png differ diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html new file mode 100644 index 000000000..3bc24a70c --- /dev/null +++ b/edi_storage_oca/static/description/index.html @@ -0,0 +1,461 @@ + + + + + + +EDI Storage backend support + + + +
+

EDI Storage backend support

+ + +

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

+

Allow exchange files using storage backends from OCA/storage.

+

This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files.

+

Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files).

+

Inside this folder you have this hierarchy:

+
+input/output folder
+    |- pending
+    |- done
+    |- error
+
+
    +
  • pending folder contains files that have been just sent
    • +
  • done folder contains files that have been processes successfully
    • +
  • error folder contains files with errors and cannot be processed
    • +
+

The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly.

+

Table of contents

+ +
+

Usage

+

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

+
+
+

Known issues / Roadmap

+
    +
  • clean deprecated methods in the storage
    • +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE
    • +
+
+
+

Contributors

+ +
+
+

Other credits

+

The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp.

+
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/edi-framework project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py new file mode 100644 index 000000000..18bc95300 --- /dev/null +++ b/edi_storage_oca/tests/__init__.py @@ -0,0 +1,5 @@ +from . import test_edi_backend_storage +from . import test_components_base +from . import test_component_match +from . import test_edi_storage_listener +from . import test_exchange_type diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py new file mode 100644 index 000000000..7db43ce57 --- /dev/null +++ b/edi_storage_oca/tests/common.py @@ -0,0 +1,167 @@ +# Copyright 2020 ACSONE SA/NV () +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). +import base64 +import functools +from unittest import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentTestCase + +FS_STORAGE_MOCK_PATH = "odoo.addons.fs_storage.models.fs_storage.FSStorage" + + +class TestEDIStorageBase(EDIBackendCommonComponentTestCase): + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + @classmethod + def _setup_records(cls): + super()._setup_records() + cls.filedata = base64.b64encode(b"This is a simple file") + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": cls.filedata, + } + cls.record = cls.backend.create_record("test_csv_output", vals) + cls.record_input = cls.backend.create_record("test_csv_input", vals) + + cls.fakepath = "/tmp/{}".format(cls._filename(cls)) + with open(cls.fakepath, "w+b") as fakefile: + fakefile.write(b"filecontent") + + cls.fakepath_ack = "/tmp/{}.ack".format(cls._filename(cls)) + with open(cls.fakepath_ack, "w+b") as fakefile: + fakefile.write(b"ACK filecontent") + + cls.fakepath_error = "/tmp/{}.error".format(cls._filename(cls)) + with open(cls.fakepath_error, "w+b") as fakefile: + fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + + cls.fakepath_input_pending_1 = "/tmp/test-input-001.csv" + with open(cls.fakepath_input_pending_1, "w+b") as fakefile: + fakefile.write(b"I received this in my storage.") + + cls.fakepath_input_pending_2 = "/tmp/test-input-002.csv" + with open(cls.fakepath_input_pending_2, "w+b") as fakefile: + fakefile.write(b"I received that in my storage.") + + cls.checker = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record}, + ) + cls.checker_input = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record_input}, + ) + cls.sender = cls.backend._find_component( + cls.partner._name, + ["storage.send"], + work_ctx={"exchange_record": cls.record}, + ) + return + + def setUp(self): + super().setUp() + self._fs_storage_calls = [] + + def _filename(self, record=None, ack=False): + record = record or self.record + if ack: + record.type_id.ack_type_id._make_exchange_filename(record) + return record.exchange_filename + + def _file_fullpath(self, state, record=None, ack=False, fname=None, checker=None): + record = record or self.record + checker = checker or self.checker + if not fname: + fname = self._filename(record, ack=ack) + if state == "error-report": + # Exception as we read from the same path but w/ error suffix + state = "error" + fname += ".error" + return checker._get_remote_file_path(state, filename=fname).as_posix() + + def _mocked_backend_get(self, mocked_paths, path, **kwargs): + self._fs_storage_calls.append(path) + if mocked_paths.get(path): + with open(mocked_paths.get(path), "rb") as remote_file: + return remote_file.read() + raise FileNotFoundError() + + def _mocked_backend_add(self, path, data, **kwargs): + self._fs_storage_calls.append(path) + + def _mocked_backend_list_files(self, mocked_paths, path, **kwargs): + files = [] + path_length = len(path) + for p in mocked_paths.keys(): + if path in p and path != p: + files.append(p[path_length:]) + return files + + def _mock_fs_storage_get(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_get, mocked_paths) + return mock.patch(FS_STORAGE_MOCK_PATH + ".get", mocked) + + def _mock_fs_storage_add(self): + return mock.patch(FS_STORAGE_MOCK_PATH + ".add", self._mocked_backend_add) + + def _mock_fs_storage_list_files(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) + return mock.patch(FS_STORAGE_MOCK_PATH + ".list_files", mocked) + + def _mock_fs_storage_find_files(self, result): + def _result(self, pattern, relative_path=None, **kw): + return result + + return mock.patch(FS_STORAGE_MOCK_PATH + ".find_files", _result) + + def _test_result( + self, + record, + expected_values, + expected_messages=None, + state_paths=None, + ): + state_paths = state_paths or ("done", "pending", "error") + # Paths will be something like: + # [ + # 'demo_out/pending/$filename.csv', + # 'demo_out/pending/$filename.csv', + # 'demo_out/error/$filename.csv', + # ] + for state in state_paths: + path = self._file_fullpath(state, record=record) + self.assertIn(path, self._fs_storage_calls) + self.assertRecordValues(record, [expected_values]) + if expected_messages: + # consider only edi related messages + messages = record.record.message_ids.filtered( + lambda x: "edi-exchange" in x.body + ) + self.assertEqual(len(messages), len(expected_messages)) + for msg_rec, expected in zip(messages, expected_messages): + self.assertIn(expected["message"], msg_rec.body) + self.assertIn("level-" + expected["level"], msg_rec.body) + # TODO: test content of file sent + + def _test_send(self, record, mocked_paths=None): + with self._mock_fs_storage_add(): + if mocked_paths: + with self._mock_fs_storage_get(mocked_paths): + self.backend.exchange_send(record) + else: + self.backend.exchange_send(record) + + def _test_run_cron(self, mocked_paths, skip_sent=True): + with self._mock_fs_storage_add(): + with self._mock_fs_storage_get(mocked_paths): + self.backend._cron_check_output_exchange_sync(skip_sent=skip_sent) + + def _test_run_cron_pending_input(self, mocked_paths): + with self._mock_fs_storage_add(): + with self._mock_fs_storage_list_files(mocked_paths): + self.backend._storage_cron_check_pending_input() diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py new file mode 100644 index 000000000..ff68e891c --- /dev/null +++ b/edi_storage_oca/tests/test_component_match.py @@ -0,0 +1,93 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def test_component_match(self): + """Lookup with special match method.""" + + class SFTPCheck(Component): + _name = "sftp.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _backend_type = "demo_backend" + _storage_type = "sftp" + + class SFTPSend(Component): + _name = "sftp.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _backend_type = "demo_backend" + _storage_type = "sftp" + + class S3Check(Component): + _name = "s3.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _exchange_type = "test_csv_output" + _storage_type = "s3" + + class S3Send(Component): + _name = "s3.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _exchange_type = "test_csv_output" + _storage_type = "s3" + + self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + + # Record not relevant for these tests + work_ctx = {"exchange_record": self.env["edi.exchange.record"].browse()} + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_type="s3", + ) + self.assertEqual(component._name, S3Check._name) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_type="sftp", + ) + self.assertEqual(component._name, SFTPCheck._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_type="sftp", + ) + self.assertEqual(component._name, SFTPSend._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_type="s3", + ) + self.assertEqual(component._name, S3Send._name) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py new file mode 100644 index 000000000..d141be3e6 --- /dev/null +++ b/edi_storage_oca/tests/test_components_base.py @@ -0,0 +1,45 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from unittest import mock + +from .common import FS_STORAGE_MOCK_PATH, TestEDIStorageBase + + +class EDIStorageComponentTestCase(TestEDIStorageBase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + direction, state, filename = _args + if direction == "input": + checker = self.checker_input + else: + checker = self.checker + path_obj = checker._get_remote_file_path(state, filename) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.checker_input._get_remote_file_path("WHATEVER", "foo.csv") + + def test_get_remote_file(self): + with mock.patch(FS_STORAGE_MOCK_PATH + ".get") as mocked: + self.checker._get_remote_file("pending") + mocked.assert_called_with( + "demo_out/pending/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("done") + mocked.assert_called_with( + "demo_out/done/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("error") + mocked.assert_called_with( + "demo_out/error/{}".format(self._filename(self.record)), binary=False + ) diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py new file mode 100644 index 000000000..e688a3b1b --- /dev/null +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -0,0 +1,239 @@ +# Copyright 2020 ACSONE SA/NV () +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). +from freezegun import freeze_time + +from odoo.tools import mute_logger + +from .common import TestEDIStorageBase + +LOGGERS = ( + "odoo.addons.edi_storage_oca.components.check", + "odoo.addons.edi_oca.models.edi_backend", +) + + +@freeze_time("2020-10-21 10:30:00") +class TestEDIBackendOutput(TestEDIStorageBase): + @mute_logger(*LOGGERS) + def test_export_file_sent(self): + """Send, no errors.""" + self.record.edi_exchange_state = "output_pending" + mocked_paths = {self._file_fullpath("pending"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + # self._test_send(self.record, mocked_paths=mocked_paths) + self._test_run_cron(mocked_paths) + self._test_result( + self.record, + {"edi_exchange_state": "output_sent"}, + expected_messages=[ + { + "message": self.record._exchange_status_message("send_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done(self): + """Already sent, successfully.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = {self._file_fullpath("done"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + self._test_run_cron(mocked_paths, skip_sent=False) + # As we simulate to find a file in `done` folder, + # we should get the final good state + # and only one call to ftp + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + # FIXME: ack should be handle as an incoming record (new machinery to be added) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_not_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # } + # self._test_run_cron(mocked_paths) + # # No ack file found, warning message is posted + # self._test_result( + # self.record, + # {"edi_exchange_state": "output_sent_and_processed"}, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_missing"), + # "level": "warning", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) + + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # self._file_fullpath("done", ack=True): self.fakepath_ack, + # } + # self._test_run_cron(mocked_paths) + # # Found ack file, set on record + # self._test_result( + # self.record, + # { + # "edi_exchange_state": "output_sent_and_processed", + # "ack_file": base64.b64encode(b"ACK filecontent"), + # }, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_received"), + # "level": "info", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) + + @mute_logger(*LOGGERS) + def test_already_sent_process_error(self): + """Already sent, error process.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = { + self._file_fullpath("error"): self.fakepath, + self._file_fullpath("error-report"): self.fakepath_error, + } + self._test_run_cron(mocked_paths, skip_sent=False) + # As we simulate to find a file in `error` folder, + # we should get a call for: done, error and then the read of the report. + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_cron_full_flow(self): + """Already sent, update the state via cron.""" + self.record.edi_exchange_state = "output_sent" + rec1 = self.record + partner2 = self.env.ref("base.res_partner_2") + partner3 = self.env.ref("base.res_partner_3") + rec2 = self.record.copy( + { + "model": partner2._name, + "res_id": partner2.id, + "exchange_filename": "rec2.csv", + "edi_exchange_state": "output_sent", + } + ) + rec3 = self.record.copy( + { + "model": partner3._name, + "res_id": partner3.id, + "exchange_filename": "rec3.csv", + "edi_exchange_state": "output_sent_and_error", + } + ) + mocked_paths = { + self._file_fullpath("done", record=rec1): self.fakepath, + self._file_fullpath("error", record=rec2): self.fakepath, + self._file_fullpath("error-report", record=rec2): self.fakepath_error, + self._file_fullpath("done", record=rec3): self.fakepath, + } + self._test_run_cron(mocked_paths, skip_sent=False) + self._test_result( + rec1, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec1._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + self._test_result( + rec2, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": rec2._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + self._test_result( + rec3, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec3._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_create_input_exchange_file_from_file_received_no_pattern(self): + exch_type = self.exchange_type_in + exch_type.exchange_filename_pattern = "" + input_dir = "/test_input/pending/" + file_names = ["some-file.csv", "another-file.csv"] + self.backend.input_dir_pending = input_dir + mocked_paths = { + input_dir: "/tmp/", + self._file_fullpath( + "pending", fname=file_names[0], checker=self.checker_input + ): self.fakepath_input_pending_1, + self._file_fullpath( + "pending", fname=file_names[1], checker=self.checker_input + ): self.fakepath_input_pending_2, + } + existing_records = self.env["edi.exchange.record"].search( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + # Run cron action: + found_files = [input_dir + fname for fname in file_names] + with self._mock_fs_storage_find_files(found_files): + self._test_run_cron_pending_input(mocked_paths) + new_records = self.env["edi.exchange.record"].search( + [ + ("backend_id", "=", self.backend.id), + ("type_id", "=", exch_type.id), + ("id", "not in", existing_records.ids), + ] + ) + self.assertEqual(len(new_records), 2) + for rec in new_records: + self.assertIn(rec.exchange_filename, file_names) + self.assertEqual(rec.edi_exchange_state, "input_pending") diff --git a/edi_storage_oca/tests/test_edi_storage_listener.py b/edi_storage_oca/tests/test_edi_storage_listener.py new file mode 100644 index 000000000..e7aa1581c --- /dev/null +++ b/edi_storage_oca/tests/test_edi_storage_listener.py @@ -0,0 +1,70 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 +from unittest import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.fake_components import FakeInputProcess + +LISTENER_MOCK_PATH = ( + "odoo.addons.edi_storage_oca.components.listener.EdiStorageListener" +) + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + cls._build_components( + cls, + FakeInputProcess, + ) + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": base64.b64encode(b"1234"), + } + cls.record = cls.backend.create_record("test_csv_input", vals) + cls.fake_move_args = None + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def setUp(self): + super().setUp() + FakeInputProcess.reset_faked() + + def _move_file_mocked(self, *args): + self.fake_move_args = [*args] + if not all([*args]): + return False + return True + + def _mock_listener_move_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._move_file", self._move_file_mocked) + + def test_01_process_record_success(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_done) + self.assertEqual(filename, self.record.exchange_filename) + + def test_02_process_record_with_error(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record._set_file_content("TEST %d" % self.record.id) + self.record.with_context( + test_break_process="OOPS! Something went wrong :(" + ).action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_error) + self.assertEqual(filename, self.record.exchange_filename) diff --git a/edi_storage_oca/tests/test_exchange_type.py b/edi_storage_oca/tests/test_exchange_type.py new file mode 100644 index 000000000..6b886eab3 --- /dev/null +++ b/edi_storage_oca/tests/test_exchange_type.py @@ -0,0 +1,67 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonTestCase + + +class EDIExchangeTypeTestCase(EDIBackendCommonTestCase): + def _check_test_storage_fullpath(self, wanted_fullpath, directory, filename): + fullpath = self.exchange_type_out._storage_fullpath(directory, filename) + self.assertEqual(fullpath.as_posix(), wanted_fullpath) + + def _do_test_storage_fullpath(self, prefix=""): + # Test with no directory and no filename + wanted_fullpath = prefix or "." + self._check_test_storage_fullpath(wanted_fullpath, None, None) + + # Test with directory + directory = "test_directory" + wanted_fullpath = f"{prefix}/{directory}" if prefix else directory + self._check_test_storage_fullpath(wanted_fullpath, directory, None) + + # Test with filename + filename = "test_filename.csv" + wanted_fullpath = f"{prefix}/{filename}" if prefix else filename + self._check_test_storage_fullpath(wanted_fullpath, None, filename) + + # Test with directory and filename + wanted_fullpath = ( + f"{prefix}/{directory}/{filename}" if prefix else f"{directory}/{filename}" + ) + self._check_test_storage_fullpath(wanted_fullpath, directory, filename) + + def test_storage_fullpath(self): + """ + Test storage fullpath defined into advanced settings. + Example of pattern: + storage: + # simple string + path: path/to/file + # name of the param containing the path + path_config_param: path/to/file + """ + + # Test without any prefix + self._do_test_storage_fullpath() + + # Force path on advanced settings + prefix = "prefix/path" + self.exchange_type_out.advanced_settings_edit = f""" + storage: + path: {prefix} + """ + self._do_test_storage_fullpath(prefix=prefix) + + # Force path on advanced settings using config param, but not defined + self.exchange_type_out.advanced_settings_edit = """ + storage: + path_config_param: prefix_path_config_param + """ + self._do_test_storage_fullpath() + + # Define config param + prefix = "prefix/path/by/config/param" + self.env["ir.config_parameter"].sudo().set_param( + "prefix_path_config_param", prefix + ) + self._do_test_storage_fullpath(prefix=prefix) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml new file mode 100644 index 000000000..ead51ef51 --- /dev/null +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -0,0 +1,22 @@ + + + + edi.backend + + + + + + + + + + + + + + + + + + diff --git a/setup/edi_storage_oca/odoo/addons/edi_storage_oca b/setup/edi_storage_oca/odoo/addons/edi_storage_oca new file mode 120000 index 000000000..7c6d68f63 --- /dev/null +++ b/setup/edi_storage_oca/odoo/addons/edi_storage_oca @@ -0,0 +1 @@ +../../../../edi_storage_oca \ No newline at end of file diff --git a/setup/edi_storage_oca/setup.py b/setup/edi_storage_oca/setup.py new file mode 100644 index 000000000..28c57bb64 --- /dev/null +++ b/setup/edi_storage_oca/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)