[16.0][MIG] edi_storage_oca: Migration to 16.0

edi_storage_oca/README.rst

@@ -0,0 +1,116 @@
+===========================
+EDI Storage backend support
+===========================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:3806d6eaf0c3ce46d7d24b5c5e002f0c49a42390385f23e4480a33dd94e85451
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Beta
+.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html
+    :alt: License: LGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi--framework-lightgray.png?logo=github
+    :target: https://github.com/OCA/edi-framework/tree/16.0/edi_storage_oca
+    :alt: OCA/edi-framework
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/edi-framework-16-0/edi-framework-16-0-edi_storage_oca
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=16.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+Allow exchange files using storage backends from `OCA/storage`.
+
+This module adds a storage backend relation on the EDI backend.
+There you can configure the backend to be used (most often and SFTP)
+and the paths where to read or put files.
+
+Often the convention when exchanging files via SFTP
+is to have one input forder (to receive files)
+and an output folder (to send files).
+
+Inside this folder you have this hierarchy::
+
+    input/output folder
+        |- pending
+        |- done
+        |- error
+
+* `pending` folder contains files that have been just sent
+* `done` folder contains files that have been processes successfully
+* `error` folder contains files with errors and cannot be processed
+
+The storage handlers take care of reading files and putting files
+in/from the right place and update exchange records data accordingly.
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Usage
+=====
+
+Go to "EDI -> EDI backend" then configure your backend to use a storage backend.
+
+Known issues / Roadmap
+======================
+
+* clean deprecated methods in the storage
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/edi-framework/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/edi-framework/issues/new?body=module:%20edi_storage_oca%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+~~~~~~~
+
+* ACSONE
+
+Contributors
+~~~~~~~~~~~~
+
+* Simone Orsi <[email protected]>
+* Foram Shah <[email protected]>
+* Lois Rilo <[email protected]>
+* Duong (Tran Quoc) <[email protected]>
+
+Other credits
+~~~~~~~~~~~~~
+
+The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp.
+
+Maintainers
+~~~~~~~~~~~
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+This module is part of the `OCA/edi-framework <https://github.com/OCA/edi-framework/tree/16.0/edi_storage_oca>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

edi_storage_oca/__init__.py

@@ -0,0 +1,2 @@
+from . import components
+from . import models

edi_storage_oca/__manifest__.py

@@ -0,0 +1,24 @@
+# Copyright 2020 ACSONE
+# @author: Simone Orsi <[email protected]>
+# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
+
+{
+    "name": "EDI Storage backend support",
+    "summary": """
+    Base module to allow exchanging files via storage backend (eg: SFTP).
+    """,
+    "version": "16.0.1.0.0",
+    "development_status": "Beta",
+    "license": "LGPL-3",
+    "website": "https://github.com/OCA/edi-framework",
+    "author": "ACSONE,Odoo Community Association (OCA)",
+    "depends": ["edi_oca", "fs_storage", "component"],
+    "data": [
+        "data/cron.xml",
+        "data/job_channel_data.xml",
+        "data/queue_job_function_data.xml",
+        "security/ir_model_access.xml",
+        "views/edi_backend_views.xml",
+    ],
+    "demo": ["demo/edi_backend_demo.xml"],
+}

edi_storage_oca/components/__init__.py

@@ -0,0 +1,5 @@
+from . import base
+from . import check
+from . import send
+from . import receive
+from . import listener

edi_storage_oca/components/base.py

@@ -0,0 +1,86 @@
+# Copyright 2020 ACSONE
+# Copyright 2022 Camptocamp
+# @author: Simone Orsi <[email protected]>
+# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
+import logging
+from pathlib import PurePath
+
+from odoo.addons.component.core import AbstractComponent
+
+_logger = logging.getLogger(__file__)
+
+
+class EDIStorageComponentMixin(AbstractComponent):
+
+    _name = "edi.storage.component.mixin"
+    _inherit = "edi.component.mixin"
+    # Components having `_storage_type` will have precedence.
+    # If the value is not set, generic components will be used.
+    _storage_type = None
+
+    @classmethod
+    def _component_match(cls, work, usage=None, model_name=None, **kw):
+        res = super()._component_match(work, usage=usage, model_name=model_name, **kw)
+        storage_type = kw.get("storage_type")
+        if storage_type and cls._storage_type:
+            return cls._storage_type == storage_type
+        return res
+
+    @property
+    def storage(self):
+        return self.backend.storage_id
+
+    def _dir_by_state(self, direction, state):
+        """Return remote directory path by direction and state.
+
+        :param direction: string stating direction of the exchange
+        :param state: string stating state of the exchange
+        :return: PurePath object
+        """
+        assert direction in ("input", "output")
+        assert state in ("pending", "done", "error")
+        return PurePath(
+            (self.backend[direction + "_dir_" + state] or "").strip().rstrip("/")
+        )
+
+    def _get_remote_file_path(self, state, filename=None):
+        """Retrieve remote path for current exchange record."""
+        filename = filename or self.exchange_record.exchange_filename
+        direction = self.exchange_record.direction
+        directory = self._dir_by_state(direction, state).as_posix()
+        path = self.exchange_record.type_id._storage_fullpath(
+            directory=directory, filename=filename
+        )
+        return path
+
+    def _get_remote_file(self, state, filename=None, binary=False):
+        """Get file for current exchange_record in the given destination state.
+
+        :param state: string ("pending", "done", "error")
+        :param filename: custom file name, exchange_record filename used by default
+        :return: remote file content as string
+        """
+        path = self._get_remote_file_path(state, filename=filename)
+        try:
+            # TODO: support match via pattern (eg: filename-prefix-*)
+            # otherwise is impossible to retrieve input files and acks
+            # (the date will never match)
+            # TODO: clean this up, .get is deprecated in fs_storage
+            return self.storage.get(path.as_posix(), binary=binary)
+        except FileNotFoundError:
+            _logger.info(
+                "Ignored FileNotFoundError when trying "
+                "to get file %s into path %s for state %s",
+                filename,
+                path,
+                state,
+            )
+            return None
+        except OSError:
+            _logger.info(
+                "Ignored OSError when trying to get file %s into path %s for state %s",
+                filename,
+                path,
+                state,
+            )
+            return None

edi_storage_oca/components/check.py

@@ -0,0 +1,71 @@
+# Copyright 2020 ACSONE
+# @author: Simone Orsi <[email protected]>
+# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
+
+import logging
+
+from odoo.tools import pycompat
+
+from odoo.addons.component.core import Component
+
+_logger = logging.getLogger(__name__)
+
+
+class EDIStorageCheckComponentMixin(Component):
+
+    _name = "edi.storage.component.check"
+    _inherit = [
+        "edi.component.check.mixin",
+        "edi.storage.component.mixin",
+    ]
+    _usage = "storage.check"
+
+    def check(self):
+        return self._exchange_output_check()
+
+    def _exchange_output_check(self):
+        """Check status output exchange and update record.
+
+        1. check if the file has been processed already (done)
+        2. if yes, post message and exit
+        3. if not, check for errors
+        4. if no errors, return
+
+        :return: boolean
+            * False if there's nothing else to be done
+            * True if file still need action
+        """
+        if self._get_remote_file("done"):
+            _logger.info(
+                "%s done",
+                self.exchange_record.identifier,
+            )
+            if (
+                not self.exchange_record.edi_exchange_state
+                == "output_sent_and_processed"
+            ):
+                self.exchange_record.edi_exchange_state = "output_sent_and_processed"
+                self.exchange_record._notify_done()
+            return False
+
+        error = self._get_remote_file("error")
+        if error:
+            _logger.info(
+                "%s error",
+                self.exchange_record.identifier,
+            )
+            # Assume a text file will be placed there w/ the same name and error suffix
+            err_filename = self.exchange_record.exchange_filename + ".error"
+            error_report = (
+                self._get_remote_file("error", filename=err_filename) or "no-report"
+            )
+            if self.exchange_record.edi_exchange_state == "output_sent":
+                self.exchange_record.update(
+                    {
+                        "edi_exchange_state": "output_sent_and_error",
+                        "exchange_error": pycompat.to_text(error_report),
+                    }
+                )
+                self.exchange_record._notify_error("process_ko")
+            return False
+        return True

edi_storage_oca/components/listener.py

@@ -0,0 +1,74 @@
+# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com)
+# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl).
+
+import functools
+import os
+from pathlib import PurePath
+
+from odoo.addons.component.core import Component
+
+
+class EdiStorageListener(Component):
+    _name = "edi.storage.component.listener"
+    _inherit = "base.event.listener"
+
+    def _move_file(self, storage, from_dir_str, to_dir_str, filename):
+        from_dir = PurePath(from_dir_str)
+        to_dir = PurePath(to_dir_str)
+        # - storage.list_files now includes path in fs_storage, breaking change
+        # - we remove path
+        # TODO: clean this up, .list_files is deprecated in fs_storage
+        files = storage.list_files(from_dir.as_posix())
+        files = [os.path.basename(f) for f in files]
+        if filename not in files:
+            return False
+        # TODO: clean this up, .move_files is deprecated in fs_storage
+        self._add_post_commit_hook(
+            storage.move_files, [(from_dir / filename).as_posix()], to_dir.as_posix()
+        )
+        return True
+
+    def _add_post_commit_hook(self, move_func, sftp_filepath, sftp_destination_path):
+        """Add hook after commit to move the file when transaction is over."""
+        self.env.cr.postcommit.add(
+            functools.partial(move_func, sftp_filepath, sftp_destination_path)
+        )
+
+    def on_edi_exchange_done(self, record):
+        storage = record.backend_id.storage_id
+        res = False
+        if record.direction == "input" and storage:
+            file = record.exchange_filename
+            pending_dir = record.type_id._storage_fullpath(
+                record.backend_id.input_dir_pending
+            ).as_posix()
+            done_dir = record.type_id._storage_fullpath(
+                record.backend_id.input_dir_done
+            ).as_posix()
+            error_dir = record.type_id._storage_fullpath(
+                record.backend_id.input_dir_error
+            ).as_posix()
+            if not done_dir:
+                return res
+            res = self._move_file(storage, pending_dir, done_dir, file)
+            if not res:
+                # If a file previously failed it should have been previously
+                # moved to the error dir, therefore it is not present in the
+                # pending dir and we need to retry from error dir.
+                res = self._move_file(storage, error_dir, done_dir, file)
+        return res
+
+    def on_edi_exchange_error(self, record):
+        storage = record.backend_id.storage_id
+        res = False
+        if record.direction == "input" and storage:
+            file = record.exchange_filename
+            pending_dir = record.type_id._storage_fullpath(
+                record.backend_id.input_dir_pending
+            ).as_posix()
+            error_dir = record.type_id._storage_fullpath(
+                record.backend_id.input_dir_error
+            ).as_posix()
+            if error_dir:
+                res = self._move_file(storage, pending_dir, error_dir, file)
+        return res