From c820a09ca7b7068477b00891123857ea6b4ac229 Mon Sep 17 00:00:00 2001 From: Parul Date: Tue, 13 Feb 2024 11:56:29 -0800 Subject: [PATCH 1/4] rename ssh to sftp and graylog optional in cli --- cli.py | 91 +++++++++++++++++++++++++++------------- pyproject.toml | 2 +- src/filepass/__init__.py | 2 +- src/filepass/filepass.py | 19 +++++---- 4 files changed, 75 insertions(+), 39 deletions(-) diff --git a/cli.py b/cli.py index 680f11e..9d039f2 100644 --- a/cli.py +++ b/cli.py @@ -1,6 +1,7 @@ import logging import os import sys +from sys import stdout import graypy @@ -32,38 +33,72 @@ def main(): to_share = os.environ.get("TOSMBSHARE") to_method = os.environ.get("TOMETHOD") to_delete = os.environ.get("TODELETE") + # Environment variable added for using local or Graylog logging + use_graylog = os.environ.get("USE_GRAYLOG") - filepass_logger = logging.getLogger("filepass_logger") - filepass_logger.setLevel(logging.DEBUG) + if use_graylog == "yes": + filepass_logger = logging.getLogger("filepass_logger") + filepass_logger.setLevel(logging.DEBUG) - handler = graypy.GELFTCPHandler( - os.environ.get("GRAYLOG_SERVER"), int(os.environ.get("GRAYLOG_PORT")) - ) - filepass_logger.addHandler(handler) + handler = graypy.GELFTCPHandler( + os.environ.get("GRAYLOG_SERVER"), int(os.environ.get("GRAYLOG_PORT")) + ) + filepass_logger.addHandler(handler) + + handler_std = logging.StreamHandler(sys.stdout) + filepass_logger.addHandler(handler_std) - handler_std = logging.StreamHandler(sys.stdout) - filepass_logger.addHandler(handler_std) + logger = logging.LoggerAdapter( + filepass_logger, + { + "from_method": from_method, + "from_user": from_user, + "from_svr": from_svr, + "from_port": from_port, + "from_share": from_share, + "from_dir": from_dir, + "from_filter": from_filter, + "to_method": to_method, + "to_user": to_user, + "to_svr": to_svr, + "to_port": to_port, + "to_share": to_share, + "to_dir": to_dir, + "integration": "filepass", + "filepass_name": os.environ.get("INTEGRATION_NAME"), + }, + ) - logger = logging.LoggerAdapter( - filepass_logger, - { - "from_method": from_method, - "from_user": from_user, - "from_svr": from_svr, - "from_port": from_port, - "from_share": from_share, - "from_dir": from_dir, - "from_filter": from_filter, - "to_method": to_method, - "to_user": to_user, - "to_svr": to_svr, - "to_port": to_port, - "to_share": to_share, - "to_dir": to_dir, - "integration": "filepass", - "filepass_name": os.environ.get("INTEGRATION_NAME"), - }, - ) + else: + # Docker logger output to stdout + logFormatter = logging.Formatter( + "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s" + ) + local_logger = logging.getLogger("local_logger") + local_logger.setLevel(logging.DEBUG) + consoleHandler = logging.StreamHandler(stdout) + consoleHandler.setFormatter(logFormatter) + local_logger.addHandler(consoleHandler) + logger = logging.LoggerAdapter( + local_logger, + { + "from_method": from_method, + "from_user": from_user, + "from_svr": from_svr, + "from_port": from_port, + "from_share": from_share, + "from_dir": from_dir, + "from_filter": from_filter, + "to_method": to_method, + "to_user": to_user, + "to_svr": to_svr, + "to_port": to_port, + "to_share": to_share, + "to_dir": to_dir, + "integration": "filepass", + "filepass_name": os.environ.get("INTEGRATION_NAME"), + }, + ) try: file_pass( logger, diff --git a/pyproject.toml b/pyproject.toml index 87aec4d..ed29138 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "filepass" -version = "0.2.0" +version = "0.3.1" authors = [ { name="Orlund Norstrom", email="onorstrom@kamloops.ca" }, { name="Marco Lussetti", email="mlussetti@kamloops.ca" }, diff --git a/src/filepass/__init__.py b/src/filepass/__init__.py index d169ea3..69d6714 100644 --- a/src/filepass/__init__.py +++ b/src/filepass/__init__.py @@ -1 +1 @@ -from .filepass import file_pass, smb_connection, ssh_connection +from .filepass import file_pass, sftp_connection, smb_connection diff --git a/src/filepass/filepass.py b/src/filepass/filepass.py index 27a6f37..d3b2ebc 100644 --- a/src/filepass/filepass.py +++ b/src/filepass/filepass.py @@ -5,9 +5,9 @@ # File Transfer Types -def ssh_connection(logger, user, pw, svr, port, dir): - logger.debug("ssh://{}:{}@{}:{}{}".format(user, "passwordhere", svr, port, dir)) - fs_conn = fs.open_fs("ssh://{}:{}@{}:{}{}".format(user, pw, svr, port, dir)) +def sftp_connection(logger, user, pw, svr, port, dir): + logger.debug("sftp://{}:{}@{}:{}{}".format(user, "passwordhere", svr, port, dir)) + fs_conn = fs.open_fs("sftp://{}:{}@{}:{}{}".format(user, pw, svr, port, dir)) return fs_conn @@ -49,9 +49,9 @@ def file_pass( to_delete, ): # From File System - if from_method == "ssh": - logger.debug("Create from SSH connection") - from_fs = ssh_connection( + if from_method == "sftp": + logger.debug("Create from SFTP connection") + from_fs = sftp_connection( logger, from_user, from_pw, from_svr, from_port, from_dir ) @@ -62,9 +62,9 @@ def file_pass( ) # To File System - if to_method == "ssh": - logger.debug("Create to SSH connection") - to_fs = ssh_connection(logger, to_user, to_pw, to_svr, to_port, to_dir) + if to_method == "sftp": + logger.debug("Create to SFTP connection") + to_fs = sftp_connection(logger, to_user, to_pw, to_svr, to_port, to_dir) if to_method == "smb": logger.debug("Create to SMB connection") to_fs = smb_connection( @@ -106,4 +106,5 @@ def file_pass( logger.debug("No delete (from): {}".format(path)) from_fs.close() + to_fs.close() From 0f9bb90533703180cc6ce47bf7420d3a8644d66d Mon Sep 17 00:00:00 2001 From: Parul Date: Wed, 14 Feb 2024 10:03:07 -0800 Subject: [PATCH 2/4] graylog optional in cli based on if env_var is set --- cli.py | 85 ++++++++++++++++++---------------------------------------- 1 file changed, 26 insertions(+), 59 deletions(-) diff --git a/cli.py b/cli.py index 9d039f2..85de688 100644 --- a/cli.py +++ b/cli.py @@ -1,7 +1,6 @@ import logging import os import sys -from sys import stdout import graypy @@ -33,72 +32,40 @@ def main(): to_share = os.environ.get("TOSMBSHARE") to_method = os.environ.get("TOMETHOD") to_delete = os.environ.get("TODELETE") - # Environment variable added for using local or Graylog logging - use_graylog = os.environ.get("USE_GRAYLOG") - if use_graylog == "yes": - filepass_logger = logging.getLogger("filepass_logger") - filepass_logger.setLevel(logging.DEBUG) + filepass_logger = logging.getLogger("filepass_logger") + filepass_logger.setLevel(logging.DEBUG) + # Check if GRAYLOG SERVER configurations are set and add handler accordingly + if os.environ.get("GRAYLOG_SERVER") and os.environ.get("GRAYLOG_PORT") is not None: handler = graypy.GELFTCPHandler( os.environ.get("GRAYLOG_SERVER"), int(os.environ.get("GRAYLOG_PORT")) ) filepass_logger.addHandler(handler) - handler_std = logging.StreamHandler(sys.stdout) - filepass_logger.addHandler(handler_std) + handler_std = logging.StreamHandler(sys.stdout) + filepass_logger.addHandler(handler_std) - logger = logging.LoggerAdapter( - filepass_logger, - { - "from_method": from_method, - "from_user": from_user, - "from_svr": from_svr, - "from_port": from_port, - "from_share": from_share, - "from_dir": from_dir, - "from_filter": from_filter, - "to_method": to_method, - "to_user": to_user, - "to_svr": to_svr, - "to_port": to_port, - "to_share": to_share, - "to_dir": to_dir, - "integration": "filepass", - "filepass_name": os.environ.get("INTEGRATION_NAME"), - }, - ) - - else: - # Docker logger output to stdout - logFormatter = logging.Formatter( - "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s" - ) - local_logger = logging.getLogger("local_logger") - local_logger.setLevel(logging.DEBUG) - consoleHandler = logging.StreamHandler(stdout) - consoleHandler.setFormatter(logFormatter) - local_logger.addHandler(consoleHandler) - logger = logging.LoggerAdapter( - local_logger, - { - "from_method": from_method, - "from_user": from_user, - "from_svr": from_svr, - "from_port": from_port, - "from_share": from_share, - "from_dir": from_dir, - "from_filter": from_filter, - "to_method": to_method, - "to_user": to_user, - "to_svr": to_svr, - "to_port": to_port, - "to_share": to_share, - "to_dir": to_dir, - "integration": "filepass", - "filepass_name": os.environ.get("INTEGRATION_NAME"), - }, - ) + logger = logging.LoggerAdapter( + filepass_logger, + { + "from_method": from_method, + "from_user": from_user, + "from_svr": from_svr, + "from_port": from_port, + "from_share": from_share, + "from_dir": from_dir, + "from_filter": from_filter, + "to_method": to_method, + "to_user": to_user, + "to_svr": to_svr, + "to_port": to_port, + "to_share": to_share, + "to_dir": to_dir, + "integration": "filepass", + "filepass_name": os.environ.get("INTEGRATION_NAME"), + }, + ) try: file_pass( logger, From 1f8bb5b49c322792f5218a466854d8ccc5ea8e05 Mon Sep 17 00:00:00 2001 From: Parul Date: Wed, 14 Feb 2024 10:34:32 -0800 Subject: [PATCH 3/4] :rename target file in single file mode --- cli.py | 4 ++++ pyproject.toml | 2 +- src/filepass/filepass.py | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cli.py b/cli.py index 85de688..2fad0f9 100644 --- a/cli.py +++ b/cli.py @@ -32,6 +32,8 @@ def main(): to_share = os.environ.get("TOSMBSHARE") to_method = os.environ.get("TOMETHOD") to_delete = os.environ.get("TODELETE") + # Environment variable added for filename change in single file mode + new_filename = os.environ.get("NEW_FILENAME") filepass_logger = logging.getLogger("filepass_logger") filepass_logger.setLevel(logging.DEBUG) @@ -64,6 +66,7 @@ def main(): "to_dir": to_dir, "integration": "filepass", "filepass_name": os.environ.get("INTEGRATION_NAME"), + "new_filename": new_filename, }, ) try: @@ -86,6 +89,7 @@ def main(): to_share, to_method, to_delete, + new_filename, ) except: logger.exception("Critical error found", stack_info=True) diff --git a/pyproject.toml b/pyproject.toml index ed29138..fe4785a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "filepass" -version = "0.3.1" +version = "0.3.9" authors = [ { name="Orlund Norstrom", email="onorstrom@kamloops.ca" }, { name="Marco Lussetti", email="mlussetti@kamloops.ca" }, diff --git a/src/filepass/filepass.py b/src/filepass/filepass.py index d3b2ebc..6a7de38 100644 --- a/src/filepass/filepass.py +++ b/src/filepass/filepass.py @@ -24,8 +24,10 @@ def smb_connection(logger, user, pw, svr, port, smbshare, dir): return fs_conn -def transfer_file(from_fs, to_fs, filename): - to_fs.writefile(filename, from_fs.open(filename, "rb")) +# Boolean parameter to add ability to rename target file in single file mode +def transfer_file(from_fs, to_fs, filename, should_rename=False, new_filename=None): + target_filename = new_filename if new_filename and should_rename else filename + to_fs.writefile(target_filename, from_fs.open(filename, "rb")) def file_pass( @@ -47,6 +49,7 @@ def file_pass( to_share, to_method, to_delete, + new_filename=None, ): # From File System if from_method == "sftp": @@ -73,7 +76,8 @@ def file_pass( # Do the move walker = Walker(filter=[from_filter], ignore_errors=True, max_depth=1) - + # Create a list of files to be transferred based on the filter. + total_files = list(walker.files(from_fs)) for path in walker.files(from_fs): logger.debug("File to move: {}".format(path)) @@ -89,8 +93,13 @@ def file_pass( else: logger.debug("No delete (to): {}".format(path)) - # todo: document, paths with files should be at the lowest level (no sub dirs) - transfer_file(from_fs, to_fs, path) + # Confirm if single file mode condition is satisfied + if len(total_files) == 1 and new_filename: + should_rename = True + else: + should_rename = False + + transfer_file(from_fs, to_fs, path, should_rename, new_filename=new_filename) if from_delete == "yes" and from_fs.exists(path): logger.debug("delete (from): {}".format(path)) From df7d1fb4fdd399a209e2995cf1ec306dca89ff5c Mon Sep 17 00:00:00 2001 From: Parul Date: Thu, 15 Feb 2024 09:43:29 -0800 Subject: [PATCH 4/4] refactor version release tag --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fe4785a..d2d82f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "filepass" -version = "0.3.9" +version = "0.3.0" authors = [ { name="Orlund Norstrom", email="onorstrom@kamloops.ca" }, { name="Marco Lussetti", email="mlussetti@kamloops.ca" },