From ee4e247cdec2bfc2bd7adf9c691a04e50cc7902b Mon Sep 17 00:00:00 2001 From: jj Date: Fri, 22 Dec 2023 11:13:39 +0100 Subject: [PATCH] special case handling in the pipeline --- ssh2ssh.py | 16 +++++++++++----- utils.py | 5 ----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ssh2ssh.py b/ssh2ssh.py index 473ab0f..7e45741 100644 --- a/ssh2ssh.py +++ b/ssh2ssh.py @@ -6,7 +6,7 @@ import pendulum from decors import get_connection -from utils import copy_streams, RFSC, walk_dir +from utils import copy_streams, RFSC, file_exist, is_dir, walk_dir @@ -52,14 +52,20 @@ def copy(**context): source_ssh_hook = get_connection(conn_id=s_con_id, params=s_params) target_ssh_hook = get_connection(conn_id=t_con_id, params=t_params) - target_client = target_ssh_hook.get_conn().open_sftp() + target_conn = target_ssh_hook.get_conn() + target_client = target_conn.open_sftp() sftp_client = source_ssh_hook.get_conn().open_sftp() sclient = RFSC(sftp_client) + - target_conn = target_ssh_hook.get_conn() - - mappings = list(walk_dir(client=sclient, path=s_params["path"], prefix="")) + if file_exist(sftp_client, s_params['path']) and not is_dir(sftp_client, s_params['path']): + print("Special case it is a file") + mappings=[s_params['path']] + s_params['path']=os.path.dirname(s_params['path']) + else: + mappings = list(walk_dir(client=sclient, path=s_params["path"], prefix="")) + for fname in mappings: target_name = fname.replace(s_params["path"], t_params["path"]) print("Processing", fname, "-->", target_name) diff --git a/utils.py b/utils.py index 99dd6db..b158fe2 100644 --- a/utils.py +++ b/utils.py @@ -221,11 +221,6 @@ def __init__(self, client, **kwargs): self.client = client def list(self, path, get_info=True): - if file_exist(self.client, path) and not is_dir(self.client, path): - print("Special case it is a file") - if not get_info: - return [path] - return [{"path": path, "isdir": False}] if not get_info: return [el.filename for el in self.client.listdir_attr(path)]