ruff linting has been applied to the code

rdsa_utils/cdp/helpers/s3_utils.py

@@ -1114,11 +1114,11 @@ def write_csv(
 ) -> bool:
     """
     Write a Pandas Dataframe to csv in an s3 bucket.
+
     Uses StringIO library as a RAM buffer, so at first Pandas writes data to the
     buffer, then the buffer returns to the beginning, and then it is sent to
     the s3 bucket using the boto3.put_object method.
 
-
     Parameters
     ----------
     client : boto3.client
@@ -1144,7 +1144,6 @@ def write_csv(
         If there is an error writing the file to s3.
 
     """
-
     try:
         # Create an Input-Output buffer
         csv_buffer = StringIO()
@@ -1158,14 +1157,14 @@ def write_csv(
         # Write the buffer into the s3 bucket. Assign the output to a mute
         # variable, so the output is not displayed in the console or log.
         _ = client.put_object(
-            Bucket=bucket_name, Body=csv_buffer.getvalue(), Key=filepath
+            Bucket=bucket_name, Body=csv_buffer.getvalue(), Key=filepath,
         )
         return True
 
     except Exception as e:
         error_message = (
             f"Error writing to csv or saving to bucket {bucket_name}, "
-            + f"filepath {filepath}: {e}"
+            f"filepath {filepath}: {e}"
         )
         logger.error(error_message)
         return False

rdsa_utils/helpers/pyspark.py

@@ -181,12 +181,18 @@ def wrapper(*args, **kwargs):
             varnames = func.__code__.co_varnames
             if args:
                 args = [
-                    (_convert_to_spark_col(arg) if varnames[i] not in exclude else arg)  # noqa: E501
+                    (
+                        _convert_to_spark_col(arg)
+                        if varnames[i] not in exclude
+                        else arg
+                    )  # noqa: E501
                     for i, arg in enumerate(args)
                 ]
             if kwargs:
                 kwargs = {
-                    k: (_convert_to_spark_col(kwarg) if k not in exclude else kwarg)  # noqa: E501
+                    k: (
+                        _convert_to_spark_col(kwarg) if k not in exclude else kwarg
+                    )  # noqa: E501
                     for k, kwarg in kwargs.items()
                 }
             return func(*args, **kwargs)

rdsa_utils/test_utils.py

@@ -172,7 +172,7 @@ def to_datetime(dt: str) -> datetime.datetime:
     return pd.to_datetime(dt).to_pydatetime()
 
 
-@pytest.fixture()
+@pytest.fixture
 def create_spark_df(spark_session):
     """Create Spark DataFrame from tuple data with first row as schema.
 
@@ -196,7 +196,7 @@ def _(data):
     return _
 
 
-@pytest.fixture()
+@pytest.fixture
 def to_spark(spark_session):
     """Convert pandas df to spark."""
 

tests/cdp/helpers/test_hdfs_utils.py

@@ -41,7 +41,7 @@ class BaseTest:
     subprocess.Popen calls.
     """
 
-    @pytest.fixture()
+    @pytest.fixture
     def mock_subprocess_popen(self, monkeypatch):  # noqa: PT004
         """Fixture to mock the subprocess.Popen function.
 
@@ -71,7 +71,7 @@ def mock_popen(*args, **kwargs):
 
         monkeypatch.setattr(subprocess, "Popen", mock_popen)
 
-    @pytest.fixture()
+    @pytest.fixture
     def mock_subprocess_popen_date_modifed(self):
         """Fixture to mock subprocess.Popen for testing get_date_modified.
 

tests/cdp/helpers/test_s3_utils.py

@@ -196,7 +196,7 @@ def test_invalid_non_s3_path_with_invalid_characters(self):
         )
 
 
-@pytest.fixture()
+@pytest.fixture
 def _aws_credentials():
     """Mock AWS Credentials for moto."""
     boto3.setup_default_session(
@@ -206,7 +206,7 @@ def _aws_credentials():
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def s3_client(_aws_credentials):
     """Provide a mocked AWS S3 client for testing
     using moto with temporary credentials.
@@ -234,7 +234,7 @@ def test_file_exists_false(self, s3_client):
         assert file_exists(s3_client, "test-bucket", "nonexistent.txt") is False
 
 
-@pytest.fixture()
+@pytest.fixture
 def setup_files(tmp_path):
     """
     Set up local files for upload and download tests.
@@ -351,7 +351,7 @@ def test_download_no_overwrite_local_file(self, s3_client, setup_files):
         )
 
 
-@pytest.fixture()
+@pytest.fixture
 def setup_folder(tmp_path):
     """
     Set up local folder and files for upload tests.
@@ -435,7 +435,7 @@ def test_upload_folder_no_overwrite_existing_files(
         )
 
 
-@pytest.fixture()
+@pytest.fixture
 def s3_client_for_list_files(_aws_credentials):
     """
     Provide a mocked AWS S3 client with temporary
@@ -496,7 +496,7 @@ def test_list_files_no_match(self, s3_client_for_list_files):
         assert len(files) == 0
 
 
-@pytest.fixture()
+@pytest.fixture
 def s3_client_for_delete_and_copy(_aws_credentials):
     """
     Provide a mocked AWS S3 client with temporary
@@ -1089,7 +1089,8 @@ def test_write_csv_success(self, s3_client):
 
     def test_write_csv_read_back(self, s3_client):
         """Test that a file wrtitten by write_csv can be read back and returns
-        the same dataframe as input. Uses kwargs."""
+        the same dataframe as input. Uses kwargs.
+        """
         data = {"name": ["John"], "age": [30], "city": ["Manchester"]}
         df = pd.DataFrame(data)
 
@@ -1099,7 +1100,8 @@ def test_write_csv_read_back(self, s3_client):
 
     def test_write_csv_failure(self, s3_client):
         """Test that write_csv returns False if unable to write.
-        Dictionary data does not have to_csv method."""
+        Dictionary data does not have to_csv method.
+        """
         data = {"name": ["John"], "age": [30], "city": ["Manchester"]}
 
         result = write_csv(s3_client, "test-bucket", data, "test_file.csv", index=False)

tests/cdp/io/test_cdsw_input.py

@@ -13,7 +13,7 @@
 class TestGetCurrentDatabase:
     """Tests for get_current_database function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def setup_and_teardown_database(  # noqa: PT004
         self,
         spark_session: SparkSession,
@@ -68,7 +68,7 @@ def test_get_current_database_after_setting(
 class TestExtractDatabaseName:
     """Tests for extract_database_name function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def dummy_database_and_table(
         self,
         spark_session: SparkSession,

tests/cdp/io/test_cdsw_output.py

@@ -14,7 +14,7 @@
 class TestInsertDataFrameToHiveTable:
     """Tests for insert_df_to_hive_table function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def test_df(self, spark_session: SparkSession, create_spark_df: Callable):
         """Fixture to create a test DataFrame with the help of `create_spark_df`
         callable.
@@ -127,12 +127,12 @@ def test_insert_df_to_hive_table_with_non_existing_table(
 class TestWriteAndReadHiveTable:
     """Tests for write_and_read_hive_table function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def mock_spark(self):
         """Fixture for mocked SparkSession."""
         return Mock(spec=SparkSession)
 
-    @pytest.fixture()
+    @pytest.fixture
     def mock_df(self):
         """Fixture for mocked DataFrame with 'run_id' and 'data' columns."""
         mock_df = Mock(spec=SparkDF)
@@ -222,7 +222,7 @@ def test_df_missing_filter_column(self, mock_spark, mock_df):
 class TestSaveCSVToHDFS:
     """Tests for save_csv_to_hdfs function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def mock_df(self) -> Mock:
         """Fixture for mocked Spark DataFrame."""
         return Mock(spec=SparkDF)

tests/gcp/helpers/test_gcp_utils.py

@@ -57,36 +57,36 @@ def test_expected(self):
         pass
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_client():
     """Mock GCS client."""
     return mock.Mock(spec=storage.Client)
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_bucket(mock_client):
     """Mock GCS bucket."""
     bucket = mock.Mock(spec=storage.Bucket)
     mock_client.bucket.return_value = bucket
     return bucket
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_blob(mock_bucket):
     """Mock GCS blob."""
     blob = mock.Mock(spec=storage.Blob)
     mock_bucket.blob.return_value = blob
     return blob
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_list_blobs(mock_client):
     """Mock list_blobs method."""
     mock_client.list_blobs.return_value = iter([mock.Mock()])
     return mock_client.list_blobs
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_path():
     """Mock Path object."""
     with mock.patch("rdsa_utils.gcp.helpers.gcp_utils.Path") as mock_path:

tests/helpers/test_pyspark.py

@@ -633,7 +633,7 @@ class TestConvertColsToStructCol:
     unusual definition of the expected dataframe in these tests.
     """
 
-    @pytest.fixture()
+    @pytest.fixture
     def input_df_fixture(self, create_spark_df) -> SparkDF:
         """Provide a basic spark dataframe."""
         return create_spark_df(
@@ -1225,7 +1225,7 @@ def test_load_csv_with_custom_quote(self, custom_spark_session, tmp_path):
 class TestTruncateExternalHiveTable:
     """Tests for truncate_external_hive_table function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def create_external_table(self, spark_session: SparkSession):
         """Create a mock external Hive table for testing."""
         spark = (
@@ -1244,7 +1244,7 @@ def create_external_table(self, spark_session: SparkSession):
         spark.sql("DROP DATABASE test_db")
         spark.stop()
 
-    @pytest.fixture()
+    @pytest.fixture
     def create_partitioned_table(self, spark_session: SparkSession):
         """Create a mock partitioned external Hive table for testing."""
         spark = (

tests/helpers/test_python.py

@@ -89,7 +89,7 @@ def test_expected(self):
 class TestOverwriteDictionary:
     """Tests for the overwrite_dictionary function."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def base_dict(self):
         """Create base dictionary used across all tests."""
         return {

tests/io/conftest.py

@@ -67,7 +67,7 @@ def yaml_config_string() -> str:
     """
 
 
-@pytest.fixture()
+@pytest.fixture
 def expected_standard_config() -> Dict[str, Any]:
     """Fixture providing the loaded config from loading the temp file."""
     return {

tests/methods/test_averaging_methods.py

@@ -4,7 +4,7 @@
 from rdsa_utils.methods.averaging_methods import *
 
 
-@pytest.fixture()
+@pytest.fixture
 def input_df(create_spark_df):
     """Fixture containing input data for tests."""
     return create_spark_df(

tests/test_logging.py

@@ -74,7 +74,7 @@ def test_expected(self, logger, expected):
 class TestPrintFullTables:
     """Tests for the print_full_table_and_raise_error."""
 
-    @pytest.fixture()
+    @pytest.fixture
     def input_df(self):
         """Input pandas dataframe."""
         return create_dataframe(