fix: creates master br if not exists, tune resources

spark-jobs/brazilian-finder/brazilian-finder_spark.py

@@ -38,6 +38,13 @@ def spark_job(spark: SparkSession, params, *args, **kwargs):
     matching_emails_df = spark.sql("SELECT email_tel FROM matching_emails")
     matching_emails_df.write.mode("overwrite").text(s3_output_path)
 
+    # Check if the master path exists, create if it doesn't
+    try:
+        spark.read.format("delta").load(s3_master_combo_path)
+    except Exception as e:
+        # If the path does not exist, create an empty Delta table
+        empty_df = spark.createDataFrame([], schema="email_tel STRING")
+        empty_df.write.format("delta").save(s3_master_combo_path)
     # Append the matching emails to the master delta table
     spark.sql(f"""
         INSERT INTO delta.`{s3_master_combo_path}`

spark-jobs/brazilian-finder/brazilian-finder_spark.yaml

@@ -45,9 +45,9 @@ spec:
     labels:
       version: "3.5.3"
   executor:
-    cores: 1
+    cores: 3
     instances: 2
-    memory: "1024m"
+    memory: "4096m"
     envFrom:
       - secretRef:
           name: s3-onprem-combopurifier