From f2eedf25b8b9c8d0c0c3678d6520e3e09e422f3d Mon Sep 17 00:00:00 2001 From: garen Date: Sat, 25 Nov 2023 22:47:17 -0600 Subject: [PATCH 1/3] done auto save --- main.py | 28 +++++++++++++++++++++------- src/JobTracker/config.py | 2 +- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 8de3bfa..ebb2aa3 100644 --- a/main.py +++ b/main.py @@ -4,11 +4,12 @@ from src.JobTracker.utils import EmailMessage from src.JobTracker.chatbot import ChatGPT +from src.JobTracker.config import AUTO_SAVE_EMAIL # Configure logging logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -def process_email(email_path): +def process_email(email_path, output_csv): ''' Process the emails at the given path and return the results. @@ -29,25 +30,38 @@ def process_email(email_path): elif k.lower() == "n": logging.info("---------Stop processing emails---------") return + email_counter = 0 for mail in mail_info: state, data = chatbot.get_content(mail) if state == 'Succeed': res.append(data) - return res + email_counter += 1 + if email_counter % AUTO_SAVE_EMAIL == 0: + export_to_csv(res, output_csv) + res = [] + return res, email_counter def export_to_csv(data, filename): - with open(filename, mode='w', newline='', encoding='utf-8') as csvfile: + file_exists = False + try: + with open(filename, 'r', newline='', encoding='utf-8') as csvfile: + file_exists = True + except FileNotFoundError: + pass + mode = 'a' if file_exists else 'w' + with open(filename, mode=mode, newline='', encoding='utf-8') as csvfile: fieldnames = data[0].keys() writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() + if not file_exists: + writer.writeheader() for row in data: writer.writerow(row) def main(email_path, output_csv): - result = process_email(email_path) - if result: + result, count = process_email(email_path, output_csv) + if count > 0: export_to_csv(result, output_csv) - logging.info(f"Processed emails successfully and exported to CSV at {output_csv}.") + logging.info(f"Processed {str(count)} emails successfully and exported to CSV at {output_csv}.") else: logging.info("No emails processed.") diff --git a/src/JobTracker/config.py b/src/JobTracker/config.py index d4762cb..7769a3e 100644 --- a/src/JobTracker/config.py +++ b/src/JobTracker/config.py @@ -1,6 +1,6 @@ import os - +AUTO_SAVE_EMAIL = 5 OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY') MODEL = 'gpt-4-1106-preview' PRICE = { From 2b6cfbae4daf21eb29fcb62daddcc336a6d109eb Mon Sep 17 00:00:00 2001 From: Garen Hu Date: Sat, 16 Dec 2023 00:07:20 -0600 Subject: [PATCH 2/3] fix issue --- .DS_Store | Bin 0 -> 6148 bytes main.py | 41 ++++++++++++++++++--------------------- src/JobTracker/config.py | 2 +- 3 files changed, 20 insertions(+), 23 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..1335fc9b654056d2445f1333c8d4656210449c51 GIT binary patch literal 6148 zcmeHKJxc>Y5S_i8aH7RkV!4&2kbiK7v#||!7Lr5~A;$%CX=SldYyX9vt-XK5(pIoj ze6zbGyIi!CATu!g_U7Xr_b%LIiAePpqb^ZSL>Zhh*~YTQIM3d)#7B04itjNUcl*P- znzbq=u&E00yG!Yi#&k(d^!}2?XPB$%vK&n70b*J6{PlQq|8f-PN5AH0li;_J8?;h~ z=5$5Fh?NfZR+*6V>FjQDdTlvp<@TUrV10!2yumAu6 literal 0 HcmV?d00001 diff --git a/main.py b/main.py index ebb2aa3..1bb8450 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ import argparse import logging import csv +import os from src.JobTracker.utils import EmailMessage from src.JobTracker.chatbot import ChatGPT @@ -18,7 +19,7 @@ def process_email(email_path, output_csv): ''' em = EmailMessage(email_path) mail_info = em.get_mail_info() - res = [] + res = 0 chatbot = ChatGPT() message = chatbot.get_cost(mail_info) logging.info(message) @@ -30,38 +31,34 @@ def process_email(email_path, output_csv): elif k.lower() == "n": logging.info("---------Stop processing emails---------") return - email_counter = 0 + cur = [] for mail in mail_info: state, data = chatbot.get_content(mail) if state == 'Succeed': - res.append(data) - email_counter += 1 - if email_counter % AUTO_SAVE_EMAIL == 0: - export_to_csv(res, output_csv) - res = [] - return res, email_counter + res += 1 + cur.append(data) + if len(cur) % AUTO_SAVE_EMAIL == 0: + export_to_csv(cur, output_csv) + cur = [] + if cur: + export_to_csv(cur, output_csv) + return res def export_to_csv(data, filename): - file_exists = False - try: - with open(filename, 'r', newline='', encoding='utf-8') as csvfile: - file_exists = True - except FileNotFoundError: - pass - mode = 'a' if file_exists else 'w' - with open(filename, mode=mode, newline='', encoding='utf-8') as csvfile: + file_exists = os.path.exists(filename) + with open(filename, mode="a", newline='', encoding='utf-8') as csvfile: fieldnames = data[0].keys() - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - if not file_exists: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + if not file_exists: writer.writeheader() for row in data: writer.writerow(row) + logging.info(f"Processed {str(len(data))} emails and exported to CSV at {filename}.") def main(email_path, output_csv): - result, count = process_email(email_path, output_csv) - if count > 0: - export_to_csv(result, output_csv) - logging.info(f"Processed {str(count)} emails successfully and exported to CSV at {output_csv}.") + result = process_email(email_path, output_csv) + if result > 0: + logging.info(f"Total {str(result)} emails Processed successfully and exported to CSV at {output_csv}.") else: logging.info("No emails processed.") diff --git a/src/JobTracker/config.py b/src/JobTracker/config.py index 7769a3e..0cebf36 100644 --- a/src/JobTracker/config.py +++ b/src/JobTracker/config.py @@ -1,6 +1,6 @@ import os -AUTO_SAVE_EMAIL = 5 +AUTO_SAVE_EMAIL = 3 OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY') MODEL = 'gpt-4-1106-preview' PRICE = { From 05cfad1fcd9c49500ca8d6789f02c5ea2d93a407 Mon Sep 17 00:00:00 2001 From: Garen Hu Date: Sat, 16 Dec 2023 00:09:07 -0600 Subject: [PATCH 3/3] change config --- src/JobTracker/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JobTracker/config.py b/src/JobTracker/config.py index 0cebf36..80c8c1f 100644 --- a/src/JobTracker/config.py +++ b/src/JobTracker/config.py @@ -1,6 +1,6 @@ import os -AUTO_SAVE_EMAIL = 3 +AUTO_SAVE_EMAIL = 20 OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY') MODEL = 'gpt-4-1106-preview' PRICE = {