Add Step to Fetch Missing Git Messages from Commits Analyzed Before Commits Message Table was Added

augur/application/db/lib.py

@@ -17,7 +17,7 @@
 
 logger = logging.getLogger("db_lib")
 
 def convert_type_of_value(config_dict, logger=None):
 
 
     data_type = config_dict["type"]
@@ -177,11 +177,27 @@
 
     try:
         working_commits = fetchall_data_from_sql_text(query)
     except:
         working_commits = []
 
     return working_commits
 
+def get_missing_commit_message_hashes(repo_id):
+
+    fetch_missing_hashes_sql = s.sql.text("""
+    SELECT DISTINCT cmt_commit_hash FROM commits
+    WHERE repo_id=:repo_id 
+    AND cmt_commit_hash NOT IN 
+    (SELECT DISTINCT cmt_hash FROM commit_messages WHERE repo_id=:repo_id);
+    """).bindparams(repo_id=repo_id)
+
+    try:
+        missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
+    except:
+        missing_commit_hashes = []
+
+    return missing_commit_hashes
+
 def get_worker_oauth_keys(platform: str):
 
     with get_session() as session:
@@ -197,7 +213,7 @@
         return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()
 
 
 def facade_bulk_insert_commits(logger, records):
 
     with get_session() as session:
 
@@ -239,7 +255,7 @@
                 raise e
 
 
 def bulk_insert_dicts(logger, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]:
 
     if isinstance(data, list) is False:
 

augur/tasks/git/facade_tasks.py

@@ -1,9 +1,11 @@
 #SPDX-License-Identifier: MIT
 
 import logging
+import datetime
 from celery import group, chain
 
-from augur.application.db.lib import get_session, get_repo_by_repo_git, get_repo_by_repo_id, remove_working_commits_by_repo_id_and_hashes, get_working_commits_by_repo_id, facade_bulk_insert_commits, bulk_insert_dicts
+from subprocess import check_output
+from augur.application.db.lib import get_session, get_repo_by_repo_git, get_repo_by_repo_id, remove_working_commits_by_repo_id_and_hashes, get_working_commits_by_repo_id, facade_bulk_insert_commits, bulk_insert_dicts, get_missing_commit_message_hashes
 
 from augur.tasks.git.util.facade_worker.facade_worker.utilitymethods import trim_commits
 from augur.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path, get_parent_commits_set, get_existing_commits_set
@@ -157,6 +159,36 @@
     facade_helper.update_status('Updating Contributors')
     facade_helper.log_activity('Info', 'Updating Contributors with commits')
 
+@celery.task(base=AugurFacadeRepoCollectionTask)
+def facade_fetch_missing_commit_messages(repo_git):
+    logger = logging.getLogger(facade_fetch_missing_commit_messages.__name__)
+    facade_helper = FacadeHelper(logger)
+
+    repo = get_repo_by_repo_git(repo_git)
+
+    logger.debug(f"Fetching missing commit message records for repo {repo_git}")
+
+    missing_message_hashes = get_missing_commit_message_hashes(repo.repo_id)
+
+    for hash in missing_message_hashes:
+
+        absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path,repo.repo_name)
+        repo_loc = (f"{absolute_path}/.git")
+
+        commit_message = check_output(
+            f"git --git-dir {repo_loc} log --format=%B -n 1 {hash}".split()
+        ).decode('utf-8').strip()
+
+        msg_record = {
+            'repo_id' : repo.repo_id,
+            'cmt_msg' : commit_message,
+            'cmt_hash' : hash,
+            'tool_source' : 'Facade',
+            'tool_version' : '0.78?',
+            'data_source' : 'git',
+            'data_collection_date' : datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        }
+
 
 #enable celery multithreading
 @celery.task(base=AugurFacadeRepoCollectionTask)
@@ -193,7 +225,7 @@
     facade_helper.log_activity('Debug',f"Commits missing from repo {repo_id}: {len(missing_commits)}")
 
 
     if not len(missing_commits) or repo_id is None:
         #session.log_activity('Info','Type of missing_commits: %s' % type(missing_commits))
         return
 
@@ -354,6 +386,7 @@
 
     analysis_sequence.append(trim_commits_post_analysis_facade_task.si(repo_git))
 
+    analysis_sequence.append(facade_fetch_missing_commit_messages.si(repo_git))
 
     analysis_sequence.append(facade_analysis_end_facade_task.si())