From 8ea88ef49d9185f68962f494a9a885796be97436 Mon Sep 17 00:00:00 2001 From: Soim Date: Tue, 3 Sep 2024 17:03:55 +0900 Subject: [PATCH] Refactoring related to FOSSLight Util --- src/fosslight_android/_common.py | 7 ++- src/fosslight_android/_write_excel.py | 62 ------------------- .../android_binary_analysis.py | 34 ++++++---- 3 files changed, 27 insertions(+), 76 deletions(-) delete mode 100644 src/fosslight_android/_write_excel.py diff --git a/src/fosslight_android/_common.py b/src/fosslight_android/_common.py index ea8eb76..879443b 100644 --- a/src/fosslight_android/_common.py +++ b/src/fosslight_android/_common.py @@ -97,7 +97,7 @@ def set_oss_name(self, value): def set_oss_version(self, value): self.oss_version = value - def get_print_items(self): + def get_print_array(self, return_excel=True): print_items_txt = [] print_items_excel = [] oss_name = self.oss_name @@ -125,7 +125,10 @@ def get_print_items(self): excel_item.extend(item.split('\t')) excel_item.extend(['', '', '', '', '', comment, need_check, self.tlsh, self.checksum]) print_items_excel.append(excel_item) - return print_items_txt, print_items_excel + if return_excel: + return print_items_excel + else: + return print_items_txt def check_empty_column(license, oss_name, directory): diff --git a/src/fosslight_android/_write_excel.py b/src/fosslight_android/_write_excel.py deleted file mode 100644 index 33fb2e7..0000000 --- a/src/fosslight_android/_write_excel.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: Copyright 2023 LG Electronics Inc. -# SPDX-License-Identifier: Apache-2.0 -import sys -import xlsxwriter -import logging -from fosslight_util.constant import LOGGER_NAME -from fosslight_util.write_excel import hide_column, write_cover_sheet - -logger = logging.getLogger(LOGGER_NAME) -HIDDEN_HEADER = ['TLSH', 'SHA1'] - - -def write_result_to_excel(out_file_name, row_list, cover=''): - header_row = ['ID', 'Binary Path', 'Source Path', 'Notice', 'OSS Name', 'OSS Version', 'License', - 'Download Location', 'Homepage', - 'Copyright Text', - 'License Text', 'Exclude', 'Comment', 'Need Check', 'TLSH', 'SHA1'] - sheet_name = "BIN (Android)" - try: - workbook = xlsxwriter.Workbook(out_file_name) - if cover: - write_cover_sheet(workbook, cover) - worksheet = create_worksheet(workbook, sheet_name, header_row) - write_result_to_sheet(worksheet, row_list) - hide_column(worksheet, header_row, HIDDEN_HEADER) - workbook.close() - except Exception as ex: - print('* Error :' + str(ex)) - - -def write_result_to_sheet(worksheet, list_to_print): - row = 1 - for row_item in list_to_print: - worksheet.write(row, 0, row) - for col_num, value in enumerate(row_item): - worksheet.write(row, col_num + 1, value) - row += 1 - - -def create_worksheet(workbook, sheet_name, header_row): - worksheet = workbook.add_worksheet(sheet_name) - for col_num, value in enumerate(header_row): - worksheet.write(0, col_num, value) - return worksheet - - -def write_result(out_excel_file, final_bin_info, cover=''): - excel_list = [] - - if final_bin_info: - for item in sorted(final_bin_info, key=lambda binary: (binary.source_code_path, binary.bin_name)): - try: - print_row, print_excel = item.get_print_items() - excel_list.extend(print_excel) - except Exception as error: - logger.error(f"Get results to print:{error}") - sys.exit(1) - write_result_to_excel(out_excel_file, excel_list, cover) - else: - logger.warning("Nothing is detected from the scanner so output file is not generated.") diff --git a/src/fosslight_android/android_binary_analysis.py b/src/fosslight_android/android_binary_analysis.py index 017388c..45e3d67 100755 --- a/src/fosslight_android/android_binary_analysis.py +++ b/src/fosslight_android/android_binary_analysis.py @@ -23,7 +23,6 @@ import numpy as np from functools import partial from fosslight_util.set_log import init_log -from ._write_excel import write_result from ._util import ( read_file, write_txt_file, @@ -54,7 +53,8 @@ from fosslight_binary.binary_analysis import return_bin_only import argparse from pathlib import Path -from fosslight_util.cover import CoverItem +from fosslight_util.oss_item import ScannerItem +from fosslight_util.output_format import write_output_file EXCEPTIONAL_PATH = [r"(/)+gen/META/lic_intermediates/"] android_log_lines = [] @@ -74,6 +74,11 @@ ANDROID_LOG_FILE_NAME = "android.log" num_cores = 1 now = "" +HIDDEN_HEADER = {'TLSH', "SHA1"} +HEADER = {'BIN (Android)': ['ID', 'Binary Path', 'Source Path', 'Notice', 'OSS Name', + 'OSS Version', 'License', 'Download Location', 'Homepage', + 'Copyright Text', 'License Text', 'Exclude', 'Comment', + 'Need Check', 'TLSH', 'SHA1']} # For checking repository's name repositories = {} @@ -635,7 +640,7 @@ def remove_duplicated_binaries_by_checking_checksum(remove_list_file): continue if skip: cnt += 1 - print_removed_str, print_removed_array = item.get_print_items() + print_removed_str = item.get_print_array(False) for row_removed in print_removed_str: str_bin_removed += f"{row_removed}\n" continue @@ -679,7 +684,7 @@ def remove_duplicated_binaries_by_checking_checksum(remove_list_file): for bin_same_name in same_name_binaries: if bin_same_name.bin_name != final_added.bin_name: - print_removed_str, print_removed_array = bin_same_name.get_print_items() + print_removed_str = bin_same_name.get_print_array(False) for row_removed in print_removed_str: str_bin_removed += f"{row_removed}\n" else: # Don't have any duplicated binaries @@ -766,6 +771,7 @@ def main(): _NOTICE_CHECKLIST_TYPE = False analyze_source = False path_to_exclude = [] + RESULT_FILE_EXTENSION = ".xlsx" num_cores = multiprocessing.cpu_count() - 1 if num_cores < 1: @@ -775,7 +781,7 @@ def main(): android_src_path = python_script_dir now = datetime.now().strftime('%y%m%d_%H%M') log_txt_file = os.path.join(python_script_dir, f"fosslight_log_android_{now}.txt") - result_excel_file = os.path.join(python_script_dir, f"fosslight_report_android_{now}.xlsx") + result_excel_file_name = os.path.join(python_script_dir, f"fosslight_report_android_{now}") remove_list_file = "" parser = argparse.ArgumentParser(description='FOSSLight Android', prog='fosslight_android', add_help=False) @@ -861,13 +867,17 @@ def main(): from ._src_analysis import find_item_to_analyze final_bin_info = find_item_to_analyze(final_bin_info, python_script_dir, now, path_to_exclude) - cover = CoverItem(tool_name=PKG_NAME, - start_time=now, - input_path=android_src_path) - cover.comment = f"Total number of binaries: {len(final_bin_info)}" - cover.comment += f"\nNotice: {notice_files}" - write_result(result_excel_file, final_bin_info, cover) - result_log["Output FOSSLight Report"] = result_excel_file + scan_item = ScannerItem(PKG_NAME, now) + scan_item.set_cover_pathinfo(android_src_path, "") + + scan_item.set_cover_comment(f"Total number of binaries: {len(final_bin_info)}") + scan_item.set_cover_comment(f"\nNotice: {notice_files}") + scan_item.append_file_items(final_bin_info, PKG_NAME) + success, msg, result_file = write_output_file(result_excel_file_name, RESULT_FILE_EXTENSION, + scan_item, HEADER, HIDDEN_HEADER) + if not success: + logger.warning(f"Failed to write result to excel:{msg}") + result_log["Output FOSSLight Report"] = f"{result_file}" if _create_additial_notice: create_additional_notice(final_bin_info, python_script_dir)