diff --git a/common/ipfs_util.py b/common/ipfs_util.py index 92594502b..ebeb90d01 100644 --- a/common/ipfs_util.py +++ b/common/ipfs_util.py @@ -8,7 +8,7 @@ class IPFSUtil(object): def __init__(self, ipfs_url, port): - self.ipfs_conn = ipfshttpclient.connect(f"/dns4/{ipfs_url}/tcp/{port}/http") + self.ipfs_conn = ipfshttpclient.connect(f"/dns/{ipfs_url}/tcp/{port}/http") def read_bytesio_from_ipfs(self, ipfs_hash): ipfs_data = self.ipfs_conn.cat(ipfs_hash) diff --git a/common/requirements.txt b/common/requirements.txt index edba321bc..fabeb9221 100644 --- a/common/requirements.txt +++ b/common/requirements.txt @@ -3,4 +3,5 @@ web3==5.10.0 requests==2.22.0 PyMySQL==0.10.1 boto3==1.15.9 -ipfsapi==0.4.3 +ipfsapi==0.4.4 +ipfshttpclient==0.4.13.2 diff --git a/common/utils.py b/common/utils.py index 1b9052de4..aaa47608c 100644 --- a/common/utils.py +++ b/common/utils.py @@ -233,21 +233,21 @@ def ipfsuri_to_bytesuri(uri): return uri.encode("ascii").ljust(32 * (len(uri) // 32 + 1), b"\0") -def publish_file_in_ipfs(file_url, file_dir, ipfs_client, wrap_with_directory=True): +def publish_file_in_ipfs(file_url, file_dir, ipfs_client, wrap_with_directory=True, ignored_files=[]): filename = download_file_from_url(file_url=file_url, file_dir=file_dir) file_type = os.path.splitext(filename)[1] logger.info(f" file type is = '{file_type.lower()}` ") if file_type.lower() == ".zip": - return publish_zip_file_in_ipfs(filename, file_dir, ipfs_client) + return publish_zip_file_in_ipfs(filename, file_dir, ipfs_client, ignored_files=ignored_files) #todo , you need to tar the folder and add that to the ipfs hash logger.info(f"writing the file on ipfs {file_dir}/{filename} ") ipfs_hash = ipfs_client.write_file_in_ipfs(f"{file_dir}/{filename}", wrap_with_directory) return ipfs_hash -def publish_zip_file_in_ipfs(filename, file_dir, ipfs_client): +def publish_zip_file_in_ipfs(filename, file_dir, ipfs_client, ignored_files=[]): logger.info(f"publish_zip_file_in_ipfs {file_dir}/{filename} ") - file_in_tar_bytes = convert_zip_file_to_tar_bytes(file_dir=file_dir, filename=filename) + file_in_tar_bytes = convert_zip_file_to_tar_bytes(file_dir=file_dir, filename=filename, ignored_files=ignored_files) logger.info(f"file_in_tar_bytes {file_in_tar_bytes} ") return ipfs_client.ipfs_conn.add_bytes(file_in_tar_bytes.getvalue()) @@ -262,13 +262,16 @@ def download_file_from_url(file_url, file_dir): return filename -def convert_zip_file_to_tar_bytes(file_dir, filename): +def convert_zip_file_to_tar_bytes(file_dir, filename, ignored_files=[]): with ZipFile(f"{file_dir}/{filename}", 'r') as zipObj: listOfFileNames = zipObj.namelist() zipObj.extractall(file_dir, listOfFileNames) if not os.path.isdir(file_dir): raise Exception("Directory %s doesn't exists" % file_dir) - files = glob.glob(os.path.join(file_dir, "*.proto")) + files = sorted( + f for f in glob.glob(os.path.join(file_dir, "*.proto")) + if os.path.basename(f) not in ignored_files + ) if len(files) == 0: raise Exception("Cannot find any %s files" % (os.path.join(file_dir, "*.proto"))) files.sort() diff --git a/lambda_layers/common_python_libraries/requirements.txt b/lambda_layers/common_python_libraries/requirements.txt index 25ed0b9c8..15dee69a6 100644 --- a/lambda_layers/common_python_libraries/requirements.txt +++ b/lambda_layers/common_python_libraries/requirements.txt @@ -9,7 +9,8 @@ sqlalchemy==1.3.5 aws-xray-sdk==2.4.2 botocore==1.19.55 boto3==1.16.1 -ipfsapi==0.4.3 +ipfsapi==0.4.4 +ipfshttpclient==0.4.13.2 pymysql==0.9.2 setuptools==57.4.0 dxlmispservice==0.1.5 diff --git a/lambda_layers/web3_and_grpc_libraries/requirements.txt b/lambda_layers/web3_and_grpc_libraries/requirements.txt index 365f82265..b9c1c75ba 100644 --- a/lambda_layers/web3_and_grpc_libraries/requirements.txt +++ b/lambda_layers/web3_and_grpc_libraries/requirements.txt @@ -1,6 +1,8 @@ rlp==1.2.0 protobuf==3.10.0 web3==5.10.0 +ipfshttpclient==0.4.13.2 +ipfsapi==0.4.4 grpcio==1.30.0 grpcio-health-checking==1.19.0 grpcio-tools==1.19.0 diff --git a/registry/application/services/service_publisher_service.py b/registry/application/services/service_publisher_service.py index aab3a75c0..89196c49d 100644 --- a/registry/application/services/service_publisher_service.py +++ b/registry/application/services/service_publisher_service.py @@ -253,9 +253,12 @@ def publish_service_data_to_ipfs(self): filename = download_file_from_url(file_url=proto_url, file_dir=f"{ASSET_DIR}/{service.org_uuid}/{service.uuid}") logger.info(f"proto file Name Retrieved = '{filename}` ") - asset_ipfs_hash = publish_zip_file_in_ipfs(filename=filename, - file_dir=f"{ASSET_DIR}/{service.org_uuid}/{service.uuid}", - ipfs_client=IPFSUtil(IPFS_URL['url'], IPFS_URL['port'])) + asset_ipfs_hash = publish_zip_file_in_ipfs( + filename=filename, + file_dir=f"{ASSET_DIR}/{service.org_uuid}/{service.uuid}", + ipfs_client=IPFSUtil(IPFS_URL['url'], IPFS_URL['port']), + ignored_files=["pricing.proto", "training.proto"] + ) service.proto = { "model_ipfs_hash": asset_ipfs_hash, "encoding": "proto", diff --git a/service_status/constant.py b/service_status/constant.py index 3dee2dfbf..5615b7fcc 100644 --- a/service_status/constant.py +++ b/service_status/constant.py @@ -1,2 +1,2 @@ SRVC_STATUS_GRPC_TIMEOUT = 10 -LIMIT = 5 \ No newline at end of file +LIMIT = 10 \ No newline at end of file diff --git a/service_status/monitor_service.py b/service_status/monitor_service.py index 9c027cbbe..69c010078 100644 --- a/service_status/monitor_service.py +++ b/service_status/monitor_service.py @@ -23,9 +23,7 @@ "Team

" CERT_EXP_EMAIL_NOTIFICATION_SUBJ = "Certificates are about to expire for service %s for %s network." CERT_EXP_SLACK_NOTIFICATION_MSG = \ - "```Alert!\n\nCertificates for service %s under organization %s for the %s network are about to expire in %s days.\n" \ - "Endpoint: %s \n\nFor any queries please email at cs-marketplace@singularitynet.io. \n\nWarmest regards, " \ - "\nSingularityNET Marketplace Team```" + "Certificates for service %s under organization %s for the %s network are about to expire in %s days." NO_OF_ENDPOINT_TO_TEST_LIMIT = 5 @@ -160,7 +158,7 @@ def _get_certificate_expiration_email_notification_message(org_id, service_id, e @staticmethod def _get_certificate_expiration_slack_notification_message(org_id, service_id, endpoint, days_left_for_expiration): - return CERT_EXP_SLACK_NOTIFICATION_MSG % (service_id, org_id, NETWORK_NAME, days_left_for_expiration, endpoint) + return CERT_EXP_SLACK_NOTIFICATION_MSG % (service_id, org_id, NETWORK_NAME, days_left_for_expiration) class MonitorServiceHealth(MonitorService): diff --git a/service_status/service_status.py b/service_status/service_status.py index 32c17b4a4..5bf7dca6c 100644 --- a/service_status/service_status.py +++ b/service_status/service_status.py @@ -135,6 +135,8 @@ def update_service_status(self): logger.info(f"error_details: {error_details}") logger.info(f"debug_error_string: {debug_error_string}") old_status = int.from_bytes(record["is_available"], "big") + logger.info(f"Service to check: row_id={record['row_id']}, status={status}, old_status={old_status}") + logger.info(f"Service endpoint: {record['endpoint']}") failed_status_count = self._calculate_failed_status_count( current_status=status, old_status=old_status, old_failed_status_count=record["failed_status_count"]) diff --git a/service_status/testcases/unit_testcases/test_monitor_service_certificates_expiry.py b/service_status/testcases/unit_testcases/test_monitor_service_certificates_expiry.py index a5ca05d18..46056617a 100644 --- a/service_status/testcases/unit_testcases/test_monitor_service_certificates_expiry.py +++ b/service_status/testcases/unit_testcases/test_monitor_service_certificates_expiry.py @@ -22,10 +22,8 @@ def test_get_certificate_expiration_slack_notification_message(self): days_left_for_expiration = 10 response = self.monitor_service_certificate._get_certificate_expiration_slack_notification_message( org_id=org_id, service_id=service_id, endpoint=endpoint, days_left_for_expiration=days_left_for_expiration) - assert (response == "```Alert!\n\nCertificates for service test_service_id under organization test_org_id for " - "the TEST network are about to expire in 10 days.\nEndpoint: https://dummyendpoint.com \n\n" - "For any queries please email at cs-marketplace@singularitynet.io. \n\nWarmest regards, \n" - "SingularityNET Marketplace Team```") + assert (response == "Certificates for service test_service_id under organization test_org_id for " + "the TEST network are about to expire in 10 days.") def test_get_certificate_expiration_email_notification_message(self): org_id = "test_org_id" diff --git a/signer/serverless.yml b/signer/serverless.yml index 86c2de349..21d1a5bf3 100644 --- a/signer/serverless.yml +++ b/signer/serverless.yml @@ -94,7 +94,9 @@ package: - License - log_setup.py - heath_check.sh + - node_modules/** include: + - node_modules/singularitynet-platform-contracts/** - resources/** diff --git a/wallets/serverless.yml b/wallets/serverless.yml index 75516cc11..a569fffbf 100644 --- a/wallets/serverless.yml +++ b/wallets/serverless.yml @@ -111,7 +111,9 @@ package: - License - log_setup.py - heath_check.sh + - node_modules/** include: + - node_modules/singularitynet-platform-contracts/** - resources/**