Merge pull request #1856 from singnet/development

Requirements, Registry, Service Status and deployment updates

common/ipfs_util.py

@@ -8,7 +8,7 @@
 class IPFSUtil(object):
 
     def __init__(self, ipfs_url, port):
-        self.ipfs_conn = ipfshttpclient.connect(f"/dns4/{ipfs_url}/tcp/{port}/http")
+        self.ipfs_conn = ipfshttpclient.connect(f"/dns/{ipfs_url}/tcp/{port}/http")
 
     def read_bytesio_from_ipfs(self, ipfs_hash):
         ipfs_data = self.ipfs_conn.cat(ipfs_hash)

common/requirements.txt

@@ -3,4 +3,5 @@ web3==5.10.0
 requests==2.22.0
 PyMySQL==0.10.1
 boto3==1.15.9
-ipfsapi==0.4.3
+ipfsapi==0.4.4
+ipfshttpclient==0.4.13.2

common/utils.py

@@ -233,21 +233,21 @@ def ipfsuri_to_bytesuri(uri):
     return uri.encode("ascii").ljust(32 * (len(uri) // 32 + 1), b"\0")
 
 
-def publish_file_in_ipfs(file_url, file_dir, ipfs_client, wrap_with_directory=True):
+def publish_file_in_ipfs(file_url, file_dir, ipfs_client, wrap_with_directory=True, ignored_files=[]):
     filename = download_file_from_url(file_url=file_url, file_dir=file_dir)
     file_type = os.path.splitext(filename)[1]
     logger.info(f" file type is  = '{file_type.lower()}` ")
     if file_type.lower() == ".zip":
-        return publish_zip_file_in_ipfs(filename, file_dir, ipfs_client)
+        return publish_zip_file_in_ipfs(filename, file_dir, ipfs_client, ignored_files=ignored_files)
     #todo , you need to tar the folder and add that to the ipfs hash
     logger.info(f"writing the file on ipfs {file_dir}/{filename} ")
     ipfs_hash = ipfs_client.write_file_in_ipfs(f"{file_dir}/{filename}", wrap_with_directory)
     return ipfs_hash
 
 
-def publish_zip_file_in_ipfs(filename, file_dir, ipfs_client):
+def publish_zip_file_in_ipfs(filename, file_dir, ipfs_client, ignored_files=[]):
     logger.info(f"publish_zip_file_in_ipfs {file_dir}/{filename} ")
-    file_in_tar_bytes = convert_zip_file_to_tar_bytes(file_dir=file_dir, filename=filename)
+    file_in_tar_bytes = convert_zip_file_to_tar_bytes(file_dir=file_dir, filename=filename, ignored_files=ignored_files)
     logger.info(f"file_in_tar_bytes {file_in_tar_bytes} ")
     return ipfs_client.ipfs_conn.add_bytes(file_in_tar_bytes.getvalue())
 
@@ -262,13 +262,16 @@ def download_file_from_url(file_url, file_dir):
     return filename
 
 
-def convert_zip_file_to_tar_bytes(file_dir, filename):
+def convert_zip_file_to_tar_bytes(file_dir, filename, ignored_files=[]):
     with ZipFile(f"{file_dir}/{filename}", 'r') as zipObj:
         listOfFileNames = zipObj.namelist()
         zipObj.extractall(file_dir, listOfFileNames)
     if not os.path.isdir(file_dir):
         raise Exception("Directory %s doesn't exists" % file_dir)
-    files = glob.glob(os.path.join(file_dir, "*.proto"))
+    files = sorted(
+        f for f in glob.glob(os.path.join(file_dir, "*.proto"))
+        if os.path.basename(f) not in ignored_files
+    )
     if len(files) == 0:
         raise Exception("Cannot find any %s files" % (os.path.join(file_dir, "*.proto")))
     files.sort()

lambda_layers/common_python_libraries/requirements.txt

@@ -9,7 +9,8 @@ sqlalchemy==1.3.5
 aws-xray-sdk==2.4.2
 botocore==1.19.55
 boto3==1.16.1
-ipfsapi==0.4.3
+ipfsapi==0.4.4
+ipfshttpclient==0.4.13.2
 pymysql==0.9.2
 setuptools==57.4.0
 dxlmispservice==0.1.5

lambda_layers/web3_and_grpc_libraries/requirements.txt

@@ -1,6 +1,8 @@
 rlp==1.2.0
 protobuf==3.10.0
 web3==5.10.0
+ipfshttpclient==0.4.13.2
+ipfsapi==0.4.4
 grpcio==1.30.0
 grpcio-health-checking==1.19.0
 grpcio-tools==1.19.0

registry/application/services/service_publisher_service.py

@@ -253,9 +253,12 @@ def publish_service_data_to_ipfs(self):
             filename = download_file_from_url(file_url=proto_url,
                                               file_dir=f"{ASSET_DIR}/{service.org_uuid}/{service.uuid}")
             logger.info(f"proto file Name Retrieved  = '{filename}` ")
-            asset_ipfs_hash = publish_zip_file_in_ipfs(filename=filename,
-                                                       file_dir=f"{ASSET_DIR}/{service.org_uuid}/{service.uuid}",
-                                                       ipfs_client=IPFSUtil(IPFS_URL['url'], IPFS_URL['port']))
+            asset_ipfs_hash = publish_zip_file_in_ipfs(
+                filename=filename,
+                file_dir=f"{ASSET_DIR}/{service.org_uuid}/{service.uuid}",
+                ipfs_client=IPFSUtil(IPFS_URL['url'], IPFS_URL['port']),
+                ignored_files=["pricing.proto", "training.proto"]
+            )
             service.proto = {
                 "model_ipfs_hash": asset_ipfs_hash,
                 "encoding": "proto",

service_status/constant.py

@@ -1,2 +1,2 @@
 SRVC_STATUS_GRPC_TIMEOUT = 10
-LIMIT = 5
+LIMIT = 10

service_status/service_status.py

@@ -135,6 +135,8 @@ def update_service_status(self):
             logger.info(f"error_details: {error_details}")
             logger.info(f"debug_error_string: {debug_error_string}")
             old_status = int.from_bytes(record["is_available"], "big")
+            logger.info(f"Service to check: row_id={record['row_id']}, status={status}, old_status={old_status}")
+            logger.info(f"Service endpoint: {record['endpoint']}")
             failed_status_count = self._calculate_failed_status_count(
                 current_status=status, old_status=old_status,
                 old_failed_status_count=record["failed_status_count"])

signer/serverless.yml

@@ -94,7 +94,9 @@ package:
     - License
     - log_setup.py
     - heath_check.sh
+    - node_modules/**
   include:
+    - node_modules/singularitynet-platform-contracts/**
     - resources/**
 
 

wallets/serverless.yml

@@ -111,7 +111,9 @@ package:
     - License
     - log_setup.py
     - heath_check.sh
+    - node_modules/**
   include:
+    - node_modules/singularitynet-platform-contracts/**
     - resources/**