From 6de062c0368ffd495a86c39e1d4348e36db30f10 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Fri, 11 Oct 2024 13:03:20 +0300
Subject: [PATCH 01/29] Start of implementation of channel caching

---
 snet/cli/arguments.py            | 109 ++++++++---------------
 snet/cli/commands/mpe_channel.py | 144 ++++++++++++++++++++++++++-----
 2 files changed, 156 insertions(+), 97 deletions(-)

diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index e9ce4b93..af55bd96 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -662,14 +662,6 @@ def add_p_open_channel_basic(p):
     p.add_argument("--open-new-anyway",
                    action="store_true",
                    help="Skip check that channel already exists and open new channel anyway")
-    add_p_from_block(p)
-
-
-def add_p_from_block(p):
-    p.add_argument("--from-block",
-                   type=int,
-                   default=0,
-                   help="Start searching from this block (for channel searching)")
 
 
 def add_mpe_channel_options(parser):
@@ -677,38 +669,17 @@ def add_mpe_channel_options(parser):
     subparsers = parser.add_subparsers(title="Commands", metavar="COMMAND")
     subparsers.required = True
 
-    p = subparsers.add_parser("init",
-                              help="Initialize channel taking org metadata from Registry")
-    p.set_defaults(fn="init_channel_from_registry")
-
-    add_p_org_id(p)
-    add_group_name(p)
-    add_p_registry_address_opt(p)
-    add_p_mpe_address_opt(p)
-    add_p_channel_id(p)
-
-    p = subparsers.add_parser("init-metadata",
-                              help="Initialize channel using organization metadata")
-    p.set_defaults(fn="init_channel_from_metadata")
-    add_p_org_id(p)
-    add_group_name(p)
-    add_p_registry_address_opt(p)
-    add_p_organization_metadata_file_opt(p)
-    add_p_mpe_address_opt(p)
-    add_p_channel_id(p)
-    add_eth_call_arguments(p)
-
-    p = subparsers.add_parser("open-init",
-                              help="Open and initialize channel using organization metadata from Registry")
-    p.set_defaults(fn="open_init_channel_from_registry")
+    p = subparsers.add_parser("open",
+                              help="Open channel using organization metadata from Registry")
+    p.set_defaults(fn="open_channel_from_registry")
 
     add_p_org_id(p)
     add_p_registry_address_opt(p)
     add_p_open_channel_basic(p)
 
-    p = subparsers.add_parser("open-init-metadata",
-                              help="Open and initialize channel using organization metadata")
-    p.set_defaults(fn="open_init_channel_from_metadata")
+    p = subparsers.add_parser("open-from-metadata",
+                              help="Open channel using existing organization metadata")
+    p.set_defaults(fn="open_channel_from_metadata")
     add_p_org_id(p)
     add_p_registry_address_opt(p)
     add_p_open_channel_basic(p)
@@ -724,14 +695,13 @@ def add_p_set_for_extend_add(_p):
         add_p_mpe_address_opt(p)
         add_transaction_arguments(p)
 
-    p = subparsers.add_parser(
-        "extend-add", help="Set new expiration for the channel and add funds")
+    p = subparsers.add_parser("extend-add", help="Set new expiration for the channel and add funds")
     p.set_defaults(fn="channel_extend_and_add_funds")
     add_p_channel_id(p)
     add_p_set_for_extend_add(p)
 
     p = subparsers.add_parser("extend-add-for-org",
-                              help="Set new expiration and add funds for the channel for the given service")
+                              help="Set new expiration and add funds for the channel for the given service (organization and group name)")
     p.set_defaults(fn="channel_extend_and_add_funds_for_org")
     add_p_org_id(p)
     add_group_name(p)
@@ -739,7 +709,6 @@ def add_p_set_for_extend_add(_p):
     add_p_set_for_extend_add(p)
     add_p_group_name(p)
     add_p_channel_id_opt(p)
-    add_p_from_block(p)
 
     p = subparsers.add_parser("block-number",
                               help="Print the last ethereum block number")
@@ -767,70 +736,63 @@ def add_p_sender(_p):
                         default=None,
                         help="Account to set as sender (by default we use the current identity)")
 
-    p = subparsers.add_parser("print-initialized",
-                              help="Print initialized channels.")
-    p.set_defaults(fn="print_initialized_channels")
-    add_p_only_id(p)
-    add_p_only_sender_signer(p)
-    add_p_mpe_address_opt(p)
-    add_eth_call_arguments(p)
-    add_p_registry_address_opt(p)
-
-    p = subparsers.add_parser("print-initialized-filter-org",
-                              help="Print initialized channels for the given org (all payment group).")
-    p.set_defaults(fn="print_initialized_channels_filter_org")
-    add_p_org_id(p)
-    add_group_name(p)
-    add_p_registry_address_opt(p)
-    add_p_only_id(p)
-    add_p_only_sender_signer(p)
-    add_p_mpe_address_opt(p)
-    add_eth_call_arguments(p)
+    def add_p_dont_sync_channels(_p):
+        _p.add_argument("--do-not-sync", "-ds",
+                        action='store_true',
+                        help="Print channels without synchronizing their state")
 
-    p = subparsers.add_parser("print-all-filter-sender",
+    p = subparsers.add_parser("print-filter-sender",
                               help="Print all channels for the given sender.")
-    p.set_defaults(fn="print_all_channels_filter_sender")
+    p.set_defaults(fn="print_channels_filter_sender")
     add_p_only_id(p)
     add_p_mpe_address_opt(p)
-    add_p_from_block(p)
     add_eth_call_arguments(p)
     add_p_sender(p)
+    add_p_dont_sync_channels(p)
 
-    p = subparsers.add_parser("print-all-filter-recipient",
+    p = subparsers.add_parser("print-filter-recipient",
                               help="Print all channels for the given recipient.")
-    p.set_defaults(fn="print_all_channels_filter_recipient")
+    p.set_defaults(fn="print_channels_filter_recipient")
     add_p_only_id(p)
     add_p_mpe_address_opt(p)
-    add_p_from_block(p)
     add_eth_call_arguments(p)
     p.add_argument("--recipient",
                    default=None,
                    help="Account to set as recipient (by default we use the current identity)")
+    add_p_dont_sync_channels(p)
 
-    p = subparsers.add_parser("print-all-filter-group",
+    p = subparsers.add_parser("print-filter-group",
                               help="Print all channels for the given service.")
-    p.set_defaults(fn="print_all_channels_filter_group")
-
+    p.set_defaults(fn="print_channels_filter_group")
     add_p_org_id(p)
     add_group_name(p)
     add_p_registry_address_opt(p)
     add_p_only_id(p)
     add_p_mpe_address_opt(p)
-    add_p_from_block(p)
     add_eth_call_arguments(p)
+    add_p_dont_sync_channels(p)
 
-    p = subparsers.add_parser("print-all-filter-group-sender",
+    p = subparsers.add_parser("print-filter-group-sender",
                               help="Print all channels for the given group and sender.")
-    p.set_defaults(fn="print_all_channels_filter_group_sender")
-
+    p.set_defaults(fn="print_channels_filter_group_sender")
     add_p_org_id(p)
     add_group_name(p)
     add_p_registry_address_opt(p)
     add_p_only_id(p)
     add_p_mpe_address_opt(p)
-    add_p_from_block(p)
     add_eth_call_arguments(p)
     add_p_sender(p)
+    add_p_dont_sync_channels(p)
+
+    p = subparsers.add_parser("print-all",
+                              help="Print all channels.")
+    p.set_defaults(fn="print_all_channels")
+    add_p_registry_address_opt(p)
+    add_p_only_id(p)
+    add_p_only_sender_signer(p)
+    add_p_mpe_address_opt(p)
+    add_eth_call_arguments(p)
+    add_p_dont_sync_channels(p)
 
     p = subparsers.add_parser("claim-timeout",
                               help="Claim timeout of the channel")
@@ -844,7 +806,6 @@ def add_p_sender(_p):
     p.set_defaults(fn="channel_claim_timeout_all")
     add_p_mpe_address_opt(p)
     add_transaction_arguments(p)
-    add_p_from_block(p)
 
 
 def add_mpe_client_options(parser):
@@ -888,9 +849,7 @@ def add_p_set1_for_call(_p):
     add_p_org_id_service_id(p)
     add_group_name(p)
     add_p_set1_for_call(p)
-
     add_p_channel_id_opt(p)
-    add_p_from_block(p)
     p.add_argument("--yes", "-y",
                    action="store_true",
                    help="Skip interactive confirmation of call price",
diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py
index 0c467c82..73bf48b0 100644
--- a/snet/cli/commands/mpe_channel.py
+++ b/snet/cli/commands/mpe_channel.py
@@ -9,7 +9,7 @@
 from eth_abi.codec import ABICodec
 from web3._utils.encoding import pad_hex
 from web3._utils.events import get_event_data
-from snet.contracts import get_contract_def
+from snet.contracts import get_contract_def, get_contract_deployment_block
 
 from snet.cli.commands.commands import OrganizationCommand
 from snet.cli.metadata.service import mpe_service_metadata_from_json, load_mpe_service_metadata
@@ -29,6 +29,52 @@ def _get_persistent_mpe_dir(self):
         registry_address = self.get_registry_address().lower()
         return Path.home().joinpath(".snet", "mpe_client", "%s_%s" % (mpe_address, registry_address))
 
+    def _get_channels_cache_file(self):
+        channels_dir = Path.home().joinpath(".snet", "cache", "mpe")
+        mpe_address = self.get_mpe_address().lower()
+        channels_file = channels_dir.joinpath(str(mpe_address), "channels.pickle")
+        return channels_file
+
+    def _update_channels_cache(self):
+        channels = []
+        last_read_block = get_contract_deployment_block(self.ident.w3, "MultiPartyEscrow")
+        channels_file = self._get_channels_cache_file()
+
+        if not channels_file.exists():
+            self._printout(f"Channels cache is empty. Caching may take some time when first accessing channels.\nCaching in progress...")
+            channels_file.parent.mkdir(parents=True, exist_ok=True)
+            with open(channels_file, "wb") as f:
+                empty_dict = {
+                    "last_read_block": last_read_block,
+                    "channels": channels
+                }
+                pickle.dump(empty_dict, f)
+        else:
+            with open(channels_file, "rb") as f:
+                load_dict = pickle.load(f)
+            last_read_block = load_dict["last_read_block"]
+            channels = load_dict["channels"]
+
+        current_block_number = self.ident.w3.eth.block_number
+
+        if last_read_block < current_block_number:
+            new_channels = self._get_all_opened_channels_from_blockchain(last_read_block, current_block_number)
+            channels = channels + new_channels
+            last_read_block = current_block_number
+
+            with open(channels_file, "wb") as f:
+                dict_to_save = {
+                    "last_read_block": last_read_block,
+                    "channels": channels
+                }
+                pickle.dump(dict_to_save, f)
+
+    def _get_channels_from_cache(self):
+        self._update_channels_cache()
+        with open(self._get_channels_cache_file(), "rb") as f:
+            load_dict = pickle.load(f)
+        return load_dict["channels"]
+
     def _get_service_base_dir(self, org_id, service_id):
         """ get persistent storage for the given service (~/.snet/mpe_client/<mpe_address>_<registry_address>/<org_id>/<service_id>/) """
         return self._get_persistent_mpe_dir().joinpath(org_id, service_id)
@@ -182,6 +228,7 @@ def _check_channel_is_mine(self, channel):
                             "(address=%s sender=%s signer=%s)" % (self.ident.address.lower(), channel["sender"].lower(), channel["signer"].lower()))
 
     def _init_channel_from_metadata(self, metadata, org_registration):
+        # TODO: delete
         channel_id = self.args.channel_id
         channel = self._get_channel_state_from_blockchain(channel_id)
         self._check_channel_is_mine(channel)
@@ -196,10 +243,14 @@ def _init_channel_from_metadata(self, metadata, org_registration):
         self._add_channel_to_initialized(self.args.org_id, channel)
 
     def init_channel_from_metadata(self):
+        # TODO: "channel init-metadata" command
+        # TODO: delete
         metadata = OrganizationMetadata.from_file(self.args.metadata_file)
         self._init_channel_from_metadata(metadata, {})
 
     def init_channel_from_registry(self):
+        # TODO: "channel init" command
+        # TODO: delete
         metadata = self._get_organization_metadata_from_registry(
             self.args.org_id)
         org_registration = self._get_organization_registration(
@@ -288,6 +339,7 @@ def _initialize_already_opened_channel(self, metadata, sender, signer):
         return None
 
     def _open_init_channel_from_metadata(self, metadata, org_registration):
+        # TODO: change to not init
         self._init_or_update_org_if_needed(metadata, org_registration)
 
         # Before open new channel we try to find already opened channel
@@ -307,11 +359,13 @@ def _open_init_channel_from_metadata(self, metadata, org_registration):
         # initialize channel
         self._add_channel_to_initialized(self.args.org_id, channel)
 
-    def open_init_channel_from_metadata(self):
+    def open_channel_from_metadata(self):
+        # TODO: "channel open-init-metadata" command
         metadata = OrganizationMetadata.from_file(self.args.metadata_file)
         self._open_init_channel_from_metadata(metadata, {})
 
-    def open_init_channel_from_registry(self):
+    def open_channel_from_registry(self):
+        # TODO: "channel open-init" command
         metadata = self._get_organization_metadata_from_registry(
             self.args.org_id)
         org_registration = self._get_organization_registration(
@@ -363,21 +417,21 @@ def check_new_expiration_from_blockchain(self, channel_id, new_expiration):
             raise Exception("New expiration (%i) is smaller then old one (%i)" % (
                 new_expiration, channel["expiration"]))
 
-    def _smart_get_initialized_channel_for_org(self, metadata, filter_by, is_try_initailize=True):
+    def _smart_get_channel_for_org(self, metadata, filter_by, is_try_initailize=True):
+        # TODO: change to not take from initialized
         '''
          - filter_by can be sender or signer
         '''
         channels = self._get_initialized_channels_for_org(self.args.org_id)
-        group_id = base64.b64decode(
-            metadata.get_group_id_by_group_name(self.args.group_name))
-        channels = [c for c in channels if c[filter_by].lower(
-        ) == self.ident.address.lower() and c["groupId"] == group_id]
+        group_id = base64.b64decode(metadata.get_group_id_by_group_name(self.args.group_name))
+        channels = [c for c in channels
+                    if c[filter_by].lower() == self.ident.address.lower() and c["groupId"] == group_id]
 
         if len(channels) == 0 and is_try_initailize:
             # this will work only in simple case where signer == sender
             self._initialize_already_opened_channel(
                 metadata, self.ident.address, self.ident.address)
-            return self._smart_get_initialized_channel_for_org(metadata, filter_by, is_try_initailize=False)
+            return self._smart_get_channel_for_org(metadata, filter_by, is_try_initailize=False)
 
         if len(channels) == 0:
             raise Exception("Cannot find initialized channel for service with org_id=%s service_id=%s and signer=%s" % (
@@ -394,16 +448,14 @@ def _smart_get_initialized_channel_for_org(self, metadata, filter_by, is_try_ini
         raise Exception(
             "Channel %i has not been initialized or your are not the sender/signer of it" % self.args.channel_id)
 
-    def _smart_get_channel_for_org(self):
+    def channel_extend_and_add_funds_for_org(self):
         self._init_or_update_registered_org_if_needed()
         metadata = self._read_metadata_for_org(self.args.org_id)
-        return self._smart_get_initialized_channel_for_org(metadata, "sender")
-
-    def channel_extend_and_add_funds_for_org(self):
-        channel_id = self._smart_get_channel_for_org()["channelId"]
+        channel_id = self._smart_get_channel_for_org(metadata, "sender")["channelId"]
         self._channel_extend_add_funds_with_channel_id(channel_id)
 
     def _get_all_initialized_channels(self):
+        # TODO: delete
         """ return dict of lists  rez[(<org_id>, <service_id>)] = [(channel_id, channel_info)] """
         channels_dict = defaultdict(list)
         for service_base_dir in self._get_persistent_mpe_dir().glob("*/*"):
@@ -490,14 +542,53 @@ def _filter_channels_sender_or_signer(self, channels):
         return good_channels
 
     def print_initialized_channels(self):
+        # TODO: "print-initialized" command
+        # TODO: delete
         channels_dict = self._get_all_initialized_channels()
         self._print_channels_dict_from_blockchain(channels_dict)
 
-    def print_initialized_channels_filter_org(self):
-        channels = self._get_initialized_channels_for_org(self.args.org_id)
-        self._print_channels_dict_from_blockchain({self.args.org_id: channels})
+    def _event_data_args_to_dict(self, event_data):
+        return {
+            "channel_id": event_data["channelId"],
+            "sender": event_data["sender"],
+            "signer": event_data["signer"],
+            "recipient": event_data["recipient"],
+            "group_id": event_data["groupId"],
+        }
+
+    def _get_all_opened_channels_from_blockchain(self, starting_block_number, to_block_number):
+        mpe_address = self.get_mpe_address()
+        event_topics = self.ident.w3.keccak(
+            text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()
+        blocks_per_batch = 5000
+        codec: ABICodec = self.ident.w3.codec
+
+        logs = []
+        from_block = starting_block_number
+        while from_block <= to_block_number:
+            to_block = min(from_block + blocks_per_batch, to_block_number)
+            logs += self.ident.w3.eth.get_logs({"fromBlock": from_block,
+                                                  "toBlock": to_block,
+                                                  "address": mpe_address,
+                                                  "topics": event_topics})
+            from_block = to_block + 1
+
+        abi = get_contract_def("MultiPartyEscrow")
+        event_abi = abi_get_element_by_name(abi, "ChannelOpen")
+
+        event_data_list = [get_event_data(codec, event_abi, l)["args"] for l in logs]
+        channels_opened = list(map(self._event_data_args_to_dict, event_data_list))
+
+        return channels_opened
+
+    def _get_filtered_channels(self, **kwargs):
+        channels = self._get_channels_from_cache()
+        for key, value in kwargs.items():
+            channels = [c for c in channels if c[key] == value]
+        return channels
 
     def _get_all_filtered_channels(self, topics_without_signature):
+        # TODO: delete
         """ get all filtered chanels from blockchain logs """
         mpe_address = self.get_mpe_address()
         event_signature = self.ident.w3.keccak(
@@ -505,7 +596,7 @@ def _get_all_filtered_channels(self, topics_without_signature):
         codec: ABICodec = self.ident.w3.codec
         topics = [event_signature] + topics_without_signature
         logs = self.ident.w3.eth.get_logs(
-            {"fromBlock": self.args.from_block, "address": mpe_address, "topics": topics})
+            {"fromBlock": get_contract_deployment_block(self.ident.w3, "MultiPartyEscrow"), "address": mpe_address, "topics": topics})
         abi = get_contract_def("MultiPartyEscrow")
         event_abi = abi_get_element_by_name(abi, "ChannelOpen")
         channels_ids = [get_event_data(codec, event_abi, l)[
@@ -517,18 +608,24 @@ def get_address_from_arg_or_ident(self, arg):
             return arg
         return self.ident.address
 
-    def print_all_channels_filter_sender(self):
+    def print_channels_filter_org(self):
+        # TODO: "print-initialized-filter-org" command
+        # TODO: change to "print-filter_org"
+        channels = self._get_initialized_channels_for_org(self.args.org_id)
+        self._print_channels_dict_from_blockchain({self.args.org_id: channels})
+
+    def print_channels_filter_sender(self):
         address = self.get_address_from_arg_or_ident(self.args.sender)
         channels_ids = self._get_all_channels_filter_sender(address)
         self._print_channels_from_blockchain(channels_ids)
 
-    def print_all_channels_filter_recipient(self):
+    def print_channels_filter_recipient(self):
         address = self.get_address_from_arg_or_ident(self.args.recipient)
         address_padded = pad_hex(address.lower(), 256)
         channels_ids = self._get_all_filtered_channels([None, address_padded])
         self._print_channels_from_blockchain(channels_ids)
 
-    def print_all_channels_filter_group(self):
+    def print_channels_filter_group(self):
         metadata = self._get_organization_metadata_from_registry(
             self.args.org_id)
         group_id = base64.b64decode(
@@ -538,7 +635,7 @@ def print_all_channels_filter_group(self):
             [None, None, group_id_hex])
         self._print_channels_from_blockchain(channels_ids)
 
-    def print_all_channels_filter_group_sender(self):
+    def print_channels_filter_group_sender(self):
         address = self.get_address_from_arg_or_ident(self.args.sender)
         address_padded = pad_hex(address.lower(), 256)
         metadata = self._get_organization_metadata_from_registry(
@@ -550,6 +647,9 @@ def print_all_channels_filter_group_sender(self):
             [address_padded, None, group_id_hex])
         self._print_channels_from_blockchain(channels_ids)
 
+    def print_all_channels(self):
+        pass
+
     def _get_all_channels_filter_sender(self, sender):
         sender_padded = pad_hex(sender.lower(), 256)
         channels_ids = self._get_all_filtered_channels([sender_padded])

From f036279ddf87cc5b3e9caf028b8a874873aadfa6 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Mon, 14 Oct 2024 16:11:51 +0300
Subject: [PATCH 02/29] Implemented cache updating and loading to output
 channels info.

---
 snet/cli/arguments.py            |  19 +--
 snet/cli/commands/mpe_channel.py | 241 ++++++++++++++++++-------------
 2 files changed, 144 insertions(+), 116 deletions(-)

diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index af55bd96..ff394aba 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -717,19 +717,8 @@ def add_p_set_for_extend_add(_p):
     def add_p_only_id(_p):
         _p.add_argument("--only-id",
                         action='store_true',
-                        help="Print only id of channels")
-
-    def add_p_only_sender_signer(_p):
-        pm = _p.add_mutually_exclusive_group(required=False)
-        pm.add_argument("--filter-sender",
-                        action='store_true',
-                        help="Print only channels in which current identity is sender")
-        pm.add_argument("--filter-signer",
-                        action='store_true',
-                        help="Print only channels in which current identity is signer")
-        pm.add_argument("--filter-my",
-                        action='store_true',
-                        help="Print only channels in which current identity is sender or signer")
+                        help="Print only id of channels",
+                        default=False)
 
     def add_p_sender(_p):
         _p.add_argument("--sender",
@@ -739,7 +728,8 @@ def add_p_sender(_p):
     def add_p_dont_sync_channels(_p):
         _p.add_argument("--do-not-sync", "-ds",
                         action='store_true',
-                        help="Print channels without synchronizing their state")
+                        help="Print channels without synchronizing their state",
+                        default=False)
 
     p = subparsers.add_parser("print-filter-sender",
                               help="Print all channels for the given sender.")
@@ -789,7 +779,6 @@ def add_p_dont_sync_channels(_p):
     p.set_defaults(fn="print_all_channels")
     add_p_registry_address_opt(p)
     add_p_only_id(p)
-    add_p_only_sender_signer(p)
     add_p_mpe_address_opt(p)
     add_eth_call_arguments(p)
     add_p_dont_sync_channels(p)
diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py
index 73bf48b0..0b85456b 100644
--- a/snet/cli/commands/mpe_channel.py
+++ b/snet/cli/commands/mpe_channel.py
@@ -4,6 +4,7 @@
 import shutil
 import tempfile
 from collections import defaultdict
+from distutils.command.check import check
 from pathlib import Path
 
 from eth_abi.codec import ABICodec
@@ -75,6 +76,68 @@ def _get_channels_from_cache(self):
             load_dict = pickle.load(f)
         return load_dict["channels"]
 
+    def _event_data_args_to_dict(self, event_data):
+        return {
+            "channel_id": event_data["channelId"],
+            "sender": event_data["sender"],
+            "signer": event_data["signer"],
+            "recipient": event_data["recipient"],
+            "group_id": event_data["groupId"],
+        }
+
+    def _get_all_opened_channels_from_blockchain(self, starting_block_number, to_block_number):
+        mpe_address = self.get_mpe_address()
+        event_topics = [self.ident.w3.keccak(
+            text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()]
+        blocks_per_batch = 5000
+        codec: ABICodec = self.ident.w3.codec
+
+        logs = []
+        from_block = starting_block_number
+        while from_block <= to_block_number:
+            to_block = min(from_block + blocks_per_batch, to_block_number)
+            logs += self.ident.w3.eth.get_logs({"fromBlock": from_block,
+                                                  "toBlock": to_block,
+                                                  "address": mpe_address,
+                                                  "topics": event_topics})
+            from_block = to_block + 1
+
+        abi = get_contract_def("MultiPartyEscrow")
+        event_abi = abi_get_element_by_name(abi, "ChannelOpen")
+
+        event_data_list = [get_event_data(codec, event_abi, l)["args"] for l in logs]
+        channels_opened = list(map(self._event_data_args_to_dict, event_data_list))
+
+        return channels_opened
+
+    def _get_filtered_channels(self, return_only_id=False, **kwargs):
+        channels = self._get_channels_from_cache()
+        for key, value in kwargs.items():
+            if key == "group_id":
+                check_ch = lambda c: base64.b64encode(c[key]).decode("utf-8") == value
+            else:
+                check_ch = lambda c: c[key] == value
+            channels = [c for c in channels if check_ch(c)]
+        if return_only_id:
+            return [c["channel_id"] for c in channels]
+        return channels
+
+    def _get_all_filtered_channels(self, topics_without_signature):
+        # TODO: delete
+        """ get all filtered chanels from blockchain logs """
+        mpe_address = self.get_mpe_address()
+        event_signature = self.ident.w3.keccak(
+            text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()
+        codec: ABICodec = self.ident.w3.codec
+        topics = [event_signature] + topics_without_signature
+        logs = self.ident.w3.eth.get_logs(
+            {"fromBlock": get_contract_deployment_block(self.ident.w3, "MultiPartyEscrow"), "address": mpe_address, "topics": topics})
+        abi = get_contract_def("MultiPartyEscrow")
+        event_abi = abi_get_element_by_name(abi, "ChannelOpen")
+        channels_ids = [get_event_data(codec, event_abi, l)[
+            "args"]["channelId"] for l in logs]
+        return channels_ids
+
     def _get_service_base_dir(self, org_id, service_id):
         """ get persistent storage for the given service (~/.snet/mpe_client/<mpe_address>_<registry_address>/<org_id>/<service_id>/) """
         return self._get_persistent_mpe_dir().joinpath(org_id, service_id)
@@ -322,12 +385,9 @@ def _open_channel_for_org(self, metadata):
 
     def _initialize_already_opened_channel(self, metadata, sender, signer):
 
-        group_id = base64.b64decode(
-            metadata.get_group_id_by_group_name(self.args.group_name))
-        recipient = metadata.get_payment_address_for_group(
-            self.args.group_name)
-        channels_ids = self._get_all_channels_filter_sender_recipient_group(
-            sender, recipient, group_id)
+        group_id = base64.b64decode(metadata.get_group_id_by_group_name(self.args.group_name))
+        recipient = metadata.get_payment_address_for_group(self.args.group_name)
+        channels_ids = self._get_all_channels_filter_sender_recipient_group(sender, recipient, group_id)
         for i in sorted(channels_ids):
             channel = self._get_channel_state_from_blockchain(i)
             if channel["signer"].lower() == signer.lower():
@@ -360,12 +420,12 @@ def _open_init_channel_from_metadata(self, metadata, org_registration):
         self._add_channel_to_initialized(self.args.org_id, channel)
 
     def open_channel_from_metadata(self):
-        # TODO: "channel open-init-metadata" command
+        # TODO: "channel open-metadata" command
         metadata = OrganizationMetadata.from_file(self.args.metadata_file)
         self._open_init_channel_from_metadata(metadata, {})
 
     def open_channel_from_registry(self):
-        # TODO: "channel open-init" command
+        # TODO: "channel open" command
         metadata = self._get_organization_metadata_from_registry(
             self.args.org_id)
         org_registration = self._get_organization_registration(
@@ -481,21 +541,55 @@ def _read_metadata_for_org(self, org_id):
                 "Service with org_id=%s is not initialized" % (org_id))
         return OrganizationMetadata.from_file(sdir.joinpath("organization_metadata.json"))
 
-    def _print_channels_from_blockchain(self, channels_ids):
-        channels_ids = sorted(channels_ids)
+    def _convert_channel_dict_to_str(self, channel, filters=None):
+        if filters is None:
+            filters = []
+        for key, value in channel.items():
+            channel[key] = str(value)
+        # converting to string not using " ".join() to always have the same order
+        channel_as_str = ""
+        channel_as_str += channel["channel_id"]
+        channel_as_str += " " + channel["nonce"] if "nonce" in channel else ""
+        channel_as_str += " " + channel["sender"] if "sender" not in filters else ""
+        channel_as_str += " " + channel["signer"]
+        channel_as_str += " " + channel["recipient"] if "recipient" not in filters else ""
+        channel_as_str += " " + channel["group_id"] if "group_id" not in filters else ""
+        channel_as_str += " " + channel["value"] if "value" in channel else ""
+        channel_as_str += " " + channel["expiration"] if "expiration" in channel else ""
+        return channel_as_str
+
+
+    def _print_channels(self, channels, filters: list[str] = None):
+        if filters is None:
+            filters = []
+
         if self.args.only_id:
-            self._printout("#channelId")
-            [self._printout(str(i)) for i in channels_ids]
+            self._printout("#channel_id")
+            [self._printout(ch_id) for ch_id in channels]
             return
-        self._printout(
-            "#channelId nonce recipient groupId(base64) value(AGIX) expiration(blocks)")
-        for i in channels_ids:
-            channel = self._get_channel_state_from_blockchain(i)
-            value_agi = cogs2stragix(channel["value"])
-            group_id_base64 = base64.b64encode(
-                channel["groupId"]).decode("ascii")
-            self._printout("%i %i %s %s %s %i" % (i, channel["nonce"], channel["recipient"], group_id_base64,
-                                                  value_agi, channel["expiration"]))
+
+        titles = ["channel_id", "nonce", "sender", "signer", "recipient", "group_id", "value", "expiration"]
+        for channel_filter in filters:
+            titles.remove(channel_filter)
+
+        if self.args.do_not_sync:
+            titles.remove("nonce")
+            titles.remove("value")
+            titles.remove("expiration")
+            self._printout("#" + " ".join(titles))
+            for channel in channels:
+                channel["group_id"] = base64.b64encode(channel["group_id"]).decode("ascii")
+                self._printout(self._convert_channel_dict_to_str(channel, filters))
+            return
+
+        channels_ids = sorted(channels)
+        self._printout("#" + " ".join(titles))
+        for channel_id in channels_ids:
+            channel = self._get_channel_state_from_blockchain(channel_id)
+            channel["channel_id"] = channel_id
+            channel["value"] = cogs2stragix(channel["value"])
+            channel["group_id"] = base64.b64encode(channel["groupId"]).decode("ascii")
+            self._printout(self._convert_channel_dict_to_str(channel, filters))
 
     def _print_channels_dict_from_blockchain(self, channels_dict):
         # print only caption
@@ -547,62 +641,6 @@ def print_initialized_channels(self):
         channels_dict = self._get_all_initialized_channels()
         self._print_channels_dict_from_blockchain(channels_dict)
 
-    def _event_data_args_to_dict(self, event_data):
-        return {
-            "channel_id": event_data["channelId"],
-            "sender": event_data["sender"],
-            "signer": event_data["signer"],
-            "recipient": event_data["recipient"],
-            "group_id": event_data["groupId"],
-        }
-
-    def _get_all_opened_channels_from_blockchain(self, starting_block_number, to_block_number):
-        mpe_address = self.get_mpe_address()
-        event_topics = self.ident.w3.keccak(
-            text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()
-        blocks_per_batch = 5000
-        codec: ABICodec = self.ident.w3.codec
-
-        logs = []
-        from_block = starting_block_number
-        while from_block <= to_block_number:
-            to_block = min(from_block + blocks_per_batch, to_block_number)
-            logs += self.ident.w3.eth.get_logs({"fromBlock": from_block,
-                                                  "toBlock": to_block,
-                                                  "address": mpe_address,
-                                                  "topics": event_topics})
-            from_block = to_block + 1
-
-        abi = get_contract_def("MultiPartyEscrow")
-        event_abi = abi_get_element_by_name(abi, "ChannelOpen")
-
-        event_data_list = [get_event_data(codec, event_abi, l)["args"] for l in logs]
-        channels_opened = list(map(self._event_data_args_to_dict, event_data_list))
-
-        return channels_opened
-
-    def _get_filtered_channels(self, **kwargs):
-        channels = self._get_channels_from_cache()
-        for key, value in kwargs.items():
-            channels = [c for c in channels if c[key] == value]
-        return channels
-
-    def _get_all_filtered_channels(self, topics_without_signature):
-        # TODO: delete
-        """ get all filtered chanels from blockchain logs """
-        mpe_address = self.get_mpe_address()
-        event_signature = self.ident.w3.keccak(
-            text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()
-        codec: ABICodec = self.ident.w3.codec
-        topics = [event_signature] + topics_without_signature
-        logs = self.ident.w3.eth.get_logs(
-            {"fromBlock": get_contract_deployment_block(self.ident.w3, "MultiPartyEscrow"), "address": mpe_address, "topics": topics})
-        abi = get_contract_def("MultiPartyEscrow")
-        event_abi = abi_get_element_by_name(abi, "ChannelOpen")
-        channels_ids = [get_event_data(codec, event_abi, l)[
-            "args"]["channelId"] for l in logs]
-        return channels_ids
-
     def get_address_from_arg_or_ident(self, arg):
         if arg:
             return arg
@@ -610,45 +648,46 @@ def get_address_from_arg_or_ident(self, arg):
 
     def print_channels_filter_org(self):
         # TODO: "print-initialized-filter-org" command
-        # TODO: change to "print-filter_org"
+        # TODO: delete
         channels = self._get_initialized_channels_for_org(self.args.org_id)
         self._print_channels_dict_from_blockchain({self.args.org_id: channels})
 
     def print_channels_filter_sender(self):
+        # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
+        return_only_id = self.args.only_id or not self.args.do_not_sync
         address = self.get_address_from_arg_or_ident(self.args.sender)
-        channels_ids = self._get_all_channels_filter_sender(address)
-        self._print_channels_from_blockchain(channels_ids)
+        channels = self._get_filtered_channels(return_only_id=return_only_id, sender=address)
+        self._print_channels(channels, ["sender"])
 
     def print_channels_filter_recipient(self):
+        # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
+        return_only_id = self.args.only_id or not self.args.do_not_sync
         address = self.get_address_from_arg_or_ident(self.args.recipient)
-        address_padded = pad_hex(address.lower(), 256)
-        channels_ids = self._get_all_filtered_channels([None, address_padded])
-        self._print_channels_from_blockchain(channels_ids)
+        channels = self._get_filtered_channels(return_only_id=return_only_id, recipient=address)
+        self._print_channels(channels, ["recipient"])
 
     def print_channels_filter_group(self):
-        metadata = self._get_organization_metadata_from_registry(
-            self.args.org_id)
-        group_id = base64.b64decode(
-            metadata.get_group_id_by_group_name(self.args.group_name))
-        group_id_hex = "0x" + group_id.hex()
-        channels_ids = self._get_all_filtered_channels(
-            [None, None, group_id_hex])
-        self._print_channels_from_blockchain(channels_ids)
+        # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
+        return_only_id = self.args.only_id or not self.args.do_not_sync
+        metadata = self._get_organization_metadata_from_registry(self.args.org_id)
+        group_id = metadata.get_group_id_by_group_name(self.args.group_name)
+        channels = self._get_filtered_channels(return_only_id=return_only_id, group_id=group_id)
+        self._print_channels(channels, ["group_id"])
 
     def print_channels_filter_group_sender(self):
+        # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
+        return_only_id = self.args.only_id or not self.args.do_not_sync
         address = self.get_address_from_arg_or_ident(self.args.sender)
-        address_padded = pad_hex(address.lower(), 256)
-        metadata = self._get_organization_metadata_from_registry(
-            self.args.org_id)
-        group_id = base64.b64decode(
-            metadata.get_group_id_by_group_name(self.args.group_name))
-        group_id_hex = "0x" + group_id.hex()
-        channels_ids = self._get_all_filtered_channels(
-            [address_padded, None, group_id_hex])
-        self._print_channels_from_blockchain(channels_ids)
+        metadata = self._get_organization_metadata_from_registry(self.args.org_id)
+        group_id = metadata.get_group_id_by_group_name(self.args.group_name)
+        channels = self._get_filtered_channels(return_only_id=return_only_id, sender=address, group_id=group_id)
+        self._print_channels(channels, ["sender", "group_id"])
 
     def print_all_channels(self):
-        pass
+        # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
+        return_only_id = self.args.only_id or not self.args.do_not_sync
+        channels = self._get_filtered_channels(return_only_id=return_only_id)
+        self._print_channels(channels)
 
     def _get_all_channels_filter_sender(self, sender):
         sender_padded = pad_hex(sender.lower(), 256)

From e8e43a73fb3c4eeab4be1608745731deb0957b12 Mon Sep 17 00:00:00 2001
From: mabredin <mbredin@bk.ru>
Date: Wed, 16 Oct 2024 12:22:29 +0300
Subject: [PATCH 03/29] Add generation of documentation in Markdown format

---
 .gitignore                       |  1 +
 docs/build-docs.sh               |  9 ++-
 docs/source/conf.py              |  4 +-
 docs/source/generate_markdown.py | 96 ++++++++++++++++++++++++++++++++
 4 files changed, 106 insertions(+), 4 deletions(-)
 create mode 100644 docs/source/generate_markdown.py

diff --git a/.gitignore b/.gitignore
index 557ba201..b2a9e7e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,5 +10,6 @@ snet.cli.egg-info/
 snet/cli/resources/node_modules
 snet/cli/resources/proto/*.py
 build/
+*.rst
 dist/
 *.pyi
diff --git a/docs/build-docs.sh b/docs/build-docs.sh
index 8eba028c..286c6458 100644
--- a/docs/build-docs.sh
+++ b/docs/build-docs.sh
@@ -1,7 +1,12 @@
 cd source
 python generate_rst.py
-
 cd ..
+
 make clean
 make html
-cp source/snet-cli-static/theme.css build/html/_static/css/ 
+cp source/snet-cli-static/theme.css build/html/_static/css/
+
+cd source
+python generate_markdown.py ../build/html/ ../build/html/clean ../build/markdown
+cd ..
+rm -rf build/html/clean
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 46711c4d..488bfa0b 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -67,7 +67,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -189,4 +189,4 @@
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+# intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/docs/source/generate_markdown.py b/docs/source/generate_markdown.py
new file mode 100644
index 00000000..be4bb6b7
--- /dev/null
+++ b/docs/source/generate_markdown.py
@@ -0,0 +1,96 @@
+import os
+import sys
+from bs4 import BeautifulSoup
+
+
+def process_html_file(file_path, output_dir, markdown_dir):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        html_content = file.read()
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+
+    footers = soup.find_all('footer')
+    for footer in footers:
+        footer.decompose()
+
+    navigations = soup.find_all('nav')
+    for navigation in navigations:
+        navigation.decompose()
+
+    hr = soup.find('hr')
+    if hr:
+        hr.decompose()
+
+    ul = soup.find('ul', class_='wy-breadcrumbs')
+    if ul:
+        ul.decompose()
+
+    headerlinks = soup.find_all('a', class_='headerlink')
+    for headerlink in headerlinks:
+        headerlink.decompose()
+
+    code_blocks = soup.find_all('div', class_='highlight-default notranslate')
+    for block in code_blocks:
+        pre_tag = soup.new_tag("p")
+        pre_tag.append("start ")
+        pre_tag.append(block.get_text().strip())
+        pre_tag.append(" finish")
+        block.replace_with(pre_tag)
+
+    clean_html = str(soup)
+
+    base_filename = os.path.basename(file_path)
+    output_path = os.path.join(output_dir, base_filename)
+
+    with open(output_path, 'w', encoding='utf-8') as file:
+        file.write(clean_html)
+    print(f"Processed: {file_path} -> {output_path}")
+
+    clean_filename = os.path.splitext(base_filename)[0] + ".md"
+    md_output_path = os.path.join(markdown_dir, clean_filename)
+
+    os.system(f"html2text --ignore-images {output_path} > {md_output_path}")
+
+    with open(md_output_path, 'r', encoding='utf-8') as file:
+        md_content = file.read()
+
+    clean_md = md_content.replace("start ", "```sh\n\t") \
+                         .replace("finish", "\n```")
+
+    with open(md_output_path, 'w', encoding='utf-8') as file:
+        file.write(clean_md)
+
+    print(f"Processed: {output_path} -> {md_output_path}\n")
+
+
+def process_html_files_in_directory(directory, output_dir, markdown_dir):
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    if not os.path.exists(markdown_dir):
+        os.makedirs(markdown_dir)
+
+    for file in os.listdir(directory):
+        if file.endswith('.html'):
+            file_path = os.path.join(directory, file)
+            process_html_file(file_path, output_dir, markdown_dir)
+
+
+def main():
+    if len(sys.argv) == 4:
+        input_directory: str = sys.argv[1]
+        output_directory: str = sys.argv[2]
+        md_directory: str = sys.argv[3]
+    else:
+        raise Exception("""
+            You can only pass 3 parameters:
+            - input_directory
+            - output_directory
+            - md_directory
+        """)
+
+    process_html_files_in_directory(input_directory, output_directory, md_directory)
+
+
+if __name__ == "__main__":
+    main()

From 7ca802e02aba874e3eebf062fd44a702899feae7 Mon Sep 17 00:00:00 2001
From: mabredin <mbredin@bk.ru>
Date: Wed, 16 Oct 2024 12:25:54 +0300
Subject: [PATCH 04/29] Clarify variable names

---
 docs/source/generate_markdown.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/docs/source/generate_markdown.py b/docs/source/generate_markdown.py
index be4bb6b7..142bf3a2 100644
--- a/docs/source/generate_markdown.py
+++ b/docs/source/generate_markdown.py
@@ -78,18 +78,22 @@ def process_html_files_in_directory(directory, output_dir, markdown_dir):
 
 def main():
     if len(sys.argv) == 4:
-        input_directory: str = sys.argv[1]
-        output_directory: str = sys.argv[2]
-        md_directory: str = sys.argv[3]
+        input_html_directory: str = sys.argv[1]
+        output_html_directory: str = sys.argv[2]
+        output_md_directory: str = sys.argv[3]
     else:
         raise Exception("""
             You can only pass 3 parameters:
-            - input_directory
-            - output_directory
-            - md_directory
+            - input_html_directory
+            - output_html_directory
+            - output_md_directory
         """)
 
-    process_html_files_in_directory(input_directory, output_directory, md_directory)
+    process_html_files_in_directory(
+        input_html_directory,
+        output_html_directory,
+        output_md_directory
+    )
 
 
 if __name__ == "__main__":

From bd00e33ab8ba8f9ae45ae91a0785d85de16e6118 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 16 Oct 2024 13:18:34 +0300
Subject: [PATCH 05/29] Finished changes in "snet channel" section.

---
 snet/cli/arguments.py            |   5 +-
 snet/cli/commands/mpe_channel.py | 101 +++++++++++++++++--------------
 2 files changed, 59 insertions(+), 47 deletions(-)

diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index d9542dbb..5a731e4c 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -1162,7 +1162,6 @@ def add_p_publish_params(_p):
         _p.add_argument("--update-mpe-address",
                         action='store_true',
                         help="Update mpe_address in metadata before publishing them")
-        add_p_storage_param(p)
         add_p_mpe_address_opt(_p)
 
     p = subparsers.add_parser("publish",
@@ -1171,7 +1170,8 @@ def add_p_publish_params(_p):
     add_p_publish_params(p)
     add_p_service_in_registry(p)
     add_transaction_arguments(p)
- 
+    add_p_storage_param(p)
+
     p = subparsers.add_parser("publish-in-ipfs",
                               help="Publish metadata only in IPFS, without publishing in Registry")
     p.set_defaults(fn="publish_metadata_in_ipfs")
@@ -1190,6 +1190,7 @@ def add_p_publish_params(_p):
     add_p_publish_params(p)
     add_p_service_in_registry(p)
     add_transaction_arguments(p)
+    add_p_storage_param(p)
 
     p = subparsers.add_parser("update-add-tags",
                               help="Add tags to existed service registration")
diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py
index 53d934e9..eb5c5412 100644
--- a/snet/cli/commands/mpe_channel.py
+++ b/snet/cli/commands/mpe_channel.py
@@ -117,7 +117,7 @@ def _get_filtered_channels(self, return_only_id=False, **kwargs):
                 check_ch = lambda c: base64.b64encode(c[key]).decode("utf-8") == value
             else:
                 check_ch = lambda c: c[key] == value
-            channels = [c for c in channels if check_ch(c)]
+            channels = [ch for ch in channels if check_ch(ch)]
         if return_only_id:
             return [c["channel_id"] for c in channels]
         return channels
@@ -351,28 +351,26 @@ def _get_expiration_from_args(self):
         return rez
 
     def _open_channel_for_org(self, metadata):
+        # TODO: needed
         mpe_cogs = self.call_contract_command(
             "MultiPartyEscrow",    "balances",  [self.ident.address])
         if mpe_cogs < self.args.amount:
             raise Exception(
                 "insufficient funds. You MPE balance is %s AGIX " % cogs2stragix(mpe_cogs))
 
-        group_id = base64.b64decode(
-            metadata.get_group_id_by_group_name(self.args.group_name))
+        group_id = base64.b64decode(metadata.get_group_id_by_group_name(self.args.group_name))
         if not group_id:
-            raise Exception(
-                "group  %s is associated with organization", self.args.group_name)
+            raise Exception("group  %s is associated with organization", self.args.group_name)
 
-        recipient = metadata.get_payment_address_for_group(
-            self.args.group_name)
+        recipient = metadata.get_payment_address_for_group(self.args.group_name)
 
         signer = self.get_address_from_arg_or_ident(self.args.signer)
 
         channel_info = {"sender": self.ident.address, "signer": signer,
-                        "recipient": recipient, "groupId": group_id}
+                        "recipient": recipient, "group_id": group_id}
         expiration = self._get_expiration_from_args()
         params = [channel_info["signer"], channel_info["recipient"],
-                  channel_info["groupId"], self.args.amount, expiration]
+                  channel_info["group_id"], self.args.amount, expiration]
         rez = self.transact_contract_command(
             "MultiPartyEscrow", "openChannel", params)
 
@@ -380,11 +378,11 @@ def _open_channel_for_org(self, metadata):
             raise Exception(
                 "We've expected only one ChannelOpen event after openChannel. Make sure that you use correct MultiPartyEscrow address")
 
-        channel_info["channelId"] = rez[1][0]["args"]["channelId"]
+        channel_info["channel_id"] = rez[1][0]["args"]["channelId"]
         return channel_info
 
     def _initialize_already_opened_channel(self, metadata, sender, signer):
-
+        # TODO: delete
         group_id = base64.b64decode(metadata.get_group_id_by_group_name(self.args.group_name))
         recipient = metadata.get_payment_address_for_group(self.args.group_name)
         channels_ids = self._get_all_channels_filter_sender_recipient_group(sender, recipient, group_id)
@@ -398,31 +396,42 @@ def _initialize_already_opened_channel(self, metadata, sender, signer):
                 return channel
         return None
 
-    def _open_init_channel_from_metadata(self, metadata, org_registration):
-        # TODO: change to not init
+    def _check_already_opened_channel(self, metadata, sender, signer):
+        group_id = metadata.get_group_id_by_group_name(self.args.group_name)
+        recipient = metadata.get_payment_address_for_group(self.args.group_name)
+        channels = self._get_filtered_channels(sender=sender, recipient=recipient, group_id=group_id)
+
+        for i in channels:
+            channel = self._get_channel_state_from_blockchain(i)
+            if channel["signer"].lower() == signer.lower():
+                self._printerr(
+                    "# Channel with given sender, signer and group_id is already exists. (channel_id = %i)"
+                    % channel["channel_id"])
+                return channel
+
+        return None
+
+    def _open_channel_from_metadata(self, metadata, org_registration):
+        # TODO: needed
         self._init_or_update_org_if_needed(metadata, org_registration)
 
         # Before open new channel we try to find already opened channel
         if not self.args.open_new_anyway:
             sender = self.ident.address
             signer = self.get_address_from_arg_or_ident(self.args.signer)
-            channel = self._initialize_already_opened_channel(
-                metadata, sender, signer)
+            channel = self._check_already_opened_channel(metadata, sender, signer)
             if channel is not None:
                 return
 
         # open payment channel
         channel = self._open_channel_for_org(metadata)
         self._printout("#channel_id")
-        self._printout(channel["channelId"])
-
-        # initialize channel
-        self._add_channel_to_initialized(self.args.org_id, channel)
+        self._printout(channel["channel_id"])
 
     def open_channel_from_metadata(self):
-        # TODO: "channel open-metadata" command
+        # TODO: "channel open-from-metadata" command
         metadata = OrganizationMetadata.from_file(self.args.metadata_file)
-        self._open_init_channel_from_metadata(metadata, {})
+        self._open_channel_from_metadata(metadata, {})
 
     def open_channel_from_registry(self):
         # TODO: "channel open" command
@@ -430,17 +439,19 @@ def open_channel_from_registry(self):
             self.args.org_id)
         org_registration = self._get_organization_registration(
             self.args.org_id)
-        self._open_init_channel_from_metadata(metadata, org_registration)
+        self._open_channel_from_metadata(metadata, org_registration)
 
     def channel_claim_timeout(self):
         rez = self._get_channel_state_from_blockchain(self.args.channel_id)
-        if rez["value"] == 0:
+        if rez["expiration"] >= self.ident.w3.eth.block_number:
+            raise Exception("Channel is not expired yet")
+        elif rez["value"] == 0:
             raise Exception("Channel has 0 value. There is nothing to claim")
         self.transact_contract_command(
             "MultiPartyEscrow", "channelClaimTimeout", [self.args.channel_id])
 
     def channel_claim_timeout_all(self):
-        channels_ids = self._get_all_channels_filter_sender(self.ident.address)
+        channels_ids = self._get_filtered_channels(return_only_id=True, sender=self.ident.address)
         for channel_id in channels_ids:
             response = self._get_channel_state_from_blockchain(channel_id)
             if response["value"] > 0 and response["expiration"] < self.ident.w3.eth.block_number:
@@ -457,7 +468,7 @@ def _channel_extend_add_funds_with_channel_id(self, channel_id):
             return
 
         expiration = self._get_expiration_from_args()
-        self.check_new_expiration_from_blockchain(channel_id, expiration)
+        self._check_new_expiration_from_blockchain(channel_id, expiration)
         # only extend channel (if --amount hasn't been specified)
         if self.args.amount is None:
             self.transact_contract_command("MultiPartyEscrow", "channelExtend", [
@@ -471,42 +482,40 @@ def _channel_extend_add_funds_with_channel_id(self, channel_id):
     def channel_extend_and_add_funds(self):
         self._channel_extend_add_funds_with_channel_id(self.args.channel_id)
 
-    def check_new_expiration_from_blockchain(self, channel_id, new_expiration):
+    def _check_new_expiration_from_blockchain(self, channel_id, new_expiration):
         channel = self._get_channel_state_from_blockchain(channel_id)
         if new_expiration < channel["expiration"]:
             raise Exception("New expiration (%i) is smaller then old one (%i)" % (
                 new_expiration, channel["expiration"]))
 
-    def _smart_get_channel_for_org(self, metadata, filter_by, is_try_initailize=True):
-        # TODO: change to not take from initialized
+    def _smart_get_channel_for_org(self, metadata, filter_by):
+        # TODO: needed
         '''
          - filter_by can be sender or signer
         '''
-        channels = self._get_initialized_channels_for_org(self.args.org_id)
-        group_id = base64.b64decode(metadata.get_group_id_by_group_name(self.args.group_name))
-        channels = [c for c in channels
-                    if c[filter_by].lower() == self.ident.address.lower() and c["groupId"] == group_id]
-
-        if len(channels) == 0 and is_try_initailize:
-            # this will work only in simple case where signer == sender
-            self._initialize_already_opened_channel(
-                metadata, self.ident.address, self.ident.address)
-            return self._smart_get_channel_for_org(metadata, filter_by, is_try_initailize=False)
+        recipient = metadata.get_payment_address_for_group(self.args.group_name)
+        group_id = metadata.get_group_id_by_group_name(self.args.group_name)
+        channels = self._get_filtered_channels(return_only_id=False, recipient=recipient, group_id=group_id)
+        channels = [c for c in channels if c[filter_by].lower() == self.ident.address.lower()]
 
         if len(channels) == 0:
-            raise Exception("Cannot find initialized channel for service with org_id=%s service_id=%s and signer=%s" % (
-                self.args.org_id, self.args.service_id, self.ident.address))
+            if self.args.service_id:
+                raise Exception("Cannot find channel for service with org_id=%s service_id=%s group_name=%s and signer=%s" % (
+                    self.args.org_id, self.args.service_id, self.args.group_name, self.ident.address))
+            else:
+                raise Exception("Cannot find channel for org_id=%s group_name=%s and signer=%s" % (
+                    self.args.org_id, self.args.group_name, self.ident.address))
         if self.args.channel_id is None:
             if len(channels) > 1:
-                channel_ids = [channel["channelId"] for channel in channels]
+                channel_ids = [channel["channel_id"] for channel in channels]
                 raise Exception(
-                    "We have several initialized channel: %s. You should use --channel-id to select one" % str(channel_ids))
+                    "We have several channels: %s. You should use --channel-id to select one" % str(channel_ids))
             return channels[0]
         for channel in channels:
-            if channel["channelId"] == self.args.channel_id:
+            if channel["channel_id"] == self.args.channel_id:
                 return channel
         raise Exception(
-            "Channel %i has not been initialized or your are not the sender/signer of it" % self.args.channel_id)
+            "Channel %i has not been opened or you are not the sender/signer of it" % self.args.channel_id)
 
     def channel_extend_and_add_funds_for_org(self):
         self._init_or_update_registered_org_if_needed()
@@ -690,11 +699,13 @@ def print_all_channels(self):
         self._print_channels(channels)
 
     def _get_all_channels_filter_sender(self, sender):
+        # TODO: delete
         sender_padded = pad_hex(sender.lower(), 256)
-        channels_ids = self._get_all_filtered_channels([sender_padded])
+        channels_ids = self._get_filtered_channels(return_only_id=True, sender=sender_padded)
         return channels_ids
 
     def _get_all_channels_filter_sender_recipient_group(self, sender, recipient, group_id):
+        # TODO: delete
         sender_padded = pad_hex(sender.lower(),    256)
         recipient_padded = pad_hex(recipient.lower(), 256)
         group_id_hex = "0x" + group_id.hex()

From d2db5f4220e8a05136f34ee1c16398ffd825b56f Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 16 Oct 2024 13:53:38 +0300
Subject: [PATCH 06/29] Changed "client call" command. Deleted unnecessary
 methods in mpe_channel.py.

---
 snet/cli/commands/mpe_channel.py   | 170 +----------------------------
 snet/cli/commands/mpe_client.py    |   7 +-
 snet/cli/commands/mpe_treasurer.py |   3 +-
 3 files changed, 4 insertions(+), 176 deletions(-)

diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py
index eb5c5412..2ec326e7 100644
--- a/snet/cli/commands/mpe_channel.py
+++ b/snet/cli/commands/mpe_channel.py
@@ -122,22 +122,6 @@ def _get_filtered_channels(self, return_only_id=False, **kwargs):
             return [c["channel_id"] for c in channels]
         return channels
 
-    def _get_all_filtered_channels(self, topics_without_signature):
-        # TODO: delete
-        """ get all filtered chanels from blockchain logs """
-        mpe_address = self.get_mpe_address()
-        event_signature = self.ident.w3.keccak(
-            text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()
-        codec: ABICodec = self.ident.w3.codec
-        topics = [event_signature] + topics_without_signature
-        logs = self.ident.w3.eth.get_logs(
-            {"fromBlock": get_contract_deployment_block(self.ident.w3, "MultiPartyEscrow"), "address": mpe_address, "topics": topics})
-        abi = get_contract_def("MultiPartyEscrow")
-        event_abi = abi_get_element_by_name(abi, "ChannelOpen")
-        channels_ids = [get_event_data(codec, event_abi, l)[
-            "args"]["channelId"] for l in logs]
-        return channels_ids
-
     def _get_service_base_dir(self, org_id, service_id):
         """ get persistent storage for the given service (~/.snet/mpe_client/<mpe_address>_<registry_address>/<org_id>/<service_id>/) """
         return self._get_persistent_mpe_dir().joinpath(org_id, service_id)
@@ -163,7 +147,6 @@ def is_service_initialized(self):
     def _get_org_base_dir(self, org_id):
         """ get persistent storage for the given service (~/.snet/mpe_client/<mpe_address>_<registry_address>/<org_id>/) """
         return self._get_persistent_mpe_dir().joinpath(org_id)
-    #
 
     def get_org_spec_dir(self, org_id):
         """ get persistent storage for the given service (~/.snet/mpe_client/<mpe_address>/<org_id>/) """
@@ -179,28 +162,6 @@ def _get_service_metadata(self):
             os.path.join(service_dir, "service_metadata.json"))
         return service_metadata
 
-    def _add_channel_to_initialized(self, org_id, channel):
-        channels_dict = self._get_initialized_channels_dict_for_org(org_id)
-        channels_dict[channel["channelId"]] = channel
-
-        # replace old file atomically (os.rename is more or less atomic)
-        tmp = tempfile.NamedTemporaryFile(delete=False)
-        pickle.dump(channels_dict, open(tmp.name, "wb"))
-        shutil.move(tmp.name, self._get_channels_info_file(org_id))
-
-    def _get_initialized_channels_dict_for_org(self, org_id):
-        '''return {channel_id: channel}'''
-        fn = self._get_channels_info_file(org_id)
-        if os.path.isfile(fn):
-            return pickle.load(open(fn, "rb"))
-        else:
-            return {}
-
-    def _get_initialized_channels_for_org(self, org_id):
-        '''return [channel]'''
-        channels_dict = self._get_initialized_channels_dict_for_org(org_id)
-        return list(channels_dict.values())
-
     def _get_org_info_file(self, org_id):
         return os.path.join(self._get_org_base_dir(org_id), "org_info.pickle")
 
@@ -290,36 +251,6 @@ def _check_channel_is_mine(self, channel):
             raise Exception("Channel does not correspond to the current Ethereum identity " +
                             "(address=%s sender=%s signer=%s)" % (self.ident.address.lower(), channel["sender"].lower(), channel["signer"].lower()))
 
-    def _init_channel_from_metadata(self, metadata, org_registration):
-        # TODO: delete
-        channel_id = self.args.channel_id
-        channel = self._get_channel_state_from_blockchain(channel_id)
-        self._check_channel_is_mine(channel)
-        group_id = metadata.get_group_id_by_group_name(self.args.group_name)
-        if group_id is None:
-            raise Exception("Channel %i does not correspond to the given metadata.\n" % channel_id +
-                            "We can't find the following group_id in metadata: " + self.args.group_name)
-
-        self._printout("#group_name")
-        # self._printout(group.group_name)
-        self._init_or_update_org_if_needed(metadata, org_registration)
-        self._add_channel_to_initialized(self.args.org_id, channel)
-
-    def init_channel_from_metadata(self):
-        # TODO: "channel init-metadata" command
-        # TODO: delete
-        metadata = OrganizationMetadata.from_file(self.args.metadata_file)
-        self._init_channel_from_metadata(metadata, {})
-
-    def init_channel_from_registry(self):
-        # TODO: "channel init" command
-        # TODO: delete
-        metadata = self._get_organization_metadata_from_registry(
-            self.args.org_id)
-        org_registration = self._get_organization_registration(
-            self.args.org_id)
-        self._init_channel_from_metadata(metadata, org_registration)
-
     def _expiration_str_to_blocks(self, expiration_str, current_block):
         s = expiration_str
         if s.startswith("+") and s.endswith("days"):
@@ -351,7 +282,7 @@ def _get_expiration_from_args(self):
         return rez
 
     def _open_channel_for_org(self, metadata):
-        # TODO: needed
+
         mpe_cogs = self.call_contract_command(
             "MultiPartyEscrow",    "balances",  [self.ident.address])
         if mpe_cogs < self.args.amount:
@@ -381,21 +312,6 @@ def _open_channel_for_org(self, metadata):
         channel_info["channel_id"] = rez[1][0]["args"]["channelId"]
         return channel_info
 
-    def _initialize_already_opened_channel(self, metadata, sender, signer):
-        # TODO: delete
-        group_id = base64.b64decode(metadata.get_group_id_by_group_name(self.args.group_name))
-        recipient = metadata.get_payment_address_for_group(self.args.group_name)
-        channels_ids = self._get_all_channels_filter_sender_recipient_group(sender, recipient, group_id)
-        for i in sorted(channels_ids):
-            channel = self._get_channel_state_from_blockchain(i)
-            if channel["signer"].lower() == signer.lower():
-                self._printerr(
-                    "# Channel with given sender, signer and group_id is already exists we simply initialize it (channel_id = %i)" % channel["channelId"])
-#                self._printerr("# Please run 'snet channel extend-add %i --expiration <EXPIRATION> --amount <AMOUNT>' if necessary"%channel["channelId"])
-                self._add_channel_to_initialized(self.args.org_id, channel)
-                return channel
-        return None
-
     def _check_already_opened_channel(self, metadata, sender, signer):
         group_id = metadata.get_group_id_by_group_name(self.args.group_name)
         recipient = metadata.get_payment_address_for_group(self.args.group_name)
@@ -412,7 +328,6 @@ def _check_already_opened_channel(self, metadata, sender, signer):
         return None
 
     def _open_channel_from_metadata(self, metadata, org_registration):
-        # TODO: needed
         self._init_or_update_org_if_needed(metadata, org_registration)
 
         # Before open new channel we try to find already opened channel
@@ -429,12 +344,10 @@ def _open_channel_from_metadata(self, metadata, org_registration):
         self._printout(channel["channel_id"])
 
     def open_channel_from_metadata(self):
-        # TODO: "channel open-from-metadata" command
         metadata = OrganizationMetadata.from_file(self.args.metadata_file)
         self._open_channel_from_metadata(metadata, {})
 
     def open_channel_from_registry(self):
-        # TODO: "channel open" command
         metadata = self._get_organization_metadata_from_registry(
             self.args.org_id)
         org_registration = self._get_organization_registration(
@@ -489,7 +402,6 @@ def _check_new_expiration_from_blockchain(self, channel_id, new_expiration):
                 new_expiration, channel["expiration"]))
 
     def _smart_get_channel_for_org(self, metadata, filter_by):
-        # TODO: needed
         '''
          - filter_by can be sender or signer
         '''
@@ -523,17 +435,6 @@ def channel_extend_and_add_funds_for_org(self):
         channel_id = self._smart_get_channel_for_org(metadata, "sender")["channelId"]
         self._channel_extend_add_funds_with_channel_id(channel_id)
 
-    def _get_all_initialized_channels(self):
-        # TODO: delete
-        """ return dict of lists  rez[(<org_id>, <service_id>)] = [(channel_id, channel_info)] """
-        channels_dict = defaultdict(list)
-        for service_base_dir in self._get_persistent_mpe_dir().glob("*/*"):
-            org_id = service_base_dir.parent.name
-            channels = self._get_initialized_channels_for_org(org_id)
-            if channels:
-                channels_dict[org_id] = channels
-        return channels_dict
-
     def _get_channel_state_from_blockchain(self, channel_id):
         abi = get_contract_def("MultiPartyEscrow")
         channel_abi = abi_get_element_by_name(abi, "channels")
@@ -600,67 +501,11 @@ def _print_channels(self, channels, filters: list[str] = None):
             channel["group_id"] = base64.b64encode(channel["groupId"]).decode("ascii")
             self._printout(self._convert_channel_dict_to_str(channel, filters))
 
-    def _print_channels_dict_from_blockchain(self, channels_dict):
-        # print only caption
-        if self.args.only_id:
-            self._printout("#organization_id service_id channelId")
-        else:
-            self._printout(
-                "#organization_id service_id group_name channel_id nonce value(AGIX) expiration(blocks)")
-        for org_id in channels_dict:
-            channels = self._filter_channels_sender_or_signer(
-                channels_dict[org_id])
-            metadata = self._read_metadata_for_org(org_id)
-            for channel in channels:
-                channel_id = channel["channelId"]
-                group_id_base64 = base64.b64encode(
-                    channel["groupId"]).decode('ascii')
-                group = metadata.get_group_by_group_id(group_id_base64)
-                if group is None:
-                    group_name = "UNDEFINED"
-                else:
-                    group_name = group.group_name
-                if self.args.only_id:
-                    self._printout("%s %s %i" %
-                                   (org_id, group_name, channel_id))
-                else:
-                    channel_blockchain = self._get_channel_state_from_blockchain(
-                        channel_id)
-                    value_agi = cogs2stragix(channel_blockchain["value"])
-                    self._printout("%s %s %i %i %s %i" % (org_id, group_name, channel_id,
-                                                          channel_blockchain["nonce"], value_agi, channel_blockchain["expiration"]))
-
-    def _filter_channels_sender_or_signer(self, channels):
-        good_channels = []
-        for channel in channels:
-            not_sender = channel["sender"] != self.ident.address
-            not_signer = channel["signer"] != self.ident.address
-            if self.args.filter_sender and not_sender:
-                continue
-            if self.args.filter_signer and not_signer:
-                continue
-            if self.args.filter_my and not_sender and not_signer:
-                continue
-            good_channels.append(channel)
-        return good_channels
-
-    def print_initialized_channels(self):
-        # TODO: "print-initialized" command
-        # TODO: delete
-        channels_dict = self._get_all_initialized_channels()
-        self._print_channels_dict_from_blockchain(channels_dict)
-
     def get_address_from_arg_or_ident(self, arg):
         if arg:
             return arg
         return self.ident.address
 
-    def print_channels_filter_org(self):
-        # TODO: "print-initialized-filter-org" command
-        # TODO: delete
-        channels = self._get_initialized_channels_for_org(self.args.org_id)
-        self._print_channels_dict_from_blockchain({self.args.org_id: channels})
-
     def print_channels_filter_sender(self):
         # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
         return_only_id = self.args.only_id or not self.args.do_not_sync
@@ -698,19 +543,6 @@ def print_all_channels(self):
         channels = self._get_filtered_channels(return_only_id=return_only_id)
         self._print_channels(channels)
 
-    def _get_all_channels_filter_sender(self, sender):
-        # TODO: delete
-        sender_padded = pad_hex(sender.lower(), 256)
-        channels_ids = self._get_filtered_channels(return_only_id=True, sender=sender_padded)
-        return channels_ids
-
-    def _get_all_channels_filter_sender_recipient_group(self, sender, recipient, group_id):
-        # TODO: delete
-        sender_padded = pad_hex(sender.lower(),    256)
-        recipient_padded = pad_hex(recipient.lower(), 256)
-        group_id_hex = "0x" + group_id.hex()
-        return self._get_all_filtered_channels([sender_padded, recipient_padded, group_id_hex])
-
     # Auxilary functions
     def print_block_number(self):
         self._printout(self.ident.w3.eth.block_number)
diff --git a/snet/cli/commands/mpe_client.py b/snet/cli/commands/mpe_client.py
index 8a508835..64b9de66 100644
--- a/snet/cli/commands/mpe_client.py
+++ b/snet/cli/commands/mpe_client.py
@@ -295,11 +295,8 @@ def call_server_statelessly_with_params(self, params, group_name):
         grpc_channel = open_grpc_channel(endpoint)
 
         # if channel was not initilized we will try to initailize it (it will work only in simple case of signer == sender)
-        channel = self._smart_get_initialized_channel_for_org(
-            org_metadata,
-            filter_by="signer"
-        )
-        channel_id = channel["channelId"]
+        channel = self._smart_get_channel_for_org(org_metadata, filter_by="signer")
+        channel_id = channel["channel_id"]
         price = self._get_price_from_metadata(service_metadata, group_name)
         server_state = self._get_channel_state_from_server(grpc_channel, channel_id)
 
diff --git a/snet/cli/commands/mpe_treasurer.py b/snet/cli/commands/mpe_treasurer.py
index 9e173414..db1d959b 100644
--- a/snet/cli/commands/mpe_treasurer.py
+++ b/snet/cli/commands/mpe_treasurer.py
@@ -151,8 +151,7 @@ def _claim_in_progress_and_claim_channels(self, grpc_channel, channels):
 
     def claim_channels(self):
         grpc_channel = open_grpc_channel(self.args.endpoint)
-        self._claim_in_progress_and_claim_channels(
-            grpc_channel, self.args.channels)
+        self._claim_in_progress_and_claim_channels(grpc_channel, self.args.channels)
 
     def claim_all_channels(self):
         grpc_channel = open_grpc_channel(self.args.endpoint)

From 8541f4d6132ede458768ebb3e9d8237b97cbe222 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 16 Oct 2024 18:21:42 +0300
Subject: [PATCH 07/29] Fixed "print" commands in "channel" section.

---
 snet/cli/commands/mpe_channel.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py
index 2ec326e7..507534a6 100644
--- a/snet/cli/commands/mpe_channel.py
+++ b/snet/cli/commands/mpe_channel.py
@@ -511,6 +511,7 @@ def print_channels_filter_sender(self):
         return_only_id = self.args.only_id or not self.args.do_not_sync
         address = self.get_address_from_arg_or_ident(self.args.sender)
         channels = self._get_filtered_channels(return_only_id=return_only_id, sender=address)
+        self._printout("Channels for sender: %s" % address)
         self._print_channels(channels, ["sender"])
 
     def print_channels_filter_recipient(self):
@@ -518,24 +519,29 @@ def print_channels_filter_recipient(self):
         return_only_id = self.args.only_id or not self.args.do_not_sync
         address = self.get_address_from_arg_or_ident(self.args.recipient)
         channels = self._get_filtered_channels(return_only_id=return_only_id, recipient=address)
+        self._printout("Channels for recipient: %s" % address)
         self._print_channels(channels, ["recipient"])
 
     def print_channels_filter_group(self):
         # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
         return_only_id = self.args.only_id or not self.args.do_not_sync
         metadata = self._get_organization_metadata_from_registry(self.args.org_id)
+        recipient = metadata.get_payment_address_for_group(self.args.group_name)
         group_id = metadata.get_group_id_by_group_name(self.args.group_name)
         channels = self._get_filtered_channels(return_only_id=return_only_id, group_id=group_id)
-        self._print_channels(channels, ["group_id"])
+        self._printout("Channels for group_id: %s and recipient: %s" % (group_id, recipient))
+        self._print_channels(channels, ["group_id", "recipient"])
 
     def print_channels_filter_group_sender(self):
         # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state
         return_only_id = self.args.only_id or not self.args.do_not_sync
-        address = self.get_address_from_arg_or_ident(self.args.sender)
+        sender = self.get_address_from_arg_or_ident(self.args.sender)
         metadata = self._get_organization_metadata_from_registry(self.args.org_id)
         group_id = metadata.get_group_id_by_group_name(self.args.group_name)
-        channels = self._get_filtered_channels(return_only_id=return_only_id, sender=address, group_id=group_id)
-        self._print_channels(channels, ["sender", "group_id"])
+        recipient = metadata.get_payment_address_for_group(self.args.group_name)
+        channels = self._get_filtered_channels(return_only_id=return_only_id, sender=sender, group_id=group_id)
+        self._printout("Channels for group_id: %s, sender: %s and recipient: %s" % (group_id, sender, recipient))
+        self._print_channels(channels, ["sender", "group_id", "recipient"])
 
     def print_all_channels(self):
         # we don't need to return other channel fields if we only need channel_id or if we'll sync channels state

From 396ed0bcf8baa04fd22cf1f425efcd5f166848c1 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 22 Oct 2024 18:08:21 +0300
Subject: [PATCH 08/29] Added script for building docs on Windows and changed
 generate_markdown.py for removing unnecessary data in .md files.

---
 docs/build-docs.ps1              | 12 +++++++
 docs/source/generate_markdown.py | 58 +++++++++++++++++++++++++++++---
 2 files changed, 66 insertions(+), 4 deletions(-)
 create mode 100644 docs/build-docs.ps1

diff --git a/docs/build-docs.ps1 b/docs/build-docs.ps1
new file mode 100644
index 00000000..59ceffe0
--- /dev/null
+++ b/docs/build-docs.ps1
@@ -0,0 +1,12 @@
+cd source
+python generate_rst.py
+cd ..
+
+make clean
+make html
+Copy-Item source/snet-cli-static/theme.css build/html/_static/css/
+
+cd source
+python generate_markdown.py ../build/html/ ../build/html/clean ../build/markdown
+cd ..
+Remove-Item -Path build/html/clean -Recurse
diff --git a/docs/source/generate_markdown.py b/docs/source/generate_markdown.py
index 142bf3a2..d0ec5eca 100644
--- a/docs/source/generate_markdown.py
+++ b/docs/source/generate_markdown.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import re
 from bs4 import BeautifulSoup
 
 
@@ -51,11 +52,19 @@ def process_html_file(file_path, output_dir, markdown_dir):
 
     os.system(f"html2text --ignore-images {output_path} > {md_output_path}")
 
-    with open(md_output_path, 'r', encoding='utf-8') as file:
-        md_content = file.read()
+    if sys.platform.startswith('win'):
+        with open(md_output_path, 'r') as file:
+            md_content = file.read()
+    else:
+        with open(md_output_path, 'r', encoding='utf-8') as file:
+            md_content = file.read()
 
-    clean_md = md_content.replace("start ", "```sh\n\t") \
-                         .replace("finish", "\n```")
+    clean_md = format_code_elements(md_content)
+    clean_md = clean_md.replace("\n### ", "\n## ")
+    clean_md = clean_md.replace("<", "\<")  # fix tags errors
+    clean_md = clean_md.replace(">", "\>")  # fix tags errors
+    clean_md = clean_md.replace("````", "```")
+    clean_md = delete_beginning(clean_md)
 
     with open(md_output_path, 'w', encoding='utf-8') as file:
         file.write(clean_md)
@@ -63,6 +72,47 @@ def process_html_file(file_path, output_dir, markdown_dir):
     print(f"Processed: {output_path} -> {md_output_path}\n")
 
 
+def format_code_elements(text: str):
+    substrings = []
+    start_index = 0
+    while True:
+        start_index = text.find("start", start_index)
+        if start_index == -1:
+            break
+
+        end_index = text.find("finish", start_index + 5)
+        if end_index == -1:
+            break
+
+        substrings.append(text[start_index + 5:end_index])
+        start_index = end_index + 6
+
+    results = []
+    for code in substrings:
+        res = re.sub(r'\s+', ' ', code).strip()
+
+        res_split = list(res.split())
+        length = len(res_split[0]) + len(res_split[1]) + len(res_split[2]) + 3
+        ind = ' ' * length
+        res = res.replace('] [', ']\n' + ind + '[')
+
+        results.append(res)
+
+    for i in range(len(results)):
+        text = text.replace("start" + substrings[i] + "finish", "```sh\n" + results[i] + "\n```")
+
+    return text
+
+
+def delete_beginning(text: str):
+    start_index = text.find("## Commands")
+    end_index = text.find("## Sub-commands")
+    if start_index == -1 or end_index == -1:
+        return text
+
+    return text.replace(text[start_index + 11:end_index + 15], "")
+
+
 def process_html_files_in_directory(directory, output_dir, markdown_dir):
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)

From f694f6aa42b3b9d7b793874a65817dce9627f254 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 22 Oct 2024 19:57:22 +0300
Subject: [PATCH 09/29] Removed deprecated functionality

---
 snet/cli/arguments.py                     | 14 +++----------
 snet/cli/commands/sdk_command.py          |  5 ++---
 snet/cli/resources/proto/merckledag.proto | 17 ---------------
 snet/cli/resources/proto/unixfs.proto     | 25 -----------------------
 snet/cli/utils/utils.py                   | 25 +++++------------------
 5 files changed, 10 insertions(+), 76 deletions(-)
 delete mode 100644 snet/cli/resources/proto/merckledag.proto
 delete mode 100644 snet/cli/resources/proto/unixfs.proto

diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index 5a731e4c..4ac9e909 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -185,9 +185,6 @@ def add_network_options(parser, config):
                    help="Name of network to create")
     p.add_argument("eth_rpc_endpoint",
                    help="Ethereum rpc endpoint")
-    p.add_argument("--default-gas-price",
-                   default="medium",
-                   help="Default gas price (in wei) or gas price strategy ('fast' ~1min, 'medium' ~5min or 'slow' ~60min), default is 'medium'")
     p.add_argument("--skip-check",
                    action="store_true",
                    help="Skip check that eth_rpc_endpoint is valid")
@@ -236,6 +233,8 @@ def add_contract_options(parser):
 
     contracts = get_all_abi_contract_files()
     for path in contracts:
+        if "TokenConversionManager" in str(path) or "TokenStake" in str(path):
+            continue
         contract_name = re.search(
             r"([^.]*)\.json", os.path.basename(path)).group(1)
         contract_p = subparsers.add_parser(
@@ -1323,16 +1322,9 @@ def add_sdk_options(parser):
     subparsers = parser.add_subparsers(title="Commands", metavar="COMMAND")
     subparsers.required = True
 
-    supported_languages = ["python", "nodejs"]
-
     p = subparsers.add_parser("generate-client-library",
-                              help="Generate compiled client libraries to call services using your language of choice")
+                              help="Generate compiled client libraries to call services using Python")
     p.set_defaults(fn="generate_client_library")
-    p.add_argument("language",
-                   choices=supported_languages,
-                   help="Choose target language for the generated client library from {}".format(
-                       supported_languages),
-                   metavar="LANGUAGE")
     add_p_service_in_registry(p)
     p.add_argument("protodir",
                    nargs="?",
diff --git a/snet/cli/commands/sdk_command.py b/snet/cli/commands/sdk_command.py
index 2a6c5cbd..fae953ac 100644
--- a/snet/cli/commands/sdk_command.py
+++ b/snet/cli/commands/sdk_command.py
@@ -17,11 +17,10 @@ def generate_client_library(self):
         os.makedirs(client_libraries_base_dir_path, exist_ok=True)
 
         # Create service client libraries path
-        library_language = self.args.language
         library_org_id = self.args.org_id
         library_service_id = self.args.service_id
 
-        library_dir_path = client_libraries_base_dir_path.joinpath(library_org_id, library_service_id, library_language)
+        library_dir_path = client_libraries_base_dir_path.joinpath(library_org_id, library_service_id, "python")
 
         metadata = self._get_service_metadata_from_registry()
         service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash")
@@ -30,7 +29,7 @@ def generate_client_library(self):
         download_and_safe_extract_proto(service_api_source, library_dir_path, self._get_ipfs_client())
 
         # Compile proto files
-        compile_proto(Path(library_dir_path), library_dir_path, target_language=self.args.language)
+        compile_proto(Path(library_dir_path), library_dir_path)
 
         self._printout(
             'client libraries for service with id "{}" in org with id "{}" generated at {}'.format(library_service_id,
diff --git a/snet/cli/resources/proto/merckledag.proto b/snet/cli/resources/proto/merckledag.proto
deleted file mode 100644
index 5af078a5..00000000
--- a/snet/cli/resources/proto/merckledag.proto
+++ /dev/null
@@ -1,17 +0,0 @@
-syntax = "proto2";
-// An IPFS MerkleDAG Link
-message MerkleLink {
-  required bytes Hash = 1;   // multihash of the target object
-  required string Name = 2;  // utf string name
-  required uint64 Tsize = 3; // cumulative size of target object
-
-  // user extensions start at 50
-}
-
-// An IPFS MerkleDAG Node
-message MerkleNode {
-  repeated MerkleLink Links = 2; // refs to other objects
-  required bytes Data = 1; // opaque user data
-
-  // user extensions start at 50
-}
\ No newline at end of file
diff --git a/snet/cli/resources/proto/unixfs.proto b/snet/cli/resources/proto/unixfs.proto
deleted file mode 100644
index c190079a..00000000
--- a/snet/cli/resources/proto/unixfs.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-syntax = "proto2";
-package unixfs.pb;
-
-message Data {
-	enum DataType {
-		Raw = 0;
-		Directory = 1;
-		File = 2;
-		Metadata = 3;
-		Symlink = 4;
-		HAMTShard = 5;
-	}
-
-	required DataType Type = 1;
-	optional bytes Data = 2;
-	optional uint64 filesize = 3;
-	repeated uint64 blocksizes = 4;
-
-	optional uint64 hashType = 5;
-	optional uint64 fanout = 6;
-}
-
-message Metadata {
-	optional string MimeType = 1;
-}
\ No newline at end of file
diff --git a/snet/cli/utils/utils.py b/snet/cli/utils/utils.py
index b69e60ba..b2b38479 100644
--- a/snet/cli/utils/utils.py
+++ b/snet/cli/utils/utils.py
@@ -146,7 +146,7 @@ def get_cli_version():
     return distribution("snet.cli").version
 
 
-def compile_proto(entry_path, codegen_dir, proto_file=None, target_language="python"):
+def compile_proto(entry_path, codegen_dir, proto_file=None):
     try:
         if not os.path.exists(codegen_dir):
             os.makedirs(codegen_dir)
@@ -157,25 +157,10 @@ def compile_proto(entry_path, codegen_dir, proto_file=None, target_language="pyt
             "-I{}".format(proto_include)
         ]
 
-        if target_language == "python":
-            compiler_args.insert(0, "protoc")
-            compiler_args.append("--python_out={}".format(codegen_dir))
-            compiler_args.append("--grpc_python_out={}".format(codegen_dir))
-            compiler = protoc
-        elif target_language == "nodejs":
-            protoc_node_compiler_path = Path(
-                RESOURCES_PATH.joinpath("node_modules").joinpath("grpc-tools").joinpath("bin").joinpath(
-                    "protoc.js")).absolute()
-            grpc_node_plugin_path = Path(
-                RESOURCES_PATH.joinpath("node_modules").joinpath("grpc-tools").joinpath("bin").joinpath(
-                    "grpc_node_plugin")).resolve()
-            if not os.path.isfile(protoc_node_compiler_path) or not os.path.isfile(grpc_node_plugin_path):
-                print("Missing required node.js protoc compiler. Retrieving from npm...")
-                subprocess.run(["npm", "install"], cwd=RESOURCES_PATH)
-            compiler_args.append("--js_out=import_style=commonjs,binary:{}".format(codegen_dir))
-            compiler_args.append("--grpc_out={}".format(codegen_dir))
-            compiler_args.append("--plugin=protoc-gen-grpc={}".format(grpc_node_plugin_path))
-            compiler = lambda args: subprocess.run([str(protoc_node_compiler_path)] + args)
+        compiler_args.insert(0, "protoc")
+        compiler_args.append("--python_out={}".format(codegen_dir))
+        compiler_args.append("--grpc_python_out={}".format(codegen_dir))
+        compiler = protoc
 
         if proto_file:
             compiler_args.append(str(proto_file))

From 50a4b1d5638387ebc13e9f1c91e7b89aaf476137 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 23 Oct 2024 17:50:24 +0300
Subject: [PATCH 10/29] Fixed "snet channel metadata-init-utility" command with
 wrong "encoding" field in metadata.

---
 snet/cli/metadata/service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/snet/cli/metadata/service.py b/snet/cli/metadata/service.py
index 895df1dc..739ae035 100644
--- a/snet/cli/metadata/service.py
+++ b/snet/cli/metadata/service.py
@@ -65,7 +65,7 @@ class MPEServiceMetadata:
     def __init__(self):
         self.m = {"version": 1,
                   "display_name": "",
-                  "encoding": "grpc",  # grpc by default
+                  "encoding": "proto",  # grpc by default
                   "service_type": "grpc",  # grpc by default
                   # one week by default (15 sec block,  24*60*60*7/15)
                   "service_api_source": "",

From 2d4f47bab507b990c2d002eee1be327bd303cc46 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Thu, 24 Oct 2024 14:23:34 +0300
Subject: [PATCH 11/29] Updated README.md for building documentation

---
 README.md | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 254bd0e4..cb21c43f 100644
--- a/README.md
+++ b/README.md
@@ -127,28 +127,49 @@ Backward compatibility for other Python versions is not guaranteed.
 $ git clone https://github.com/singnet/snet-cli.git
 $ cd snet-cli/packages/snet_cli
 ```
+
+* 
   
 * Install the package in development/editable mode  
+
 ```bash  
 $ pip3 install -e .
 ```
 
-#### Building the Documentation
+### Building the Documentation in Markdown files
+
+* Clone repository and install dependencies
 
-* Install sphinx, sphinx-argparse and the rtd theme
 ```bash
+$ git clone https://github.com/singnet/snet-cli.git
+$ cd snet-cli/packages/snet_cli
 $ pip install sphinx
 $ pip install sphinx-argparse
 $ pip install sphinx-rtd-theme
-``` 
+$ pip install bs4
+$ pip install html2text
+```
+
+#### On Linux
 
 * Run the build-docs.sh in the docs directory
+
 ```bash
 $ cd docs
 $ sh build-docs.sh
 ```
 
-The documentation is generated under the docs/build/html folder
+#### On Windows
+
+* Install `make` utility and run the build-docs.ps1 in the docs directory
+
+```powershell
+choco install make # install choco if it is not already installed
+cd docs
+powershell -file build-docs.ps1
+```
+
+The documentation is generated under the docs/build/markdown folder
 
 ### Release  
   

From 9ca7975a9ee2f90433d1461cab283ee7ae05b3ee Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Thu, 24 Oct 2024 15:03:29 +0300
Subject: [PATCH 12/29] Got rid of the "goerli" network in the default config.

---
 snet/cli/config.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/snet/cli/config.py b/snet/cli/config.py
index 95dd3b31..79a002cf 100644
--- a/snet/cli/config.py
+++ b/snet/cli/config.py
@@ -200,9 +200,6 @@ def create_default_config(self):
         self["network.mainnet"] = {
             "default_eth_rpc_endpoint": "https://mainnet.infura.io/v3/09027f4a13e841d48dbfefc67e7685d5"
         }
-        self["network.goerli"] = {
-            "default_eth_rpc_endpoint": "https://goerli.infura.io/v3/09027f4a13e841d48dbfefc67e7685d5",
-        }
         self["network.sepolia"] = {
             "default_eth_rpc_endpoint": "https://sepolia.infura.io/v3/09027f4a13e841d48dbfefc67e7685d5",
         }

From c2d04c38a3f297705e0d057eec0754485077d276 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Fri, 25 Oct 2024 13:45:34 +0300
Subject: [PATCH 13/29] Fixed contract names in "snet contract" command.

---
 snet/cli/arguments.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index 4ac9e909..0336a9c7 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -233,13 +233,12 @@ def add_contract_options(parser):
 
     contracts = get_all_abi_contract_files()
     for path in contracts:
-        if "TokenConversionManager" in str(path) or "TokenStake" in str(path):
-            continue
-        contract_name = re.search(
-            r"([^.]*)\.json", os.path.basename(path)).group(1)
-        contract_p = subparsers.add_parser(
-            contract_name, help="{} contract".format(contract_name))
-        add_contract_function_options(contract_p, contract_name)
+        if "MultiPartyEscrow" in str(path) or "Registry" in str(path) or "SingularityNetToken" in str(path):
+            contract_name = re.search(
+                r"([^.]*)\.json", os.path.basename(path)).group(1)
+            contract_p = subparsers.add_parser(
+                contract_name, help="{} contract".format(contract_name))
+            add_contract_function_options(contract_p, contract_name)
 
 
 def add_organization_arguments(parser):

From 5cea4726f235f16af679a37b207caffcea07efac Mon Sep 17 00:00:00 2001
From: Kirill <65371121+kiruxaspb@users.noreply.github.com>
Date: Mon, 28 Oct 2024 22:56:37 +0300
Subject: [PATCH 14/29] header level fix

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 254bd0e4..7ce701dc 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-SingularityNET CLI
+# SingularityNET CLI
 
 ## Package
 

From d3b8ff1967d1cb96319e2a02fe6d408ca3cd9699 Mon Sep 17 00:00:00 2001
From: Kirill <65371121+kiruxaspb@users.noreply.github.com>
Date: Tue, 29 Oct 2024 10:44:03 +0300
Subject: [PATCH 15/29] hide UserWarnings at the logs processing

---
 snet/cli/contract.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/snet/cli/contract.py b/snet/cli/contract.py
index 6c79d1fe..0934b2c0 100644
--- a/snet/cli/contract.py
+++ b/snet/cli/contract.py
@@ -1,3 +1,6 @@
+from web3.logs import DISCARD
+
+
 class Contract:
     def __init__(self, w3, address, abi):
         self.w3 = w3
@@ -22,6 +25,6 @@ def process_receipt(self, receipt):
 
         contract_events = map(lambda e: e["name"], filter(lambda e: e["type"] == "event", self.abi))
         for contract_event in contract_events:
-            events.extend(getattr(self.contract.events, contract_event)().process_receipt(receipt))
+            events.extend(getattr(self.contract.events, contract_event)().process_receipt(receipt, errors=DISCARD))
 
         return events

From 788cc2ffc144c0dc4518d6babceccdaa0570a3fa Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 29 Oct 2024 10:50:20 +0300
Subject: [PATCH 16/29] Partially fixed bug with "snet service
 validate-metadata" command.

---
 snet/cli/commands/mpe_service.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/snet/cli/commands/mpe_service.py b/snet/cli/commands/mpe_service.py
index 9b30cfd1..dc36e4bf 100644
--- a/snet/cli/commands/mpe_service.py
+++ b/snet/cli/commands/mpe_service.py
@@ -394,7 +394,7 @@ def metadata_validate(self):
         """
         # Set path to `service_schema` stored in the `resources` directory from cwd of `mpe_service.py`
         current_path = Path(__file__).parent
-        relative_path = '../../snet/snet_cli/resources/service_schema'
+        relative_path = '../../cli/resources/service_schema'
         path_to_schema = (current_path / relative_path).resolve()
         with open(path_to_schema, 'r') as f:
             schema = json.load(f)
@@ -405,21 +405,24 @@ def metadata_validate(self):
             docs = "http://snet-cli-docs.singularitynet.io/service.html"
             error_message = f"\nVisit {docs} for more information."
             if e.validator == 'required':
-                raise ValidationError(e.message + error_message)
+                self._printout(e.message + error_message)
             elif e.validator == 'minLength':
-                raise ValidationError(f"`{e.path[-1]}` -> cannot be empty." + error_message)
+                self._printout(f"`{e.path[-1]}` -> cannot be empty." + error_message)
             elif e.validator == 'minItems':
-                raise ValidationError(f"`{e.path[-1]}` -> minimum 1 item required." + error_message)
+                self._printout(f"`{e.path[-1]}` -> minimum 1 item required." + error_message)
             elif e.validator == 'type':
-                raise ValidationError(f"`{e.path[-1]}` -> {e.message}" + error_message)
+                self._printout(f"`{e.path[-1]}` -> {e.message}" + error_message)
             elif e.validator == 'enum':
-                raise ValidationError(f"`{e.path[-1]}` -> {e.message}" + error_message)
+                self._printout(f"`{e.path[-1]}` -> {e.message}" + error_message)
             elif e.validator == 'additionalProperties':
                 if len(e.path) != 0:
-                    raise ValidationError(f"{e.message} in `{e.path[-2]}`." + error_message)
+                    self._printout(f"{e.message} in `{e.path[-2]}`." + error_message)
                 else:
-                    raise ValidationError(f"{e.message} in main object." + error_message)
+                    self._printout(f"{e.message} in main object." + error_message)
+            else:
+                self._printout(e)
         else:
+            self._printout("Service metadata is valid.")
             exit("OK. Ready to publish.")
 
     def _prepare_to_publish_metadata(self, metadata_file):

From 9a7feba0269331810819b8eac9ce093f0629aec0 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 29 Oct 2024 14:01:56 +0300
Subject: [PATCH 17/29] Added requirements.txt for generating docs. Changed
 README.md.

---
 README.md             | 8 ++------
 docs/requirements.txt | 5 +++++
 2 files changed, 7 insertions(+), 6 deletions(-)
 create mode 100644 docs/requirements.txt

diff --git a/README.md b/README.md
index cb21c43f..1f8eedab 100644
--- a/README.md
+++ b/README.md
@@ -142,12 +142,8 @@ $ pip3 install -e .
 
 ```bash
 $ git clone https://github.com/singnet/snet-cli.git
-$ cd snet-cli/packages/snet_cli
-$ pip install sphinx
-$ pip install sphinx-argparse
-$ pip install sphinx-rtd-theme
-$ pip install bs4
-$ pip install html2text
+$ cd snet-cli
+$ pip install -r docs/requirements.txt
 ```
 
 #### On Linux
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 00000000..c15a5798
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,5 @@
+Sphinx~=8.1.3
+sphinx-argparse~=0.5.2
+sphinx-rtd-theme~=3.0.1
+bs4~=0.0.2
+html2text~=2024.2.26

From 5a4203356da325d91926b1199445158875921ec9 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Thu, 31 Oct 2024 18:43:30 +0300
Subject: [PATCH 18/29] Completely fixed "service validate-metadata" command.

---
 snet/cli/commands/mpe_service.py              | 54 ++++++++++--------
 snet/cli/metadata/service.py                  |  2 +-
 .../{service_schema => service_schema.json}   | 57 +++++++++++++++----
 3 files changed, 78 insertions(+), 35 deletions(-)
 rename snet/cli/resources/{service_schema => service_schema.json} (83%)

diff --git a/snet/cli/commands/mpe_service.py b/snet/cli/commands/mpe_service.py
index dc36e4bf..61e3ea89 100644
--- a/snet/cli/commands/mpe_service.py
+++ b/snet/cli/commands/mpe_service.py
@@ -7,7 +7,7 @@
 
 from grpc_health.v1 import health_pb2 as heartb_pb2
 from grpc_health.v1 import health_pb2_grpc as heartb_pb2_grpc
-from jsonschema import validate, ValidationError
+import jsonschema
 
 from snet.cli.commands.commands import BlockchainCommand
 from snet.cli.metadata.organization import OrganizationMetadata
@@ -394,33 +394,41 @@ def metadata_validate(self):
         """
         # Set path to `service_schema` stored in the `resources` directory from cwd of `mpe_service.py`
         current_path = Path(__file__).parent
-        relative_path = '../../cli/resources/service_schema'
+        relative_path = '../resources/service_schema.json'
         path_to_schema = (current_path / relative_path).resolve()
         with open(path_to_schema, 'r') as f:
             schema = json.load(f)
         metadata = load_mpe_service_metadata(self.args.metadata_file)
-        try:
-            validate(instance=metadata.m, schema=schema)
-        except Exception as e:
-            docs = "http://snet-cli-docs.singularitynet.io/service.html"
-            error_message = f"\nVisit {docs} for more information."
-            if e.validator == 'required':
-                self._printout(e.message + error_message)
-            elif e.validator == 'minLength':
-                self._printout(f"`{e.path[-1]}` -> cannot be empty." + error_message)
-            elif e.validator == 'minItems':
-                self._printout(f"`{e.path[-1]}` -> minimum 1 item required." + error_message)
-            elif e.validator == 'type':
-                self._printout(f"`{e.path[-1]}` -> {e.message}" + error_message)
-            elif e.validator == 'enum':
-                self._printout(f"`{e.path[-1]}` -> {e.message}" + error_message)
-            elif e.validator == 'additionalProperties':
-                if len(e.path) != 0:
-                    self._printout(f"{e.message} in `{e.path[-2]}`." + error_message)
+
+        validator = jsonschema.Draft7Validator(schema)
+        errors = list(validator.iter_errors(metadata.m))
+
+        def get_path(err):
+            return " -> ".join(f"`{el}`" for el in err.path)
+
+        if len(errors) > 0:
+            self._printout("\nErrors found in the metadata file: ")
+            for i, e in enumerate(errors):
+                if e.validator == 'required':
+                    self._printout(str(i + 1) + ". " + e.message)
+                elif e.validator == 'minLength':
+                    self._printout(f"{i + 1}. {get_path(e)} - cannot be empty.")
+                elif e.validator == 'minItems':
+                    self._printout(f"{i + 1}. {get_path(e)} - minimum 1 item required.")
+                elif e.validator == 'type':
+                    self._printout(f"{i + 1}. {get_path(e)} - {e.message}")
+                elif e.validator == 'enum':
+                    self._printout(f"{i + 1}. {get_path(e)} - {e.message}")
+                elif e.validator == 'additionalProperties':
+                    if len(e.path) != 0:
+                        self._printout(f"{i + 1}. {e.message} in `{e.path[-1]}`.")
+                    else:
+                        self._printout(f"{i + 1}. {e.message} in main object.")
                 else:
-                    self._printout(f"{e.message} in main object." + error_message)
-            else:
-                self._printout(e)
+                    self._printout(str(i + 1) + ". " + e.message)
+            docs = "https://dev.singularitynet.io/docs/products/DecentralizedAIPlatform/CLI/Manual/Service/"
+            error_message = f"\nVisit {docs} for more information."
+            self._printout(f"\n{len(errors)} errors found." + error_message + "\n")
         else:
             self._printout("Service metadata is valid.")
             exit("OK. Ready to publish.")
diff --git a/snet/cli/metadata/service.py b/snet/cli/metadata/service.py
index 739ae035..6184afd9 100644
--- a/snet/cli/metadata/service.py
+++ b/snet/cli/metadata/service.py
@@ -65,7 +65,7 @@ class MPEServiceMetadata:
     def __init__(self):
         self.m = {"version": 1,
                   "display_name": "",
-                  "encoding": "proto",  # grpc by default
+                  "encoding": "proto",  # proto by default
                   "service_type": "grpc",  # grpc by default
                   # one week by default (15 sec block,  24*60*60*7/15)
                   "service_api_source": "",
diff --git a/snet/cli/resources/service_schema b/snet/cli/resources/service_schema.json
similarity index 83%
rename from snet/cli/resources/service_schema
rename to snet/cli/resources/service_schema.json
index 27ea583f..94fa5bd4 100644
--- a/snet/cli/resources/service_schema
+++ b/snet/cli/resources/service_schema.json
@@ -1,5 +1,5 @@
 {
-    "$schema": "http://json-schema.org/draft/2019-09/schema#",
+    "$schema": "https://json-schema.org/draft-07/schema#",
     "$id": "https://dev.singularitynet.io/docs/ai-developers/service/#metadata-overview",
     "title": "Service Metadata",
     "description": "Schema of a correct service metadata file",
@@ -27,8 +27,7 @@
         },
         "model_ipfs_hash": {
             "description": "Hash of directory which contains protobuf files",
-            "type": "string",
-            "minLength": 1
+            "type": "string"
         },
         "mpe_address": {
             "description": "Address of MultiPartyEscrow contract",
@@ -97,10 +96,15 @@
                         "description": "Public key address used for validating signatures requested specially for free call",
                         "type": "string",
                         "minLength": 1
+                    },
+                    "group_id": {
+                        "description": "Group ID",
+                        "type": "string",
+                        "minLength": 1
                     }
                 },
                 "additionalProperties": false,
-                "required": [ "group_name", "pricing", "endpoints", "daemon_addresses" ]
+                "required": [ "group_name", "pricing", "endpoints", "daemon_addresses", "group_id" ]
             },
             "minItems": 1
         },
@@ -119,10 +123,18 @@
                 "short_description": {
                     "description": "Service short description",
                     "type": "string"
+                },
+                "description": {
+                    "description": "Service description",
+                    "type": "string"
                 }
             },
-            "additionalProperties": false,
-            "required": [ "url", "long_description", "short_description" ]
+            "additionalProperties": true,
+            "required": ["url", "short_description"],
+            "anyOf": [
+                {"required": ["long_description"]},
+                {"required": ["description"]}
+            ]
         },
         "media": {
             "description": "Media assets with IPFS hash",
@@ -147,7 +159,11 @@
                     "alt_text": {
                         "description": "Alternate to display if media doesn't load",
                         "type": "string",
-                        "enum": [ "hover_on_the_image_text", "hover_on_the_video_url" ]
+                        "enum": [ "hover_on_the_image_text", "hover_on_the_video_url", ""]
+                    },
+                    "asset_type": {
+                        "description": "Asset type",
+                        "type": "string"
                     }
                 },
                 "additionalProperties": false,
@@ -167,19 +183,38 @@
                     },
                     "email_id": {
                         "description": "Email of contributor",
-                        "type": "string",
-                        "minLength": 1
+                        "type": "string"
                     }
                 },
                 "additionalProperties": false,
-                "required": [ "name", "email_id" ]
+                "required": ["name", "email_id"]
             },
             "minItems": 1
+        },
+        "tags": {
+            "description": "Service tags",
+            "type": "array",
+            "items": {
+                "type": "string",
+                "minLength": 1
+            }
+        },
+        "assets": {
+            "description": "Service assets",
+            "type": "object",
+            "additionalProperties": true
+        },
+        "service_api_source": {
+            "description": "Source of service API",
+            "type": "string",
+            "minLength": 1
         }
     },
     "additionalProperties": false,
     "required": [
         "version", "display_name", "encoding", "service_type",
         "model_ipfs_hash", "mpe_address", "groups", "contributors"
-    ]
+    ],
+    "if": {"properties": {"model_ipfs_hash": {"const": ""}}},
+    "then": {"required": ["service_api_source"]}
 }
\ No newline at end of file

From b42a6ea40540e534ed2f17f3f4dc2c1411592154 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Fri, 1 Nov 2024 18:19:12 +0300
Subject: [PATCH 19/29] Removed "default_gas_price" option from "set" and
 "unset" commands. Added "filecoin_api_key" option to "unset" command.

---
 snet/cli/config.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/snet/cli/config.py b/snet/cli/config.py
index 79a002cf..3a01344c 100644
--- a/snet/cli/config.py
+++ b/snet/cli/config.py
@@ -110,7 +110,10 @@ def set_session_field(self, key, value, out_f):
     def unset_session_field(self, key, out_f):
         if key in get_session_network_keys_removable():
             print("unset %s from network %s" % (key, self["session"]["network"]), file=out_f)
-            del self._get_network_section(self["session"]["network"])[key]
+            if key == "filecoin_api_key":
+                self.unset_filecoin_key()
+            else:
+                del self._get_network_section(self["session"]["network"])[key]
         self._persist()
 
     def session_to_dict(self):
@@ -176,6 +179,9 @@ def set_filecoin_key(self, filecoin_key: str):
         self["filecoin"]["filecoin_api_key"] = filecoin_key
         self._persist()
 
+    def unset_filecoin_key(self):
+        del self["filecoin"]["filecoin_api_key"]
+        self._persist()
 
     def get_all_identities_names(self):
         return [x[len("identity."):] for x in self.sections() if x.startswith("identity.")]
@@ -282,12 +288,12 @@ def get_session_identity_keys():
 
 
 def get_session_network_keys():
-    return ["default_gas_price", "current_registry_at", "current_multipartyescrow_at", "current_singularitynettoken_at",
+    return ["current_registry_at", "current_multipartyescrow_at", "current_singularitynettoken_at",
             "default_eth_rpc_endpoint"]
 
 
 def get_session_network_keys_removable():
-    return ["default_gas_price", "current_registry_at", "current_multipartyescrow_at", "current_singularitynettoken_at"]
+    return ["current_registry_at", "current_multipartyescrow_at", "current_singularitynettoken_at", "filecoin_api_key"]
 
 
 def get_session_keys():

From 9b1567e38325843cc4516fd1376c85c5902d4086 Mon Sep 17 00:00:00 2001
From: kiruxaspb <dvoreckovkirill@gmail.com>
Date: Fri, 8 Nov 2024 13:46:44 +0300
Subject: [PATCH 20/29] Added backward compatibility

with configs created older CLI versions
---
 snet/cli/config.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/snet/cli/config.py b/snet/cli/config.py
index 3a01344c..140fcff7 100644
--- a/snet/cli/config.py
+++ b/snet/cli/config.py
@@ -171,11 +171,13 @@ def set_ipfs_endpoint(self, ipfs_endpoint):
         self._persist()
 
     def get_filecoin_key(self):
-        if not self["filecoin"].get("filecoin_api_key"):
+        if "filecoin" not in self or not self["filecoin"].get("filecoin_api_key"):
             raise Exception("Use [snet set filecoin_api_key <YOUR_LIGHTHOUSE_API_KEY>] to set filecoin key")
         return self["filecoin"]["filecoin_api_key"]
 
     def set_filecoin_key(self, filecoin_key: str):
+        if "filecoin" not in self:
+            self["filecoin"] = {"filecoin_api_key": ""}
         self["filecoin"]["filecoin_api_key"] = filecoin_key
         self._persist()
 

From 489854b57121ec52b3b171f711ed13fbb300e6e8 Mon Sep 17 00:00:00 2001
From: Kirill <65371121+kiruxaspb@users.noreply.github.com>
Date: Fri, 8 Nov 2024 17:12:15 +0300
Subject: [PATCH 21/29] Delete blockchain directory

---
 blockchain/package-lock.json | 34 ----------------------------------
 blockchain/package.json      |  6 ------
 2 files changed, 40 deletions(-)
 delete mode 100644 blockchain/package-lock.json
 delete mode 100644 blockchain/package.json

diff --git a/blockchain/package-lock.json b/blockchain/package-lock.json
deleted file mode 100644
index a7a60efd..00000000
--- a/blockchain/package-lock.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-  "name": "blockchain",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "dependencies": {
-        "singularitynet-platform-contracts": "1.0.2",
-        "singularitynet-token-contracts": "3.0.1"
-      }
-    },
-    "node_modules/@openzeppelin/contracts": {
-      "version": "3.4.2",
-      "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-3.4.2.tgz",
-      "integrity": "sha512-z0zMCjyhhp4y7XKAcDAi3Vgms4T2PstwBdahiO0+9NaGICQKjynK3wduSRplTgk4LXmoO1yfDGO5RbjKYxtuxA=="
-    },
-    "node_modules/singularitynet-platform-contracts": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/singularitynet-platform-contracts/-/singularitynet-platform-contracts-1.0.2.tgz",
-      "integrity": "sha512-5+0j1+UhPuFRD37XG7UxaqekYVvbEJ1xmfqBG0VlMnl+1khAXWw8C1+i3FYdyoZvMPtcIEH5iAoFsouBW91GQA==",
-      "dependencies": {
-        "singularitynet-token-contracts": "^3.0.1"
-      }
-    },
-    "node_modules/singularitynet-token-contracts": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/singularitynet-token-contracts/-/singularitynet-token-contracts-3.0.1.tgz",
-      "integrity": "sha512-SBKlHkwQOLNma3H+jNXqh2dK5BZgHeECfpxurHBrwLQpe0wkePpWVAlYIZZTsdmwsPA4OpJR1nim6ON3RxGtQg==",
-      "dependencies": {
-        "@openzeppelin/contracts": "^3.2.0"
-      }
-    }
-  }
-}
diff --git a/blockchain/package.json b/blockchain/package.json
deleted file mode 100644
index d9837494..00000000
--- a/blockchain/package.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "dependencies": {
-    "singularitynet-platform-contracts": "1.0.2",
-    "singularitynet-token-contracts": "3.0.1"
-  }
-}

From 97ad6ca50746e6e29255b9edfe001e8fd57b38ce Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 12 Nov 2024 12:15:12 +0300
Subject: [PATCH 22/29] Added "organization validate-metadata" command. Changed
 and extended metadata validation in "organization create" and "organization
 update-metadata" commands.

---
 snet/cli/arguments.py                  |   7 +-
 snet/cli/commands/commands.py          | 129 +++++++++++++++---
 snet/cli/metadata/organization.py      |  66 +++------
 snet/cli/resources/org_schema.json     | 180 +++++++++++++++++++++++++
 snet/cli/resources/service_schema.json |   1 -
 5 files changed, 314 insertions(+), 69 deletions(-)
 create mode 100644 snet/cli/resources/org_schema.json

diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index 0336a9c7..8819f42f 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -255,7 +255,7 @@ def add_p_org_id(p):
 def add_metadatafile_argument_for_org(p):
     p.add_argument("--metadata-file",
                    default="organization_metadata.json",
-                   help="Service metadata json file (default organization_metadata.json)")
+                   help="Organization metadata json file (default organization_metadata.json)")
 
 
 def add_p_storage_param(_p):
@@ -435,6 +435,11 @@ def add_organization_options(parser):
     add_p_org_id(p)
     add_organization_arguments(p)
 
+    p = subparsers.add_parser("validate-metadata",
+                              help="Validates if created metadata is consistent")
+    p.set_defaults(fn="metadata_validate")
+    add_metadatafile_argument_for_org(p)
+
 
 def add_contract_function_options(parser, contract_name):
     add_contract_identity_arguments(parser)
diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py
index c94baa25..4c76786c 100644
--- a/snet/cli/commands/commands.py
+++ b/snet/cli/commands/commands.py
@@ -3,10 +3,13 @@
 import json
 import secrets
 import sys
+from idlelib.iomenu import errors
+from pathlib import Path
 from textwrap import indent
 from urllib.parse import urljoin
 
 import ipfshttpclient
+import jsonschema
 from lighthouseweb3 import Lighthouse
 import yaml
 from rfc3986 import urlparse
@@ -480,7 +483,7 @@ def initialize_metadata(self):
 
     def print_metadata(self):
         org_id = self.args.org_id
-        org_metadata = self._get_organization_metadata_from_registry(org_id)
+        org_metadata = self._get_organization_metadata_from_registry(org_id, False)
         self._printout(org_metadata.get_json_pretty())
 
     def _get_organization_registration(self, org_id):
@@ -492,7 +495,7 @@ def _get_organization_registration(self, org_id):
                 self.args.org_id))
         return {"orgMetadataURI": rez[2]}
 
-    def _get_organization_metadata_from_registry(self, org_id):
+    def _get_organization_metadata_from_registry(self, org_id, check_url=True):
         rez = self._get_organization_registration(org_id)
         storage_type, metadata_hash = bytesuri_to_hash(rez["orgMetadataURI"])
         if storage_type == "ipfs":
@@ -500,7 +503,7 @@ def _get_organization_metadata_from_registry(self, org_id):
         else:
             metadata = get_file_from_filecoin(metadata_hash)
         metadata = metadata.decode("utf-8")
-        return OrganizationMetadata.from_json(json.loads(metadata))
+        return OrganizationMetadata.from_json(json.loads(metadata), check_url)
 
     def _get_organization_by_id(self, org_id):
         org_id_bytes32 = type_converter("bytes32")(org_id)
@@ -573,20 +576,108 @@ def info(self):
             for idx, service in enumerate(serviceNames):
                 self._printout(" - {}".format(bytes32_to_str(service)))
 
-    def create(self):
+    def metadata_validate(self):
+        self._metadata_validate(as_exception=False)
+
+    def _metadata_validate(self, as_exception=True):
+        validation_res = self._metadata_validate_with_schema()
+        if validation_res["status"] == 2:
+            if as_exception:
+                raise Exception(validation_res["msg"])
+            else:
+                self._printout(validation_res["msg"])
+                exit(1)
+        with open(self.args.metadata_file, 'r') as f:
+            org_metadata = OrganizationMetadata.from_json(json.load(f))
+
+        occurred_errors = []
+        unique_group_names = set([group.group_name for group in org_metadata.groups])
+        if len(unique_group_names) != len(org_metadata.groups):
+            occurred_errors.append("There should be no groups with duplicated names in the metadata file.")
+
+        for group in org_metadata.groups:
+            for i, endpoint in enumerate(group.payment.payment_channel_storage_client.endpoints):
+                if not is_valid_url(endpoint):
+                    occurred_errors.append(f"Invalid endpoint `{endpoint}` at index {i} in group `{group.group_name}`.")
+
+        existing_errors = validation_res.get("n_errors", 0)
+
+        docs = "https://dev.singularitynet.io/docs/products/DecentralizedAIPlatform/CLI/Manual/Organization/"
+        hint_message = f"\nVisit {docs} for more information."
+        hint_message = f"\n{len(occurred_errors) + existing_errors} errors found." + hint_message + "\n"
+
+        res_msg = ""
+        for i in range(len(occurred_errors)):
+            res_msg += str(existing_errors + i + 1) + ". " + occurred_errors[i] + "\n"
+
+        if res_msg:
+            if validation_res["status"] == 0:
+                res_msg = "\nErrors found in the metadata file:\n" + res_msg
+            else:
+                res_msg += validation_res["msg"] + res_msg
+            res_msg += hint_message
+        elif validation_res["status"] == 0:
+            res_msg = validation_res["msg"]
+        else:
+            res_msg = validation_res["msg"] + hint_message
+
+        if as_exception:
+            raise Exception(res_msg)
+        else:
+            self._printout(res_msg)
 
-        metadata_file = self.args.metadata_file
 
+    def _metadata_validate_with_schema(self):
+        current_path = Path(__file__).parent
+        relative_path = '../resources/org_schema.json'
+        path_to_schema = (current_path / relative_path).resolve()
+        with open(path_to_schema, 'r') as f:
+            schema = json.load(f)
+
+        metadata_file = self.args.metadata_file
         try:
             with open(metadata_file, 'r') as f:
-                org_metadata = OrganizationMetadata.from_json(json.load(f))
+                metadata_dict = json.load(f)
         except Exception as e:
-            print(
-                "Organization metadata json file not found ,Please check --metadata-file path ")
-            raise e
+                return {"status": 2, "msg": "Organization metadata json file not found, please check --metadata-file path"}
+
+        validator = jsonschema.Draft7Validator(schema)
+        occurred_errors = list(validator.iter_errors(metadata_dict))
+
+        def get_path(err):
+            return " -> ".join(f"`{el}`" for el in err.path)
+
+        if len(occurred_errors) > 0:
+            res_msg = f"\nErrors found in the metadata file:\n"
+            for i, e in enumerate(occurred_errors):
+                res_msg += str(i + 1) + ". "
+                if e.validator == 'additionalProperties':
+                    if len(e.path) != 0:
+                        res_msg += f"{e.message} in {get_path(e)}."
+                    else:
+                        res_msg += f"{e.message} in main object."
+                elif e.validator in ['required', 'type', 'enum', 'pattern', 'minLength', 'minItems']:
+                    res_msg += f"{get_path(e)} - {e.message}"
+                    if e.validator == 'minItems':
+                        res_msg += f" (minimum 1 item required)"
+                else:
+                    res_msg += e.message
+                res_msg += "\n"
+
+            return {"status": 1, "msg": res_msg, "n_errors": len(occurred_errors)}
+        else:
+            return {"status": 0, "msg": "Organization metadata is valid and ready to publish."}
+
+    def create(self):
+
+        self._metadata_validate()
+
+        metadata_file = self.args.metadata_file
+        with open(metadata_file, 'r') as f:
+            org_metadata = OrganizationMetadata.from_json(json.load(f))
+        org_metadata.check_remove_groups()
+
         org_id = self.args.org_id
-        # validate the metadata before creating
-        org_metadata.validate()
 
         # R Check if Organization already exists
         found = self._get_organization_by_id(org_id)[0]
@@ -630,19 +721,17 @@ def delete(self):
             raise
 
     def update_metadata(self):
-        metadata_file = self.args.metadata_file
 
-        try:
-            with open(metadata_file, 'r') as f:
-                org_metadata = OrganizationMetadata.from_json(json.load(f))
-        except Exception as e:
-            print("Organization metadata JSON file not found. Please check --metadata-file path.")
-            raise e
+        self._metadata_validate()
+
+        metadata_file = self.args.metadata_file
+        with open(metadata_file, 'r') as f:
+            org_metadata = OrganizationMetadata.from_json(json.load(f))
+        org_metadata.check_remove_groups()
 
-        # Validate the metadata before updating
         org_id = self.args.org_id
         existing_registry_org_metadata = self._get_organization_metadata_from_registry(org_id)
-        org_metadata.validate(existing_registry_org_metadata)
+        org_metadata.check_remove_groups(existing_registry_org_metadata)
 
         # Check if Organization already exists
         found = self._get_organization_by_id(org_id)[0]
diff --git a/snet/cli/metadata/organization.py b/snet/cli/metadata/organization.py
index 1de99c39..c6bc7857 100644
--- a/snet/cli/metadata/organization.py
+++ b/snet/cli/metadata/organization.py
@@ -2,6 +2,7 @@
 from enum import Enum
 from json import JSONEncoder
 import json
+from tabnanny import check
 
 from snet.cli.utils.utils import is_valid_url
 
@@ -30,20 +31,15 @@ def add_payment_storage_client_details(self, connection_time_out, request_timeou
         self.endpoints = endpoints
 
     @classmethod
-    def from_json(cls, json_data: dict):
-        endpoints = json_data["endpoints"]
-        if endpoints:
-            for endpoint in endpoints:
-                if not is_valid_url(endpoint):
-                    raise Exception("Invalid endpoint passed in json file")
+    def from_json(cls, json_data: dict, check_url=True):
+        if check_url:
+            endpoints = json_data["endpoints"]
+            if endpoints:
+                for endpoint in endpoints:
+                    if not is_valid_url(endpoint):
+                        raise Exception("Invalid endpoint passed in json file")
         return cls(**json_data)
 
-    def validate(self):
-        if len(self.endpoints) < 1:
-            raise Exception(
-                "At least one endpoint is required for payment channel ")
-
-
 class Payment(object):
 
     def __init__(self, payment_address="", payment_expiration_threshold="", payment_channel_storage_type="",
@@ -54,9 +50,9 @@ def __init__(self, payment_address="", payment_expiration_threshold="", payment_
         self.payment_channel_storage_client = payment_channel_storage_client
 
     @classmethod
-    def from_json(cls, json_data: dict):
+    def from_json(cls, json_data: dict, check_url=True):
         payment_channel_storage_client = PaymentStorageClient.from_json(
-            json_data['payment_channel_storage_client'])
+            json_data['payment_channel_storage_client'], check_url)
         return cls(json_data['payment_address'], json_data['payment_expiration_threshold'],
                    json_data['payment_channel_storage_type'], payment_channel_storage_client)
 
@@ -91,10 +87,10 @@ def __init__(self, group_name="", group_id="", payment=Payment()):
         self.payment = payment
 
     @classmethod
-    def from_json(cls, json_data: dict):
+    def from_json(cls, json_data: dict, check_url=True):
         payment = Payment()
         if 'payment' in json_data:
-            payment = Payment.from_json(json_data['payment'])
+            payment = Payment.from_json(json_data['payment'], check_url)
         return cls(json_data['group_name'], json_data['group_id'], payment)
 
     def add_group_details(self, group_name, group_id, payment):
@@ -219,10 +215,10 @@ def save_pretty(self, file_name):
             f.write(self.get_json_pretty())
 
     @classmethod
-    def from_json(cls, json_data: dict):
+    def from_json(cls, json_data: dict, check_url=True):
         groups = []
         if 'groups' in json_data:
-            groups = list(map(Group.from_json, json_data["groups"]))
+            groups = list(map(lambda j_d: Group.from_json(j_d, check_url), json_data["groups"]))
             if "contacts" not in json_data:
                 json_data["contacts"] = []
             if "description" not in json_data:
@@ -252,46 +248,22 @@ def from_file(cls, filepath):
             raise e
 
     def is_removing_existing_group_from_org(self, current_group_name, existing_registry_metadata_group_names):
-        if len(existing_registry_metadata_group_names-current_group_name) == 0:
+        if len(existing_registry_metadata_group_names - current_group_name) == 0:
             pass
         else:
             removed_groups = existing_registry_metadata_group_names - current_group_name
             raise Exception("Cannot remove existing group from organization as it might be attached"
                             " to services, groups you are removing are  %s" % removed_groups)
 
-    def validate(self, existing_registry_metadata=None):
-
-        if self.org_id is None:
-            raise Exception("Org_id cannot be null")
-        if self.org_name is None:
-            raise Exception("Org_name cannot be null")
-        if self.org_type is None:
-            raise Exception("Org_type cannot be null")
-        if self.contacts is None:
-            raise Exception("contact_details can not be null")
-        if self.description is None:
-            raise Exception("description can not be null")
-        if self.groups:
-            unique_group_names = set()
-            for group in self.groups:
-                unique_group_names.add(group.group_name)
-
-            if len(unique_group_names) < len(self.groups):
-                raise Exception("Cannot create group with duplicate names")
-        if len(self.groups) < 1:
-            raise Exception(
-                "At least One group is required to create an organization")
-        else:
-            for group in self.groups:
-                group.validate()
-
+    def check_remove_groups(self, existing_registry_metadata):
+        unique_group_names = set([group.group_name for group in self.groups])
         existing_registry_metadata_group_names = set()
+
         if existing_registry_metadata:
             for group in existing_registry_metadata.groups:
                 existing_registry_metadata_group_names.add(group.group_name)
 
-        self.is_removing_existing_group_from_org(
-            unique_group_names, existing_registry_metadata_group_names)
+        self.is_removing_existing_group_from_org(unique_group_names, existing_registry_metadata_group_names)
 
     def get_payment_address_for_group(self, group_name):
         for group in self.groups:
diff --git a/snet/cli/resources/org_schema.json b/snet/cli/resources/org_schema.json
new file mode 100644
index 00000000..6e427a04
--- /dev/null
+++ b/snet/cli/resources/org_schema.json
@@ -0,0 +1,180 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "Organization Metadata",
+    "description": "Schema of a correct organization metadata file",
+    "type": "object",
+    "properties": {
+        "org_name": {
+            "description": "Organization name",
+            "type": "string",
+            "minLength": 1
+        },
+        "org_id": {
+            "description": "Organization id",
+            "type": "string",
+            "minLength": 1
+        },
+        "org_type": {
+            "description": "Organization type [organization, individual]",
+            "type": "string",
+            "enum": [ "organization", "individual" ]
+        },
+        "description": {
+            "description": "Organization description",
+            "type": "object",
+            "properties": {
+                "description": {
+                    "description": "Organization full description",
+                    "type": "string",
+                    "minLength": 1
+                },
+                "short_description": {
+                    "description": "Organization short description",
+                    "type": "string",
+                    "minLength": 1
+                },
+                "url": {
+                    "description": "Organization url",
+                    "type": "string",
+                    "format": "url"
+                }
+            },
+            "additionalProperties": false,
+            "required": ["description", "short_description", "url"]
+        },
+        "assets": {
+            "description": "Organization assets",
+            "type": "object",
+            "properties": {
+                "hero_image": {
+                    "description": "Organization hero image",
+                    "type": "string",
+                    "minLength": 1
+                }
+            },
+            "additionalProperties": true
+        },
+        "contacts": {
+            "description": "Organization contacts",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "email": {
+                        "description": "Contact email",
+                        "type": "string"
+                    },
+                    "phone": {
+                        "description": "Contact phone",
+                        "type": "string"
+                    },
+                    "contact_type": {
+                        "description": "Contact type [general, support]",
+                        "type": "string",
+                        "enum": [ "general", "support" ]
+                    }
+                },
+                "additionalProperties": false,
+                "required": ["contact_type", "email", "phone"]
+            }
+        },
+        "groups": {
+            "description": "Organization groups",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "group_name": {
+                        "description": "Group name",
+                        "type": "string",
+                        "minLength": 1
+                    },
+                    "group_id": {
+                        "description": "Group id",
+                        "type": "string",
+                        "minLength": 1
+                    },
+                    "payment": {
+                        "description": "Group payment",
+                        "type": "object",
+                        "properties": {
+                            "payment_address": {
+                                "description": "Payment address",
+                                "type": "string",
+                                "minLength": 1
+                            },
+                            "payment_expiration_threshold": {
+                                "description": "Payment expiration threshold",
+                                "type": "integer",
+                                "minimum": 1
+                            },
+                            "payment_channel_storage_type": {
+                                "description": "Payment channel storage type (only 'etcd' is supported)",
+                                "type": "string",
+                                "minLength": 1,
+                                "enum": [
+                                    "etcd"
+                                ]
+                            },
+                            "payment_channel_storage_client": {
+                                "description": "Payment channel storage client",
+                                "type": "object",
+                                "properties": {
+                                    "connection_timeout": {
+                                        "description": "Payment channel storage connection timeout",
+                                        "type": "string",
+                                        "pattern": "^\\d{1,3}(s|ms)$"
+                                    },
+                                    "request_timeout": {
+                                        "description": "Payment channel storage request timeout",
+                                        "type": "string",
+                                        "pattern": "^\\d{1,3}(s|ms)$"
+                                    },
+                                    "endpoints": {
+                                        "description": "Payment channel storage endpoints",
+                                        "type": "array",
+                                        "items": {
+                                            "type": "string",
+                                            "minLength": 1,
+                                            "format": "url"
+                                        },
+                                        "minItems": 1
+                                    }
+                                },
+                                "additionalProperties": false,
+                                "required": [
+                                    "connection_timeout",
+                                    "request_timeout",
+                                    "endpoints"
+                                ]
+                            }
+                        },
+                        "additionalProperties": false,
+                        "required": [
+                            "payment_address",
+                            "payment_expiration_threshold",
+                            "payment_channel_storage_type",
+                            "payment_channel_storage_client"
+                        ]
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "group_name",
+                    "group_id",
+                    "payment"
+                ]
+            },
+            "minItems": 1
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "org_name",
+        "org_type",
+        "description",
+        "assets",
+        "contacts",
+        "groups"
+    ]
+}
\ No newline at end of file
diff --git a/snet/cli/resources/service_schema.json b/snet/cli/resources/service_schema.json
index 94fa5bd4..8ab85338 100644
--- a/snet/cli/resources/service_schema.json
+++ b/snet/cli/resources/service_schema.json
@@ -1,6 +1,5 @@
 {
     "$schema": "https://json-schema.org/draft-07/schema#",
-    "$id": "https://dev.singularitynet.io/docs/ai-developers/service/#metadata-overview",
     "title": "Service Metadata",
     "description": "Schema of a correct service metadata file",
     "type": "object",

From d352bba9c8ea6781d02c10933d66264780feb49d Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 12 Nov 2024 12:28:24 +0300
Subject: [PATCH 23/29] Small fix for org metadata validation.

---
 snet/cli/commands/commands.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py
index 4c76786c..179f59fa 100644
--- a/snet/cli/commands/commands.py
+++ b/snet/cli/commands/commands.py
@@ -621,12 +621,11 @@ def _metadata_validate(self, as_exception=True):
         else:
             res_msg = validation_res["msg"] + hint_message
 
-        if as_exception:
+        if as_exception and not res_msg.startswith("Organization metadata is valid and ready to publish."):
             raise Exception(res_msg)
-        else:
+        elif not as_exception:
             self._printout(res_msg)
 
-
     def _metadata_validate_with_schema(self):
         current_path = Path(__file__).parent
         relative_path = '../resources/org_schema.json'

From 39f618dbe1cf106f9d03aacf4aca3ddee872c7c6 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 12 Nov 2024 15:09:29 +0300
Subject: [PATCH 24/29] Removed unnecessary methods.

---
 snet/cli/metadata/organization.py | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/snet/cli/metadata/organization.py b/snet/cli/metadata/organization.py
index c6bc7857..46b07273 100644
--- a/snet/cli/metadata/organization.py
+++ b/snet/cli/metadata/organization.py
@@ -56,19 +56,6 @@ def from_json(cls, json_data: dict, check_url=True):
         return cls(json_data['payment_address'], json_data['payment_expiration_threshold'],
                    json_data['payment_channel_storage_type'], payment_channel_storage_client)
 
-    def validate(self):
-        if self.payment_address is None:
-            raise Exception("Payment address cannot be null")
-        if self.payment_channel_storage_type is None:
-            raise Exception("Payment channel storage type cannot be null")
-        if self.payment_expiration_threshold is None:
-            raise Exception("Payment expiration threshold cannot be null")
-
-        if self.payment_channel_storage_client is None:
-            raise Exception("Payment channel storage client cannot be null")
-        else:
-            self.payment_channel_storage_client.validate()
-
     def update_connection_timeout(self, connection_timeout):
         self.payment_channel_storage_client.connection_timeout = connection_timeout
 
@@ -98,18 +85,6 @@ def add_group_details(self, group_name, group_id, payment):
         self.group_id = group_id
         self.payment = payment
 
-    def validate(self):
-        if self.group_name is None:
-            raise Exception("group name cannot be null")
-        if self.group_id is None:
-            raise Exception("group_id is cannot be null")
-
-        if self.payment is None:
-            raise Exception(
-                "payment details cannot be null for group_name %s", self.group_name)
-        else:
-            self.payment.validate()
-
     def update_payment_expiration_threshold(self, payment_expiration_threshold):
         self.payment.payment_expiration_threshold = payment_expiration_threshold
 

From ec99dde9650f17355c6e69d09275b9f78cd87ddb Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Tue, 12 Nov 2024 18:43:13 +0300
Subject: [PATCH 25/29] Fixed libraries conflicts

---
 requirements.txt                  | 1 -
 snet/cli/arguments.py             | 2 --
 snet/cli/commands/commands.py     | 3 ---
 snet/cli/metadata/organization.py | 1 -
 4 files changed, 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 73d7a945..5e61ba38 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,6 @@ mnemonic==0.20
 pycoin==0.92.20230326
 pyyaml==6.0.1
 ipfshttpclient==0.4.13.2
-rfc3986==2.0.0
 pymultihash==0.8.2
 base58==2.1.1
 argcomplete==3.1.2
diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py
index 8819f42f..02f0eb8d 100644
--- a/snet/cli/arguments.py
+++ b/snet/cli/arguments.py
@@ -2,8 +2,6 @@
 import os
 import re
 import sys
-from email.policy import default
-from random import choices
 
 from snet.contracts import get_all_abi_contract_files, get_contract_def
 
diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py
index 179f59fa..5200ab9a 100644
--- a/snet/cli/commands/commands.py
+++ b/snet/cli/commands/commands.py
@@ -3,16 +3,13 @@
 import json
 import secrets
 import sys
-from idlelib.iomenu import errors
 from pathlib import Path
 from textwrap import indent
-from urllib.parse import urljoin
 
 import ipfshttpclient
 import jsonschema
 from lighthouseweb3 import Lighthouse
 import yaml
-from rfc3986 import urlparse
 import web3
 from snet.contracts import get_contract_def
 
diff --git a/snet/cli/metadata/organization.py b/snet/cli/metadata/organization.py
index 46b07273..28a5c1af 100644
--- a/snet/cli/metadata/organization.py
+++ b/snet/cli/metadata/organization.py
@@ -2,7 +2,6 @@
 from enum import Enum
 from json import JSONEncoder
 import json
-from tabnanny import check
 
 from snet.cli.utils.utils import is_valid_url
 

From c33c698ff78cf2f69be7c93336118461584f99e7 Mon Sep 17 00:00:00 2001
From: Kirill <65371121+kiruxaspb@users.noreply.github.com>
Date: Wed, 13 Nov 2024 16:31:32 +0300
Subject: [PATCH 26/29] Update version to 2.3.0

---
 version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version.py b/version.py
index 8a124bf6..55e47090 100644
--- a/version.py
+++ b/version.py
@@ -1 +1 @@
-__version__ = "2.2.0"
+__version__ = "2.3.0"

From fec3ba9d19b3c15167d3ae434c46ac014b5c8aaf Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 13 Nov 2024 17:55:21 +0300
Subject: [PATCH 27/29] Fixed error in "organization update-metadata" and
 "organization create" commands.

---
 snet/cli/commands/commands.py      | 2 --
 snet/cli/resources/org_schema.json | 3 ++-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py
index 5200ab9a..28d440fd 100644
--- a/snet/cli/commands/commands.py
+++ b/snet/cli/commands/commands.py
@@ -671,7 +671,6 @@ def create(self):
         metadata_file = self.args.metadata_file
         with open(metadata_file, 'r') as f:
             org_metadata = OrganizationMetadata.from_json(json.load(f))
-        org_metadata.check_remove_groups()
 
         org_id = self.args.org_id
 
@@ -723,7 +722,6 @@ def update_metadata(self):
         metadata_file = self.args.metadata_file
         with open(metadata_file, 'r') as f:
             org_metadata = OrganizationMetadata.from_json(json.load(f))
-        org_metadata.check_remove_groups()
 
         org_id = self.args.org_id
         existing_registry_org_metadata = self._get_organization_metadata_from_registry(org_id)
diff --git a/snet/cli/resources/org_schema.json b/snet/cli/resources/org_schema.json
index 6e427a04..1b3d6b59 100644
--- a/snet/cli/resources/org_schema.json
+++ b/snet/cli/resources/org_schema.json
@@ -31,7 +31,8 @@
                 "short_description": {
                     "description": "Organization short description",
                     "type": "string",
-                    "minLength": 1
+                    "minLength": 1,
+                    "maxLength": 160
                 },
                 "url": {
                     "description": "Organization url",

From 3b9b34ba9f90521fb1e1cdb11ee6f8cd11cd021e Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 13 Nov 2024 18:23:12 +0300
Subject: [PATCH 28/29] Fixed org metadata validation

---
 snet/cli/commands/commands.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py
index 28d440fd..e72c8a38 100644
--- a/snet/cli/commands/commands.py
+++ b/snet/cli/commands/commands.py
@@ -585,7 +585,7 @@ def _metadata_validate(self, as_exception=True):
                 self._printout(validation_res["msg"])
                 exit(1)
         with open(self.args.metadata_file, 'r') as f:
-            org_metadata = OrganizationMetadata.from_json(json.load(f))
+            org_metadata = OrganizationMetadata.from_json(json.load(f), False)
 
         occurred_errors = []
         unique_group_names = set([group.group_name for group in org_metadata.groups])
@@ -656,6 +656,8 @@ def get_path(err):
                     res_msg += f"{get_path(e)} - {e.message}"
                     if e.validator == 'minItems':
                         res_msg += f" (minimum 1 item required)"
+                elif e.validator == 'maxLength':
+                    res_msg += f"{get_path(e)} - string is too long"
                 else:
                     res_msg += e.message
                 res_msg += "\n"

From 10d2d7ea074e7cb5200afa32013ab72bdcb16cd0 Mon Sep 17 00:00:00 2001
From: Arondondon <andreypravdin52@gmail.com>
Date: Wed, 13 Nov 2024 18:41:13 +0300
Subject: [PATCH 29/29] Fixed a bug when displaying errors while checking the
 organization metadata.

---
 snet/cli/commands/commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py
index e72c8a38..df0d38cd 100644
--- a/snet/cli/commands/commands.py
+++ b/snet/cli/commands/commands.py
@@ -611,7 +611,7 @@ def _metadata_validate(self, as_exception=True):
             if validation_res["status"] == 0:
                 res_msg = "\nErrors found in the metadata file:\n" + res_msg
             else:
-                res_msg += validation_res["msg"] + res_msg
+                res_msg = validation_res["msg"] + res_msg
             res_msg += hint_message
         elif validation_res["status"] == 0:
             res_msg = validation_res["msg"]