Merge pull request #951 from netenglabs/lib-upd8s-23

update libs for 0.23 release

pyproject.toml

@@ -17,23 +17,23 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">3.8.1, < 3.10"
-aiohttp = "==3.7.4"
-asyncssh = "~2.11"
+aiohttp = ">3.8, <4.0"
+asyncssh = "~2.14"
 textfsm = "*"
-async-timeout = "<4.0"
+async-timeout = ">4.0, <5.0"
 aiofiles = "*"
 prompt-toolkit = ">2"
 pyarrow = "~11.0"
 pandas = "~1.5"
 PyYAML = "*"
 python-nubia = "0.2b5"
 uvloop = "*"
-networkx = "^2.4"
-matplotlib = "^3.2.2"
-faker = "^4.1.1"
-netconan = "^0.11.2"
-jsonpath-ng = "^1.5.1"
-fastapi = "~0.95"
+networkx = "~2.4"
+matplotlib = "~3.7"
+faker = "~4.18"
+netconan = "~0.11.2"
+jsonpath-ng = "~1.6"
+fastapi = "~0.95.2"
 uvicorn = "^0.14.0"
 tabulate = "^0.8.7"
 streamlit = "1.11.1"
@@ -43,15 +43,17 @@ natsort = "^8.0.0"
 colorama = "^0.4.4"
 tzlocal = "< 3.0"
 pyparsing = ">=2.2.0, <3.0"
-xmltodict = "^0.12.0"
-streamlit-aggrid = "^0.2.2"
-ciscoconfparse = "^1.6.21"
+xmltodict = "~0.12.0"
+streamlit-aggrid = "~0.2.2"
+ciscoconfparse = "~1.6.21"
 notebook = "6.4.12"
-urllib3 = "^1.26.12"
-packaging = "^21.3"
-psutil = "^5.9.4"
+urllib3 = "~1.26.12"
+packaging = "~21.3"
+psutil = "~5.9.4"
 jellyfish = "~0.10"
 altair = '>3.2, <5.0'
+pydantic = '< 2.0'
+numpy = '~1.20'
 
 [tool.poetry.dev-dependencies]
 pylint = "*"

suzieq/db/parquet/parquetdb.py

@@ -1,7 +1,7 @@
 import os
 import re
 from time import time
-from typing import List, Optional
+from typing import Any, Callable, List, Optional
 import logging
 from pathlib import Path
 from datetime import datetime, timedelta, timezone
@@ -130,8 +130,7 @@ def read(self, table_name: str, data_format: str,
                         continue
                     if need_sqvers:
                         vers = float(str(elem).split('=')[-1])
-                        if vers > max_vers:
-                            max_vers = vers
+                        max_vers = max(vers, max_vers)
 
                     dataset = ds.dataset(elem, format='parquet',
                                          partitioning='hive')
@@ -226,6 +225,9 @@ def write(self, table_name: str, data_format: str,
                 df = pd.DataFrame.from_dict(data["records"])
                 table = pa.Table.from_pandas(df, schema=schema,
                                              preserve_index=False)
+            else:
+                raise ValueError('Unknown format of data provided:'
+                                 f'{type(data)}')
 
             pq.write_to_dataset(table,
                                 root_path=folder,
@@ -539,8 +541,7 @@ def _get_cp_dataset(self, table_name: str, need_sqvers: bool,
                 continue
             if need_sqvers:
                 vers = float(str(elem).split('=')[-1])
-                if vers > max_vers:
-                    max_vers = vers
+                max_vers = max(vers, max_vers)
 
             dataset = ds.dataset(elem, format='parquet', partitioning='hive')
 
@@ -618,7 +619,7 @@ def _get_filtered_fileset(self, dataset: ds, namespaces: list) -> ds:
             ds: pyarrow dataset of only the files that match filter
         """
         def check_ns_conds(ns_to_test: str, filter_list: List[str],
-                           op: operator.or_) -> bool:
+                           op: Callable[[Any, Any], Any]) -> bool:
             """Concat the expressions with the provided (AND or OR) operator
             and return the result of the resulting expression tested on the
             provided namespace.
@@ -631,11 +632,10 @@ def check_ns_conds(ns_to_test: str, filter_list: List[str],
             Returns:
                 bool: the result of the expression
             """
-            # pylint: disable=comparison-with-callable
             # We would like to init the result to False if we concat the
             # expressions with OR, while with True if we use AND.
             res = False
-            if operator.and_ == op:
+            if operator.and_ is op:
                 res = True
             for filter_val in filter_list:
                 ns_to_test = ns_to_test.split('namespace=')[-1]

suzieq/engines/pandas/routes.py

@@ -35,7 +35,6 @@ def _cons_addnl_fields(self, columns: List[str], addnl_fields: List[str],
 
         return addnl_fields
 
-    # pylint: disable=too-many-statements
     def get(self, **kwargs):
         '''Return the routes table for the given filters'''
 

suzieq/gui/stlit/path.py

@@ -259,6 +259,14 @@ def _get_path(self, forward_dir: bool = True) -> Tuple[pd.DataFrame,
                                    vrf=self._state.vrf)
 
             summ_df = self._path_summarize(df)
+        else:
+            df = self._pathobj.get(namespace=[self._state.namespace],
+                                   src=self._state.dest,
+                                   dest=self._state.source,
+                                   vrf=self._state.vrf)
+
+            summ_df = self._path_summarize(df)
+
         return df, summ_df
 
     def _get_failed_data(self, namespace: str, pgbar) -> None:

suzieq/gui/stlit/xplore.py

@@ -389,8 +389,7 @@ def _draw_aggrid_df(self, df) -> AgGrid:
 
         start_row = self._state.start_row
         end_row = self._state.end_row
-        if end_row > df.shape[0]:
-            end_row = df.shape[0]
+        end_row = min(end_row, df.shape[0])
 
         if end_row - start_row > MAX_ROW_WARN:
             st.warning('Printing too many rows can fail or be very slow')

suzieq/poller/controller/source/ansible.py

@@ -23,6 +23,7 @@ class AnsibleSourceModel(SourceModel):
     def validate_and_set(cls, path: str):
         """checks if the path is valid
         """
+        inventory = None
         if isinstance(path, str):
             if not Path(path).is_file():
                 raise ValueError(
@@ -38,6 +39,10 @@ def validate_and_set(cls, path: str):
 
         elif isinstance(path, Dict):
             inventory = path
+
+        if not inventory:
+            raise ValueError('Unknown inventory format: expect path or dict')
+
         if '_meta' not in inventory \
                 or 'hostvars' not in inventory['_meta']:
             if isinstance(inventory, list) and 'namespace' in inventory[0]:

suzieq/poller/worker/nodes/node.py

@@ -344,6 +344,7 @@ async def _post_result(self, service_callback: asyncio.coroutine,
     async def _parse_device_type_hostname(self, output, _) -> None:
         devtype = ""
         hostname = None
+        version_str = '0.0.0'   # default that's possibly never used
 
         if output[0]["status"] == 0:
             # don't keep trying if we're connected to an unsupported dev

suzieq/poller/worker/services/evpnVni.py

@@ -198,7 +198,7 @@ def _clean_junos_data(self, processed_data, _):
                             'vrf': vrf,
                             'os': 'junos'
                         }
-                    newntries[vni] = vni_entry
+                        newntries[vni] = vni_entry
                     continue
             elif entry['_entryType'] == 'l3':
                 vni = int(entry.get('vni', '0'))

suzieq/poller/worker/services/ospfIf.py

@@ -79,6 +79,7 @@ def _clean_eos_data(self, processed_data, _):
     def _clean_junos_data(self, processed_data, _):
 
         drop_indices = []
+        routerId = ''
 
         for i, entry in enumerate(processed_data):
             if entry['_entryType'] == 'overview':

suzieq/poller/worker/services/routes.py

@@ -136,6 +136,7 @@ def _clean_junos_data(self, processed_data, raw_data):
             if entry.get('protocol', '') == 'direct':
                 entry['protocol'] = 'connected'
             vrf = entry.pop("vrf")[0]['data']
+            vers = 0
             if vrf == "inet.0":
                 vrf = "default"
                 vers = 4

suzieq/poller/worker/services/service_manager.py

@@ -188,9 +188,8 @@ async def init_services(self) -> List[Service]:
         svc_classes = Service.get_plugins()
 
         schemas = Schema(self.schema_dir)
-        if schemas:
-            poller_schema = schemas.get_arrow_schema('sqPoller')
-            poller_schema_version = SchemaForTable('sqPoller', schemas).version
+        poller_schema = schemas.get_arrow_schema('sqPoller')
+        poller_schema_version = SchemaForTable('sqPoller', schemas).version
 
         db_access = self._get_db_access(self.cfg)
 

suzieq/poller/worker/services/svcparser.py

@@ -415,6 +415,7 @@ def cons_recs_from_json_template(tmplt_str, in_data):
         # Process default value processing of the form <key>?|<def_val> or
         # <key>?<expected_val>|<def_val>
         op = None  # pylint: disable=redefined-outer-name
+        exp_val = None
         if "?" in rval:
             rval, op = rval.split("?")
             exp_val, def_val = op.split("|")