move logging to debug

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "scrapework"
-version = "0.5.1"
+version = "0.5.3"
 description = "simple scraping framework"
 authors = ["Stéphane Busso <[email protected]>"]
 license = "MIT"

scrapework/module.py

@@ -6,4 +6,4 @@
 class Module(ABC):
     def __init__(self) -> None:
         self.logger = Logger().get_logger()
-        self.logger.info(f"Using {self.__class__.__name__}")
+        self.logger.debug(f"Using {self.__class__.__name__}")

scrapework/processors.py

@@ -15,7 +15,7 @@ class Processor(ABC):
 
     def __init__(self) -> None:
         self.logger = Logger().get_logger()
-        self.logger.info(f"Using processor: {self.__class__.__name__}")
+        self.logger.debug(f"Using processor: {self.__class__.__name__}")
 
     @abstractmethod
     def process_items(

scrapework/scraper.py

@@ -144,11 +144,12 @@ def run(self, start_urls: Optional[List[str]] = None, input: Optional[Any] = Non
 
             # TODO: self.process(ctx, new_items)
 
-            # append new_items to items, Items can be a list or a dict
-            if isinstance(new_items, dict):
-                items.append(new_items)
-            else:
-                items += new_items
+            # append new_items to items, Items can be a list or a dict or None
+            if new_items:
+                if isinstance(new_items, dict):
+                    items.append(new_items)
+                else:
+                    items += new_items
 
             iter_end_time = datetime.datetime.now()
             items_count = len(items)