Remove static HTML based webscrape (#930)

refuel-ai · Nov 6, 2024 · 1ffc02d · 1ffc02d
1 parent 3561097
commit 1ffc02d
Show file tree

Hide file tree

Showing 4 changed files with 0 additions and 256 deletions.
diff --git a/src/autolabel/transforms/__init__.py b/src/autolabel/transforms/__init__.py
@@ -7,7 +7,6 @@
 from .serper_maps import SerperMaps
 from .custom_api import CustomApi
 from .webpage_transform import WebpageTransform
-from .webpage_scrape import WebpageScrape
 from .image import ImageTransform
 from typing import Dict
 from autolabel.transforms.schema import TransformType
@@ -18,7 +17,6 @@
 TRANSFORM_REGISTRY = {
     TransformType.PDF: PDFTransform,
     TransformType.WEBPAGE_TRANSFORM: WebpageTransform,
-    TransformType.WEBPAGE_SCRAPE: WebpageScrape,
     TransformType.IMAGE: ImageTransform,
     TransformType.WEB_SEARCH_SERP_API: SerpApi,
     TransformType.WEB_SEARCH_SERPER: SerperApi,

diff --git a/src/autolabel/transforms/schema.py b/src/autolabel/transforms/schema.py
@@ -9,7 +9,6 @@ class TransformType(str, Enum):
     """Enum containing all Transforms supported by autolabel"""
 
     WEBPAGE_TRANSFORM = "webpage_transform"
-    WEBPAGE_SCRAPE = "webpage_scrape"
     PDF = "pdf"
     IMAGE = "image"
     WEB_SEARCH_SERP_API = "web_search_serp_api"

diff --git a/src/autolabel/transforms/webpage_scrape.py b/src/autolabel/transforms/webpage_scrape.py
diff --git a/tests/unit/transforms/test_webpage_scrape.py b/tests/unit/transforms/test_webpage_scrape.py