feat: implement worker backend

.github/workflows/worker-lint-tests.yaml

@@ -0,0 +1,61 @@
+name: Worker - lint and tests
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "worker/**"
+  push:
+    branches:
+      - "!release"
+    paths:
+      - "worker/**"
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  BE_DIR: worker
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    strategy:
+      matrix:
+        python: [ "3.10", "3.11", "3.12" ]
+    defaults:
+      run:
+        working-directory: ${{ env.BE_DIR }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+        pip install .[dev]
+        pip install .[test]
+
+    - name: Run tests with coverage
+      run: |
+        set -o pipefail
+        pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=device_discovery/ | tee pytest-coverage.txt
+
+    - name: Pytest coverage comment
+      uses: MishaKav/pytest-coverage-comment@81882822c5b22af01f91bd3eacb1cefb6ad73dc2 #v1.1.53
+      with:
+        pytest-coverage-path: ${{ env.BE_DIR }}/pytest-coverage.txt
+        junitxml-path: ${{ env.BE_DIR }}/pytest.xml
+
+    - name: Lint with Ruff
+      run: |
+        ruff check --output-format=github worker/ tests/
+      continue-on-error: true

worker/README.md

@@ -0,0 +1,154 @@
+# worker
+Orb custom discovery backend
+
+### Usage
+```bash
+usage: worker [-h] [-V] [-s HOST] [-p PORT] -t DIODE_TARGET -k DIODE_API_KEY
+
+Orb Worker Backend
+
+options:
+  -h, --help            show this help message and exit
+  -V, --version         Display Discovery, NAPALM and Diode SDK versions
+  -s HOST, --host HOST  Server host
+  -p PORT, --port PORT  Server port
+  -t DIODE_TARGET, --diode-target DIODE_TARGET
+                        Diode target
+  -k DIODE_API_KEY, --diode-api-key DIODE_API_KEY
+                        Diode API key. Environment variables can be used by wrapping them in ${} (e.g.
+                        ${MY_API_KEY})
+  -a DIODE_APP_NAME_PREFIX, --diode-app-name-prefix DIODE_APP_NAME_PREFIX
+                        Diode producer_app_name prefix
+```
+
+### Policy RFC
+```yaml
+policies:
+  worker_policy:
+    config:
+      package: my_custom_package
+      schedule: "* * * * *" #Cron expression
+      custom_config: custom value
+    scope:
+      any_key: any_value
+```
+## Run worker
+worker can be run by installing it with pip
+```sh
+git clone https://github.com/netboxlabs/orb-discovery.git
+cd orb-discovery/
+pip install --no-cache-dir ./worker/
+worker -t 'grpc://192.168.0.10:8080/diode' -k '${DIODE_API_KEY}'
+```
+
+## Docker Image
+worker can be build and run using docker:
+```sh
+cd worker
+docker build --no-cache -t worker:develop -f docker/Dockerfile .
+docker run  -e DIODE_API_KEY={YOUR_API_KEY} -p 8072:8072 worker:develop \
+ worker -t 'grpc://192.168.0.10:8080/diode' -k '${DIODE_API_KEY}'
+```
+
+### Routes (v1)
+
+#### Get runtime and capabilities information
+
+<details>
+ <summary><code>GET</code> <code><b>/api/v1/status</b></code> <code>(gets discovery runtime data)</code></summary>
+
+##### Parameters
+
+> None
+
+##### Responses
+
+> | http code     | content-type                      | response                                                            |
+> |---------------|-----------------------------------|---------------------------------------------------------------------|
+> | `200`         | `application/json; charset=utf-8` |  `{"version": "0.1.0","up_time_seconds": 3678 }`                    |
+
+##### Example cURL
+
+> ```sh
+>  curl -X GET -H "Content-Type: application/json" http://localhost:8072/api/v1/status
+> ```
+
+</details>
+
+<details>
+ <summary><code>GET</code> <code><b>/api/v1/capabilities</b></code> <code>(gets worker capabilities)</code></summary>
+
+##### Parameters
+
+> None
+
+##### Responses
+
+> | http code     | content-type                      | response                                                            |
+> |---------------|-----------------------------------|---------------------------------------------------------------------|
+> | `200`         | `application/json; charset=utf-8` | `{"supported_drivers":["ios","eos","junos","nxos","cumulus"]}`      |
+
+##### Example cURL
+
+> ```sh
+>  curl -X GET -H "Content-Type: application/json" http://localhost:8072/api/v1/capabilities
+> ```
+
+</details>
+
+#### Policies Management
+
+
+<details>
+ <summary><code>POST</code> <code><b>/api/v1/policies</b></code> <code>(Creates a new policy)</code></summary>
+
+##### Parameters
+
+> | name      |  type     | data type               | description                                                           |
+> |-----------|-----------|-------------------------|-----------------------------------------------------------------------|
+> | None      |  required | YAML object             | yaml format specified in [Policy RFC](#policy-rfc)                    |
+ 
+
+##### Responses
+
+> | http code     | content-type                       | response                                                            |
+> |---------------|------------------------------------|---------------------------------------------------------------------|
+> | `201`         | `application/json; charset=UTF-8`  | `{"detail":"policy 'policy_name' was started"}`                     |
+> | `400`         | `application/json; charset=UTF-8`  | `{ "detail": "invalid Content-Type. Only 'application/x-yaml' is supported" }`|
+> | `400`         | `application/json; charset=UTF-8`  | Any other policy error                                              |
+> | `403`         | `application/json; charset=UTF-8`  | `{ "detail": "config field is required" }`                          |
+> | `409`         | `application/json; charset=UTF-8`  | `{ "detail": "policy 'policy_name' already exists" }`               |
+ 
+
+##### Example cURL
+
+> ```sh
+>  curl -X POST -H "Content-Type: application/x-yaml" --data-binary @policy.yaml http://localhost:8072/api/v1/policies
+> ```
+
+</details>
+
+<details>
+ <summary><code>DELETE</code> <code><b>/api/v1/policies/{policy_name}</b></code> <code>(delete a existing policy)</code></summary>
+
+##### Parameters
+
+> | name              |  type     | data type      | description                         |
+> |-------------------|-----------|----------------|-------------------------------------|
+> |   `policy_name`   |  required | string         | The unique policy name              |
+
+##### Responses
+
+> | http code     | content-type                      | response                                                            |
+> |---------------|-----------------------------------|---------------------------------------------------------------------|
+> | `200`         | `application/json; charset=UTF-8` | `{ "detail": "policy 'policy_name' was deleted" }`                  |
+> | `400`         | `application/json; charset=UTF-8` | Any other policy deletion error                                     |
+> | `404`         | `application/json; charset=UTF-8` | `{ "detail": "policy 'policy_name' not found" }`                    |
+
+##### Example cURL
+
+> ```sh
+>  curl -X DELETE http://localhost:8072/api/v1/policies/policy_name
+> ```
+
+</details>

worker/docker/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim-bullseye AS builder
+
+WORKDIR /usr/src/app
+
+COPY . .
+
+RUN python -m venv /opt/venv && \
+    . /opt/venv/bin/activate && \
+    pip install --no-cache-dir . && \
+    deactivate
+
+FROM python:3.12-slim-bullseye
+
+RUN addgroup --system netdev && useradd -m --shell /bin/bash -G netdev appuser && echo "appuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+COPY --from=builder /opt/venv /opt/venv
+
+# Activate virtual environment for all users
+ENV PATH="/opt/venv/bin:$PATH"
+
+USER appuser
+
+CMD [ "worker" ]

worker/pyproject.toml

@@ -0,0 +1,69 @@
+[project]
+name = "netboxlabs-worker"
+version = "0.0.1"  # Overwritten during the build process
+description = "NetBox Labs, Worker backend for Orb Agent, part of NetBox Discovery"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+authors = [
+    {name = "NetBox Labs", email = "[email protected]" }
+]
+maintainers = [
+    {name = "NetBox Labs", email = "[email protected]" }
+]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Build Tools",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
+]
+
+dependencies = [
+    "APScheduler~=3.10",
+    "croniter~=5.0",
+    "fastapi~=0.115",
+    "httpx~=0.27",
+    "netboxlabs-diode-sdk~=0.4",
+    "pydantic~=2.9",
+    "uvicorn~=0.32",
+]
+
+[project.optional-dependencies]
+dev = ["black", "check-manifest", "ruff"]
+test = ["coverage", "pytest", "pytest-cov"]
+
+[project.urls]
+"Homepage" = "https://netboxlabs.com/"
+
+[project.scripts]
+worker = "worker.main:main"
+
+[tool.setuptools]
+packages = [
+    "worker",
+    "worker.policy",
+]
+package-data = {"worker" = ["**/*", "policy/**"]}
+exclude-package-data = {worker = ["tests/*"]}
+
+[build-system]
+requires = ["setuptools>=43.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+
+[tool.ruff]
+line-length = 140
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.ruff.lint]
+select = ["C", "D", "E", "F", "I", "R", "UP", "W"]
+ignore = ["F401", "D203", "D212", "D400", "D401", "D404", "RET504"]

worker/tests/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# Copyright 2025 NetBox Labs Inc
+"""NetBox Labs - Tests namespace."""

worker/tests/mock_custom/README.md

@@ -0,0 +1,2 @@
+### Mock Custom
+A simple implementation for testing.

worker/tests/mock_custom/nbl_custom/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# Copyright 2025 NetBox Labs Inc
+"""NetBox Labs - Mock custom namespace."""
+
+from nbl_custom.impl import MockBackend
+
+__all__ = ["MockBackend"]

worker/tests/mock_custom/nbl_custom/impl.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# Copyright 2025 NetBox Labs Inc
+"""NetBox Labs - Mock Impl."""
+
+from collections.abc import Iterable
+from typing import Any
+
+from netboxlabs.diode.sdk.ingester import Device, Entity
+from worker.backend import Backend
+from worker.models import Config, Metadata
+
+
+class MockBackend(Backend):
+    """Mock backend class."""
+
+    def setup(self) -> Metadata:
+        """Mock setup method."""
+        return Metadata(name="mock_custom", app_name="mock_app", app_version="1.0.0")
+
+    def run(self, _: Config, __: Any) -> Iterable[Entity]:
+        """Mock run method."""
+        entities = []
+
+        device = Device(
+            name="Device A",
+            device_type="Device Type A",
+            platform="Platform A",
+            manufacturer="Manufacturer A",
+            site="Site ABC",
+            role="Role ABC",
+            serial="123456",
+            asset_tag="123456",
+            status="active",
+            tags=["tag 1", "tag 2"],
+        )
+
+        entities.append(Entity(device=device))
+        return entities