From 82e350591f790bffa7037a4ffd6f1fea81ef390b Mon Sep 17 00:00:00 2001 From: Leonardo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:20:18 -0300 Subject: [PATCH] feat: replace config file with command args for network discovery (#33) --- network-discovery/Makefile | 2 +- network-discovery/README.md | 61 ++++++++++-------- network-discovery/cmd/main.go | 68 ++++++++++---------- network-discovery/config/config.go | 30 +-------- network-discovery/go.mod | 1 - network-discovery/go.sum | 2 - network-discovery/policy/manager.go | 6 +- network-discovery/policy/manager_test.go | 42 ++++++------ network-discovery/policy/runner.go | 2 +- network-discovery/server/server.go | 39 ++++++----- network-discovery/server/server_test.go | 82 ++++++++++-------------- 11 files changed, 151 insertions(+), 184 deletions(-) diff --git a/network-discovery/Makefile b/network-discovery/Makefile index b5989f6..b16a3fa 100644 --- a/network-discovery/Makefile +++ b/network-discovery/Makefile @@ -8,7 +8,7 @@ COMMIT_SHA := $(shell git rev-parse --short HEAD) .PHONY: build build: - CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=$(GOARCH) GOARM=$(GOARM) go build -mod=mod -o ${BUILD_DIR}/network-discovery cmd/main.go + CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) go build -mod=mod -o ${BUILD_DIR}/network-discovery cmd/main.go .PHONY: lint lint: diff --git a/network-discovery/README.md b/network-discovery/README.md index ceac7d2..228ada6 100644 --- a/network-discovery/README.md +++ b/network-discovery/README.md @@ -1,35 +1,39 @@ # network-discovery Orb network discovery backend -### Config RFC -```yaml -diode: - config: - target: grpc://localhost:8080/diode - api_key: ${DIODE_API_KEY} -network_discovery: - config: - host: 0.0.0.0 - port: 8072 - log_level: info - log_format: json +### Usage +```sh +Usage of network-discovery: + -diode-api-key string + diode api key (REQUIRED). Environment variables can be used by wrapping them in ${} (e.g. ${MY_API_KEY}) + -diode-target string + diode target (REQUIRED) + -help + show this help + -host string + server host (default "0.0.0.0") + -log-format string + log format (default "TEXT") + -log-level string + log level (default "INFO") + -port int + server port (default 8073) ``` ### Policy RFC ```yaml -network_discovery: - policies: - network_1: - config: - schedule: "* * * * *" #Cron expression - timeout: 5 #default 2 minutes - scope: - targets: [192.168.1.0/24] - discover_once: # will run only once - scope: - targets: - - 192.168.0.34/24 - - google.com +policies: + network_1: + config: + schedule: "* * * * *" #Cron expression + timeout: 5 #default 2 minutes + scope: + targets: [192.168.1.0/24] + discover_once: # will run only once + scope: + targets: + - 192.168.0.34/24 + - google.com ``` ## Run device-discovery device-discovery can be run by installing it with pip @@ -37,7 +41,7 @@ device-discovery can be run by installing it with pip git clone https://github.com/netboxlabs/orb-discovery.git cd network-discovery/ make bin -build/network-discovery -c config.yaml +build/network-discovery --diode-target grpc://192.168.31.114:8080/diode --diode-api-key '${DIODE_API_KEY}' ``` ## Docker Image @@ -45,7 +49,10 @@ device-discovery can be build and run using docker: ```sh cd network-discovery/ docker build --no-cache -t network-discovery:develop -f docker/Dockerfile . -docker run -v /local/orb:/usr/local/orb/ --net=host device-discovery:develop network-discovery -c /usr/local/orb/config.yaml +docker run --net=host -e DIODE_API_KEY={YOUR_API_KEY} \ + network-discovery:develop network-discovery \ + --diode-target grpc://192.168.31.114:8080/diode \ + --diode-api-key '${DIODE_API_KEY}' ``` ### Routes (v1) diff --git a/network-discovery/cmd/main.go b/network-discovery/cmd/main.go index 1eeaeb9..ef54682 100644 --- a/network-discovery/cmd/main.go +++ b/network-discovery/cmd/main.go @@ -6,11 +6,10 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" - "github.com/a8m/envsubst" "github.com/netboxlabs/diode-sdk-go/diode" - "gopkg.in/yaml.v3" "github.com/netboxlabs/orb-discovery/network-discovery/config" "github.com/netboxlabs/orb-discovery/network-discovery/policy" @@ -21,47 +20,49 @@ import ( // AppName is the application name const AppName = "network-discovery" +func resolveEnv(value string) string { + // Check if the value starts with ${ and ends with } + if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") { + // Extract the environment variable name + envVar := value[2 : len(value)-1] + // Get the value of the environment variable + envValue := os.Getenv(envVar) + if envValue != "" { + return envValue + } + fmt.Printf("error: environment variable %s is not set\n", envVar) + os.Exit(1) + } + // Return the original value if no substitution occurs + return value +} + func main() { - configPath := flag.String("config", "", "path to the configuration file (required)") + host := flag.String("host", "0.0.0.0", "server host") + port := flag.Int("port", 8073, "server port") + diodeTarget := flag.String("diode-target", "", "diode target (REQUIRED)") + diodeAPIKey := flag.String("diode-api-key", "", "diode api key (REQUIRED)."+ + " Environment variables can be used by wrapping them in ${} (e.g. ${MY_API_KEY})") + logLevel := flag.String("log-level", "INFO", "log level") + logFormat := flag.String("log-format", "TEXT", "log format") + help := flag.Bool("help", false, "show this help") flag.Parse() - if *configPath == "" { + if *help || *diodeTarget == "" || *diodeAPIKey == "" { fmt.Fprintf(os.Stderr, "Usage of network-discovery:\n") flag.PrintDefaults() - os.Exit(1) - - } - if _, err := os.Stat(*configPath); os.IsNotExist(err) { - fmt.Printf("configuration file '%s' does not exist", *configPath) - os.Exit(1) - } - fileData, err := envsubst.ReadFile(*configPath) - if err != nil { - fmt.Printf("error reading configuration file: %v", err) - os.Exit(1) - } - - cfg := config.Config{ - Network: config.Network{ - Config: config.StartupConfig{ - Host: "0.0.0.0", - Port: 8073, - LogLevel: "INFO", - LogFormat: "TEXT", - }}, - } - - if err = yaml.Unmarshal(fileData, &cfg); err != nil { - fmt.Printf("error parsing configuration file: %v\n", err) + if *help { + os.Exit(0) + } os.Exit(1) } client, err := diode.NewClient( - cfg.Diode.Config.Target, + *diodeTarget, AppName, version.GetBuildVersion(), - diode.WithAPIKey(cfg.Diode.Config.APIKey), + diode.WithAPIKey(resolveEnv(*diodeAPIKey)), ) if err != nil { fmt.Printf("error creating diode client: %v\n", err) @@ -69,11 +70,10 @@ func main() { } ctx := context.Background() - logger := config.NewLogger(cfg.Network.Config.LogLevel, cfg.Network.Config.LogFormat) + logger := config.NewLogger(*logLevel, *logFormat) policyManager := policy.NewManager(ctx, logger, client) - server := server.Server{} - server.Configure(logger, policyManager, version.GetBuildVersion(), cfg.Network.Config) + server := server.NewServer(*host, *port, logger, policyManager, version.GetBuildVersion()) // handle signals done := make(chan bool, 1) diff --git a/network-discovery/config/config.go b/network-discovery/config/config.go index 86024d5..c1854ef 100644 --- a/network-discovery/config/config.go +++ b/network-discovery/config/config.go @@ -27,33 +27,7 @@ type Policy struct { Scope Scope `yaml:"scope"` } -// StartupConfig represents the configuration of the network-discovery service -type StartupConfig struct { - Host string `yaml:"host"` - Port int32 `yaml:"port"` - LogLevel string `yaml:"log_level"` - LogFormat string `yaml:"log_format"` -} - -// Network represents the network-discovery configuration -type Network struct { - Config StartupConfig `yaml:"config"` +// Policies represents a collection of network-discovery policies +type Policies struct { Policies map[string]Policy `mapstructure:"policies"` } - -// DiodeConfig represents the configuration of diode service -type DiodeConfig struct { - Target string `yaml:"target"` - APIKey string `yaml:"api_key"` -} - -// Diode represents the root configuration of diode service -type Diode struct { - Config DiodeConfig `yaml:"config"` -} - -// Config represents the root configuration of the network-discovery service -type Config struct { - Diode Diode `yaml:"diode"` - Network Network `yaml:"network_discovery"` -} diff --git a/network-discovery/go.mod b/network-discovery/go.mod index 1899010..4347b80 100644 --- a/network-discovery/go.mod +++ b/network-discovery/go.mod @@ -4,7 +4,6 @@ go 1.23.3 require ( github.com/Ullaakut/nmap/v3 v3.0.4 - github.com/a8m/envsubst v1.4.2 github.com/gin-gonic/gin v1.10.0 github.com/go-co-op/gocron/v2 v2.12.4 github.com/netboxlabs/diode-sdk-go v0.2.0 diff --git a/network-discovery/go.sum b/network-discovery/go.sum index 4e8637f..5ba1fa7 100644 --- a/network-discovery/go.sum +++ b/network-discovery/go.sum @@ -1,7 +1,5 @@ github.com/Ullaakut/nmap/v3 v3.0.4 h1:ZwRM4gbE0gtoXipC7uTHs2wvf3r4S8d9mtqfTabXE34= github.com/Ullaakut/nmap/v3 v3.0.4/go.mod h1:dd5K68P7LHc5nKrFwQx6EdTt61O9UN5x3zn1R4SLcco= -github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg= -github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= diff --git a/network-discovery/policy/manager.go b/network-discovery/policy/manager.go index 3bc52b9..60568cc 100644 --- a/network-discovery/policy/manager.go +++ b/network-discovery/policy/manager.go @@ -32,16 +32,16 @@ func NewManager(ctx context.Context, logger *slog.Logger, client diode.Client) * // ParsePolicies parses the policies from the request func (m *Manager) ParsePolicies(data []byte) (map[string]config.Policy, error) { - var payload config.Config + var payload config.Policies if err := yaml.Unmarshal(data, &payload); err != nil { return nil, err } - if payload.Network.Policies == nil { + if len(payload.Policies) == 0 { return nil, errors.New("no policies found in the request") } - return payload.Network.Policies, nil + return payload.Policies, nil } // HasPolicy checks if the policy exists diff --git a/network-discovery/policy/manager_test.go b/network-discovery/policy/manager_test.go index 4b6175e..2da326f 100644 --- a/network-discovery/policy/manager_test.go +++ b/network-discovery/policy/manager_test.go @@ -38,15 +38,14 @@ func TestManagerParsePolicies(t *testing.T) { t.Run("Valid Policies", func(t *testing.T) { yamlData := []byte(` - network_discovery: - policies: - policy1: - config: - defaults: - site: New York NY - scope: - targets: - - 192.168.1.1/24 + policies: + policy1: + config: + defaults: + site: New York NY + scope: + targets: + - 192.168.1.1/24 `) policies, err := manager.ParsePolicies(yamlData) @@ -67,19 +66,18 @@ func TestManagerPolicyLifecycle(t *testing.T) { logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) manager := policy.NewManager(context.Background(), logger, nil) yamlData := []byte(` - network_discovery: - policies: - policy1: - scope: - targets: - - 192.168.1.1/24 - policy2: - scope: - targets: - - 192.168.2.1/24 - policy3: - scope: - targets: [] + policies: + policy1: + scope: + targets: + - 192.168.1.1/24 + policy2: + scope: + targets: + - 192.168.2.1/24 + policy3: + scope: + targets: [] `) policies, err := manager.ParsePolicies(yamlData) diff --git a/network-discovery/policy/runner.go b/network-discovery/policy/runner.go index 63cab96..0616a6b 100644 --- a/network-discovery/policy/runner.go +++ b/network-discovery/policy/runner.go @@ -81,7 +81,7 @@ func (r *Runner) run() { r.logger.Error("error creating scanner", slog.Any("error", err), slog.Any("policy", r.ctx.Value(policyKey))) return } - + r.logger.Info("running scanner", slog.Any("targets", r.scope.Targets), slog.Any("policy", r.ctx.Value(policyKey))) result, warnings, err := scanner.Run() if len(*warnings) > 0 { r.logger.Warn("run finished with warnings", slog.String("warnings", fmt.Sprintf("%v", *warnings))) diff --git a/network-discovery/server/server.go b/network-discovery/server/server.go index f73a600..1fd8d97 100644 --- a/network-discovery/server/server.go +++ b/network-discovery/server/server.go @@ -31,30 +31,37 @@ type Server struct { manager *policy.Manager stat config.Status logger *slog.Logger - config config.StartupConfig + host string + port int } func init() { gin.SetMode(gin.ReleaseMode) } -// Configure configures the network-discovery server -func (s *Server) Configure(logger *slog.Logger, manager *policy.Manager, version string, config config.StartupConfig) { - s.stat.Version = version - s.stat.StartTime = time.Now() - s.manager = manager - s.logger = logger - s.config = config - - s.router = gin.New() +// NewServer returns a new network-discovery server +func NewServer(host string, port int, logger *slog.Logger, manager *policy.Manager, version string) *Server { + server := &Server{ + router: gin.New(), + manager: manager, + stat: config.Status{ + Version: version, + StartTime: time.Now(), + }, + logger: logger, + host: host, + port: port, + } - v1 := s.router.Group("/api/v1") + v1 := server.router.Group("/api/v1") { - v1.GET("/status", s.getStatus) - v1.GET("/capabilities", s.getCapabilities) - v1.POST("/policies", s.createPolicy) - v1.DELETE("/policies/:policy", s.deletePolicy) + v1.GET("/status", server.getStatus) + v1.GET("/capabilities", server.getCapabilities) + v1.POST("/policies", server.createPolicy) + v1.DELETE("/policies/:policy", server.deletePolicy) } + + return server } // Router returns the router @@ -65,7 +72,7 @@ func (s *Server) Router() *gin.Engine { // Start starts the network-discovery server func (s *Server) Start() { go func() { - serv := fmt.Sprintf("%s:%d", s.config.Host, s.config.Port) + serv := fmt.Sprintf("%s:%d", s.host, s.port) s.logger.Info("starting network-discovery server at: " + serv) if err := s.router.Run(serv); err != nil { s.logger.Error("shutting down the server", "error", err) diff --git a/network-discovery/server/server_test.go b/network-discovery/server/server_test.go index daa076b..54b216c 100644 --- a/network-discovery/server/server_test.go +++ b/network-discovery/server/server_test.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/netboxlabs/orb-discovery/network-discovery/config" "github.com/netboxlabs/orb-discovery/network-discovery/policy" "github.com/netboxlabs/orb-discovery/network-discovery/server" ) @@ -40,15 +39,7 @@ func TestServerConfigureAndStart(t *testing.T) { client := new(MockClient) policyManager := policy.NewManager(ctx, logger, client) - srv := &server.Server{} - - config := config.StartupConfig{ - Host: "localhost", - Port: 8080, - } - version := "1.0.0" - - srv.Configure(logger, policyManager, version, config) + srv := server.NewServer("localhost", 8080, logger, policyManager, "1.0.0") srv.Start() // Check /status endpoint @@ -72,8 +63,7 @@ func TestServerGetCapabilities(t *testing.T) { client := new(MockClient) policyManager := policy.NewManager(ctx, logger, client) - srv := &server.Server{} - srv.Configure(logger, policyManager, "1.0.0", config.StartupConfig{}) + srv := server.NewServer("localhost", 8073, logger, policyManager, "1.0.0") // Check /capabilities endpoint w := httptest.NewRecorder() @@ -93,19 +83,17 @@ func TestServerCreateDeletePolicy(t *testing.T) { client := new(MockClient) policyManager := policy.NewManager(ctx, logger, client) - srv := &server.Server{} - srv.Configure(logger, policyManager, "1.0.0", config.StartupConfig{}) + srv := server.NewServer("localhost", 8073, logger, policyManager, "1.0.0") body := []byte(` - network_discovery: - policies: - test-policy: - config: - defaults: - site: New York NY - scope: - targets: - - 192.168.31.1/24 + policies: + test-policy: + config: + defaults: + site: New York NY + scope: + targets: + - 192.168.31.1/24 `) w := httptest.NewRecorder() @@ -120,16 +108,15 @@ func TestServerCreateDeletePolicy(t *testing.T) { // Try to create the same policy again body = []byte(` - network_discovery: - policies: - test-pol: - scope: - targets: - - 192.168.31.1/24 - test-policy: - scope: - targets: - - 192.168.31.1/24 + policies: + test-pol: + scope: + targets: + - 192.168.31.1/24 + test-policy: + scope: + targets: + - 192.168.31.1/24 `) w = httptest.NewRecorder() request, _ = http.NewRequest(http.MethodPost, "/api/v1/policies", bytes.NewReader(body)) @@ -183,8 +170,7 @@ func TestServerCreateInvalidPolicy(t *testing.T) { desc: "no policies found", contentType: "application/x-yaml", body: []byte(` - network_discovery: - config: {} + policies: {} `), returnCode: http.StatusBadRequest, returnMessage: `no policies found in the request`, @@ -193,18 +179,17 @@ func TestServerCreateInvalidPolicy(t *testing.T) { desc: "no targets found", contentType: "application/x-yaml", body: []byte(` - network_discovery: - policies: - test-policy: - scope: - targets: - - 192.168.31.1/24 - test-policy-invalid: - config: - defaults: - site: New York NY - scope: - ports: [80, 443] + policies: + test-policy: + scope: + targets: + - 192.168.31.1/24 + test-policy-invalid: + config: + defaults: + site: New York NY + scope: + ports: [80, 443] `), returnCode: http.StatusBadRequest, returnMessage: `test-policy-invalid : no targets found in the policy`, @@ -217,8 +202,7 @@ func TestServerCreateInvalidPolicy(t *testing.T) { client := new(MockClient) policyManager := policy.NewManager(ctx, logger, client) - srv := &server.Server{} - srv.Configure(logger, policyManager, "1.0.0", config.StartupConfig{}) + srv := server.NewServer("localhost", 8073, logger, policyManager, "1.0.0") // Create invalid policy request w := httptest.NewRecorder()