From c500e640af6a2c4827b223583a0ca7da5eda7b63 Mon Sep 17 00:00:00 2001 From: brmcdoug Date: Wed, 27 Nov 2024 23:52:25 -0800 Subject: [PATCH 1/5] igp-graph transferred from github.com/jalapeno --- Makefile.igp-graph | 29 + build/Dockerfile.igp-graph | 4 + cmd/igp-graph/Makefile | 2 + cmd/igp-graph/main.go | 200 +++++++ deployment/igp-graph.yaml | 36 ++ igp-graph/arango-conn.go | 80 +++ igp-graph/arangodb.go | 542 ++++++++++++++++++ igp-graph/arangodb/arango-conn.go | 80 +++ igp-graph/arangodb/arangodb.go | 542 ++++++++++++++++++ igp-graph/arangodb/handler.go | 185 ++++++ igp-graph/arangodb/lsnode-processor.go | 341 +++++++++++ igp-graph/arangodb/lsv4link.go | 150 +++++ igp-graph/arangodb/lsv4prefix.go | 123 ++++ igp-graph/arangodb/lsv6link.go | 151 +++++ igp-graph/arangodb/lsv6prefix.go | 121 ++++ igp-graph/arangodb/types.go | 108 ++++ igp-graph/handler.go | 185 ++++++ igp-graph/kafkamessenger/kafkamessenger.go | 117 ++++ igp-graph/kafkanotifier/kafkanotifier.go | 199 +++++++ igp-graph/lsnode-processor.go | 341 +++++++++++ igp-graph/lsv4link.go | 150 +++++ igp-graph/lsv4prefix.go | 123 ++++ igp-graph/lsv6link.go | 151 +++++ igp-graph/lsv6prefix.go | 121 ++++ igp-graph/messenger/msg-server.go | 7 + igp-graph/types.go | 108 ++++ .../telegraf_ingress_cfg.yaml | 5 + install/processors/igp-graph/igp-graph.yaml | 36 ++ 28 files changed, 4237 insertions(+) create mode 100755 Makefile.igp-graph create mode 100755 build/Dockerfile.igp-graph create mode 100755 cmd/igp-graph/Makefile create mode 100755 cmd/igp-graph/main.go create mode 100755 deployment/igp-graph.yaml create mode 100755 igp-graph/arango-conn.go create mode 100755 igp-graph/arangodb.go create mode 100755 igp-graph/arangodb/arango-conn.go create mode 100755 igp-graph/arangodb/arangodb.go create mode 100644 igp-graph/arangodb/handler.go create mode 100755 igp-graph/arangodb/lsnode-processor.go create mode 100644 igp-graph/arangodb/lsv4link.go create mode 100644 igp-graph/arangodb/lsv4prefix.go create mode 100644 igp-graph/arangodb/lsv6link.go create mode 100644 igp-graph/arangodb/lsv6prefix.go create mode 100644 igp-graph/arangodb/types.go create mode 100644 igp-graph/handler.go create mode 100755 igp-graph/kafkamessenger/kafkamessenger.go create mode 100644 igp-graph/kafkanotifier/kafkanotifier.go create mode 100755 igp-graph/lsnode-processor.go create mode 100644 igp-graph/lsv4link.go create mode 100644 igp-graph/lsv4prefix.go create mode 100644 igp-graph/lsv6link.go create mode 100644 igp-graph/lsv6prefix.go create mode 100755 igp-graph/messenger/msg-server.go create mode 100644 igp-graph/types.go create mode 100644 install/processors/igp-graph/igp-graph.yaml diff --git a/Makefile.igp-graph b/Makefile.igp-graph new file mode 100755 index 00000000..3093134e --- /dev/null +++ b/Makefile.igp-graph @@ -0,0 +1,29 @@ +REGISTRY_NAME?=docker.io/iejalapeno +IMAGE_VERSION?=latest + +.PHONY: all igp-graph container push clean test + +ifdef V +TESTARGS = -v -args -alsologtostderr -v 5 +else +TESTARGS = +endif + +all: igp-graph + +igp-graph: + mkdir -p bin + $(MAKE) -C ./cmd compile-igp-graph + +igp-graph-container: igp-graph + docker build -t $(REGISTRY_NAME)/igp-graph:$(IMAGE_VERSION) -f ./build/Dockerfile.igp-graph . + +push: igp-graph-container + docker push $(REGISTRY_NAME)/igp-graph:$(IMAGE_VERSION) + +clean: + rm -rf bin + +test: + GO111MODULE=on go test `go list ./... | grep -v 'vendor'` $(TESTARGS) + GO111MODULE=on go vet `go list ./... | grep -v vendor` diff --git a/build/Dockerfile.igp-graph b/build/Dockerfile.igp-graph new file mode 100755 index 00000000..0bb964c8 --- /dev/null +++ b/build/Dockerfile.igp-graph @@ -0,0 +1,4 @@ +FROM scratch + +COPY ./bin/igp-graph /igp-graph +ENTRYPOINT ["/igp-graph"] diff --git a/cmd/igp-graph/Makefile b/cmd/igp-graph/Makefile new file mode 100755 index 00000000..688697db --- /dev/null +++ b/cmd/igp-graph/Makefile @@ -0,0 +1,2 @@ +compile-igp-graph: + CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -a -ldflags '-extldflags "-static"' -o ../../bin/igp-graph ./main.go diff --git a/cmd/igp-graph/main.go b/cmd/igp-graph/main.go new file mode 100755 index 00000000..aebb4571 --- /dev/null +++ b/cmd/igp-graph/main.go @@ -0,0 +1,200 @@ +// Copyright (c) 2022 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +package main + +import ( + "flag" + "fmt" + "io" + "os" + "os/signal" + "runtime" + + "github.com/cisco-open/jalapeno/igp-graph/arangodb" + "github.com/cisco-open/jalapeno/igp-graph/kafkamessenger" + "github.com/cisco-open/jalapeno/topology/kafkanotifier" + "github.com/golang/glog" + + _ "net/http/pprof" +) + +const ( + // userFile defines the name of file containing base64 encoded user name + userFile = "./credentials/.username" + // passFile defines the name of file containing base64 encoded password + passFile = "./credentials/.password" + // MAXUSERNAME defines maximum length of ArangoDB user name + MAXUSERNAME = 256 + // MAXPASS defines maximum length of ArangoDB password + MAXPASS = 256 +) + +var ( + msgSrvAddr string + dbSrvAddr string + dbName string + dbUser string + dbPass string + lsprefix string + lslink string + lssrv6sid string + lsnode string + lsnodeExt string + igpDomain string + lsv4Graph string + lsv6Graph string +) + +func init() { + runtime.GOMAXPROCS(1) + flag.StringVar(&msgSrvAddr, "message-server", "", "URL to the messages supplying server") + flag.StringVar(&dbSrvAddr, "database-server", "", "{dns name}:port or X.X.X.X:port of the graph database") + flag.StringVar(&dbName, "database-name", "", "DB name") + flag.StringVar(&dbUser, "database-user", "", "DB User name") + flag.StringVar(&dbPass, "database-pass", "", "DB User's password") + + flag.StringVar(&lsprefix, "ls_prefix", "ls_prefix", "ls_prefix Collection name, default: \"ls_prefix\"") + flag.StringVar(&lslink, "ls_link", "ls_link", "ls_link Collection name, default \"ls_link\"") + flag.StringVar(&lssrv6sid, "ls_srv6_sid", "ls_srv6_sid", "ls_srv6_sid Collection name, default: \"ls_srv6_sid\"") + flag.StringVar(&lsnode, "ls_node", "ls_node", "ls_node Collection name, default \"ls_node\"") + flag.StringVar(&lsnodeExt, "ls_node_extended", "ls_node_extended", "ls_node_extended Collection name, default \"ls_node_extended\"") + flag.StringVar(&igpDomain, "igp_domain", "igp_domain", "igp_domain Collection name, default \"igp_domain\"") + flag.StringVar(&lsv4Graph, "lsv4_graph", "lsv4_graph", "lsv4_graph Collection name, default \"lsv4_graph\"") + flag.StringVar(&lsv6Graph, "lsv6_graph", "lsv6_graph", "lsv6_graph Collection name, default \"lsv6_graph\"") +} + +var ( + onlyOneSignalHandler = make(chan struct{}) + shutdownSignals = []os.Signal{os.Interrupt} +) + +func setupSignalHandler() (stopCh <-chan struct{}) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + <-c + close(stop) + <-c + os.Exit(1) // second signal. Exit directly. + }() + + return stop +} + +func main() { + flag.Parse() + _ = flag.Set("logtostderr", "true") + + // validateDBCreds check if the user name and the password are provided either as + // command line parameters or via files. If both are provided command line parameters + // will be used, if neither, processor will fail. + if err := validateDBCreds(); err != nil { + glog.Errorf("failed to validate the database credentials with error: %+v", err) + os.Exit(1) + } + + // initialize kafkanotifier to write back processed events into ls_node_edge_events topic + notifier, err := kafkanotifier.NewKafkaNotifier(msgSrvAddr) + if err != nil { + glog.Errorf("failed to initialize events notifier with error: %+v", err) + os.Exit(1) + } + + dbSrv, err := arangodb.NewDBSrvClient(dbSrvAddr, dbUser, dbPass, dbName, lsprefix, lslink, lssrv6sid, lsnode, + lsnodeExt, igpDomain, lsv4Graph, lsv6Graph, notifier) + if err != nil { + glog.Errorf("failed to initialize databse client with error: %+v", err) + os.Exit(1) + } + + if err := dbSrv.Start(); err != nil { + if err != nil { + glog.Errorf("failed to connect to database with error: %+v", err) + os.Exit(1) + } + } + + // Initializing messenger process + msgSrv, err := kafkamessenger.NewKafkaMessenger(msgSrvAddr, dbSrv.GetInterface()) + if err != nil { + glog.Errorf("failed to initialize message server with error: %+v", err) + os.Exit(1) + } + + msgSrv.Start() + + stopCh := setupSignalHandler() + <-stopCh + + msgSrv.Stop() + dbSrv.Stop() + + os.Exit(0) +} + +func validateDBCreds() error { + // Attempting to access username and password files. + u, err := readAndDecode(userFile, MAXUSERNAME) + if err != nil { + if dbUser != "" && dbPass != "" { + return nil + } + return fmt.Errorf("failed to access %s with error: %+v and no username and password provided via command line arguments", userFile, err) + } + p, err := readAndDecode(passFile, MAXPASS) + if err != nil { + if dbUser != "" && dbPass != "" { + return nil + } + return fmt.Errorf("failed to access %s with error: %+v and no username and password provided via command line arguments", passFile, err) + } + dbUser, dbPass = u, p + + return nil +} + +func readAndDecode(fn string, max int) (string, error) { + f, err := os.Open(fn) + if err != nil { + return "", err + } + defer f.Close() + l, err := f.Stat() + if err != nil { + return "", err + } + b := make([]byte, int(l.Size())) + n, err := io.ReadFull(f, b) + if err != nil { + return "", err + } + if n > max { + return "", fmt.Errorf("length of data %d exceeds maximum acceptable length: %d", n, max) + } + b = b[:n] + + return string(b), nil +} diff --git a/deployment/igp-graph.yaml b/deployment/igp-graph.yaml new file mode 100755 index 00000000..26a77723 --- /dev/null +++ b/deployment/igp-graph.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +spec: + replicas: 1 + selector: + matchLabels: + app: igp-graph + template: + metadata: + labels: + app: igp-graph + spec: + containers: + - args: + - --v + - "5" + - --message-server + - "broker.jalapeno:9092" + - --database-server + - "http://arangodb.jalapeno:8529" + - --database-name + - "jalapeno" + image: docker.io/iejalapeno/igp-graph:latest + imagePullPolicy: Always + name: igp-graph + volumeMounts: + - name: credentials + mountPath: /credentials + volumes: + - name: credentials + secret: + secretName: jalapeno +metadata: + name: igp-graph + namespace: jalapeno diff --git a/igp-graph/arango-conn.go b/igp-graph/arango-conn.go new file mode 100755 index 00000000..12c2579d --- /dev/null +++ b/igp-graph/arango-conn.go @@ -0,0 +1,80 @@ +// Borrowed from https://github.com/cisco-ie/jalapeno + +package arangodb + +import ( + "context" + "errors" + "fmt" + "strings" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" + "github.com/golang/glog" +) + +var ( + ErrEmptyConfig = errors.New("ArangoDB Config has an empty field") + ErrUpSafe = errors.New("Failed to UpdateSafe. Requires *DBObjects") + ErrNilObject = errors.New("Failed to operate on NIL object") + ErrNotFound = errors.New("Document not found") +) + +type ArangoConfig struct { + URL string `desc:"Arangodb server URL (http://127.0.0.1:8529)"` + User string `desc:"Arangodb server username"` + Password string `desc:"Arangodb server user password"` + Database string `desc:"Arangodb database name"` +} + +func NewConfig() ArangoConfig { + return ArangoConfig{} +} + +type ArangoConn struct { + db driver.Database +} + +var ( + ErrCollectionNotFound = fmt.Errorf("Could not find collection") +) + +func NewArango(cfg ArangoConfig) (*ArangoConn, error) { + // Connect to DB + if cfg.URL == "" || cfg.User == "" || cfg.Password == "" || cfg.Database == "" { + return nil, ErrEmptyConfig + } + if !strings.Contains(cfg.URL, "http") { + cfg.URL = "http://" + cfg.URL + } + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{cfg.URL}, + }) + if err != nil { + glog.Errorf("Failed to create HTTP connection: %v", err) + return nil, err + } + + // Authenticate with DB + conn, err = conn.SetAuthentication(driver.BasicAuthentication(cfg.User, cfg.Password)) + if err != nil { + glog.Errorf("Failed to authenticate with arango: %v", err) + return nil, err + } + + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + glog.Errorf("Failed to create client: %v", err) + return nil, err + } + + // If Jalapeno databse does not exist, the topology is not running, goig into a crash loop + db, err := c.Database(context.Background(), cfg.Database) + if err != nil { + return nil, err + } + + return &ArangoConn{db: db}, nil +} diff --git a/igp-graph/arangodb.go b/igp-graph/arangodb.go new file mode 100755 index 00000000..7046e138 --- /dev/null +++ b/igp-graph/arangodb.go @@ -0,0 +1,542 @@ +package arangodb + +import ( + "context" + "encoding/json" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/jalapeno/topology/pkg/dbclient" + "github.com/jalapeno/topology/pkg/kafkanotifier" + notifier "github.com/jalapeno/topology/pkg/kafkanotifier" + "github.com/sbezverk/gobmp/pkg/bmp" + "github.com/sbezverk/gobmp/pkg/message" + "github.com/sbezverk/gobmp/pkg/tools" +) + +type arangoDB struct { + dbclient.DB + *ArangoConn + stop chan struct{} + lsprefix driver.Collection + lslink driver.Collection + lssrv6sid driver.Collection + lsnode driver.Collection + lsnodeExt driver.Collection + igpDomain driver.Collection + graphv4 driver.Collection + graphv6 driver.Collection + lsv4Graph driver.Graph + lsv6Graph driver.Graph + notifier kafkanotifier.Event +} + +// NewDBSrvClient returns an instance of a DB server client process +func NewDBSrvClient(arangoSrv, user, pass, dbname, lsprefix, lslink, lssrv6sid, lsnode, + lsnodeExt string, igpDomain string, lsv4Graph string, lsv6Graph string, notifier kafkanotifier.Event) (dbclient.Srv, error) { + if err := tools.URLAddrValidation(arangoSrv); err != nil { + return nil, err + } + arangoConn, err := NewArango(ArangoConfig{ + URL: arangoSrv, + User: user, + Password: pass, + Database: dbname, + }) + if err != nil { + return nil, err + } + arango := &arangoDB{ + stop: make(chan struct{}), + } + arango.DB = arango + arango.ArangoConn = arangoConn + + // Check if base link state collections exist, if not fail as Jalapeno topology is not running + arango.lsprefix, err = arango.db.Collection(context.TODO(), lsprefix) + if err != nil { + return nil, err + } + arango.lslink, err = arango.db.Collection(context.TODO(), lslink) + if err != nil { + return nil, err + } + arango.lssrv6sid, err = arango.db.Collection(context.TODO(), lssrv6sid) + if err != nil { + return nil, err + } + arango.lsnode, err = arango.db.Collection(context.TODO(), lsnode) + if err != nil { + return nil, err + } + + // check for ls_node_extended collection + found, err := arango.db.CollectionExists(context.TODO(), lsnodeExt) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Collection(context.TODO(), lsnodeExt) + if err != nil { + return nil, err + } + if err := c.Remove(context.TODO()); err != nil { + return nil, err + } + } + // create ls_node_extended collection + var lsnodeExt_options = &driver.CreateCollectionOptions{ /* ... */ } + glog.V(5).Infof("ls_node_extended not found, creating") + arango.lsnodeExt, err = arango.db.CreateCollection(context.TODO(), "ls_node_extended", lsnodeExt_options) + if err != nil { + return nil, err + } + // check if collection exists, if not fail as processor has failed to create collection + arango.lsnodeExt, err = arango.db.Collection(context.TODO(), lsnodeExt) + if err != nil { + return nil, err + } + + // check for igp_domain collection + found, err = arango.db.CollectionExists(context.TODO(), igpDomain) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Collection(context.TODO(), igpDomain) + if err != nil { + return nil, err + } + if err := c.Remove(context.TODO()); err != nil { + return nil, err + } + } + // create igp_domain collection + var igpdomain_options = &driver.CreateCollectionOptions{ /* ... */ } + glog.V(5).Infof("igp_domain collection not found, creating") + arango.igpDomain, err = arango.db.CreateCollection(context.TODO(), "igp_domain", igpdomain_options) + if err != nil { + return nil, err + } + // check if collection exists, if not fail as processor has failed to create collection + arango.igpDomain, err = arango.db.Collection(context.TODO(), igpDomain) + if err != nil { + return nil, err + } + + // check for lsv4 topology graph + found, err = arango.db.GraphExists(context.TODO(), lsv4Graph) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Graph(context.TODO(), lsv4Graph) + if err != nil { + return nil, err + } + glog.Infof("found graph %s", c) + + } else { + // create graph + var edgeDefinition driver.EdgeDefinition + edgeDefinition.Collection = "lsv4_graph" + edgeDefinition.From = []string{"ls_node_extended"} + edgeDefinition.To = []string{"ls_node_extended"} + var options driver.CreateGraphOptions + options.OrphanVertexCollections = []string{"ls_srv6_sid", "ls_prefix"} + options.EdgeDefinitions = []driver.EdgeDefinition{edgeDefinition} + + arango.lsv4Graph, err = arango.db.CreateGraph(context.TODO(), lsv4Graph, &options) + if err != nil { + return nil, err + } + } + + // check if lsv4_graph exists, if not fail as processor has failed to create graph + arango.graphv4, err = arango.db.Collection(context.TODO(), "lsv4_graph") + if err != nil { + return nil, err + } + + // check for lsv6 topology graph + found, err = arango.db.GraphExists(context.TODO(), lsv6Graph) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Graph(context.TODO(), lsv6Graph) + if err != nil { + return nil, err + } + glog.Infof("found graph %s", c) + + } else { + // create graph + var edgeDefinition driver.EdgeDefinition + edgeDefinition.Collection = "lsv6_graph" + edgeDefinition.From = []string{"ls_node_extended"} + edgeDefinition.To = []string{"ls_node_extended"} + var options driver.CreateGraphOptions + options.OrphanVertexCollections = []string{"ls_srv6_sid", "ls_prefix"} + options.EdgeDefinitions = []driver.EdgeDefinition{edgeDefinition} + + arango.lsv6Graph, err = arango.db.CreateGraph(context.TODO(), lsv6Graph, &options) + if err != nil { + return nil, err + } + } + + // check if lsv6_graph exists, if not fail as processor has failed to create graph + arango.graphv6, err = arango.db.Collection(context.TODO(), "lsv6_graph") + if err != nil { + return nil, err + } + + return arango, nil +} + +func (a *arangoDB) Start() error { + if err := a.loadCollections(); err != nil { + return err + } + glog.Infof("Connected to arango database, starting monitor") + go a.monitor() + + return nil +} + +func (a *arangoDB) Stop() error { + close(a.stop) + + return nil +} + +func (a *arangoDB) GetInterface() dbclient.DB { + return a.DB +} + +func (a *arangoDB) GetArangoDBInterface() *ArangoConn { + return a.ArangoConn +} + +func (a *arangoDB) StoreMessage(msgType dbclient.CollectionType, msg []byte) error { + event := ¬ifier.EventMessage{} + if err := json.Unmarshal(msg, event); err != nil { + return err + } + event.TopicType = msgType + switch msgType { + case bmp.LSSRv6SIDMsg: + return a.lsSRv6SIDHandler(event) + case bmp.LSNodeMsg: + return a.lsNodeHandler(event) + case bmp.LSPrefixMsg: + return a.lsPrefixHandler(event) + case bmp.LSLinkMsg: + return a.lsLinkHandler(event) + } + return nil +} + +func (a *arangoDB) monitor() { + for { + select { + case <-a.stop: + // TODO Add clean up of connection with Arango DB + return + } + } +} + +// loadCollections calls a series of subfunctions to perform ArangoDB operations including populating +// ls_node_extended collection and querying other link state collections to build the lsv4_graph and lsv6_graph + +func (a *arangoDB) loadCollections() error { + ctx := context.TODO() + + if err := a.lsExtendedNodes(ctx); err != nil { + return err + } + if err := a.processDuplicateNodes(ctx); err != nil { + return err + } + if err := a.loadPrefixSIDs(ctx); err != nil { + return err + } + if err := a.loadSRv6SIDs(ctx); err != nil { + return err + } + if err := a.processIBGPv6Peering(ctx); err != nil { + return err + } + if err := a.createIGPDomains(ctx); err != nil { + return err + } + if err := a.lsv4LinkEdges(ctx); err != nil { + return err + } + if err := a.lsv4PrefixEdges(ctx); err != nil { + return err + } + if err := a.lsv6LinkEdges(ctx); err != nil { + return err + } + if err := a.lsv6PrefixEdges(ctx); err != nil { + return err + } + + return nil +} +func (a *arangoDB) lsExtendedNodes(ctx context.Context) error { + lsn_query := "for l in " + a.lsnode.Name() + " insert l in " + a.lsnodeExt.Name() + "" + cursor, err := a.db.Query(ctx, lsn_query, nil) + if err != nil { + return err + } + defer cursor.Close() + return nil +} + +func (a *arangoDB) processDuplicateNodes(ctx context.Context) error { + // BGP-LS generates both a level-1 and a level-2 entry for level-1-2 nodes + // Here we remove duplicate entries in the ls_node_extended collection + dup_query := "LET duplicates = ( FOR d IN " + a.lsnodeExt.Name() + + " COLLECT id = d.igp_router_id, domain = d.domain_id, area = d.area_id WITH COUNT INTO count " + + " FILTER count > 1 RETURN { id: id, domain: domain, area: area, count: count }) " + + "FOR d IN duplicates FOR m IN ls_node_extended " + + "FILTER d.id == m.igp_router_id filter d.domain == m.domain_id RETURN m " + pcursor, err := a.db.Query(ctx, dup_query, nil) + if err != nil { + return err + } + defer pcursor.Close() + for { + var doc duplicateNode + dupe, err := pcursor.ReadDocument(ctx, &doc) + + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.Infof("Got doc with key '%s' from query\n", dupe.Key) + + if doc.ProtocolID == 1 { + glog.Infof("remove level-1 duplicate node: %s + igp id: %s protocol id: %v + ", doc.Key, doc.IGPRouterID, doc.ProtocolID) + if _, err := a.lsnodeExt.RemoveDocument(ctx, doc.Key); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + if doc.ProtocolID == 2 { + update_query := "for l in " + a.lsnodeExt.Name() + " filter l._key == " + "\"" + doc.Key + "\"" + + " UPDATE l with { protocol: " + "\"" + "ISIS Level 1-2" + "\"" + " } in " + a.lsnodeExt.Name() + "" + cursor, err := a.db.Query(ctx, update_query, nil) + glog.Infof("update query: %s ", update_query) + if err != nil { + return err + } + defer cursor.Close() + } + } + + return nil +} + +func (a *arangoDB) loadPrefixSIDs(ctx context.Context) error { + // Find and add sr-mpls prefix sids to nodes in the ls_node_extended collection + sr_query := "for p in " + a.lsprefix.Name() + + " filter p.mt_id_tlv.mt_id != 2 && p.prefix_attr_tlvs.ls_prefix_sid != null return p " + cursor, err := a.db.Query(ctx, sr_query, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSPrefix + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processPrefixSID(ctx, meta.Key, meta.ID.String(), p); err != nil { + glog.Errorf("Failed to process ls_prefix_sid %s with error: %+v", p.ID, err) + } + } + + return nil +} + +func (a *arangoDB) loadSRv6SIDs(ctx context.Context) error { + // Find and add srv6 sids to nodes in the ls_node_extended collection + srv6_query := "for s in " + a.lssrv6sid.Name() + " return s " + cursor, err := a.db.Query(ctx, srv6_query, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSSRv6SID + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSSRv6SID(ctx, meta.Key, meta.ID.String(), &p); err != nil { + glog.Errorf("Failed to process ls_srv6_sid %s with error: %+v", p.ID, err) + } + } + + return nil +} + +func (a *arangoDB) processIBGPv6Peering(ctx context.Context) error { + // add ipv6 iBGP peering address and ipv4 bgp router-id + ibgp6_query := "for s in peer filter s.remote_ip like " + "\"%:%\"" + " return s " + cursor, err := a.db.Query(ctx, ibgp6_query, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.PeerStateChange + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processbgp6(ctx, meta.Key, meta.ID.String(), &p); err != nil { + glog.Errorf("Failed to process ibgp peering %s with error: %+v", p.ID, err) + } + } + + return nil +} + +func (a *arangoDB) createIGPDomains(ctx context.Context) error { + // create igp_domain collection - useful in scaled multi-domain environments + igpdomain_query := "for l in ls_node_extended insert " + + "{ _key: CONCAT_SEPARATOR(" + "\"_\", l.protocol_id, l.domain_id, l.asn), " + + "asn: l.asn, protocol_id: l.protocol_id, domain_id: l.domain_id, protocol: l.protocol } " + + "into igp_domain OPTIONS { ignoreErrors: true } return l" + cursor, err := a.db.Query(ctx, igpdomain_query, nil) + if err != nil { + return err + } + defer cursor.Close() + + return nil +} + +func (a *arangoDB) lsv4LinkEdges(ctx context.Context) error { + // Find ipv4 ls_link entries to create edges in the lsv4_graph + lsv4linkquery := "for l in " + a.lslink.Name() + " filter l.protocol_id != 7 RETURN l" + cursor, err := a.db.Query(ctx, lsv4linkquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSLink + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSLinkEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil +} + +func (a *arangoDB) lsv4PrefixEdges(ctx context.Context) error { + // Find ls_prefix entries to create prefix or subnet edges in the lsv4_graph + lsv4pfxquery := "for l in " + a.lsprefix.Name() + //" filter l.mt_id_tlv == null return l" + " filter l.mt_id_tlv.mt_id != 2 && l.prefix_len != 30 && " + + "l.prefix_len != 31 && l.prefix_len != 32 return l" + cursor, err := a.db.Query(ctx, lsv4pfxquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSPrefix + meta, err := cursor.ReadDocument(ctx, &p) + //glog.Infof("processing lsprefix document: %+v", p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSPrefixEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil +} + +func (a *arangoDB) lsv6LinkEdges(ctx context.Context) error { + // Find ipv6 ls_link entries to create edges in the lsv6_graph + lsv6linkquery := "for l in " + a.lslink.Name() + " filter l.protocol_id != 7 RETURN l" + cursor, err := a.db.Query(ctx, lsv6linkquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSLink + meta, err := cursor.ReadDocument(ctx, &p) + //glog.Infof("processing lslink document: %+v", p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSv6LinkEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil +} + +func (a *arangoDB) lsv6PrefixEdges(ctx context.Context) error { + // Find ipv6 ls_prefix entries to create prefix or subnet edges in the lsv6_graph + lsv6pfxquery := "for l in " + a.lsprefix.Name() + + " filter l.mt_id_tlv.mt_id == 2 && l.prefix_len != 126 && " + + "l.prefix_len != 127 && l.prefix_len != 128 return l" + cursor, err := a.db.Query(ctx, lsv6pfxquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSPrefix + meta, err := cursor.ReadDocument(ctx, &p) + //glog.Infof("processing lsprefix document: %+v", p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSv6PrefixEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil + +} diff --git a/igp-graph/arangodb/arango-conn.go b/igp-graph/arangodb/arango-conn.go new file mode 100755 index 00000000..12c2579d --- /dev/null +++ b/igp-graph/arangodb/arango-conn.go @@ -0,0 +1,80 @@ +// Borrowed from https://github.com/cisco-ie/jalapeno + +package arangodb + +import ( + "context" + "errors" + "fmt" + "strings" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" + "github.com/golang/glog" +) + +var ( + ErrEmptyConfig = errors.New("ArangoDB Config has an empty field") + ErrUpSafe = errors.New("Failed to UpdateSafe. Requires *DBObjects") + ErrNilObject = errors.New("Failed to operate on NIL object") + ErrNotFound = errors.New("Document not found") +) + +type ArangoConfig struct { + URL string `desc:"Arangodb server URL (http://127.0.0.1:8529)"` + User string `desc:"Arangodb server username"` + Password string `desc:"Arangodb server user password"` + Database string `desc:"Arangodb database name"` +} + +func NewConfig() ArangoConfig { + return ArangoConfig{} +} + +type ArangoConn struct { + db driver.Database +} + +var ( + ErrCollectionNotFound = fmt.Errorf("Could not find collection") +) + +func NewArango(cfg ArangoConfig) (*ArangoConn, error) { + // Connect to DB + if cfg.URL == "" || cfg.User == "" || cfg.Password == "" || cfg.Database == "" { + return nil, ErrEmptyConfig + } + if !strings.Contains(cfg.URL, "http") { + cfg.URL = "http://" + cfg.URL + } + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{cfg.URL}, + }) + if err != nil { + glog.Errorf("Failed to create HTTP connection: %v", err) + return nil, err + } + + // Authenticate with DB + conn, err = conn.SetAuthentication(driver.BasicAuthentication(cfg.User, cfg.Password)) + if err != nil { + glog.Errorf("Failed to authenticate with arango: %v", err) + return nil, err + } + + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + glog.Errorf("Failed to create client: %v", err) + return nil, err + } + + // If Jalapeno databse does not exist, the topology is not running, goig into a crash loop + db, err := c.Database(context.Background(), cfg.Database) + if err != nil { + return nil, err + } + + return &ArangoConn{db: db}, nil +} diff --git a/igp-graph/arangodb/arangodb.go b/igp-graph/arangodb/arangodb.go new file mode 100755 index 00000000..301babfc --- /dev/null +++ b/igp-graph/arangodb/arangodb.go @@ -0,0 +1,542 @@ +package arangodb + +import ( + "context" + "encoding/json" + + driver "github.com/arangodb/go-driver" + "github.com/cisco-open/jalapeno/topology/dbclient" + "github.com/cisco-open/jalapeno/topology/kafkanotifier" + notifier "github.com/cisco-open/jalapeno/topology/kafkanotifier" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/bmp" + "github.com/sbezverk/gobmp/pkg/message" + "github.com/sbezverk/gobmp/pkg/tools" +) + +type arangoDB struct { + dbclient.DB + *ArangoConn + stop chan struct{} + lsprefix driver.Collection + lslink driver.Collection + lssrv6sid driver.Collection + lsnode driver.Collection + lsnodeExt driver.Collection + igpDomain driver.Collection + graphv4 driver.Collection + graphv6 driver.Collection + lsv4Graph driver.Graph + lsv6Graph driver.Graph + notifier kafkanotifier.Event +} + +// NewDBSrvClient returns an instance of a DB server client process +func NewDBSrvClient(arangoSrv, user, pass, dbname, lsprefix, lslink, lssrv6sid, lsnode, + lsnodeExt string, igpDomain string, lsv4Graph string, lsv6Graph string, notifier kafkanotifier.Event) (dbclient.Srv, error) { + if err := tools.URLAddrValidation(arangoSrv); err != nil { + return nil, err + } + arangoConn, err := NewArango(ArangoConfig{ + URL: arangoSrv, + User: user, + Password: pass, + Database: dbname, + }) + if err != nil { + return nil, err + } + arango := &arangoDB{ + stop: make(chan struct{}), + } + arango.DB = arango + arango.ArangoConn = arangoConn + + // Check if base link state collections exist, if not fail as Jalapeno topology is not running + arango.lsprefix, err = arango.db.Collection(context.TODO(), lsprefix) + if err != nil { + return nil, err + } + arango.lslink, err = arango.db.Collection(context.TODO(), lslink) + if err != nil { + return nil, err + } + arango.lssrv6sid, err = arango.db.Collection(context.TODO(), lssrv6sid) + if err != nil { + return nil, err + } + arango.lsnode, err = arango.db.Collection(context.TODO(), lsnode) + if err != nil { + return nil, err + } + + // check for ls_node_extended collection + found, err := arango.db.CollectionExists(context.TODO(), lsnodeExt) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Collection(context.TODO(), lsnodeExt) + if err != nil { + return nil, err + } + if err := c.Remove(context.TODO()); err != nil { + return nil, err + } + } + // create ls_node_extended collection + var lsnodeExt_options = &driver.CreateCollectionOptions{ /* ... */ } + glog.V(5).Infof("ls_node_extended not found, creating") + arango.lsnodeExt, err = arango.db.CreateCollection(context.TODO(), "ls_node_extended", lsnodeExt_options) + if err != nil { + return nil, err + } + // check if collection exists, if not fail as processor has failed to create collection + arango.lsnodeExt, err = arango.db.Collection(context.TODO(), lsnodeExt) + if err != nil { + return nil, err + } + + // check for igp_domain collection + found, err = arango.db.CollectionExists(context.TODO(), igpDomain) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Collection(context.TODO(), igpDomain) + if err != nil { + return nil, err + } + if err := c.Remove(context.TODO()); err != nil { + return nil, err + } + } + // create igp_domain collection + var igpdomain_options = &driver.CreateCollectionOptions{ /* ... */ } + glog.V(5).Infof("igp_domain collection not found, creating") + arango.igpDomain, err = arango.db.CreateCollection(context.TODO(), "igp_domain", igpdomain_options) + if err != nil { + return nil, err + } + // check if collection exists, if not fail as processor has failed to create collection + arango.igpDomain, err = arango.db.Collection(context.TODO(), igpDomain) + if err != nil { + return nil, err + } + + // check for lsv4 topology graph + found, err = arango.db.GraphExists(context.TODO(), lsv4Graph) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Graph(context.TODO(), lsv4Graph) + if err != nil { + return nil, err + } + glog.Infof("found graph %s", c) + + } else { + // create graph + var edgeDefinition driver.EdgeDefinition + edgeDefinition.Collection = "lsv4_graph" + edgeDefinition.From = []string{"ls_node_extended"} + edgeDefinition.To = []string{"ls_node_extended"} + var options driver.CreateGraphOptions + options.OrphanVertexCollections = []string{"ls_srv6_sid", "ls_prefix"} + options.EdgeDefinitions = []driver.EdgeDefinition{edgeDefinition} + + arango.lsv4Graph, err = arango.db.CreateGraph(context.TODO(), lsv4Graph, &options) + if err != nil { + return nil, err + } + } + + // check if lsv4_graph exists, if not fail as processor has failed to create graph + arango.graphv4, err = arango.db.Collection(context.TODO(), "lsv4_graph") + if err != nil { + return nil, err + } + + // check for lsv6 topology graph + found, err = arango.db.GraphExists(context.TODO(), lsv6Graph) + if err != nil { + return nil, err + } + if found { + c, err := arango.db.Graph(context.TODO(), lsv6Graph) + if err != nil { + return nil, err + } + glog.Infof("found graph %s", c) + + } else { + // create graph + var edgeDefinition driver.EdgeDefinition + edgeDefinition.Collection = "lsv6_graph" + edgeDefinition.From = []string{"ls_node_extended"} + edgeDefinition.To = []string{"ls_node_extended"} + var options driver.CreateGraphOptions + options.OrphanVertexCollections = []string{"ls_srv6_sid", "ls_prefix"} + options.EdgeDefinitions = []driver.EdgeDefinition{edgeDefinition} + + arango.lsv6Graph, err = arango.db.CreateGraph(context.TODO(), lsv6Graph, &options) + if err != nil { + return nil, err + } + } + + // check if lsv6_graph exists, if not fail as processor has failed to create graph + arango.graphv6, err = arango.db.Collection(context.TODO(), "lsv6_graph") + if err != nil { + return nil, err + } + + return arango, nil +} + +func (a *arangoDB) Start() error { + if err := a.loadCollections(); err != nil { + return err + } + glog.Infof("Connected to arango database, starting monitor") + go a.monitor() + + return nil +} + +func (a *arangoDB) Stop() error { + close(a.stop) + + return nil +} + +func (a *arangoDB) GetInterface() dbclient.DB { + return a.DB +} + +func (a *arangoDB) GetArangoDBInterface() *ArangoConn { + return a.ArangoConn +} + +func (a *arangoDB) StoreMessage(msgType dbclient.CollectionType, msg []byte) error { + event := ¬ifier.EventMessage{} + if err := json.Unmarshal(msg, event); err != nil { + return err + } + event.TopicType = msgType + switch msgType { + case bmp.LSSRv6SIDMsg: + return a.lsSRv6SIDHandler(event) + case bmp.LSNodeMsg: + return a.lsNodeHandler(event) + case bmp.LSPrefixMsg: + return a.lsPrefixHandler(event) + case bmp.LSLinkMsg: + return a.lsLinkHandler(event) + } + return nil +} + +func (a *arangoDB) monitor() { + for { + select { + case <-a.stop: + // TODO Add clean up of connection with Arango DB + return + } + } +} + +// loadCollections calls a series of subfunctions to perform ArangoDB operations including populating +// ls_node_extended collection and querying other link state collections to build the lsv4_graph and lsv6_graph + +func (a *arangoDB) loadCollections() error { + ctx := context.TODO() + + if err := a.lsExtendedNodes(ctx); err != nil { + return err + } + if err := a.processDuplicateNodes(ctx); err != nil { + return err + } + if err := a.loadPrefixSIDs(ctx); err != nil { + return err + } + if err := a.loadSRv6SIDs(ctx); err != nil { + return err + } + if err := a.processIBGPv6Peering(ctx); err != nil { + return err + } + if err := a.createIGPDomains(ctx); err != nil { + return err + } + if err := a.lsv4LinkEdges(ctx); err != nil { + return err + } + if err := a.lsv4PrefixEdges(ctx); err != nil { + return err + } + if err := a.lsv6LinkEdges(ctx); err != nil { + return err + } + if err := a.lsv6PrefixEdges(ctx); err != nil { + return err + } + + return nil +} +func (a *arangoDB) lsExtendedNodes(ctx context.Context) error { + lsn_query := "for l in " + a.lsnode.Name() + " insert l in " + a.lsnodeExt.Name() + "" + cursor, err := a.db.Query(ctx, lsn_query, nil) + if err != nil { + return err + } + defer cursor.Close() + return nil +} + +func (a *arangoDB) processDuplicateNodes(ctx context.Context) error { + // BGP-LS generates both a level-1 and a level-2 entry for level-1-2 nodes + // Here we remove duplicate entries in the ls_node_extended collection + dup_query := "LET duplicates = ( FOR d IN " + a.lsnodeExt.Name() + + " COLLECT id = d.igp_router_id, domain = d.domain_id, area = d.area_id WITH COUNT INTO count " + + " FILTER count > 1 RETURN { id: id, domain: domain, area: area, count: count }) " + + "FOR d IN duplicates FOR m IN ls_node_extended " + + "FILTER d.id == m.igp_router_id filter d.domain == m.domain_id RETURN m " + pcursor, err := a.db.Query(ctx, dup_query, nil) + if err != nil { + return err + } + defer pcursor.Close() + for { + var doc duplicateNode + dupe, err := pcursor.ReadDocument(ctx, &doc) + + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.Infof("Got doc with key '%s' from query\n", dupe.Key) + + if doc.ProtocolID == 1 { + glog.Infof("remove level-1 duplicate node: %s + igp id: %s protocol id: %v + ", doc.Key, doc.IGPRouterID, doc.ProtocolID) + if _, err := a.lsnodeExt.RemoveDocument(ctx, doc.Key); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + if doc.ProtocolID == 2 { + update_query := "for l in " + a.lsnodeExt.Name() + " filter l._key == " + "\"" + doc.Key + "\"" + + " UPDATE l with { protocol: " + "\"" + "ISIS Level 1-2" + "\"" + " } in " + a.lsnodeExt.Name() + "" + cursor, err := a.db.Query(ctx, update_query, nil) + glog.Infof("update query: %s ", update_query) + if err != nil { + return err + } + defer cursor.Close() + } + } + + return nil +} + +func (a *arangoDB) loadPrefixSIDs(ctx context.Context) error { + // Find and add sr-mpls prefix sids to nodes in the ls_node_extended collection + sr_query := "for p in " + a.lsprefix.Name() + + " filter p.mt_id_tlv.mt_id != 2 && p.prefix_attr_tlvs.ls_prefix_sid != null return p " + cursor, err := a.db.Query(ctx, sr_query, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSPrefix + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processPrefixSID(ctx, meta.Key, meta.ID.String(), p); err != nil { + glog.Errorf("Failed to process ls_prefix_sid %s with error: %+v", p.ID, err) + } + } + + return nil +} + +func (a *arangoDB) loadSRv6SIDs(ctx context.Context) error { + // Find and add srv6 sids to nodes in the ls_node_extended collection + srv6_query := "for s in " + a.lssrv6sid.Name() + " return s " + cursor, err := a.db.Query(ctx, srv6_query, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSSRv6SID + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSSRv6SID(ctx, meta.Key, meta.ID.String(), &p); err != nil { + glog.Errorf("Failed to process ls_srv6_sid %s with error: %+v", p.ID, err) + } + } + + return nil +} + +func (a *arangoDB) processIBGPv6Peering(ctx context.Context) error { + // add ipv6 iBGP peering address and ipv4 bgp router-id + ibgp6_query := "for s in peer filter s.remote_ip like " + "\"%:%\"" + " return s " + cursor, err := a.db.Query(ctx, ibgp6_query, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.PeerStateChange + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processbgp6(ctx, meta.Key, meta.ID.String(), &p); err != nil { + glog.Errorf("Failed to process ibgp peering %s with error: %+v", p.ID, err) + } + } + + return nil +} + +func (a *arangoDB) createIGPDomains(ctx context.Context) error { + // create igp_domain collection - useful in scaled multi-domain environments + igpdomain_query := "for l in ls_node_extended insert " + + "{ _key: CONCAT_SEPARATOR(" + "\"_\", l.protocol_id, l.domain_id, l.asn), " + + "asn: l.asn, protocol_id: l.protocol_id, domain_id: l.domain_id, protocol: l.protocol } " + + "into igp_domain OPTIONS { ignoreErrors: true } return l" + cursor, err := a.db.Query(ctx, igpdomain_query, nil) + if err != nil { + return err + } + defer cursor.Close() + + return nil +} + +func (a *arangoDB) lsv4LinkEdges(ctx context.Context) error { + // Find ipv4 ls_link entries to create edges in the lsv4_graph + lsv4linkquery := "for l in " + a.lslink.Name() + " filter l.protocol_id != 7 RETURN l" + cursor, err := a.db.Query(ctx, lsv4linkquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSLink + meta, err := cursor.ReadDocument(ctx, &p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSLinkEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil +} + +func (a *arangoDB) lsv4PrefixEdges(ctx context.Context) error { + // Find ls_prefix entries to create prefix or subnet edges in the lsv4_graph + lsv4pfxquery := "for l in " + a.lsprefix.Name() + //" filter l.mt_id_tlv == null return l" + " filter l.mt_id_tlv.mt_id != 2 && l.prefix_len != 30 && " + + "l.prefix_len != 31 && l.prefix_len != 32 return l" + cursor, err := a.db.Query(ctx, lsv4pfxquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSPrefix + meta, err := cursor.ReadDocument(ctx, &p) + //glog.Infof("processing lsprefix document: %+v", p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSPrefixEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil +} + +func (a *arangoDB) lsv6LinkEdges(ctx context.Context) error { + // Find ipv6 ls_link entries to create edges in the lsv6_graph + lsv6linkquery := "for l in " + a.lslink.Name() + " filter l.protocol_id != 7 RETURN l" + cursor, err := a.db.Query(ctx, lsv6linkquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSLink + meta, err := cursor.ReadDocument(ctx, &p) + //glog.Infof("processing lslink document: %+v", p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSv6LinkEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil +} + +func (a *arangoDB) lsv6PrefixEdges(ctx context.Context) error { + // Find ipv6 ls_prefix entries to create prefix or subnet edges in the lsv6_graph + lsv6pfxquery := "for l in " + a.lsprefix.Name() + + " filter l.mt_id_tlv.mt_id == 2 && l.prefix_len != 126 && " + + "l.prefix_len != 127 && l.prefix_len != 128 return l" + cursor, err := a.db.Query(ctx, lsv6pfxquery, nil) + if err != nil { + return err + } + defer cursor.Close() + for { + var p message.LSPrefix + meta, err := cursor.ReadDocument(ctx, &p) + //glog.Infof("processing lsprefix document: %+v", p) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if err := a.processLSv6PrefixEdge(ctx, meta.Key, &p); err != nil { + glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) + continue + } + } + + return nil + +} diff --git a/igp-graph/arangodb/handler.go b/igp-graph/arangodb/handler.go new file mode 100644 index 00000000..76a5bc53 --- /dev/null +++ b/igp-graph/arangodb/handler.go @@ -0,0 +1,185 @@ +package arangodb + +import ( + "context" + "fmt" + "strings" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + + "github.com/cisco-open/jalapeno/topology/kafkanotifier" + notifier "github.com/cisco-open/jalapeno/topology/kafkanotifier" + "github.com/sbezverk/gobmp/pkg/message" +) + +func (a *arangoDB) lsNodeHandler(obj *notifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + // Check if Collection encoded in ls_node message ID exists + //glog.Infof("handler received: %+v", obj) + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lsnode.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lsnode.Name(), c) + } + glog.Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + var o message.LSNode + _, err := a.lsnode.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a LSNode removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + return a.processLSNodeExtRemoval(ctx, obj.Key) + } + switch obj.Action { + case "add": + if err := a.processLSNodeExt(ctx, obj.Key, &o); err != nil { + return fmt.Errorf("failed to process action %s for vertex %s with error: %+v", obj.Action, obj.Key, err) + } + default: + // NOOP for update + } + + return nil +} + +func (a *arangoDB) lsSRv6SIDHandler(obj *notifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + // Check if Collection encoded in ls_srv6_sid message ID exists + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lssrv6sid.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lssrv6sid.Name(), c) + } + glog.V(6).Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + var o message.LSSRv6SID + _, err := a.lssrv6sid.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a ls_srv6_sid removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + glog.V(6).Infof("SRv6 SID deleted: %s for ls_node_extended key: %s ", obj.Action, obj.Key) + return nil + } + switch obj.Action { + case "add": + if err := a.processLSSRv6SID(ctx, obj.Key, obj.ID, &o); err != nil { + return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) + } + default: + // NOOP + } + + return nil +} + +func (a *arangoDB) lsPrefixHandler(obj *kafkanotifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + // Check if Collection encoded in ls_prefix message ID exists + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lsprefix.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lsprefix.Name(), c) + } + //glog.V(5).Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + var o message.LSPrefix + _, err := a.lsprefix.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a ls_link removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + + // Detect IPv6 link by checking for ":" in the key + if strings.Contains(obj.Key, ":") { + return a.processv6PrefixRemoval(ctx, obj.Key, obj.Action) + } + + err := a.processPrefixRemoval(ctx, obj.Key, obj.Action) + if err != nil { + return err + } + // write event into ls_node_edge topic + // a.notifier.EventNotification(obj) + // return nil + } + switch obj.Action { + case "add": + fallthrough + case "update": + if err := a.processPrefixSID(ctx, obj.Key, obj.ID, o); err != nil { + return fmt.Errorf("failed to process action %s for vertex %s with error: %+v", obj.Action, obj.Key, err) + } + if err := a.processLSPrefixEdge(ctx, obj.Key, &o); err != nil { + return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) + } + } + + return nil +} + +func (a *arangoDB) lsLinkHandler(obj *kafkanotifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + glog.Infof("Processing eventmessage: %+v", obj) + // Check if Collection encoded in ls_link message ID exists + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lslink.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lslink.Name(), c) + } + var o message.LSLink + _, err := a.lslink.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a ls_link removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + + // Detect IPv6 link by checking for ":" in the key + if strings.Contains(obj.Key, ":") { + return a.processv6LinkRemoval(ctx, obj.Key, obj.Action) + } + + return a.processLinkRemoval(ctx, obj.Key, obj.Action) + } + switch obj.Action { + case "add": + fallthrough + case "update": + if err := a.processLSLinkEdge(ctx, obj.Key, &o); err != nil { + return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) + } + } + //glog.V(5).Infof("Complete processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + + // write event into ls_topoogy_v4 topic + //a.notifier.EventNotification(obj) + + return nil +} diff --git a/igp-graph/arangodb/lsnode-processor.go b/igp-graph/arangodb/lsnode-processor.go new file mode 100755 index 00000000..a1b4477c --- /dev/null +++ b/igp-graph/arangodb/lsnode-processor.go @@ -0,0 +1,341 @@ +package arangodb + +import ( + "context" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// Add srv6 sids / locators to nodes in the ls_node_extended collection +func (a *arangoDB) processLSSRv6SID(ctx context.Context, key, id string, e *message.LSSRv6SID) error { + query := "for l in " + a.lsnodeExt.Name() + + " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + + " filter l.domain_id == " + strconv.Itoa(int(e.DomainID)) + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var sn LSNodeExt + ns, err := ncursor.ReadDocument(ctx, &sn) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + // glog.Infof("ls_node_extended %s + srv6sid %s", ns.Key, e.SRv6SID) + // glog.Infof("existing sids: %+v", &sn.SIDS) + + newsid := SID{ + SRv6SID: e.SRv6SID, + SRv6EndpointBehavior: e.SRv6EndpointBehavior, + SRv6BGPPeerNodeSID: e.SRv6BGPPeerNodeSID, + SRv6SIDStructure: e.SRv6SIDStructure, + } + var result bool = false + for _, x := range sn.SIDS { + if x == newsid { + result = true + break + } + } + if result { + glog.Infof("sid %+v exists in ls_node_extended document", e.SRv6SID) + } else { + + sn.SIDS = append(sn.SIDS, newsid) + srn := LSNodeExt{ + SIDS: sn.SIDS, + } + // glog.Infof("appending sid %+v ", e.Key) + + if _, err := a.lsnodeExt.UpdateDocument(ctx, ns.Key, &srn); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + return nil +} + +// Find and add sr-mpls prefix sids to nodes in the ls_node_extended collection +func (a *arangoDB) processPrefixSID(ctx context.Context, key, id string, e message.LSPrefix) error { + query := "for l in " + a.lsnodeExt.Name() + + " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + query += " return l" + pcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer pcursor.Close() + for { + var ln LSNodeExt + nl, err := pcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.V(6).Infof("ls_node_extended: %s + prefix sid %v + ", ln.Key, e.PrefixAttrTLVs.LSPrefixSID) + + obj := srObject{ + PrefixAttrTLVs: e.PrefixAttrTLVs, + } + + if _, err := a.lsnodeExt.UpdateDocument(ctx, nl.Key, &obj); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + return nil +} + +// Find and add ls_node entries to the ls_node_extended collection +func (a *arangoDB) processLSNodeExt(ctx context.Context, key string, e *message.LSNode) error { + if e.ProtocolID == base.BGP { + // EPE Case cannot be processed because LS Node collection does not have BGP routers + return nil + } + query := "for l in " + a.lsnode.Name() + + " filter l._key == " + "\"" + e.Key + "\"" + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var sn LSNodeExt + ns, err := ncursor.ReadDocument(ctx, &sn) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + + if _, err := a.lsnodeExt.CreateDocument(ctx, &sn); err != nil { + glog.Infof("adding ls_node_extended: %s with area_id %s ", sn.Key, e.AreaID) + if !driver.IsConflict(err) { + return err + } + if err := a.findPrefixSID(ctx, sn.Key, e); err != nil { + if err != nil { + return err + } + } + // The document already exists, updating it with the latest info + if _, err := a.lsnodeExt.UpdateDocument(ctx, ns.Key, e); err != nil { + return err + } + return nil + } + + if err := a.processLSNodeExt(ctx, ns.Key, e); err != nil { + glog.Errorf("Failed to process ls_node_extended %s with error: %+v", ns.Key, err) + } + + if err := a.processIgpDomain(ctx, ns.Key, e); err != nil { + if err != nil { + return err + } + } + return nil +} + +// Find sr-mpls prefix sids and add them to newly added node ls_node_extended collection +func (a *arangoDB) findPrefixSID(ctx context.Context, key string, e *message.LSNode) error { + query := "for l in " + a.lsprefix.Name() + + " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + + " filter l.prefix_attr_tlvs.ls_prefix_sid != null" + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var lp message.LSPrefix + pl, err := ncursor.ReadDocument(ctx, &lp) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + obj := srObject{ + PrefixAttrTLVs: lp.PrefixAttrTLVs, + } + if _, err := a.lsnodeExt.UpdateDocument(ctx, e.Key, &obj); err != nil { + glog.V(5).Infof("adding prefix sid: %s ", pl.Key) + return err + } + if err := a.dedupeLSNodeExt(); err != nil { + if err != nil { + return err + } + } + return nil +} + +// BGP-LS generates a level-1 and a level-2 entry for level-1-2 nodes +// remove duplicate entries in the lsnodeExt collection +func (a *arangoDB) dedupeLSNodeExt() error { + ctx := context.TODO() + dup_query := "LET duplicates = ( FOR d IN " + a.lsnodeExt.Name() + + " COLLECT id = d.igp_router_id, domain = d.domain_id WITH COUNT INTO count " + + " FILTER count > 1 RETURN { id: id, domain: domain, count: count }) " + + "FOR d IN duplicates FOR m IN ls_node_extended " + + "FILTER d.id == m.igp_router_id filter d.domain == m.domain_id RETURN m " + pcursor, err := a.db.Query(ctx, dup_query, nil) + glog.Infof("dedup query: %+v", dup_query) + if err != nil { + return err + } + defer pcursor.Close() + for { + var doc duplicateNode + dupe, err := pcursor.ReadDocument(ctx, &doc) + + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.Infof("Got doc with key '%s' from query\n", dupe.Key) + + if doc.ProtocolID == 1 { + glog.Infof("remove level-1 duplicate node: %s + igp id: %s protocol id: %v + ", doc.Key, doc.IGPRouterID, doc.ProtocolID) + if _, err := a.lsnodeExt.RemoveDocument(ctx, doc.Key); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + if doc.ProtocolID == 2 { + update_query := "for l in " + a.lsnodeExt.Name() + " filter l._key == " + "\"" + doc.Key + "\"" + + " UPDATE l with { protocol: " + "\"" + "ISIS Level 1-2" + "\"" + " } in " + a.lsnodeExt.Name() + "" + cursor, err := a.db.Query(ctx, update_query, nil) + glog.Infof("update query: %s ", update_query) + if err != nil { + return err + } + defer cursor.Close() + } + } + return nil +} + +// Nov 10 2024 - find ipv6 lsnode's ipv4 bgp router-id +func (a *arangoDB) processbgp6(ctx context.Context, key, id string, e *message.PeerStateChange) error { + query := "for l in " + a.lsnodeExt.Name() + + " filter l.router_id == " + "\"" + e.RemoteIP + "\"" + query += " return l" + pcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer pcursor.Close() + for { + var ln LSNodeExt + nl, err := pcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.Infof("ls_node_extended: %s + peer %v + ", ln.Key, e.RemoteBGPID) + + obj := peerObject{ + BGPRouterID: e.RemoteBGPID, + } + + if _, err := a.lsnodeExt.UpdateDocument(ctx, nl.Key, &obj); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + return nil +} + +// processLSNodeExtRemoval removes records from the sn_node collection which are referring to deleted LSNode +func (a *arangoDB) processLSNodeExtRemoval(ctx context.Context, key string) error { + query := "FOR d IN " + a.lsnodeExt.Name() + + " filter d._key == " + "\"" + key + "\"" + query += " return d" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + + for { + var nm LSNodeExt + m, err := ncursor.ReadDocument(ctx, &nm) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + if _, err := a.lsnodeExt.RemoveDocument(ctx, m.ID.Key()); err != nil { + if !driver.IsNotFound(err) { + return err + } + } + } + + return nil +} + +// when a new igp domain is detected, create a new entry in the igp_domain collection +func (a *arangoDB) processIgpDomain(ctx context.Context, key string, e *message.LSNode) error { + if e.ProtocolID == base.BGP { + // EPE Case cannot be processed because LS Node collection does not have BGP routers + return nil + } + query := "for l in ls_node_extended insert " + + "{ _key: CONCAT_SEPARATOR(" + "\"_\", l.protocol_id, l.domain_id, l.asn), " + + "asn: l.asn, protocol_id: l.protocol_id, domain_id: l.domain_id, protocol: l.protocol } " + + "into igp_domain OPTIONS { ignoreErrors: true } " + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var sn LSNodeExt + ns, err := ncursor.ReadDocument(ctx, &sn) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + + if _, err := a.igpDomain.CreateDocument(ctx, &sn); err != nil { + glog.Infof("adding igp_domain: %s with area_id %v ", sn.Key, e.ASN) + if !driver.IsConflict(err) { + return err + } + if err := a.findPrefixSID(ctx, sn.Key, e); err != nil { + if err != nil { + return err + } + } + // The document already exists, updating it with the latest info + if _, err := a.igpDomain.UpdateDocument(ctx, ns.Key, e); err != nil { + return err + } + return nil + } + if err := a.processIgpDomain(ctx, ns.Key, e); err != nil { + glog.Errorf("Failed to process igp_domain %s with error: %+v", ns.Key, err) + } + return nil +} diff --git a/igp-graph/arangodb/lsv4link.go b/igp-graph/arangodb/lsv4link.go new file mode 100644 index 00000000..4a905a2b --- /dev/null +++ b/igp-graph/arangodb/lsv4link.go @@ -0,0 +1,150 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processEdge processes a single ls_link connection which is a unidirectional edge between two nodes (vertices). +func (a *arangoDB) processLSLinkEdge(ctx context.Context, key string, l *message.LSLink) error { + if l.ProtocolID == base.BGP { + return nil + } + if l.MTID != nil { + return a.processLSv6LinkEdge(ctx, key, l) + } + glog.Infof("processEdge processing lslink: %s", l.ID) + // get local node from ls_link entry + ln, err := a.getv4Node(ctx, l, true) + if err != nil { + glog.Errorf("processEdge failed to get local lsnode %s for link: %s with error: %+v", l.IGPRouterID, l.ID, err) + return err + } + + // get remote node from ls_link entry + rn, err := a.getv4Node(ctx, l, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", l.RemoteIGPRouterID, l.ID, err) + return err + } + glog.V(6).Infof("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.V(6).Infof("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + if err := a.createv4EdgeObject(ctx, l, ln, rn); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + glog.Errorf("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.Errorf("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + return err + } + //glog.V(9).Infof("processEdge completed processing lslink: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processLinkRemoval removes a record from Node's graph collection +// since the key matches in both collections (LS Links and Nodes' Graph) deleting the record directly. +func (a *arangoDB) processLinkRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv4.RemoveDocument(ctx, key); err != nil { + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getv4Node(ctx context.Context, e *message.LSLink, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_link's IGP Router ID + query := "FOR d IN " + a.lsnodeExt.Name() + if local { + //glog.Infof("getNode local node per link: %s, %s, %v", e.IGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + } else { + //glog.Infof("getNode remote node per link: %s, %s, %v", e.RemoteIGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.RemoteIGPRouterID + "\"" + } + query += " filter d.domain_id == " + strconv.Itoa(int(e.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if e.ProtocolID == base.OSPFv2 || e.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + e.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createv4EdgeObject(ctx context.Context, l *message.LSLink, ln, rn *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: rn.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: rn.Protocol, + LocalLinkID: l.LocalLinkID, + RemoteLinkID: l.RemoteLinkID, + LocalLinkIP: l.LocalLinkIP, + RemoteLinkIP: l.RemoteLinkIP, + LocalNodeASN: l.LocalNodeASN, + RemoteNodeASN: l.RemoteNodeASN, + PeerNodeSID: l.PeerNodeSID, + SRv6BGPPeerNodeSID: l.SRv6BGPPeerNodeSID, + SRv6ENDXSID: l.SRv6ENDXSID, + LSAdjacencySID: l.LSAdjacencySID, + UnidirLinkDelay: l.UnidirLinkDelay, + UnidirLinkDelayMinMax: l.UnidirLinkDelayMinMax, + UnidirDelayVariation: l.UnidirDelayVariation, + UnidirPacketLoss: l.UnidirPacketLoss, + UnidirResidualBW: l.UnidirResidualBW, + UnidirAvailableBW: l.UnidirAvailableBW, + UnidirBWUtilization: l.UnidirBWUtilization, + } + if _, err := a.graphv4.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv4.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/arangodb/lsv4prefix.go b/igp-graph/arangodb/lsv4prefix.go new file mode 100644 index 00000000..e995e3cc --- /dev/null +++ b/igp-graph/arangodb/lsv4prefix.go @@ -0,0 +1,123 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processLSPrefixEdge processes a single ls_prefix entry which is connected to a node +func (a *arangoDB) processLSPrefixEdge(ctx context.Context, key string, p *message.LSPrefix) error { + //glog.V(9).Infof("processEdge processing lsprefix: %s", l.ID) + + if p.MTID != nil { + return a.processLSv6PrefixEdge(ctx, key, p) + } + // filter out IPv6, ls link, and loopback prefixes + if p.PrefixLen == 30 || p.PrefixLen == 31 || p.PrefixLen == 32 { + return nil + } + + // get remote node from ls_link entry + lsnode, err := a.getLSv4Node(ctx, p, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", p.IGPRouterID, p.ID, err) + return err + } + if err := a.createLSv4PrefixEdgeObject(ctx, p, lsnode); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + return err + } + //glog.V(9).Infof("processEdge completed processing lsprefix: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processPrefixRemoval removes a record from Node's graph collection +func (a *arangoDB) processPrefixRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv4.RemoveDocument(ctx, key); err != nil { + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getLSv4Node(ctx context.Context, p *message.LSPrefix, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_prefix's IGP Router ID + query := "FOR d IN " + a.lsnodeExt.Name() + query += " filter d.igp_router_id == " + "\"" + p.IGPRouterID + "\"" + query += " filter d.domain_id == " + strconv.Itoa(int(p.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if p.ProtocolID == base.OSPFv2 || p.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + p.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createLSv4PrefixEdgeObject(ctx context.Context, l *message.LSPrefix, ln *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: l.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: l.Protocol, + LocalNodeASN: ln.ASN, + Prefix: l.Prefix, + PrefixLen: l.PrefixLen, + PrefixMetric: l.PrefixMetric, + PrefixAttrTLVs: l.PrefixAttrTLVs, + } + if _, err := a.graphv4.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv4.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/arangodb/lsv6link.go b/igp-graph/arangodb/lsv6link.go new file mode 100644 index 00000000..85f79f85 --- /dev/null +++ b/igp-graph/arangodb/lsv6link.go @@ -0,0 +1,151 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processEdge processes a single ipvls_link connection which is a unidirectional edge between two nodes (vertices). +func (a *arangoDB) processLSv6LinkEdge(ctx context.Context, key string, l *message.LSLink) error { + if l.ProtocolID == base.BGP { + return nil + } + if l.MTID == nil { + return nil + } + //glog.Infof("processEdge processing lslink: %s", l.ID) + // get local node from ls_link entry + ln, err := a.getv6Node(ctx, l, true) + if err != nil { + glog.Errorf("processEdge failed to get local lsnode %s for link: %s with error: %+v", l.IGPRouterID, l.ID, err) + return err + } + + // get remote node from ls_link entry + rn, err := a.getv6Node(ctx, l, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", l.RemoteIGPRouterID, l.ID, err) + return err + } + glog.V(6).Infof("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.V(6).Infof("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + if err := a.createv6EdgeObject(ctx, l, ln, rn); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + glog.Errorf("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.Errorf("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + return err + } + //glog.V(9).Infof("processEdge completed processing lslink: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processEdgeRemoval removes a record from Node's graph collection +// since the key matches in both collections (LS Links and Nodes' Graph) deleting the record directly. +func (a *arangoDB) processv6LinkRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv6.RemoveDocument(ctx, key); err != nil { + glog.Infof("removing edge %s", key) + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getv6Node(ctx context.Context, e *message.LSLink, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_link's IGP Router ID + query := "FOR d IN ls_node_extended " //+ a.lsnodeExt.Name() + if local { + //glog.Infof("getNode local node per link: %s, %s, %v", e.IGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + } else { + //glog.Infof("getNode remote node per link: %s, %s, %v", e.RemoteIGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.RemoteIGPRouterID + "\"" + } + query += " filter d.domain_id == " + strconv.Itoa(int(e.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if e.ProtocolID == base.OSPFv2 || e.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + e.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createv6EdgeObject(ctx context.Context, l *message.LSLink, ln, rn *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: rn.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: rn.Protocol, + LocalLinkID: l.LocalLinkID, + RemoteLinkID: l.RemoteLinkID, + LocalLinkIP: l.LocalLinkIP, + RemoteLinkIP: l.RemoteLinkIP, + LocalNodeASN: l.LocalNodeASN, + RemoteNodeASN: l.RemoteNodeASN, + PeerNodeSID: l.PeerNodeSID, + SRv6BGPPeerNodeSID: l.SRv6BGPPeerNodeSID, + SRv6ENDXSID: l.SRv6ENDXSID, + LSAdjacencySID: l.LSAdjacencySID, + UnidirLinkDelay: l.UnidirLinkDelay, + UnidirLinkDelayMinMax: l.UnidirLinkDelayMinMax, + UnidirDelayVariation: l.UnidirDelayVariation, + UnidirPacketLoss: l.UnidirPacketLoss, + UnidirResidualBW: l.UnidirResidualBW, + UnidirAvailableBW: l.UnidirAvailableBW, + UnidirBWUtilization: l.UnidirBWUtilization, + } + if _, err := a.graphv6.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv6.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/arangodb/lsv6prefix.go b/igp-graph/arangodb/lsv6prefix.go new file mode 100644 index 00000000..6958a8db --- /dev/null +++ b/igp-graph/arangodb/lsv6prefix.go @@ -0,0 +1,121 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processEdge processes a single ipv6 ls_prefix entry which is connected to a node +func (a *arangoDB) processLSv6PrefixEdge(ctx context.Context, key string, p *message.LSPrefix) error { + //glog.V(9).Infof("processEdge processing lsprefix: %s", l.ID) + + // filter out IPv6, ls link, and loopback prefixes + if p.MTID == nil || p.PrefixLen == 126 || p.PrefixLen == 127 || p.PrefixLen == 128 { + return nil + } + + // get remote node from ls_link entry + lsnode, err := a.getLSv6Node(ctx, p, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", p.IGPRouterID, p.ID, err) + return err + } + if err := a.createLSv6PrefixEdgeObject(ctx, p, lsnode); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + return err + } + //glog.V(9).Infof("processEdge completed processing lsprefix: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processEdgeRemoval removes a record from Node's graph collection +func (a *arangoDB) processv6PrefixRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv6.RemoveDocument(ctx, key); err != nil { + glog.Infof("removing edge %s", key) + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getLSv6Node(ctx context.Context, p *message.LSPrefix, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_prefix's IGP Router ID + query := "FOR d IN ls_node_extended" //+ a.lsnodeExt.Name() + query += " filter d.igp_router_id == " + "\"" + p.IGPRouterID + "\"" + query += " filter d.domain_id == " + strconv.Itoa(int(p.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if p.ProtocolID == base.OSPFv2 || p.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + p.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createLSv6PrefixEdgeObject(ctx context.Context, l *message.LSPrefix, ln *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: l.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: l.Protocol, + LocalNodeASN: ln.ASN, + Prefix: l.Prefix, + PrefixLen: l.PrefixLen, + PrefixMetric: l.PrefixMetric, + PrefixAttrTLVs: l.PrefixAttrTLVs, + } + if _, err := a.graphv6.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv6.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/arangodb/types.go b/igp-graph/arangodb/types.go new file mode 100644 index 00000000..8aa63fb3 --- /dev/null +++ b/igp-graph/arangodb/types.go @@ -0,0 +1,108 @@ +package arangodb + +import ( + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/bgpls" + "github.com/sbezverk/gobmp/pkg/sr" + "github.com/sbezverk/gobmp/pkg/srv6" +) + +type duplicateNode struct { + Key string `json:"_key,omitempty"` + DomainID int64 `json:"domain_id"` + IGPRouterID string `json:"igp_router_id,omitempty"` + //AreaID string `json:"area_id"` + Protocol string `json:"protocol,omitempty"` + ProtocolID base.ProtoID `json:"protocol_id,omitempty"` + Name string `json:"name,omitempty"` +} + +type srObject struct { + PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs,omitempty"` +} + +type LSNodeExt struct { + Key string `json:"_key,omitempty"` + ID string `json:"_id,omitempty"` + Rev string `json:"_rev,omitempty"` + Action string `json:"action,omitempty"` // Action can be "add" or "del" + Sequence int `json:"sequence,omitempty"` + Hash string `json:"hash,omitempty"` + RouterHash string `json:"router_hash,omitempty"` + DomainID int64 `json:"domain_id"` + RouterIP string `json:"router_ip,omitempty"` + PeerHash string `json:"peer_hash,omitempty"` + PeerIP string `json:"peer_ip,omitempty"` + PeerASN uint32 `json:"peer_asn,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + IGPRouterID string `json:"igp_router_id,omitempty"` + RouterID string `json:"router_id,omitempty"` + ASN uint32 `json:"asn,omitempty"` + LSID uint32 `json:"ls_id,omitempty"` + MTID []*base.MultiTopologyIdentifier `json:"mt_id_tlv,omitempty"` + Protocol string `json:"protocol,omitempty"` + ProtocolID base.ProtoID `json:"protocol_id,omitempty"` + NodeFlags *bgpls.NodeAttrFlags `json:"node_flags,omitempty"` + Name string `json:"name,omitempty"` + SRCapabilities *sr.Capability `json:"ls_sr_capabilities,omitempty"` + SRAlgorithm []int `json:"sr_algorithm,omitempty"` + SRLocalBlock *sr.LocalBlock `json:"sr_local_block,omitempty"` + SRv6CapabilitiesTLV *srv6.CapabilityTLV `json:"srv6_capabilities_tlv,omitempty"` + NodeMSD []*base.MSDTV `json:"node_msd,omitempty"` + FlexAlgoDefinition []*bgpls.FlexAlgoDefinition `json:"flex_algo_definition,omitempty"` + IsPrepolicy bool `json:"is_prepolicy"` + IsAdjRIBIn bool `json:"is_adj_rib_in"` + Prefix string `json:"prefix,omitempty"` + PrefixLen int32 `json:"prefix_len,omitempty"` + PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs,omitempty"` + PrefixSID []*sr.PrefixSIDTLV `json:"prefix_sid_tlv,omitempty"` + FlexAlgoPrefixMetric []*bgpls.FlexAlgoPrefixMetric `json:"flex_algo_prefix_metric,omitempty"` + SRv6SID string `json:"srv6_sid,omitempty"` + SIDS []SID `json:"sids,omitempty"` +} + +type SID struct { + SRv6SID string `json:"srv6_sid,omitempty"` + SRv6EndpointBehavior *srv6.EndpointBehavior `json:"srv6_endpoint_behavior,omitempty"` + SRv6BGPPeerNodeSID *srv6.BGPPeerNodeSID `json:"srv6_bgp_peer_node_sid,omitempty"` + SRv6SIDStructure *srv6.SIDStructure `json:"srv6_sid_structure,omitempty"` +} + +type peerObject struct { + BGPRouterID string `json:"bgp_router_id,omitempty"` +} + +type lsTopologyObject struct { + Key string `json:"_key"` + From string `json:"_from"` + To string `json:"_to"` + Link string `json:"link"` + ProtocolID base.ProtoID `json:"protocol_id"` + DomainID int64 `json:"domain_id"` + MTID uint16 `json:"mt_id"` + AreaID string `json:"area_id"` + Protocol string `json:"protocol"` + LocalLinkID uint32 `json:"local_link_id"` + RemoteLinkID uint32 `json:"remote_link_id"` + LocalLinkIP string `json:"local_link_ip"` + RemoteLinkIP string `json:"remote_link_ip"` + LocalNodeASN uint32 `json:"local_node_asn"` + RemoteNodeASN uint32 `json:"remote_node_asn"` + PeerNodeSID *sr.PeerSID `json:"peer_node_sid,omitempty"` + PeerAdjSID *sr.PeerSID `json:"peer_adj_sid,omitempty"` + PeerSetSID *sr.PeerSID `json:"peer_set_sid,omitempty"` + SRv6BGPPeerNodeSID *srv6.BGPPeerNodeSID `json:"srv6_bgp_peer_node_sid,omitempty"` + SRv6ENDXSID []*srv6.EndXSIDTLV `json:"srv6_endx_sid,omitempty"` + LSAdjacencySID []*sr.AdjacencySIDTLV `json:"ls_adjacency_sid,omitempty"` + UnidirLinkDelay uint32 `json:"unidir_link_delay"` + UnidirLinkDelayMinMax []uint32 `json:"unidir_link_delay_min_max"` + UnidirDelayVariation uint32 `json:"unidir_delay_variation,omitempty"` + UnidirPacketLoss uint32 `json:"unidir_packet_loss,omitempty"` + UnidirResidualBW uint32 `json:"unidir_residual_bw,omitempty"` + UnidirAvailableBW uint32 `json:"unidir_available_bw,omitempty"` + UnidirBWUtilization uint32 `json:"unidir_bw_utilization,omitempty"` + Prefix string `json:"prefix"` + PrefixLen int32 `json:"prefix_len"` + PrefixMetric uint32 `json:"prefix_metric"` + PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs"` +} diff --git a/igp-graph/handler.go b/igp-graph/handler.go new file mode 100644 index 00000000..c93030be --- /dev/null +++ b/igp-graph/handler.go @@ -0,0 +1,185 @@ +package arangodb + +import ( + "context" + "fmt" + "strings" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + + "github.com/jalapeno/topology/pkg/kafkanotifier" + notifier "github.com/jalapeno/topology/pkg/kafkanotifier" + "github.com/sbezverk/gobmp/pkg/message" +) + +func (a *arangoDB) lsNodeHandler(obj *notifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + // Check if Collection encoded in ls_node message ID exists + //glog.Infof("handler received: %+v", obj) + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lsnode.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lsnode.Name(), c) + } + glog.Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + var o message.LSNode + _, err := a.lsnode.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a LSNode removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + return a.processLSNodeExtRemoval(ctx, obj.Key) + } + switch obj.Action { + case "add": + if err := a.processLSNodeExt(ctx, obj.Key, &o); err != nil { + return fmt.Errorf("failed to process action %s for vertex %s with error: %+v", obj.Action, obj.Key, err) + } + default: + // NOOP for update + } + + return nil +} + +func (a *arangoDB) lsSRv6SIDHandler(obj *notifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + // Check if Collection encoded in ls_srv6_sid message ID exists + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lssrv6sid.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lssrv6sid.Name(), c) + } + glog.V(6).Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + var o message.LSSRv6SID + _, err := a.lssrv6sid.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a ls_srv6_sid removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + glog.V(6).Infof("SRv6 SID deleted: %s for ls_node_extended key: %s ", obj.Action, obj.Key) + return nil + } + switch obj.Action { + case "add": + if err := a.processLSSRv6SID(ctx, obj.Key, obj.ID, &o); err != nil { + return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) + } + default: + // NOOP + } + + return nil +} + +func (a *arangoDB) lsPrefixHandler(obj *kafkanotifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + // Check if Collection encoded in ls_prefix message ID exists + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lsprefix.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lsprefix.Name(), c) + } + //glog.V(5).Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + var o message.LSPrefix + _, err := a.lsprefix.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a ls_link removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + + // Detect IPv6 link by checking for ":" in the key + if strings.Contains(obj.Key, ":") { + return a.processv6PrefixRemoval(ctx, obj.Key, obj.Action) + } + + err := a.processPrefixRemoval(ctx, obj.Key, obj.Action) + if err != nil { + return err + } + // write event into ls_node_edge topic + // a.notifier.EventNotification(obj) + // return nil + } + switch obj.Action { + case "add": + fallthrough + case "update": + if err := a.processPrefixSID(ctx, obj.Key, obj.ID, o); err != nil { + return fmt.Errorf("failed to process action %s for vertex %s with error: %+v", obj.Action, obj.Key, err) + } + if err := a.processLSPrefixEdge(ctx, obj.Key, &o); err != nil { + return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) + } + } + + return nil +} + +func (a *arangoDB) lsLinkHandler(obj *kafkanotifier.EventMessage) error { + ctx := context.TODO() + if obj == nil { + return fmt.Errorf("event message is nil") + } + glog.Infof("Processing eventmessage: %+v", obj) + // Check if Collection encoded in ls_link message ID exists + c := strings.Split(obj.ID, "/")[0] + if strings.Compare(c, a.lslink.Name()) != 0 { + return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lslink.Name(), c) + } + var o message.LSLink + _, err := a.lslink.ReadDocument(ctx, obj.Key, &o) + if err != nil { + // In case of a ls_link removal notification, reading it will return Not Found error + if !driver.IsNotFound(err) { + return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) + } + // If operation matches to "del" then it is confirmed delete operation, otherwise return error + if obj.Action != "del" { + return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) + } + + // Detect IPv6 link by checking for ":" in the key + if strings.Contains(obj.Key, ":") { + return a.processv6LinkRemoval(ctx, obj.Key, obj.Action) + } + + return a.processLinkRemoval(ctx, obj.Key, obj.Action) + } + switch obj.Action { + case "add": + fallthrough + case "update": + if err := a.processLSLinkEdge(ctx, obj.Key, &o); err != nil { + return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) + } + } + //glog.V(5).Infof("Complete processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) + + // write event into ls_topoogy_v4 topic + //a.notifier.EventNotification(obj) + + return nil +} diff --git a/igp-graph/kafkamessenger/kafkamessenger.go b/igp-graph/kafkamessenger/kafkamessenger.go new file mode 100755 index 00000000..349852f1 --- /dev/null +++ b/igp-graph/kafkamessenger/kafkamessenger.go @@ -0,0 +1,117 @@ +package kafkamessenger + +import ( + "time" + + "github.com/Shopify/sarama" + "github.com/cisco-open/jalapeno/topology/dbclient" + "github.com/cisco-open/jalapeno/topology/kafkanotifier" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/bmp" + "github.com/sbezverk/gobmp/pkg/tools" +) + +var ( + topics = map[string]dbclient.CollectionType{ + kafkanotifier.LSSRv6SIDEventTopic: bmp.LSSRv6SIDMsg, + kafkanotifier.LSNodeEventTopic: bmp.LSNodeMsg, + kafkanotifier.LSPrefixEventTopic: bmp.LSPrefixMsg, + kafkanotifier.LSLinkEventTopic: bmp.LSLinkMsg, + } +) + +// Srv defines required method of a processor server +type Srv interface { + Start() error + Stop() error +} + +type kafka struct { + stopCh chan struct{} + brokers []string + db dbclient.DB + config *sarama.Config + master sarama.Consumer +} + +// NewKafkaMessenger returns an instance of a kafka consumer acting as a messenger server +func NewKafkaMessenger(kafkaSrv string, db dbclient.DB) (Srv, error) { + glog.Infof("ls events kafka reader") + if err := tools.HostAddrValidator(kafkaSrv); err != nil { + return nil, err + } + + config := sarama.NewConfig() + config.ClientID = "ls-node-collection" + config.Consumer.Return.Errors = true + config.Version = sarama.V0_11_0_0 + + brokers := []string{kafkaSrv} + + // Create new consumer + master, err := sarama.NewConsumer(brokers, config) + if err != nil { + return nil, err + } + k := &kafka{ + stopCh: make(chan struct{}), + config: config, + master: master, + db: db, + } + + return k, nil +} + +func (k *kafka) Start() error { + // Starting readers for each topic name and type defined in topics map + for topicName, topicType := range topics { + go k.topicReader(topicType, topicName) + } + + return nil +} + +func (k *kafka) Stop() error { + close(k.stopCh) + k.master.Close() + return nil +} + +func (k *kafka) topicReader(topicType dbclient.CollectionType, topicName string) { + ticker := time.NewTicker(200 * time.Millisecond) + for { + partitions, _ := k.master.Partitions(topicName) + // this only consumes partition no 1, you would probably want to consume all partitions + consumer, err := k.master.ConsumePartition(topicName, partitions[0], sarama.OffsetOldest) + if nil != err { + glog.Infof("Consumer error: %+v", err) + select { + case <-ticker.C: + case <-k.stopCh: + return + } + continue + } + glog.Infof("Starting Kafka reader for topic: %s topicType: %d", topicName, topicType) + for { + select { + case msg := <-consumer.Messages(): + if msg == nil { + continue + } + //glog.Infof("event msg received %+v for topic %+v", msg, topicName) + if err := k.db.StoreMessage(topicType, msg.Value); err != nil { + glog.Errorf("failed to process a message from topic %s with error: %+v", topicName, err) + } + case consumerError := <-consumer.Errors(): + if consumerError == nil { + break + } + glog.Errorf("error %+v for topic: %s, partition: %s ", consumerError.Err, string(consumerError.Topic), string(consumerError.Partition)) + case <-k.stopCh: + return + } + } + } +} diff --git a/igp-graph/kafkanotifier/kafkanotifier.go b/igp-graph/kafkanotifier/kafkanotifier.go new file mode 100644 index 00000000..62c125f2 --- /dev/null +++ b/igp-graph/kafkanotifier/kafkanotifier.go @@ -0,0 +1,199 @@ +package kafkanotifier + +import ( + "encoding/json" + "fmt" + "math" + "net" + "strconv" + "time" + + "github.com/Shopify/sarama" + "github.com/cisco-open/jalapeno/topology/dbclient" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/bmp" +) + +const ( + LinkStateEdgeV4EventTopic = "jalapeno.linkstate_edge_v4_events" +) + +var ( + brockerConnectTimeout = 10 * time.Second + topicCreateTimeout = 1 * time.Second + // topic Retention for events is 5 minutes + topicRetention = "300000" +) + +var ( + // topics defines a list of topic to initialize and connect, + // initialization is done as a part of NewKafkaPublisher func. + topicNames = []string{ + LinkStateEdgeV4EventTopic, + } +) + +type EventMessage struct { + TopicType dbclient.CollectionType + Key string `json:"_key"` + ID string `json:"_id"` + Action string `json:"action"` +} + +type Event interface { + EventNotification(*EventMessage) error +} + +type notifier struct { + broker *sarama.Broker + config *sarama.Config + producer sarama.SyncProducer +} + +func (n *notifier) EventNotification(msg *EventMessage) error { + switch msg.TopicType { + case bmp.LSNodeMsg: + return n.triggerNotification(LinkStateEdgeV4EventTopic, msg) + case bmp.LSLinkMsg: + return n.triggerNotification(LinkStateEdgeV4EventTopic, msg) + case bmp.LSPrefixMsg: + return n.triggerNotification(LinkStateEdgeV4EventTopic, msg) + } + + return fmt.Errorf("unknown topic type %d", msg.TopicType) +} + +func NewKafkaNotifier(kafkaSrv string) (Event, error) { + glog.Infof("Initializing Kafka events producer client") + if err := validator(kafkaSrv); err != nil { + glog.Errorf("Failed to validate Kafka server address %s with error: %+v", kafkaSrv, err) + return nil, err + } + config := sarama.NewConfig() + config.Producer.Return.Successes = true + config.Version = sarama.V0_11_0_0 + + br := sarama.NewBroker(kafkaSrv) + if err := br.Open(config); err != nil { + if err != sarama.ErrAlreadyConnected { + return nil, err + } + } + + if err := waitForBrokerConnection(br, brockerConnectTimeout); err != nil { + glog.Errorf("failed to open connection to the broker with error: %+v\n", err) + return nil, err + } + glog.V(5).Infof("Connected to broker: %s id: %d\n", br.Addr(), br.ID()) + + for _, t := range topicNames { + if err := ensureTopic(br, topicCreateTimeout, t); err != nil { + return nil, err + } + } + producer, err := sarama.NewSyncProducer([]string{kafkaSrv}, config) + if err != nil { + return nil, err + } + glog.V(5).Infof("Initialized Kafka Sync producer") + + return ¬ifier{ + broker: br, + config: config, + producer: producer, + }, nil +} + +func (n *notifier) triggerNotification(topic string, msg *EventMessage) error { + k := sarama.ByteEncoder{} + k = []byte(msg.Key) + m := sarama.ByteEncoder{} + m, _ = json.Marshal(msg) + _, _, err := n.producer.SendMessage(&sarama.ProducerMessage{ + Topic: topic, + Key: k, + Value: m, + }) + + return err +} + +func validator(addr string) error { + host, port, _ := net.SplitHostPort(addr) + if host == "" || port == "" { + return fmt.Errorf("host or port cannot be ''") + } + // Try to resolve if the hostname was used in the address + if ip, err := net.LookupIP(host); err != nil || ip == nil { + // Check if IP address was used in address instead of a host name + if net.ParseIP(host) == nil { + return fmt.Errorf("fail to parse host part of address") + } + } + np, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("fail to parse port with error: %w", err) + } + if np == 0 || np > math.MaxUint16 { + return fmt.Errorf("the value of port is invalid") + } + return nil +} + +func ensureTopic(br *sarama.Broker, timeout time.Duration, topicName string) error { + ticker := time.NewTicker(100 * time.Millisecond) + tout := time.NewTimer(timeout) + topic := &sarama.CreateTopicsRequest{ + TopicDetails: map[string]*sarama.TopicDetail{ + topicName: { + NumPartitions: 1, + ReplicationFactor: 1, + ConfigEntries: map[string]*string{ + "retention.ms": &topicRetention, + }, + }, + }, + } + + for { + t, err := br.CreateTopics(topic) + if err != nil { + return err + } + if e, ok := t.TopicErrors[topicName]; ok { + if e.Err == sarama.ErrTopicAlreadyExists || e.Err == sarama.ErrNoError { + return nil + } + if e.Err != sarama.ErrRequestTimedOut { + return e + } + } + select { + case <-ticker.C: + continue + case <-tout.C: + return fmt.Errorf("timeout waiting for topic %s", topicName) + } + } +} + +func waitForBrokerConnection(br *sarama.Broker, timeout time.Duration) error { + ticker := time.NewTicker(100 * time.Millisecond) + tout := time.NewTimer(timeout) + for { + ok, err := br.Connected() + if ok { + return nil + } + if err != nil { + return err + } + select { + case <-ticker.C: + continue + case <-tout.C: + return fmt.Errorf("timeout waiting for the connection to the broker %s", br.Addr()) + } + } + +} diff --git a/igp-graph/lsnode-processor.go b/igp-graph/lsnode-processor.go new file mode 100755 index 00000000..a1b4477c --- /dev/null +++ b/igp-graph/lsnode-processor.go @@ -0,0 +1,341 @@ +package arangodb + +import ( + "context" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// Add srv6 sids / locators to nodes in the ls_node_extended collection +func (a *arangoDB) processLSSRv6SID(ctx context.Context, key, id string, e *message.LSSRv6SID) error { + query := "for l in " + a.lsnodeExt.Name() + + " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + + " filter l.domain_id == " + strconv.Itoa(int(e.DomainID)) + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var sn LSNodeExt + ns, err := ncursor.ReadDocument(ctx, &sn) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + // glog.Infof("ls_node_extended %s + srv6sid %s", ns.Key, e.SRv6SID) + // glog.Infof("existing sids: %+v", &sn.SIDS) + + newsid := SID{ + SRv6SID: e.SRv6SID, + SRv6EndpointBehavior: e.SRv6EndpointBehavior, + SRv6BGPPeerNodeSID: e.SRv6BGPPeerNodeSID, + SRv6SIDStructure: e.SRv6SIDStructure, + } + var result bool = false + for _, x := range sn.SIDS { + if x == newsid { + result = true + break + } + } + if result { + glog.Infof("sid %+v exists in ls_node_extended document", e.SRv6SID) + } else { + + sn.SIDS = append(sn.SIDS, newsid) + srn := LSNodeExt{ + SIDS: sn.SIDS, + } + // glog.Infof("appending sid %+v ", e.Key) + + if _, err := a.lsnodeExt.UpdateDocument(ctx, ns.Key, &srn); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + return nil +} + +// Find and add sr-mpls prefix sids to nodes in the ls_node_extended collection +func (a *arangoDB) processPrefixSID(ctx context.Context, key, id string, e message.LSPrefix) error { + query := "for l in " + a.lsnodeExt.Name() + + " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + query += " return l" + pcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer pcursor.Close() + for { + var ln LSNodeExt + nl, err := pcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.V(6).Infof("ls_node_extended: %s + prefix sid %v + ", ln.Key, e.PrefixAttrTLVs.LSPrefixSID) + + obj := srObject{ + PrefixAttrTLVs: e.PrefixAttrTLVs, + } + + if _, err := a.lsnodeExt.UpdateDocument(ctx, nl.Key, &obj); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + return nil +} + +// Find and add ls_node entries to the ls_node_extended collection +func (a *arangoDB) processLSNodeExt(ctx context.Context, key string, e *message.LSNode) error { + if e.ProtocolID == base.BGP { + // EPE Case cannot be processed because LS Node collection does not have BGP routers + return nil + } + query := "for l in " + a.lsnode.Name() + + " filter l._key == " + "\"" + e.Key + "\"" + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var sn LSNodeExt + ns, err := ncursor.ReadDocument(ctx, &sn) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + + if _, err := a.lsnodeExt.CreateDocument(ctx, &sn); err != nil { + glog.Infof("adding ls_node_extended: %s with area_id %s ", sn.Key, e.AreaID) + if !driver.IsConflict(err) { + return err + } + if err := a.findPrefixSID(ctx, sn.Key, e); err != nil { + if err != nil { + return err + } + } + // The document already exists, updating it with the latest info + if _, err := a.lsnodeExt.UpdateDocument(ctx, ns.Key, e); err != nil { + return err + } + return nil + } + + if err := a.processLSNodeExt(ctx, ns.Key, e); err != nil { + glog.Errorf("Failed to process ls_node_extended %s with error: %+v", ns.Key, err) + } + + if err := a.processIgpDomain(ctx, ns.Key, e); err != nil { + if err != nil { + return err + } + } + return nil +} + +// Find sr-mpls prefix sids and add them to newly added node ls_node_extended collection +func (a *arangoDB) findPrefixSID(ctx context.Context, key string, e *message.LSNode) error { + query := "for l in " + a.lsprefix.Name() + + " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + + " filter l.prefix_attr_tlvs.ls_prefix_sid != null" + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var lp message.LSPrefix + pl, err := ncursor.ReadDocument(ctx, &lp) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + obj := srObject{ + PrefixAttrTLVs: lp.PrefixAttrTLVs, + } + if _, err := a.lsnodeExt.UpdateDocument(ctx, e.Key, &obj); err != nil { + glog.V(5).Infof("adding prefix sid: %s ", pl.Key) + return err + } + if err := a.dedupeLSNodeExt(); err != nil { + if err != nil { + return err + } + } + return nil +} + +// BGP-LS generates a level-1 and a level-2 entry for level-1-2 nodes +// remove duplicate entries in the lsnodeExt collection +func (a *arangoDB) dedupeLSNodeExt() error { + ctx := context.TODO() + dup_query := "LET duplicates = ( FOR d IN " + a.lsnodeExt.Name() + + " COLLECT id = d.igp_router_id, domain = d.domain_id WITH COUNT INTO count " + + " FILTER count > 1 RETURN { id: id, domain: domain, count: count }) " + + "FOR d IN duplicates FOR m IN ls_node_extended " + + "FILTER d.id == m.igp_router_id filter d.domain == m.domain_id RETURN m " + pcursor, err := a.db.Query(ctx, dup_query, nil) + glog.Infof("dedup query: %+v", dup_query) + if err != nil { + return err + } + defer pcursor.Close() + for { + var doc duplicateNode + dupe, err := pcursor.ReadDocument(ctx, &doc) + + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.Infof("Got doc with key '%s' from query\n", dupe.Key) + + if doc.ProtocolID == 1 { + glog.Infof("remove level-1 duplicate node: %s + igp id: %s protocol id: %v + ", doc.Key, doc.IGPRouterID, doc.ProtocolID) + if _, err := a.lsnodeExt.RemoveDocument(ctx, doc.Key); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + if doc.ProtocolID == 2 { + update_query := "for l in " + a.lsnodeExt.Name() + " filter l._key == " + "\"" + doc.Key + "\"" + + " UPDATE l with { protocol: " + "\"" + "ISIS Level 1-2" + "\"" + " } in " + a.lsnodeExt.Name() + "" + cursor, err := a.db.Query(ctx, update_query, nil) + glog.Infof("update query: %s ", update_query) + if err != nil { + return err + } + defer cursor.Close() + } + } + return nil +} + +// Nov 10 2024 - find ipv6 lsnode's ipv4 bgp router-id +func (a *arangoDB) processbgp6(ctx context.Context, key, id string, e *message.PeerStateChange) error { + query := "for l in " + a.lsnodeExt.Name() + + " filter l.router_id == " + "\"" + e.RemoteIP + "\"" + query += " return l" + pcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer pcursor.Close() + for { + var ln LSNodeExt + nl, err := pcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + glog.Infof("ls_node_extended: %s + peer %v + ", ln.Key, e.RemoteBGPID) + + obj := peerObject{ + BGPRouterID: e.RemoteBGPID, + } + + if _, err := a.lsnodeExt.UpdateDocument(ctx, nl.Key, &obj); err != nil { + if !driver.IsConflict(err) { + return err + } + } + } + return nil +} + +// processLSNodeExtRemoval removes records from the sn_node collection which are referring to deleted LSNode +func (a *arangoDB) processLSNodeExtRemoval(ctx context.Context, key string) error { + query := "FOR d IN " + a.lsnodeExt.Name() + + " filter d._key == " + "\"" + key + "\"" + query += " return d" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + + for { + var nm LSNodeExt + m, err := ncursor.ReadDocument(ctx, &nm) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + break + } + if _, err := a.lsnodeExt.RemoveDocument(ctx, m.ID.Key()); err != nil { + if !driver.IsNotFound(err) { + return err + } + } + } + + return nil +} + +// when a new igp domain is detected, create a new entry in the igp_domain collection +func (a *arangoDB) processIgpDomain(ctx context.Context, key string, e *message.LSNode) error { + if e.ProtocolID == base.BGP { + // EPE Case cannot be processed because LS Node collection does not have BGP routers + return nil + } + query := "for l in ls_node_extended insert " + + "{ _key: CONCAT_SEPARATOR(" + "\"_\", l.protocol_id, l.domain_id, l.asn), " + + "asn: l.asn, protocol_id: l.protocol_id, domain_id: l.domain_id, protocol: l.protocol } " + + "into igp_domain OPTIONS { ignoreErrors: true } " + query += " return l" + ncursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return err + } + defer ncursor.Close() + var sn LSNodeExt + ns, err := ncursor.ReadDocument(ctx, &sn) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return err + } + } + + if _, err := a.igpDomain.CreateDocument(ctx, &sn); err != nil { + glog.Infof("adding igp_domain: %s with area_id %v ", sn.Key, e.ASN) + if !driver.IsConflict(err) { + return err + } + if err := a.findPrefixSID(ctx, sn.Key, e); err != nil { + if err != nil { + return err + } + } + // The document already exists, updating it with the latest info + if _, err := a.igpDomain.UpdateDocument(ctx, ns.Key, e); err != nil { + return err + } + return nil + } + if err := a.processIgpDomain(ctx, ns.Key, e); err != nil { + glog.Errorf("Failed to process igp_domain %s with error: %+v", ns.Key, err) + } + return nil +} diff --git a/igp-graph/lsv4link.go b/igp-graph/lsv4link.go new file mode 100644 index 00000000..4a905a2b --- /dev/null +++ b/igp-graph/lsv4link.go @@ -0,0 +1,150 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processEdge processes a single ls_link connection which is a unidirectional edge between two nodes (vertices). +func (a *arangoDB) processLSLinkEdge(ctx context.Context, key string, l *message.LSLink) error { + if l.ProtocolID == base.BGP { + return nil + } + if l.MTID != nil { + return a.processLSv6LinkEdge(ctx, key, l) + } + glog.Infof("processEdge processing lslink: %s", l.ID) + // get local node from ls_link entry + ln, err := a.getv4Node(ctx, l, true) + if err != nil { + glog.Errorf("processEdge failed to get local lsnode %s for link: %s with error: %+v", l.IGPRouterID, l.ID, err) + return err + } + + // get remote node from ls_link entry + rn, err := a.getv4Node(ctx, l, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", l.RemoteIGPRouterID, l.ID, err) + return err + } + glog.V(6).Infof("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.V(6).Infof("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + if err := a.createv4EdgeObject(ctx, l, ln, rn); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + glog.Errorf("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.Errorf("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + return err + } + //glog.V(9).Infof("processEdge completed processing lslink: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processLinkRemoval removes a record from Node's graph collection +// since the key matches in both collections (LS Links and Nodes' Graph) deleting the record directly. +func (a *arangoDB) processLinkRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv4.RemoveDocument(ctx, key); err != nil { + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getv4Node(ctx context.Context, e *message.LSLink, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_link's IGP Router ID + query := "FOR d IN " + a.lsnodeExt.Name() + if local { + //glog.Infof("getNode local node per link: %s, %s, %v", e.IGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + } else { + //glog.Infof("getNode remote node per link: %s, %s, %v", e.RemoteIGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.RemoteIGPRouterID + "\"" + } + query += " filter d.domain_id == " + strconv.Itoa(int(e.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if e.ProtocolID == base.OSPFv2 || e.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + e.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createv4EdgeObject(ctx context.Context, l *message.LSLink, ln, rn *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: rn.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: rn.Protocol, + LocalLinkID: l.LocalLinkID, + RemoteLinkID: l.RemoteLinkID, + LocalLinkIP: l.LocalLinkIP, + RemoteLinkIP: l.RemoteLinkIP, + LocalNodeASN: l.LocalNodeASN, + RemoteNodeASN: l.RemoteNodeASN, + PeerNodeSID: l.PeerNodeSID, + SRv6BGPPeerNodeSID: l.SRv6BGPPeerNodeSID, + SRv6ENDXSID: l.SRv6ENDXSID, + LSAdjacencySID: l.LSAdjacencySID, + UnidirLinkDelay: l.UnidirLinkDelay, + UnidirLinkDelayMinMax: l.UnidirLinkDelayMinMax, + UnidirDelayVariation: l.UnidirDelayVariation, + UnidirPacketLoss: l.UnidirPacketLoss, + UnidirResidualBW: l.UnidirResidualBW, + UnidirAvailableBW: l.UnidirAvailableBW, + UnidirBWUtilization: l.UnidirBWUtilization, + } + if _, err := a.graphv4.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv4.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/lsv4prefix.go b/igp-graph/lsv4prefix.go new file mode 100644 index 00000000..e995e3cc --- /dev/null +++ b/igp-graph/lsv4prefix.go @@ -0,0 +1,123 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processLSPrefixEdge processes a single ls_prefix entry which is connected to a node +func (a *arangoDB) processLSPrefixEdge(ctx context.Context, key string, p *message.LSPrefix) error { + //glog.V(9).Infof("processEdge processing lsprefix: %s", l.ID) + + if p.MTID != nil { + return a.processLSv6PrefixEdge(ctx, key, p) + } + // filter out IPv6, ls link, and loopback prefixes + if p.PrefixLen == 30 || p.PrefixLen == 31 || p.PrefixLen == 32 { + return nil + } + + // get remote node from ls_link entry + lsnode, err := a.getLSv4Node(ctx, p, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", p.IGPRouterID, p.ID, err) + return err + } + if err := a.createLSv4PrefixEdgeObject(ctx, p, lsnode); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + return err + } + //glog.V(9).Infof("processEdge completed processing lsprefix: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processPrefixRemoval removes a record from Node's graph collection +func (a *arangoDB) processPrefixRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv4.RemoveDocument(ctx, key); err != nil { + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getLSv4Node(ctx context.Context, p *message.LSPrefix, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_prefix's IGP Router ID + query := "FOR d IN " + a.lsnodeExt.Name() + query += " filter d.igp_router_id == " + "\"" + p.IGPRouterID + "\"" + query += " filter d.domain_id == " + strconv.Itoa(int(p.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if p.ProtocolID == base.OSPFv2 || p.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + p.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createLSv4PrefixEdgeObject(ctx context.Context, l *message.LSPrefix, ln *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: l.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: l.Protocol, + LocalNodeASN: ln.ASN, + Prefix: l.Prefix, + PrefixLen: l.PrefixLen, + PrefixMetric: l.PrefixMetric, + PrefixAttrTLVs: l.PrefixAttrTLVs, + } + if _, err := a.graphv4.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv4.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/lsv6link.go b/igp-graph/lsv6link.go new file mode 100644 index 00000000..85f79f85 --- /dev/null +++ b/igp-graph/lsv6link.go @@ -0,0 +1,151 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processEdge processes a single ipvls_link connection which is a unidirectional edge between two nodes (vertices). +func (a *arangoDB) processLSv6LinkEdge(ctx context.Context, key string, l *message.LSLink) error { + if l.ProtocolID == base.BGP { + return nil + } + if l.MTID == nil { + return nil + } + //glog.Infof("processEdge processing lslink: %s", l.ID) + // get local node from ls_link entry + ln, err := a.getv6Node(ctx, l, true) + if err != nil { + glog.Errorf("processEdge failed to get local lsnode %s for link: %s with error: %+v", l.IGPRouterID, l.ID, err) + return err + } + + // get remote node from ls_link entry + rn, err := a.getv6Node(ctx, l, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", l.RemoteIGPRouterID, l.ID, err) + return err + } + glog.V(6).Infof("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.V(6).Infof("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + if err := a.createv6EdgeObject(ctx, l, ln, rn); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + glog.Errorf("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) + glog.Errorf("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) + return err + } + //glog.V(9).Infof("processEdge completed processing lslink: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processEdgeRemoval removes a record from Node's graph collection +// since the key matches in both collections (LS Links and Nodes' Graph) deleting the record directly. +func (a *arangoDB) processv6LinkRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv6.RemoveDocument(ctx, key); err != nil { + glog.Infof("removing edge %s", key) + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getv6Node(ctx context.Context, e *message.LSLink, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_link's IGP Router ID + query := "FOR d IN ls_node_extended " //+ a.lsnodeExt.Name() + if local { + //glog.Infof("getNode local node per link: %s, %s, %v", e.IGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + } else { + //glog.Infof("getNode remote node per link: %s, %s, %v", e.RemoteIGPRouterID, e.ID, e.ProtocolID) + query += " filter d.igp_router_id == " + "\"" + e.RemoteIGPRouterID + "\"" + } + query += " filter d.domain_id == " + strconv.Itoa(int(e.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if e.ProtocolID == base.OSPFv2 || e.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + e.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createv6EdgeObject(ctx context.Context, l *message.LSLink, ln, rn *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: rn.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: rn.Protocol, + LocalLinkID: l.LocalLinkID, + RemoteLinkID: l.RemoteLinkID, + LocalLinkIP: l.LocalLinkIP, + RemoteLinkIP: l.RemoteLinkIP, + LocalNodeASN: l.LocalNodeASN, + RemoteNodeASN: l.RemoteNodeASN, + PeerNodeSID: l.PeerNodeSID, + SRv6BGPPeerNodeSID: l.SRv6BGPPeerNodeSID, + SRv6ENDXSID: l.SRv6ENDXSID, + LSAdjacencySID: l.LSAdjacencySID, + UnidirLinkDelay: l.UnidirLinkDelay, + UnidirLinkDelayMinMax: l.UnidirLinkDelayMinMax, + UnidirDelayVariation: l.UnidirDelayVariation, + UnidirPacketLoss: l.UnidirPacketLoss, + UnidirResidualBW: l.UnidirResidualBW, + UnidirAvailableBW: l.UnidirAvailableBW, + UnidirBWUtilization: l.UnidirBWUtilization, + } + if _, err := a.graphv6.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv6.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/lsv6prefix.go b/igp-graph/lsv6prefix.go new file mode 100644 index 00000000..6958a8db --- /dev/null +++ b/igp-graph/lsv6prefix.go @@ -0,0 +1,121 @@ +package arangodb + +import ( + "context" + "fmt" + "strconv" + + driver "github.com/arangodb/go-driver" + "github.com/golang/glog" + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/message" +) + +// processEdge processes a single ipv6 ls_prefix entry which is connected to a node +func (a *arangoDB) processLSv6PrefixEdge(ctx context.Context, key string, p *message.LSPrefix) error { + //glog.V(9).Infof("processEdge processing lsprefix: %s", l.ID) + + // filter out IPv6, ls link, and loopback prefixes + if p.MTID == nil || p.PrefixLen == 126 || p.PrefixLen == 127 || p.PrefixLen == 128 { + return nil + } + + // get remote node from ls_link entry + lsnode, err := a.getLSv6Node(ctx, p, false) + if err != nil { + glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", p.IGPRouterID, p.ID, err) + return err + } + if err := a.createLSv6PrefixEdgeObject(ctx, p, lsnode); err != nil { + glog.Errorf("processEdge failed to create Edge object with error: %+v", err) + return err + } + //glog.V(9).Infof("processEdge completed processing lsprefix: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) + + return nil +} + +// processEdgeRemoval removes a record from Node's graph collection +func (a *arangoDB) processv6PrefixRemoval(ctx context.Context, key string, action string) error { + if _, err := a.graphv6.RemoveDocument(ctx, key); err != nil { + glog.Infof("removing edge %s", key) + if !driver.IsNotFound(err) { + return err + } + return nil + } + + return nil +} + +func (a *arangoDB) getLSv6Node(ctx context.Context, p *message.LSPrefix, local bool) (*message.LSNode, error) { + // Need to find ls_node object matching ls_prefix's IGP Router ID + query := "FOR d IN ls_node_extended" //+ a.lsnodeExt.Name() + query += " filter d.igp_router_id == " + "\"" + p.IGPRouterID + "\"" + query += " filter d.domain_id == " + strconv.Itoa(int(p.DomainID)) + + // If OSPFv2 or OSPFv3, then query must include AreaID + if p.ProtocolID == base.OSPFv2 || p.ProtocolID == base.OSPFv3 { + query += " filter d.area_id == " + "\"" + p.AreaID + "\"" + } + query += " return d" + //glog.Infof("query: %s", query) + lcursor, err := a.db.Query(ctx, query, nil) + if err != nil { + return nil, err + } + defer lcursor.Close() + var ln message.LSNode + i := 0 + for ; ; i++ { + _, err := lcursor.ReadDocument(ctx, &ln) + if err != nil { + if !driver.IsNoMoreDocuments(err) { + return nil, err + } + break + } + } + if i == 0 { + return nil, fmt.Errorf("query %s returned 0 results", query) + } + if i > 1 { + return nil, fmt.Errorf("query %s returned more than 1 result", query) + } + + return &ln, nil +} + +func (a *arangoDB) createLSv6PrefixEdgeObject(ctx context.Context, l *message.LSPrefix, ln *message.LSNode) error { + mtid := 0 + if l.MTID != nil { + mtid = int(l.MTID.MTID) + } + ne := lsTopologyObject{ + Key: l.Key, + From: ln.ID, + To: l.ID, + Link: l.Key, + ProtocolID: l.ProtocolID, + DomainID: l.DomainID, + MTID: uint16(mtid), + AreaID: l.AreaID, + Protocol: l.Protocol, + LocalNodeASN: ln.ASN, + Prefix: l.Prefix, + PrefixLen: l.PrefixLen, + PrefixMetric: l.PrefixMetric, + PrefixAttrTLVs: l.PrefixAttrTLVs, + } + if _, err := a.graphv6.CreateDocument(ctx, &ne); err != nil { + if !driver.IsConflict(err) { + return err + } + // The document already exists, updating it with the latest info + if _, err := a.graphv6.UpdateDocument(ctx, ne.Key, &ne); err != nil { + return err + } + } + + return nil +} diff --git a/igp-graph/messenger/msg-server.go b/igp-graph/messenger/msg-server.go new file mode 100755 index 00000000..087a9445 --- /dev/null +++ b/igp-graph/messenger/msg-server.go @@ -0,0 +1,7 @@ +package messenger + +// Srv defines required method of a message server +type Srv interface { + Start() error + Stop() error +} diff --git a/igp-graph/types.go b/igp-graph/types.go new file mode 100644 index 00000000..8aa63fb3 --- /dev/null +++ b/igp-graph/types.go @@ -0,0 +1,108 @@ +package arangodb + +import ( + "github.com/sbezverk/gobmp/pkg/base" + "github.com/sbezverk/gobmp/pkg/bgpls" + "github.com/sbezverk/gobmp/pkg/sr" + "github.com/sbezverk/gobmp/pkg/srv6" +) + +type duplicateNode struct { + Key string `json:"_key,omitempty"` + DomainID int64 `json:"domain_id"` + IGPRouterID string `json:"igp_router_id,omitempty"` + //AreaID string `json:"area_id"` + Protocol string `json:"protocol,omitempty"` + ProtocolID base.ProtoID `json:"protocol_id,omitempty"` + Name string `json:"name,omitempty"` +} + +type srObject struct { + PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs,omitempty"` +} + +type LSNodeExt struct { + Key string `json:"_key,omitempty"` + ID string `json:"_id,omitempty"` + Rev string `json:"_rev,omitempty"` + Action string `json:"action,omitempty"` // Action can be "add" or "del" + Sequence int `json:"sequence,omitempty"` + Hash string `json:"hash,omitempty"` + RouterHash string `json:"router_hash,omitempty"` + DomainID int64 `json:"domain_id"` + RouterIP string `json:"router_ip,omitempty"` + PeerHash string `json:"peer_hash,omitempty"` + PeerIP string `json:"peer_ip,omitempty"` + PeerASN uint32 `json:"peer_asn,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + IGPRouterID string `json:"igp_router_id,omitempty"` + RouterID string `json:"router_id,omitempty"` + ASN uint32 `json:"asn,omitempty"` + LSID uint32 `json:"ls_id,omitempty"` + MTID []*base.MultiTopologyIdentifier `json:"mt_id_tlv,omitempty"` + Protocol string `json:"protocol,omitempty"` + ProtocolID base.ProtoID `json:"protocol_id,omitempty"` + NodeFlags *bgpls.NodeAttrFlags `json:"node_flags,omitempty"` + Name string `json:"name,omitempty"` + SRCapabilities *sr.Capability `json:"ls_sr_capabilities,omitempty"` + SRAlgorithm []int `json:"sr_algorithm,omitempty"` + SRLocalBlock *sr.LocalBlock `json:"sr_local_block,omitempty"` + SRv6CapabilitiesTLV *srv6.CapabilityTLV `json:"srv6_capabilities_tlv,omitempty"` + NodeMSD []*base.MSDTV `json:"node_msd,omitempty"` + FlexAlgoDefinition []*bgpls.FlexAlgoDefinition `json:"flex_algo_definition,omitempty"` + IsPrepolicy bool `json:"is_prepolicy"` + IsAdjRIBIn bool `json:"is_adj_rib_in"` + Prefix string `json:"prefix,omitempty"` + PrefixLen int32 `json:"prefix_len,omitempty"` + PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs,omitempty"` + PrefixSID []*sr.PrefixSIDTLV `json:"prefix_sid_tlv,omitempty"` + FlexAlgoPrefixMetric []*bgpls.FlexAlgoPrefixMetric `json:"flex_algo_prefix_metric,omitempty"` + SRv6SID string `json:"srv6_sid,omitempty"` + SIDS []SID `json:"sids,omitempty"` +} + +type SID struct { + SRv6SID string `json:"srv6_sid,omitempty"` + SRv6EndpointBehavior *srv6.EndpointBehavior `json:"srv6_endpoint_behavior,omitempty"` + SRv6BGPPeerNodeSID *srv6.BGPPeerNodeSID `json:"srv6_bgp_peer_node_sid,omitempty"` + SRv6SIDStructure *srv6.SIDStructure `json:"srv6_sid_structure,omitempty"` +} + +type peerObject struct { + BGPRouterID string `json:"bgp_router_id,omitempty"` +} + +type lsTopologyObject struct { + Key string `json:"_key"` + From string `json:"_from"` + To string `json:"_to"` + Link string `json:"link"` + ProtocolID base.ProtoID `json:"protocol_id"` + DomainID int64 `json:"domain_id"` + MTID uint16 `json:"mt_id"` + AreaID string `json:"area_id"` + Protocol string `json:"protocol"` + LocalLinkID uint32 `json:"local_link_id"` + RemoteLinkID uint32 `json:"remote_link_id"` + LocalLinkIP string `json:"local_link_ip"` + RemoteLinkIP string `json:"remote_link_ip"` + LocalNodeASN uint32 `json:"local_node_asn"` + RemoteNodeASN uint32 `json:"remote_node_asn"` + PeerNodeSID *sr.PeerSID `json:"peer_node_sid,omitempty"` + PeerAdjSID *sr.PeerSID `json:"peer_adj_sid,omitempty"` + PeerSetSID *sr.PeerSID `json:"peer_set_sid,omitempty"` + SRv6BGPPeerNodeSID *srv6.BGPPeerNodeSID `json:"srv6_bgp_peer_node_sid,omitempty"` + SRv6ENDXSID []*srv6.EndXSIDTLV `json:"srv6_endx_sid,omitempty"` + LSAdjacencySID []*sr.AdjacencySIDTLV `json:"ls_adjacency_sid,omitempty"` + UnidirLinkDelay uint32 `json:"unidir_link_delay"` + UnidirLinkDelayMinMax []uint32 `json:"unidir_link_delay_min_max"` + UnidirDelayVariation uint32 `json:"unidir_delay_variation,omitempty"` + UnidirPacketLoss uint32 `json:"unidir_packet_loss,omitempty"` + UnidirResidualBW uint32 `json:"unidir_residual_bw,omitempty"` + UnidirAvailableBW uint32 `json:"unidir_available_bw,omitempty"` + UnidirBWUtilization uint32 `json:"unidir_bw_utilization,omitempty"` + Prefix string `json:"prefix"` + PrefixLen int32 `json:"prefix_len"` + PrefixMetric uint32 `json:"prefix_metric"` + PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs"` +} diff --git a/install/collectors/telegraf-ingress/telegraf_ingress_cfg.yaml b/install/collectors/telegraf-ingress/telegraf_ingress_cfg.yaml index 398c0e46..50c328fb 100644 --- a/install/collectors/telegraf-ingress/telegraf_ingress_cfg.yaml +++ b/install/collectors/telegraf-ingress/telegraf_ingress_cfg.yaml @@ -29,3 +29,8 @@ data: brokers = ["broker.jalapeno.svc:9092"] topic = "jalapeno.telemetry" + [[outputs.kafka]] + brokers = ["broker.jalapeno.svc:9092"] + topic = "jalapeno.srv6" + data_format = "json" + namepass = ["Cisco-IOS-XR-segment-routing-srv6-oper*"] \ No newline at end of file diff --git a/install/processors/igp-graph/igp-graph.yaml b/install/processors/igp-graph/igp-graph.yaml new file mode 100644 index 00000000..5ce7b7a8 --- /dev/null +++ b/install/processors/igp-graph/igp-graph.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +spec: + replicas: 1 + selector: + matchLabels: + app: igp-graph + template: + metadata: + labels: + app: igp-graph + spec: + containers: + - args: + - --v + - "5" + - --message-server + - "broker.jalapeno:9092" + - --database-server + - "http://arangodb.jalapeno:8529" + - --database-name + - "jalapeno" + image: docker.io/iejalapeno/igp-processor:latest + imagePullPolicy: Always + name: igp-graph + volumeMounts: + - name: credentials + mountPath: /credentials + volumes: + - name: credentials + secret: + secretName: jalapeno +metadata: + name: igp-graph + namespace: jalapeno From ebd7e217343c27148e7e5a8f6f4c7630f8a22ed1 Mon Sep 17 00:00:00 2001 From: brmcdoug Date: Wed, 27 Nov 2024 23:58:21 -0800 Subject: [PATCH 2/5] deploy --- install/processors/deploy_processors.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/processors/deploy_processors.sh b/install/processors/deploy_processors.sh index 792f2bab..8e57dfec 100755 --- a/install/processors/deploy_processors.sh +++ b/install/processors/deploy_processors.sh @@ -14,3 +14,6 @@ ${KUBE} create -f ${PWD}/${BASEDIR}/telegraf-egress/. echo "Deploying LS Link-Node Edge Processor" ${KUBE} create -f ${PWD}/${BASEDIR}/lslinknode-edge/lslinknode-edge.yaml + +echo "Deploying IGP Graph Processor" +${KUBE} create -f ${PWD}/${BASEDIR}/igp-graph/igp-graph.yaml From 9f5402339be1f0674c915d73f700c19b04b3d7b1 Mon Sep 17 00:00:00 2001 From: brmcdoug Date: Thu, 28 Nov 2024 00:09:50 -0800 Subject: [PATCH 3/5] typo --- install/processors/igp-graph/igp-graph.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/processors/igp-graph/igp-graph.yaml b/install/processors/igp-graph/igp-graph.yaml index 5ce7b7a8..26a77723 100644 --- a/install/processors/igp-graph/igp-graph.yaml +++ b/install/processors/igp-graph/igp-graph.yaml @@ -21,7 +21,7 @@ spec: - "http://arangodb.jalapeno:8529" - --database-name - "jalapeno" - image: docker.io/iejalapeno/igp-processor:latest + image: docker.io/iejalapeno/igp-graph:latest imagePullPolicy: Always name: igp-graph volumeMounts: From b9a62c68cab4b5835d783d8ea112e7a43d25ad73 Mon Sep 17 00:00:00 2001 From: brmcdoug Date: Thu, 28 Nov 2024 10:36:21 -0800 Subject: [PATCH 4/5] makefile --- Makefile.igp-graph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.igp-graph b/Makefile.igp-graph index 3093134e..b2f27161 100755 --- a/Makefile.igp-graph +++ b/Makefile.igp-graph @@ -13,7 +13,7 @@ all: igp-graph igp-graph: mkdir -p bin - $(MAKE) -C ./cmd compile-igp-graph + $(MAKE) -C ./cmd/igp-graph compile-igp-graph igp-graph-container: igp-graph docker build -t $(REGISTRY_NAME)/igp-graph:$(IMAGE_VERSION) -f ./build/Dockerfile.igp-graph . From 27948a9bf3bbbada767b062d31f4ac54e9a2b584 Mon Sep 17 00:00:00 2001 From: brmcdoug Date: Thu, 28 Nov 2024 10:52:59 -0800 Subject: [PATCH 5/5] apache license --- cmd/igp-graph/main.go | 2 +- igp-graph/arango-conn.go | 80 --- igp-graph/arangodb.go | 542 --------------------- igp-graph/arangodb/arango-conn.go | 22 +- igp-graph/arangodb/arangodb.go | 22 + igp-graph/arangodb/handler.go | 23 + igp-graph/arangodb/lsnode-processor.go | 22 + igp-graph/arangodb/lsv4link.go | 22 + igp-graph/arangodb/lsv4prefix.go | 22 + igp-graph/arangodb/lsv6link.go | 22 + igp-graph/arangodb/lsv6prefix.go | 22 + igp-graph/arangodb/types.go | 22 + igp-graph/handler.go | 185 ------- igp-graph/kafkamessenger/kafkamessenger.go | 22 + igp-graph/kafkanotifier/kafkanotifier.go | 22 + igp-graph/lsnode-processor.go | 341 ------------- igp-graph/lsv4link.go | 150 ------ igp-graph/lsv4prefix.go | 123 ----- igp-graph/lsv6link.go | 151 ------ igp-graph/lsv6prefix.go | 121 ----- igp-graph/messenger/msg-server.go | 22 + igp-graph/types.go | 108 ---- 22 files changed, 265 insertions(+), 1803 deletions(-) delete mode 100755 igp-graph/arango-conn.go delete mode 100755 igp-graph/arangodb.go delete mode 100644 igp-graph/handler.go delete mode 100755 igp-graph/lsnode-processor.go delete mode 100644 igp-graph/lsv4link.go delete mode 100644 igp-graph/lsv4prefix.go delete mode 100644 igp-graph/lsv6link.go delete mode 100644 igp-graph/lsv6prefix.go delete mode 100644 igp-graph/types.go diff --git a/cmd/igp-graph/main.go b/cmd/igp-graph/main.go index aebb4571..4d987bdd 100755 --- a/cmd/igp-graph/main.go +++ b/cmd/igp-graph/main.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Cisco Systems, Inc. and its affiliates +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates // All rights reserved. // // Redistribution and use in source and binary forms, with or without diff --git a/igp-graph/arango-conn.go b/igp-graph/arango-conn.go deleted file mode 100755 index 12c2579d..00000000 --- a/igp-graph/arango-conn.go +++ /dev/null @@ -1,80 +0,0 @@ -// Borrowed from https://github.com/cisco-ie/jalapeno - -package arangodb - -import ( - "context" - "errors" - "fmt" - "strings" - - driver "github.com/arangodb/go-driver" - "github.com/arangodb/go-driver/http" - "github.com/golang/glog" -) - -var ( - ErrEmptyConfig = errors.New("ArangoDB Config has an empty field") - ErrUpSafe = errors.New("Failed to UpdateSafe. Requires *DBObjects") - ErrNilObject = errors.New("Failed to operate on NIL object") - ErrNotFound = errors.New("Document not found") -) - -type ArangoConfig struct { - URL string `desc:"Arangodb server URL (http://127.0.0.1:8529)"` - User string `desc:"Arangodb server username"` - Password string `desc:"Arangodb server user password"` - Database string `desc:"Arangodb database name"` -} - -func NewConfig() ArangoConfig { - return ArangoConfig{} -} - -type ArangoConn struct { - db driver.Database -} - -var ( - ErrCollectionNotFound = fmt.Errorf("Could not find collection") -) - -func NewArango(cfg ArangoConfig) (*ArangoConn, error) { - // Connect to DB - if cfg.URL == "" || cfg.User == "" || cfg.Password == "" || cfg.Database == "" { - return nil, ErrEmptyConfig - } - if !strings.Contains(cfg.URL, "http") { - cfg.URL = "http://" + cfg.URL - } - conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{cfg.URL}, - }) - if err != nil { - glog.Errorf("Failed to create HTTP connection: %v", err) - return nil, err - } - - // Authenticate with DB - conn, err = conn.SetAuthentication(driver.BasicAuthentication(cfg.User, cfg.Password)) - if err != nil { - glog.Errorf("Failed to authenticate with arango: %v", err) - return nil, err - } - - c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, - }) - if err != nil { - glog.Errorf("Failed to create client: %v", err) - return nil, err - } - - // If Jalapeno databse does not exist, the topology is not running, goig into a crash loop - db, err := c.Database(context.Background(), cfg.Database) - if err != nil { - return nil, err - } - - return &ArangoConn{db: db}, nil -} diff --git a/igp-graph/arangodb.go b/igp-graph/arangodb.go deleted file mode 100755 index 7046e138..00000000 --- a/igp-graph/arangodb.go +++ /dev/null @@ -1,542 +0,0 @@ -package arangodb - -import ( - "context" - "encoding/json" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - "github.com/jalapeno/topology/pkg/dbclient" - "github.com/jalapeno/topology/pkg/kafkanotifier" - notifier "github.com/jalapeno/topology/pkg/kafkanotifier" - "github.com/sbezverk/gobmp/pkg/bmp" - "github.com/sbezverk/gobmp/pkg/message" - "github.com/sbezverk/gobmp/pkg/tools" -) - -type arangoDB struct { - dbclient.DB - *ArangoConn - stop chan struct{} - lsprefix driver.Collection - lslink driver.Collection - lssrv6sid driver.Collection - lsnode driver.Collection - lsnodeExt driver.Collection - igpDomain driver.Collection - graphv4 driver.Collection - graphv6 driver.Collection - lsv4Graph driver.Graph - lsv6Graph driver.Graph - notifier kafkanotifier.Event -} - -// NewDBSrvClient returns an instance of a DB server client process -func NewDBSrvClient(arangoSrv, user, pass, dbname, lsprefix, lslink, lssrv6sid, lsnode, - lsnodeExt string, igpDomain string, lsv4Graph string, lsv6Graph string, notifier kafkanotifier.Event) (dbclient.Srv, error) { - if err := tools.URLAddrValidation(arangoSrv); err != nil { - return nil, err - } - arangoConn, err := NewArango(ArangoConfig{ - URL: arangoSrv, - User: user, - Password: pass, - Database: dbname, - }) - if err != nil { - return nil, err - } - arango := &arangoDB{ - stop: make(chan struct{}), - } - arango.DB = arango - arango.ArangoConn = arangoConn - - // Check if base link state collections exist, if not fail as Jalapeno topology is not running - arango.lsprefix, err = arango.db.Collection(context.TODO(), lsprefix) - if err != nil { - return nil, err - } - arango.lslink, err = arango.db.Collection(context.TODO(), lslink) - if err != nil { - return nil, err - } - arango.lssrv6sid, err = arango.db.Collection(context.TODO(), lssrv6sid) - if err != nil { - return nil, err - } - arango.lsnode, err = arango.db.Collection(context.TODO(), lsnode) - if err != nil { - return nil, err - } - - // check for ls_node_extended collection - found, err := arango.db.CollectionExists(context.TODO(), lsnodeExt) - if err != nil { - return nil, err - } - if found { - c, err := arango.db.Collection(context.TODO(), lsnodeExt) - if err != nil { - return nil, err - } - if err := c.Remove(context.TODO()); err != nil { - return nil, err - } - } - // create ls_node_extended collection - var lsnodeExt_options = &driver.CreateCollectionOptions{ /* ... */ } - glog.V(5).Infof("ls_node_extended not found, creating") - arango.lsnodeExt, err = arango.db.CreateCollection(context.TODO(), "ls_node_extended", lsnodeExt_options) - if err != nil { - return nil, err - } - // check if collection exists, if not fail as processor has failed to create collection - arango.lsnodeExt, err = arango.db.Collection(context.TODO(), lsnodeExt) - if err != nil { - return nil, err - } - - // check for igp_domain collection - found, err = arango.db.CollectionExists(context.TODO(), igpDomain) - if err != nil { - return nil, err - } - if found { - c, err := arango.db.Collection(context.TODO(), igpDomain) - if err != nil { - return nil, err - } - if err := c.Remove(context.TODO()); err != nil { - return nil, err - } - } - // create igp_domain collection - var igpdomain_options = &driver.CreateCollectionOptions{ /* ... */ } - glog.V(5).Infof("igp_domain collection not found, creating") - arango.igpDomain, err = arango.db.CreateCollection(context.TODO(), "igp_domain", igpdomain_options) - if err != nil { - return nil, err - } - // check if collection exists, if not fail as processor has failed to create collection - arango.igpDomain, err = arango.db.Collection(context.TODO(), igpDomain) - if err != nil { - return nil, err - } - - // check for lsv4 topology graph - found, err = arango.db.GraphExists(context.TODO(), lsv4Graph) - if err != nil { - return nil, err - } - if found { - c, err := arango.db.Graph(context.TODO(), lsv4Graph) - if err != nil { - return nil, err - } - glog.Infof("found graph %s", c) - - } else { - // create graph - var edgeDefinition driver.EdgeDefinition - edgeDefinition.Collection = "lsv4_graph" - edgeDefinition.From = []string{"ls_node_extended"} - edgeDefinition.To = []string{"ls_node_extended"} - var options driver.CreateGraphOptions - options.OrphanVertexCollections = []string{"ls_srv6_sid", "ls_prefix"} - options.EdgeDefinitions = []driver.EdgeDefinition{edgeDefinition} - - arango.lsv4Graph, err = arango.db.CreateGraph(context.TODO(), lsv4Graph, &options) - if err != nil { - return nil, err - } - } - - // check if lsv4_graph exists, if not fail as processor has failed to create graph - arango.graphv4, err = arango.db.Collection(context.TODO(), "lsv4_graph") - if err != nil { - return nil, err - } - - // check for lsv6 topology graph - found, err = arango.db.GraphExists(context.TODO(), lsv6Graph) - if err != nil { - return nil, err - } - if found { - c, err := arango.db.Graph(context.TODO(), lsv6Graph) - if err != nil { - return nil, err - } - glog.Infof("found graph %s", c) - - } else { - // create graph - var edgeDefinition driver.EdgeDefinition - edgeDefinition.Collection = "lsv6_graph" - edgeDefinition.From = []string{"ls_node_extended"} - edgeDefinition.To = []string{"ls_node_extended"} - var options driver.CreateGraphOptions - options.OrphanVertexCollections = []string{"ls_srv6_sid", "ls_prefix"} - options.EdgeDefinitions = []driver.EdgeDefinition{edgeDefinition} - - arango.lsv6Graph, err = arango.db.CreateGraph(context.TODO(), lsv6Graph, &options) - if err != nil { - return nil, err - } - } - - // check if lsv6_graph exists, if not fail as processor has failed to create graph - arango.graphv6, err = arango.db.Collection(context.TODO(), "lsv6_graph") - if err != nil { - return nil, err - } - - return arango, nil -} - -func (a *arangoDB) Start() error { - if err := a.loadCollections(); err != nil { - return err - } - glog.Infof("Connected to arango database, starting monitor") - go a.monitor() - - return nil -} - -func (a *arangoDB) Stop() error { - close(a.stop) - - return nil -} - -func (a *arangoDB) GetInterface() dbclient.DB { - return a.DB -} - -func (a *arangoDB) GetArangoDBInterface() *ArangoConn { - return a.ArangoConn -} - -func (a *arangoDB) StoreMessage(msgType dbclient.CollectionType, msg []byte) error { - event := ¬ifier.EventMessage{} - if err := json.Unmarshal(msg, event); err != nil { - return err - } - event.TopicType = msgType - switch msgType { - case bmp.LSSRv6SIDMsg: - return a.lsSRv6SIDHandler(event) - case bmp.LSNodeMsg: - return a.lsNodeHandler(event) - case bmp.LSPrefixMsg: - return a.lsPrefixHandler(event) - case bmp.LSLinkMsg: - return a.lsLinkHandler(event) - } - return nil -} - -func (a *arangoDB) monitor() { - for { - select { - case <-a.stop: - // TODO Add clean up of connection with Arango DB - return - } - } -} - -// loadCollections calls a series of subfunctions to perform ArangoDB operations including populating -// ls_node_extended collection and querying other link state collections to build the lsv4_graph and lsv6_graph - -func (a *arangoDB) loadCollections() error { - ctx := context.TODO() - - if err := a.lsExtendedNodes(ctx); err != nil { - return err - } - if err := a.processDuplicateNodes(ctx); err != nil { - return err - } - if err := a.loadPrefixSIDs(ctx); err != nil { - return err - } - if err := a.loadSRv6SIDs(ctx); err != nil { - return err - } - if err := a.processIBGPv6Peering(ctx); err != nil { - return err - } - if err := a.createIGPDomains(ctx); err != nil { - return err - } - if err := a.lsv4LinkEdges(ctx); err != nil { - return err - } - if err := a.lsv4PrefixEdges(ctx); err != nil { - return err - } - if err := a.lsv6LinkEdges(ctx); err != nil { - return err - } - if err := a.lsv6PrefixEdges(ctx); err != nil { - return err - } - - return nil -} -func (a *arangoDB) lsExtendedNodes(ctx context.Context) error { - lsn_query := "for l in " + a.lsnode.Name() + " insert l in " + a.lsnodeExt.Name() + "" - cursor, err := a.db.Query(ctx, lsn_query, nil) - if err != nil { - return err - } - defer cursor.Close() - return nil -} - -func (a *arangoDB) processDuplicateNodes(ctx context.Context) error { - // BGP-LS generates both a level-1 and a level-2 entry for level-1-2 nodes - // Here we remove duplicate entries in the ls_node_extended collection - dup_query := "LET duplicates = ( FOR d IN " + a.lsnodeExt.Name() + - " COLLECT id = d.igp_router_id, domain = d.domain_id, area = d.area_id WITH COUNT INTO count " + - " FILTER count > 1 RETURN { id: id, domain: domain, area: area, count: count }) " + - "FOR d IN duplicates FOR m IN ls_node_extended " + - "FILTER d.id == m.igp_router_id filter d.domain == m.domain_id RETURN m " - pcursor, err := a.db.Query(ctx, dup_query, nil) - if err != nil { - return err - } - defer pcursor.Close() - for { - var doc duplicateNode - dupe, err := pcursor.ReadDocument(ctx, &doc) - - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - break - } - glog.Infof("Got doc with key '%s' from query\n", dupe.Key) - - if doc.ProtocolID == 1 { - glog.Infof("remove level-1 duplicate node: %s + igp id: %s protocol id: %v + ", doc.Key, doc.IGPRouterID, doc.ProtocolID) - if _, err := a.lsnodeExt.RemoveDocument(ctx, doc.Key); err != nil { - if !driver.IsConflict(err) { - return err - } - } - } - if doc.ProtocolID == 2 { - update_query := "for l in " + a.lsnodeExt.Name() + " filter l._key == " + "\"" + doc.Key + "\"" + - " UPDATE l with { protocol: " + "\"" + "ISIS Level 1-2" + "\"" + " } in " + a.lsnodeExt.Name() + "" - cursor, err := a.db.Query(ctx, update_query, nil) - glog.Infof("update query: %s ", update_query) - if err != nil { - return err - } - defer cursor.Close() - } - } - - return nil -} - -func (a *arangoDB) loadPrefixSIDs(ctx context.Context) error { - // Find and add sr-mpls prefix sids to nodes in the ls_node_extended collection - sr_query := "for p in " + a.lsprefix.Name() + - " filter p.mt_id_tlv.mt_id != 2 && p.prefix_attr_tlvs.ls_prefix_sid != null return p " - cursor, err := a.db.Query(ctx, sr_query, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.LSPrefix - meta, err := cursor.ReadDocument(ctx, &p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processPrefixSID(ctx, meta.Key, meta.ID.String(), p); err != nil { - glog.Errorf("Failed to process ls_prefix_sid %s with error: %+v", p.ID, err) - } - } - - return nil -} - -func (a *arangoDB) loadSRv6SIDs(ctx context.Context) error { - // Find and add srv6 sids to nodes in the ls_node_extended collection - srv6_query := "for s in " + a.lssrv6sid.Name() + " return s " - cursor, err := a.db.Query(ctx, srv6_query, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.LSSRv6SID - meta, err := cursor.ReadDocument(ctx, &p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processLSSRv6SID(ctx, meta.Key, meta.ID.String(), &p); err != nil { - glog.Errorf("Failed to process ls_srv6_sid %s with error: %+v", p.ID, err) - } - } - - return nil -} - -func (a *arangoDB) processIBGPv6Peering(ctx context.Context) error { - // add ipv6 iBGP peering address and ipv4 bgp router-id - ibgp6_query := "for s in peer filter s.remote_ip like " + "\"%:%\"" + " return s " - cursor, err := a.db.Query(ctx, ibgp6_query, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.PeerStateChange - meta, err := cursor.ReadDocument(ctx, &p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processbgp6(ctx, meta.Key, meta.ID.String(), &p); err != nil { - glog.Errorf("Failed to process ibgp peering %s with error: %+v", p.ID, err) - } - } - - return nil -} - -func (a *arangoDB) createIGPDomains(ctx context.Context) error { - // create igp_domain collection - useful in scaled multi-domain environments - igpdomain_query := "for l in ls_node_extended insert " + - "{ _key: CONCAT_SEPARATOR(" + "\"_\", l.protocol_id, l.domain_id, l.asn), " + - "asn: l.asn, protocol_id: l.protocol_id, domain_id: l.domain_id, protocol: l.protocol } " + - "into igp_domain OPTIONS { ignoreErrors: true } return l" - cursor, err := a.db.Query(ctx, igpdomain_query, nil) - if err != nil { - return err - } - defer cursor.Close() - - return nil -} - -func (a *arangoDB) lsv4LinkEdges(ctx context.Context) error { - // Find ipv4 ls_link entries to create edges in the lsv4_graph - lsv4linkquery := "for l in " + a.lslink.Name() + " filter l.protocol_id != 7 RETURN l" - cursor, err := a.db.Query(ctx, lsv4linkquery, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.LSLink - meta, err := cursor.ReadDocument(ctx, &p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processLSLinkEdge(ctx, meta.Key, &p); err != nil { - glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) - continue - } - } - - return nil -} - -func (a *arangoDB) lsv4PrefixEdges(ctx context.Context) error { - // Find ls_prefix entries to create prefix or subnet edges in the lsv4_graph - lsv4pfxquery := "for l in " + a.lsprefix.Name() + //" filter l.mt_id_tlv == null return l" - " filter l.mt_id_tlv.mt_id != 2 && l.prefix_len != 30 && " + - "l.prefix_len != 31 && l.prefix_len != 32 return l" - cursor, err := a.db.Query(ctx, lsv4pfxquery, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.LSPrefix - meta, err := cursor.ReadDocument(ctx, &p) - //glog.Infof("processing lsprefix document: %+v", p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processLSPrefixEdge(ctx, meta.Key, &p); err != nil { - glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) - continue - } - } - - return nil -} - -func (a *arangoDB) lsv6LinkEdges(ctx context.Context) error { - // Find ipv6 ls_link entries to create edges in the lsv6_graph - lsv6linkquery := "for l in " + a.lslink.Name() + " filter l.protocol_id != 7 RETURN l" - cursor, err := a.db.Query(ctx, lsv6linkquery, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.LSLink - meta, err := cursor.ReadDocument(ctx, &p) - //glog.Infof("processing lslink document: %+v", p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processLSv6LinkEdge(ctx, meta.Key, &p); err != nil { - glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) - continue - } - } - - return nil -} - -func (a *arangoDB) lsv6PrefixEdges(ctx context.Context) error { - // Find ipv6 ls_prefix entries to create prefix or subnet edges in the lsv6_graph - lsv6pfxquery := "for l in " + a.lsprefix.Name() + - " filter l.mt_id_tlv.mt_id == 2 && l.prefix_len != 126 && " + - "l.prefix_len != 127 && l.prefix_len != 128 return l" - cursor, err := a.db.Query(ctx, lsv6pfxquery, nil) - if err != nil { - return err - } - defer cursor.Close() - for { - var p message.LSPrefix - meta, err := cursor.ReadDocument(ctx, &p) - //glog.Infof("processing lsprefix document: %+v", p) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - return err - } - if err := a.processLSv6PrefixEdge(ctx, meta.Key, &p); err != nil { - glog.Errorf("failed to process key: %s with error: %+v", meta.Key, err) - continue - } - } - - return nil - -} diff --git a/igp-graph/arangodb/arango-conn.go b/igp-graph/arangodb/arango-conn.go index 12c2579d..307d6c68 100755 --- a/igp-graph/arangodb/arango-conn.go +++ b/igp-graph/arangodb/arango-conn.go @@ -1,4 +1,24 @@ -// Borrowed from https://github.com/cisco-ie/jalapeno +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. package arangodb diff --git a/igp-graph/arangodb/arangodb.go b/igp-graph/arangodb/arangodb.go index 301babfc..926a55a7 100755 --- a/igp-graph/arangodb/arangodb.go +++ b/igp-graph/arangodb/arangodb.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/handler.go b/igp-graph/arangodb/handler.go index 76a5bc53..dc8c40ea 100644 --- a/igp-graph/arangodb/handler.go +++ b/igp-graph/arangodb/handler.go @@ -1,3 +1,26 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// - Redistributions of source code must retain the above copyright +// +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/lsnode-processor.go b/igp-graph/arangodb/lsnode-processor.go index a1b4477c..0eef567d 100755 --- a/igp-graph/arangodb/lsnode-processor.go +++ b/igp-graph/arangodb/lsnode-processor.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/lsv4link.go b/igp-graph/arangodb/lsv4link.go index 4a905a2b..45e99524 100644 --- a/igp-graph/arangodb/lsv4link.go +++ b/igp-graph/arangodb/lsv4link.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/lsv4prefix.go b/igp-graph/arangodb/lsv4prefix.go index e995e3cc..d58f6c96 100644 --- a/igp-graph/arangodb/lsv4prefix.go +++ b/igp-graph/arangodb/lsv4prefix.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/lsv6link.go b/igp-graph/arangodb/lsv6link.go index 85f79f85..0ae8c596 100644 --- a/igp-graph/arangodb/lsv6link.go +++ b/igp-graph/arangodb/lsv6link.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/lsv6prefix.go b/igp-graph/arangodb/lsv6prefix.go index 6958a8db..c6b1cbff 100644 --- a/igp-graph/arangodb/lsv6prefix.go +++ b/igp-graph/arangodb/lsv6prefix.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/arangodb/types.go b/igp-graph/arangodb/types.go index 8aa63fb3..79846bad 100644 --- a/igp-graph/arangodb/types.go +++ b/igp-graph/arangodb/types.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package arangodb import ( diff --git a/igp-graph/handler.go b/igp-graph/handler.go deleted file mode 100644 index c93030be..00000000 --- a/igp-graph/handler.go +++ /dev/null @@ -1,185 +0,0 @@ -package arangodb - -import ( - "context" - "fmt" - "strings" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - - "github.com/jalapeno/topology/pkg/kafkanotifier" - notifier "github.com/jalapeno/topology/pkg/kafkanotifier" - "github.com/sbezverk/gobmp/pkg/message" -) - -func (a *arangoDB) lsNodeHandler(obj *notifier.EventMessage) error { - ctx := context.TODO() - if obj == nil { - return fmt.Errorf("event message is nil") - } - // Check if Collection encoded in ls_node message ID exists - //glog.Infof("handler received: %+v", obj) - c := strings.Split(obj.ID, "/")[0] - if strings.Compare(c, a.lsnode.Name()) != 0 { - return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lsnode.Name(), c) - } - glog.Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) - var o message.LSNode - _, err := a.lsnode.ReadDocument(ctx, obj.Key, &o) - if err != nil { - // In case of a LSNode removal notification, reading it will return Not Found error - if !driver.IsNotFound(err) { - return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) - } - // If operation matches to "del" then it is confirmed delete operation, otherwise return error - if obj.Action != "del" { - return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) - } - return a.processLSNodeExtRemoval(ctx, obj.Key) - } - switch obj.Action { - case "add": - if err := a.processLSNodeExt(ctx, obj.Key, &o); err != nil { - return fmt.Errorf("failed to process action %s for vertex %s with error: %+v", obj.Action, obj.Key, err) - } - default: - // NOOP for update - } - - return nil -} - -func (a *arangoDB) lsSRv6SIDHandler(obj *notifier.EventMessage) error { - ctx := context.TODO() - if obj == nil { - return fmt.Errorf("event message is nil") - } - // Check if Collection encoded in ls_srv6_sid message ID exists - c := strings.Split(obj.ID, "/")[0] - if strings.Compare(c, a.lssrv6sid.Name()) != 0 { - return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lssrv6sid.Name(), c) - } - glog.V(6).Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) - var o message.LSSRv6SID - _, err := a.lssrv6sid.ReadDocument(ctx, obj.Key, &o) - if err != nil { - // In case of a ls_srv6_sid removal notification, reading it will return Not Found error - if !driver.IsNotFound(err) { - return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) - } - // If operation matches to "del" then it is confirmed delete operation, otherwise return error - if obj.Action != "del" { - return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) - } - glog.V(6).Infof("SRv6 SID deleted: %s for ls_node_extended key: %s ", obj.Action, obj.Key) - return nil - } - switch obj.Action { - case "add": - if err := a.processLSSRv6SID(ctx, obj.Key, obj.ID, &o); err != nil { - return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) - } - default: - // NOOP - } - - return nil -} - -func (a *arangoDB) lsPrefixHandler(obj *kafkanotifier.EventMessage) error { - ctx := context.TODO() - if obj == nil { - return fmt.Errorf("event message is nil") - } - // Check if Collection encoded in ls_prefix message ID exists - c := strings.Split(obj.ID, "/")[0] - if strings.Compare(c, a.lsprefix.Name()) != 0 { - return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lsprefix.Name(), c) - } - //glog.V(5).Infof("Processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) - var o message.LSPrefix - _, err := a.lsprefix.ReadDocument(ctx, obj.Key, &o) - if err != nil { - // In case of a ls_link removal notification, reading it will return Not Found error - if !driver.IsNotFound(err) { - return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) - } - // If operation matches to "del" then it is confirmed delete operation, otherwise return error - if obj.Action != "del" { - return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) - } - - // Detect IPv6 link by checking for ":" in the key - if strings.Contains(obj.Key, ":") { - return a.processv6PrefixRemoval(ctx, obj.Key, obj.Action) - } - - err := a.processPrefixRemoval(ctx, obj.Key, obj.Action) - if err != nil { - return err - } - // write event into ls_node_edge topic - // a.notifier.EventNotification(obj) - // return nil - } - switch obj.Action { - case "add": - fallthrough - case "update": - if err := a.processPrefixSID(ctx, obj.Key, obj.ID, o); err != nil { - return fmt.Errorf("failed to process action %s for vertex %s with error: %+v", obj.Action, obj.Key, err) - } - if err := a.processLSPrefixEdge(ctx, obj.Key, &o); err != nil { - return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) - } - } - - return nil -} - -func (a *arangoDB) lsLinkHandler(obj *kafkanotifier.EventMessage) error { - ctx := context.TODO() - if obj == nil { - return fmt.Errorf("event message is nil") - } - glog.Infof("Processing eventmessage: %+v", obj) - // Check if Collection encoded in ls_link message ID exists - c := strings.Split(obj.ID, "/")[0] - if strings.Compare(c, a.lslink.Name()) != 0 { - return fmt.Errorf("configured collection name %s and received in event collection name %s do not match", a.lslink.Name(), c) - } - var o message.LSLink - _, err := a.lslink.ReadDocument(ctx, obj.Key, &o) - if err != nil { - // In case of a ls_link removal notification, reading it will return Not Found error - if !driver.IsNotFound(err) { - return fmt.Errorf("failed to read existing document %s with error: %+v", obj.Key, err) - } - // If operation matches to "del" then it is confirmed delete operation, otherwise return error - if obj.Action != "del" { - return fmt.Errorf("document %s not found but Action is not \"del\", possible stale event", obj.Key) - } - - // Detect IPv6 link by checking for ":" in the key - if strings.Contains(obj.Key, ":") { - return a.processv6LinkRemoval(ctx, obj.Key, obj.Action) - } - - return a.processLinkRemoval(ctx, obj.Key, obj.Action) - } - switch obj.Action { - case "add": - fallthrough - case "update": - if err := a.processLSLinkEdge(ctx, obj.Key, &o); err != nil { - return fmt.Errorf("failed to process action %s for edge %s with error: %+v", obj.Action, obj.Key, err) - } - } - //glog.V(5).Infof("Complete processing action: %s for key: %s ID: %s", obj.Action, obj.Key, obj.ID) - - // write event into ls_topoogy_v4 topic - //a.notifier.EventNotification(obj) - - return nil -} diff --git a/igp-graph/kafkamessenger/kafkamessenger.go b/igp-graph/kafkamessenger/kafkamessenger.go index 349852f1..a0d839da 100755 --- a/igp-graph/kafkamessenger/kafkamessenger.go +++ b/igp-graph/kafkamessenger/kafkamessenger.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package kafkamessenger import ( diff --git a/igp-graph/kafkanotifier/kafkanotifier.go b/igp-graph/kafkanotifier/kafkanotifier.go index 62c125f2..2c35eda1 100644 --- a/igp-graph/kafkanotifier/kafkanotifier.go +++ b/igp-graph/kafkanotifier/kafkanotifier.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package kafkanotifier import ( diff --git a/igp-graph/lsnode-processor.go b/igp-graph/lsnode-processor.go deleted file mode 100755 index a1b4477c..00000000 --- a/igp-graph/lsnode-processor.go +++ /dev/null @@ -1,341 +0,0 @@ -package arangodb - -import ( - "context" - "strconv" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - "github.com/sbezverk/gobmp/pkg/base" - "github.com/sbezverk/gobmp/pkg/message" -) - -// Add srv6 sids / locators to nodes in the ls_node_extended collection -func (a *arangoDB) processLSSRv6SID(ctx context.Context, key, id string, e *message.LSSRv6SID) error { - query := "for l in " + a.lsnodeExt.Name() + - " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + - " filter l.domain_id == " + strconv.Itoa(int(e.DomainID)) - query += " return l" - ncursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer ncursor.Close() - var sn LSNodeExt - ns, err := ncursor.ReadDocument(ctx, &sn) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - } - // glog.Infof("ls_node_extended %s + srv6sid %s", ns.Key, e.SRv6SID) - // glog.Infof("existing sids: %+v", &sn.SIDS) - - newsid := SID{ - SRv6SID: e.SRv6SID, - SRv6EndpointBehavior: e.SRv6EndpointBehavior, - SRv6BGPPeerNodeSID: e.SRv6BGPPeerNodeSID, - SRv6SIDStructure: e.SRv6SIDStructure, - } - var result bool = false - for _, x := range sn.SIDS { - if x == newsid { - result = true - break - } - } - if result { - glog.Infof("sid %+v exists in ls_node_extended document", e.SRv6SID) - } else { - - sn.SIDS = append(sn.SIDS, newsid) - srn := LSNodeExt{ - SIDS: sn.SIDS, - } - // glog.Infof("appending sid %+v ", e.Key) - - if _, err := a.lsnodeExt.UpdateDocument(ctx, ns.Key, &srn); err != nil { - if !driver.IsConflict(err) { - return err - } - } - } - return nil -} - -// Find and add sr-mpls prefix sids to nodes in the ls_node_extended collection -func (a *arangoDB) processPrefixSID(ctx context.Context, key, id string, e message.LSPrefix) error { - query := "for l in " + a.lsnodeExt.Name() + - " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" - query += " return l" - pcursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer pcursor.Close() - for { - var ln LSNodeExt - nl, err := pcursor.ReadDocument(ctx, &ln) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - break - } - glog.V(6).Infof("ls_node_extended: %s + prefix sid %v + ", ln.Key, e.PrefixAttrTLVs.LSPrefixSID) - - obj := srObject{ - PrefixAttrTLVs: e.PrefixAttrTLVs, - } - - if _, err := a.lsnodeExt.UpdateDocument(ctx, nl.Key, &obj); err != nil { - if !driver.IsConflict(err) { - return err - } - } - } - return nil -} - -// Find and add ls_node entries to the ls_node_extended collection -func (a *arangoDB) processLSNodeExt(ctx context.Context, key string, e *message.LSNode) error { - if e.ProtocolID == base.BGP { - // EPE Case cannot be processed because LS Node collection does not have BGP routers - return nil - } - query := "for l in " + a.lsnode.Name() + - " filter l._key == " + "\"" + e.Key + "\"" - query += " return l" - ncursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer ncursor.Close() - var sn LSNodeExt - ns, err := ncursor.ReadDocument(ctx, &sn) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - } - - if _, err := a.lsnodeExt.CreateDocument(ctx, &sn); err != nil { - glog.Infof("adding ls_node_extended: %s with area_id %s ", sn.Key, e.AreaID) - if !driver.IsConflict(err) { - return err - } - if err := a.findPrefixSID(ctx, sn.Key, e); err != nil { - if err != nil { - return err - } - } - // The document already exists, updating it with the latest info - if _, err := a.lsnodeExt.UpdateDocument(ctx, ns.Key, e); err != nil { - return err - } - return nil - } - - if err := a.processLSNodeExt(ctx, ns.Key, e); err != nil { - glog.Errorf("Failed to process ls_node_extended %s with error: %+v", ns.Key, err) - } - - if err := a.processIgpDomain(ctx, ns.Key, e); err != nil { - if err != nil { - return err - } - } - return nil -} - -// Find sr-mpls prefix sids and add them to newly added node ls_node_extended collection -func (a *arangoDB) findPrefixSID(ctx context.Context, key string, e *message.LSNode) error { - query := "for l in " + a.lsprefix.Name() + - " filter l.igp_router_id == " + "\"" + e.IGPRouterID + "\"" + - " filter l.prefix_attr_tlvs.ls_prefix_sid != null" - query += " return l" - ncursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer ncursor.Close() - var lp message.LSPrefix - pl, err := ncursor.ReadDocument(ctx, &lp) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - } - obj := srObject{ - PrefixAttrTLVs: lp.PrefixAttrTLVs, - } - if _, err := a.lsnodeExt.UpdateDocument(ctx, e.Key, &obj); err != nil { - glog.V(5).Infof("adding prefix sid: %s ", pl.Key) - return err - } - if err := a.dedupeLSNodeExt(); err != nil { - if err != nil { - return err - } - } - return nil -} - -// BGP-LS generates a level-1 and a level-2 entry for level-1-2 nodes -// remove duplicate entries in the lsnodeExt collection -func (a *arangoDB) dedupeLSNodeExt() error { - ctx := context.TODO() - dup_query := "LET duplicates = ( FOR d IN " + a.lsnodeExt.Name() + - " COLLECT id = d.igp_router_id, domain = d.domain_id WITH COUNT INTO count " + - " FILTER count > 1 RETURN { id: id, domain: domain, count: count }) " + - "FOR d IN duplicates FOR m IN ls_node_extended " + - "FILTER d.id == m.igp_router_id filter d.domain == m.domain_id RETURN m " - pcursor, err := a.db.Query(ctx, dup_query, nil) - glog.Infof("dedup query: %+v", dup_query) - if err != nil { - return err - } - defer pcursor.Close() - for { - var doc duplicateNode - dupe, err := pcursor.ReadDocument(ctx, &doc) - - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - break - } - glog.Infof("Got doc with key '%s' from query\n", dupe.Key) - - if doc.ProtocolID == 1 { - glog.Infof("remove level-1 duplicate node: %s + igp id: %s protocol id: %v + ", doc.Key, doc.IGPRouterID, doc.ProtocolID) - if _, err := a.lsnodeExt.RemoveDocument(ctx, doc.Key); err != nil { - if !driver.IsConflict(err) { - return err - } - } - } - if doc.ProtocolID == 2 { - update_query := "for l in " + a.lsnodeExt.Name() + " filter l._key == " + "\"" + doc.Key + "\"" + - " UPDATE l with { protocol: " + "\"" + "ISIS Level 1-2" + "\"" + " } in " + a.lsnodeExt.Name() + "" - cursor, err := a.db.Query(ctx, update_query, nil) - glog.Infof("update query: %s ", update_query) - if err != nil { - return err - } - defer cursor.Close() - } - } - return nil -} - -// Nov 10 2024 - find ipv6 lsnode's ipv4 bgp router-id -func (a *arangoDB) processbgp6(ctx context.Context, key, id string, e *message.PeerStateChange) error { - query := "for l in " + a.lsnodeExt.Name() + - " filter l.router_id == " + "\"" + e.RemoteIP + "\"" - query += " return l" - pcursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer pcursor.Close() - for { - var ln LSNodeExt - nl, err := pcursor.ReadDocument(ctx, &ln) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - break - } - glog.Infof("ls_node_extended: %s + peer %v + ", ln.Key, e.RemoteBGPID) - - obj := peerObject{ - BGPRouterID: e.RemoteBGPID, - } - - if _, err := a.lsnodeExt.UpdateDocument(ctx, nl.Key, &obj); err != nil { - if !driver.IsConflict(err) { - return err - } - } - } - return nil -} - -// processLSNodeExtRemoval removes records from the sn_node collection which are referring to deleted LSNode -func (a *arangoDB) processLSNodeExtRemoval(ctx context.Context, key string) error { - query := "FOR d IN " + a.lsnodeExt.Name() + - " filter d._key == " + "\"" + key + "\"" - query += " return d" - ncursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer ncursor.Close() - - for { - var nm LSNodeExt - m, err := ncursor.ReadDocument(ctx, &nm) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - break - } - if _, err := a.lsnodeExt.RemoveDocument(ctx, m.ID.Key()); err != nil { - if !driver.IsNotFound(err) { - return err - } - } - } - - return nil -} - -// when a new igp domain is detected, create a new entry in the igp_domain collection -func (a *arangoDB) processIgpDomain(ctx context.Context, key string, e *message.LSNode) error { - if e.ProtocolID == base.BGP { - // EPE Case cannot be processed because LS Node collection does not have BGP routers - return nil - } - query := "for l in ls_node_extended insert " + - "{ _key: CONCAT_SEPARATOR(" + "\"_\", l.protocol_id, l.domain_id, l.asn), " + - "asn: l.asn, protocol_id: l.protocol_id, domain_id: l.domain_id, protocol: l.protocol } " + - "into igp_domain OPTIONS { ignoreErrors: true } " - query += " return l" - ncursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return err - } - defer ncursor.Close() - var sn LSNodeExt - ns, err := ncursor.ReadDocument(ctx, &sn) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return err - } - } - - if _, err := a.igpDomain.CreateDocument(ctx, &sn); err != nil { - glog.Infof("adding igp_domain: %s with area_id %v ", sn.Key, e.ASN) - if !driver.IsConflict(err) { - return err - } - if err := a.findPrefixSID(ctx, sn.Key, e); err != nil { - if err != nil { - return err - } - } - // The document already exists, updating it with the latest info - if _, err := a.igpDomain.UpdateDocument(ctx, ns.Key, e); err != nil { - return err - } - return nil - } - if err := a.processIgpDomain(ctx, ns.Key, e); err != nil { - glog.Errorf("Failed to process igp_domain %s with error: %+v", ns.Key, err) - } - return nil -} diff --git a/igp-graph/lsv4link.go b/igp-graph/lsv4link.go deleted file mode 100644 index 4a905a2b..00000000 --- a/igp-graph/lsv4link.go +++ /dev/null @@ -1,150 +0,0 @@ -package arangodb - -import ( - "context" - "fmt" - "strconv" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - "github.com/sbezverk/gobmp/pkg/base" - "github.com/sbezverk/gobmp/pkg/message" -) - -// processEdge processes a single ls_link connection which is a unidirectional edge between two nodes (vertices). -func (a *arangoDB) processLSLinkEdge(ctx context.Context, key string, l *message.LSLink) error { - if l.ProtocolID == base.BGP { - return nil - } - if l.MTID != nil { - return a.processLSv6LinkEdge(ctx, key, l) - } - glog.Infof("processEdge processing lslink: %s", l.ID) - // get local node from ls_link entry - ln, err := a.getv4Node(ctx, l, true) - if err != nil { - glog.Errorf("processEdge failed to get local lsnode %s for link: %s with error: %+v", l.IGPRouterID, l.ID, err) - return err - } - - // get remote node from ls_link entry - rn, err := a.getv4Node(ctx, l, false) - if err != nil { - glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", l.RemoteIGPRouterID, l.ID, err) - return err - } - glog.V(6).Infof("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) - glog.V(6).Infof("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) - if err := a.createv4EdgeObject(ctx, l, ln, rn); err != nil { - glog.Errorf("processEdge failed to create Edge object with error: %+v", err) - glog.Errorf("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) - glog.Errorf("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) - return err - } - //glog.V(9).Infof("processEdge completed processing lslink: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) - - return nil -} - -// processLinkRemoval removes a record from Node's graph collection -// since the key matches in both collections (LS Links and Nodes' Graph) deleting the record directly. -func (a *arangoDB) processLinkRemoval(ctx context.Context, key string, action string) error { - if _, err := a.graphv4.RemoveDocument(ctx, key); err != nil { - if !driver.IsNotFound(err) { - return err - } - return nil - } - - return nil -} - -func (a *arangoDB) getv4Node(ctx context.Context, e *message.LSLink, local bool) (*message.LSNode, error) { - // Need to find ls_node object matching ls_link's IGP Router ID - query := "FOR d IN " + a.lsnodeExt.Name() - if local { - //glog.Infof("getNode local node per link: %s, %s, %v", e.IGPRouterID, e.ID, e.ProtocolID) - query += " filter d.igp_router_id == " + "\"" + e.IGPRouterID + "\"" - } else { - //glog.Infof("getNode remote node per link: %s, %s, %v", e.RemoteIGPRouterID, e.ID, e.ProtocolID) - query += " filter d.igp_router_id == " + "\"" + e.RemoteIGPRouterID + "\"" - } - query += " filter d.domain_id == " + strconv.Itoa(int(e.DomainID)) - - // If OSPFv2 or OSPFv3, then query must include AreaID - if e.ProtocolID == base.OSPFv2 || e.ProtocolID == base.OSPFv3 { - query += " filter d.area_id == " + "\"" + e.AreaID + "\"" - } - query += " return d" - //glog.Infof("query: %s", query) - lcursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return nil, err - } - defer lcursor.Close() - var ln message.LSNode - i := 0 - for ; ; i++ { - _, err := lcursor.ReadDocument(ctx, &ln) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return nil, err - } - break - } - } - if i == 0 { - return nil, fmt.Errorf("query %s returned 0 results", query) - } - if i > 1 { - return nil, fmt.Errorf("query %s returned more than 1 result", query) - } - - return &ln, nil -} - -func (a *arangoDB) createv4EdgeObject(ctx context.Context, l *message.LSLink, ln, rn *message.LSNode) error { - mtid := 0 - if l.MTID != nil { - mtid = int(l.MTID.MTID) - } - ne := lsTopologyObject{ - Key: l.Key, - From: ln.ID, - To: rn.ID, - Link: l.Key, - ProtocolID: l.ProtocolID, - DomainID: l.DomainID, - MTID: uint16(mtid), - AreaID: l.AreaID, - Protocol: rn.Protocol, - LocalLinkID: l.LocalLinkID, - RemoteLinkID: l.RemoteLinkID, - LocalLinkIP: l.LocalLinkIP, - RemoteLinkIP: l.RemoteLinkIP, - LocalNodeASN: l.LocalNodeASN, - RemoteNodeASN: l.RemoteNodeASN, - PeerNodeSID: l.PeerNodeSID, - SRv6BGPPeerNodeSID: l.SRv6BGPPeerNodeSID, - SRv6ENDXSID: l.SRv6ENDXSID, - LSAdjacencySID: l.LSAdjacencySID, - UnidirLinkDelay: l.UnidirLinkDelay, - UnidirLinkDelayMinMax: l.UnidirLinkDelayMinMax, - UnidirDelayVariation: l.UnidirDelayVariation, - UnidirPacketLoss: l.UnidirPacketLoss, - UnidirResidualBW: l.UnidirResidualBW, - UnidirAvailableBW: l.UnidirAvailableBW, - UnidirBWUtilization: l.UnidirBWUtilization, - } - if _, err := a.graphv4.CreateDocument(ctx, &ne); err != nil { - if !driver.IsConflict(err) { - return err - } - // The document already exists, updating it with the latest info - if _, err := a.graphv4.UpdateDocument(ctx, ne.Key, &ne); err != nil { - return err - } - } - - return nil -} diff --git a/igp-graph/lsv4prefix.go b/igp-graph/lsv4prefix.go deleted file mode 100644 index e995e3cc..00000000 --- a/igp-graph/lsv4prefix.go +++ /dev/null @@ -1,123 +0,0 @@ -package arangodb - -import ( - "context" - "fmt" - "strconv" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - "github.com/sbezverk/gobmp/pkg/base" - "github.com/sbezverk/gobmp/pkg/message" -) - -// processLSPrefixEdge processes a single ls_prefix entry which is connected to a node -func (a *arangoDB) processLSPrefixEdge(ctx context.Context, key string, p *message.LSPrefix) error { - //glog.V(9).Infof("processEdge processing lsprefix: %s", l.ID) - - if p.MTID != nil { - return a.processLSv6PrefixEdge(ctx, key, p) - } - // filter out IPv6, ls link, and loopback prefixes - if p.PrefixLen == 30 || p.PrefixLen == 31 || p.PrefixLen == 32 { - return nil - } - - // get remote node from ls_link entry - lsnode, err := a.getLSv4Node(ctx, p, false) - if err != nil { - glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", p.IGPRouterID, p.ID, err) - return err - } - if err := a.createLSv4PrefixEdgeObject(ctx, p, lsnode); err != nil { - glog.Errorf("processEdge failed to create Edge object with error: %+v", err) - return err - } - //glog.V(9).Infof("processEdge completed processing lsprefix: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) - - return nil -} - -// processPrefixRemoval removes a record from Node's graph collection -func (a *arangoDB) processPrefixRemoval(ctx context.Context, key string, action string) error { - if _, err := a.graphv4.RemoveDocument(ctx, key); err != nil { - if !driver.IsNotFound(err) { - return err - } - return nil - } - - return nil -} - -func (a *arangoDB) getLSv4Node(ctx context.Context, p *message.LSPrefix, local bool) (*message.LSNode, error) { - // Need to find ls_node object matching ls_prefix's IGP Router ID - query := "FOR d IN " + a.lsnodeExt.Name() - query += " filter d.igp_router_id == " + "\"" + p.IGPRouterID + "\"" - query += " filter d.domain_id == " + strconv.Itoa(int(p.DomainID)) - - // If OSPFv2 or OSPFv3, then query must include AreaID - if p.ProtocolID == base.OSPFv2 || p.ProtocolID == base.OSPFv3 { - query += " filter d.area_id == " + "\"" + p.AreaID + "\"" - } - query += " return d" - //glog.Infof("query: %s", query) - lcursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return nil, err - } - defer lcursor.Close() - var ln message.LSNode - i := 0 - for ; ; i++ { - _, err := lcursor.ReadDocument(ctx, &ln) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return nil, err - } - break - } - } - if i == 0 { - return nil, fmt.Errorf("query %s returned 0 results", query) - } - if i > 1 { - return nil, fmt.Errorf("query %s returned more than 1 result", query) - } - - return &ln, nil -} - -func (a *arangoDB) createLSv4PrefixEdgeObject(ctx context.Context, l *message.LSPrefix, ln *message.LSNode) error { - mtid := 0 - if l.MTID != nil { - mtid = int(l.MTID.MTID) - } - ne := lsTopologyObject{ - Key: l.Key, - From: ln.ID, - To: l.ID, - Link: l.Key, - ProtocolID: l.ProtocolID, - DomainID: l.DomainID, - MTID: uint16(mtid), - AreaID: l.AreaID, - Protocol: l.Protocol, - LocalNodeASN: ln.ASN, - Prefix: l.Prefix, - PrefixLen: l.PrefixLen, - PrefixMetric: l.PrefixMetric, - PrefixAttrTLVs: l.PrefixAttrTLVs, - } - if _, err := a.graphv4.CreateDocument(ctx, &ne); err != nil { - if !driver.IsConflict(err) { - return err - } - // The document already exists, updating it with the latest info - if _, err := a.graphv4.UpdateDocument(ctx, ne.Key, &ne); err != nil { - return err - } - } - - return nil -} diff --git a/igp-graph/lsv6link.go b/igp-graph/lsv6link.go deleted file mode 100644 index 85f79f85..00000000 --- a/igp-graph/lsv6link.go +++ /dev/null @@ -1,151 +0,0 @@ -package arangodb - -import ( - "context" - "fmt" - "strconv" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - "github.com/sbezverk/gobmp/pkg/base" - "github.com/sbezverk/gobmp/pkg/message" -) - -// processEdge processes a single ipvls_link connection which is a unidirectional edge between two nodes (vertices). -func (a *arangoDB) processLSv6LinkEdge(ctx context.Context, key string, l *message.LSLink) error { - if l.ProtocolID == base.BGP { - return nil - } - if l.MTID == nil { - return nil - } - //glog.Infof("processEdge processing lslink: %s", l.ID) - // get local node from ls_link entry - ln, err := a.getv6Node(ctx, l, true) - if err != nil { - glog.Errorf("processEdge failed to get local lsnode %s for link: %s with error: %+v", l.IGPRouterID, l.ID, err) - return err - } - - // get remote node from ls_link entry - rn, err := a.getv6Node(ctx, l, false) - if err != nil { - glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", l.RemoteIGPRouterID, l.ID, err) - return err - } - glog.V(6).Infof("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) - glog.V(6).Infof("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) - if err := a.createv6EdgeObject(ctx, l, ln, rn); err != nil { - glog.Errorf("processEdge failed to create Edge object with error: %+v", err) - glog.Errorf("Local node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", ln.ProtocolID, ln.DomainID, ln.IGPRouterID) - glog.Errorf("Remote node -> Protocol: %+v Domain ID: %+v IGP Router ID: %+v", rn.ProtocolID, rn.DomainID, rn.IGPRouterID) - return err - } - //glog.V(9).Infof("processEdge completed processing lslink: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) - - return nil -} - -// processEdgeRemoval removes a record from Node's graph collection -// since the key matches in both collections (LS Links and Nodes' Graph) deleting the record directly. -func (a *arangoDB) processv6LinkRemoval(ctx context.Context, key string, action string) error { - if _, err := a.graphv6.RemoveDocument(ctx, key); err != nil { - glog.Infof("removing edge %s", key) - if !driver.IsNotFound(err) { - return err - } - return nil - } - - return nil -} - -func (a *arangoDB) getv6Node(ctx context.Context, e *message.LSLink, local bool) (*message.LSNode, error) { - // Need to find ls_node object matching ls_link's IGP Router ID - query := "FOR d IN ls_node_extended " //+ a.lsnodeExt.Name() - if local { - //glog.Infof("getNode local node per link: %s, %s, %v", e.IGPRouterID, e.ID, e.ProtocolID) - query += " filter d.igp_router_id == " + "\"" + e.IGPRouterID + "\"" - } else { - //glog.Infof("getNode remote node per link: %s, %s, %v", e.RemoteIGPRouterID, e.ID, e.ProtocolID) - query += " filter d.igp_router_id == " + "\"" + e.RemoteIGPRouterID + "\"" - } - query += " filter d.domain_id == " + strconv.Itoa(int(e.DomainID)) - - // If OSPFv2 or OSPFv3, then query must include AreaID - if e.ProtocolID == base.OSPFv2 || e.ProtocolID == base.OSPFv3 { - query += " filter d.area_id == " + "\"" + e.AreaID + "\"" - } - query += " return d" - //glog.Infof("query: %s", query) - lcursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return nil, err - } - defer lcursor.Close() - var ln message.LSNode - i := 0 - for ; ; i++ { - _, err := lcursor.ReadDocument(ctx, &ln) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return nil, err - } - break - } - } - if i == 0 { - return nil, fmt.Errorf("query %s returned 0 results", query) - } - if i > 1 { - return nil, fmt.Errorf("query %s returned more than 1 result", query) - } - - return &ln, nil -} - -func (a *arangoDB) createv6EdgeObject(ctx context.Context, l *message.LSLink, ln, rn *message.LSNode) error { - mtid := 0 - if l.MTID != nil { - mtid = int(l.MTID.MTID) - } - ne := lsTopologyObject{ - Key: l.Key, - From: ln.ID, - To: rn.ID, - Link: l.Key, - ProtocolID: l.ProtocolID, - DomainID: l.DomainID, - MTID: uint16(mtid), - AreaID: l.AreaID, - Protocol: rn.Protocol, - LocalLinkID: l.LocalLinkID, - RemoteLinkID: l.RemoteLinkID, - LocalLinkIP: l.LocalLinkIP, - RemoteLinkIP: l.RemoteLinkIP, - LocalNodeASN: l.LocalNodeASN, - RemoteNodeASN: l.RemoteNodeASN, - PeerNodeSID: l.PeerNodeSID, - SRv6BGPPeerNodeSID: l.SRv6BGPPeerNodeSID, - SRv6ENDXSID: l.SRv6ENDXSID, - LSAdjacencySID: l.LSAdjacencySID, - UnidirLinkDelay: l.UnidirLinkDelay, - UnidirLinkDelayMinMax: l.UnidirLinkDelayMinMax, - UnidirDelayVariation: l.UnidirDelayVariation, - UnidirPacketLoss: l.UnidirPacketLoss, - UnidirResidualBW: l.UnidirResidualBW, - UnidirAvailableBW: l.UnidirAvailableBW, - UnidirBWUtilization: l.UnidirBWUtilization, - } - if _, err := a.graphv6.CreateDocument(ctx, &ne); err != nil { - if !driver.IsConflict(err) { - return err - } - // The document already exists, updating it with the latest info - if _, err := a.graphv6.UpdateDocument(ctx, ne.Key, &ne); err != nil { - return err - } - } - - return nil -} diff --git a/igp-graph/lsv6prefix.go b/igp-graph/lsv6prefix.go deleted file mode 100644 index 6958a8db..00000000 --- a/igp-graph/lsv6prefix.go +++ /dev/null @@ -1,121 +0,0 @@ -package arangodb - -import ( - "context" - "fmt" - "strconv" - - driver "github.com/arangodb/go-driver" - "github.com/golang/glog" - "github.com/sbezverk/gobmp/pkg/base" - "github.com/sbezverk/gobmp/pkg/message" -) - -// processEdge processes a single ipv6 ls_prefix entry which is connected to a node -func (a *arangoDB) processLSv6PrefixEdge(ctx context.Context, key string, p *message.LSPrefix) error { - //glog.V(9).Infof("processEdge processing lsprefix: %s", l.ID) - - // filter out IPv6, ls link, and loopback prefixes - if p.MTID == nil || p.PrefixLen == 126 || p.PrefixLen == 127 || p.PrefixLen == 128 { - return nil - } - - // get remote node from ls_link entry - lsnode, err := a.getLSv6Node(ctx, p, false) - if err != nil { - glog.Errorf("processEdge failed to get remote lsnode %s for link: %s with error: %+v", p.IGPRouterID, p.ID, err) - return err - } - if err := a.createLSv6PrefixEdgeObject(ctx, p, lsnode); err != nil { - glog.Errorf("processEdge failed to create Edge object with error: %+v", err) - return err - } - //glog.V(9).Infof("processEdge completed processing lsprefix: %s for ls nodes: %s - %s", l.ID, ln.ID, rn.ID) - - return nil -} - -// processEdgeRemoval removes a record from Node's graph collection -func (a *arangoDB) processv6PrefixRemoval(ctx context.Context, key string, action string) error { - if _, err := a.graphv6.RemoveDocument(ctx, key); err != nil { - glog.Infof("removing edge %s", key) - if !driver.IsNotFound(err) { - return err - } - return nil - } - - return nil -} - -func (a *arangoDB) getLSv6Node(ctx context.Context, p *message.LSPrefix, local bool) (*message.LSNode, error) { - // Need to find ls_node object matching ls_prefix's IGP Router ID - query := "FOR d IN ls_node_extended" //+ a.lsnodeExt.Name() - query += " filter d.igp_router_id == " + "\"" + p.IGPRouterID + "\"" - query += " filter d.domain_id == " + strconv.Itoa(int(p.DomainID)) - - // If OSPFv2 or OSPFv3, then query must include AreaID - if p.ProtocolID == base.OSPFv2 || p.ProtocolID == base.OSPFv3 { - query += " filter d.area_id == " + "\"" + p.AreaID + "\"" - } - query += " return d" - //glog.Infof("query: %s", query) - lcursor, err := a.db.Query(ctx, query, nil) - if err != nil { - return nil, err - } - defer lcursor.Close() - var ln message.LSNode - i := 0 - for ; ; i++ { - _, err := lcursor.ReadDocument(ctx, &ln) - if err != nil { - if !driver.IsNoMoreDocuments(err) { - return nil, err - } - break - } - } - if i == 0 { - return nil, fmt.Errorf("query %s returned 0 results", query) - } - if i > 1 { - return nil, fmt.Errorf("query %s returned more than 1 result", query) - } - - return &ln, nil -} - -func (a *arangoDB) createLSv6PrefixEdgeObject(ctx context.Context, l *message.LSPrefix, ln *message.LSNode) error { - mtid := 0 - if l.MTID != nil { - mtid = int(l.MTID.MTID) - } - ne := lsTopologyObject{ - Key: l.Key, - From: ln.ID, - To: l.ID, - Link: l.Key, - ProtocolID: l.ProtocolID, - DomainID: l.DomainID, - MTID: uint16(mtid), - AreaID: l.AreaID, - Protocol: l.Protocol, - LocalNodeASN: ln.ASN, - Prefix: l.Prefix, - PrefixLen: l.PrefixLen, - PrefixMetric: l.PrefixMetric, - PrefixAttrTLVs: l.PrefixAttrTLVs, - } - if _, err := a.graphv6.CreateDocument(ctx, &ne); err != nil { - if !driver.IsConflict(err) { - return err - } - // The document already exists, updating it with the latest info - if _, err := a.graphv6.UpdateDocument(ctx, ne.Key, &ne); err != nil { - return err - } - } - - return nil -} diff --git a/igp-graph/messenger/msg-server.go b/igp-graph/messenger/msg-server.go index 087a9445..16c09f16 100755 --- a/igp-graph/messenger/msg-server.go +++ b/igp-graph/messenger/msg-server.go @@ -1,3 +1,25 @@ +// Copyright (c) 2024 Cisco Systems, Inc. and its affiliates +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// The contents of this file are licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + package messenger // Srv defines required method of a message server diff --git a/igp-graph/types.go b/igp-graph/types.go deleted file mode 100644 index 8aa63fb3..00000000 --- a/igp-graph/types.go +++ /dev/null @@ -1,108 +0,0 @@ -package arangodb - -import ( - "github.com/sbezverk/gobmp/pkg/base" - "github.com/sbezverk/gobmp/pkg/bgpls" - "github.com/sbezverk/gobmp/pkg/sr" - "github.com/sbezverk/gobmp/pkg/srv6" -) - -type duplicateNode struct { - Key string `json:"_key,omitempty"` - DomainID int64 `json:"domain_id"` - IGPRouterID string `json:"igp_router_id,omitempty"` - //AreaID string `json:"area_id"` - Protocol string `json:"protocol,omitempty"` - ProtocolID base.ProtoID `json:"protocol_id,omitempty"` - Name string `json:"name,omitempty"` -} - -type srObject struct { - PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs,omitempty"` -} - -type LSNodeExt struct { - Key string `json:"_key,omitempty"` - ID string `json:"_id,omitempty"` - Rev string `json:"_rev,omitempty"` - Action string `json:"action,omitempty"` // Action can be "add" or "del" - Sequence int `json:"sequence,omitempty"` - Hash string `json:"hash,omitempty"` - RouterHash string `json:"router_hash,omitempty"` - DomainID int64 `json:"domain_id"` - RouterIP string `json:"router_ip,omitempty"` - PeerHash string `json:"peer_hash,omitempty"` - PeerIP string `json:"peer_ip,omitempty"` - PeerASN uint32 `json:"peer_asn,omitempty"` - Timestamp string `json:"timestamp,omitempty"` - IGPRouterID string `json:"igp_router_id,omitempty"` - RouterID string `json:"router_id,omitempty"` - ASN uint32 `json:"asn,omitempty"` - LSID uint32 `json:"ls_id,omitempty"` - MTID []*base.MultiTopologyIdentifier `json:"mt_id_tlv,omitempty"` - Protocol string `json:"protocol,omitempty"` - ProtocolID base.ProtoID `json:"protocol_id,omitempty"` - NodeFlags *bgpls.NodeAttrFlags `json:"node_flags,omitempty"` - Name string `json:"name,omitempty"` - SRCapabilities *sr.Capability `json:"ls_sr_capabilities,omitempty"` - SRAlgorithm []int `json:"sr_algorithm,omitempty"` - SRLocalBlock *sr.LocalBlock `json:"sr_local_block,omitempty"` - SRv6CapabilitiesTLV *srv6.CapabilityTLV `json:"srv6_capabilities_tlv,omitempty"` - NodeMSD []*base.MSDTV `json:"node_msd,omitempty"` - FlexAlgoDefinition []*bgpls.FlexAlgoDefinition `json:"flex_algo_definition,omitempty"` - IsPrepolicy bool `json:"is_prepolicy"` - IsAdjRIBIn bool `json:"is_adj_rib_in"` - Prefix string `json:"prefix,omitempty"` - PrefixLen int32 `json:"prefix_len,omitempty"` - PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs,omitempty"` - PrefixSID []*sr.PrefixSIDTLV `json:"prefix_sid_tlv,omitempty"` - FlexAlgoPrefixMetric []*bgpls.FlexAlgoPrefixMetric `json:"flex_algo_prefix_metric,omitempty"` - SRv6SID string `json:"srv6_sid,omitempty"` - SIDS []SID `json:"sids,omitempty"` -} - -type SID struct { - SRv6SID string `json:"srv6_sid,omitempty"` - SRv6EndpointBehavior *srv6.EndpointBehavior `json:"srv6_endpoint_behavior,omitempty"` - SRv6BGPPeerNodeSID *srv6.BGPPeerNodeSID `json:"srv6_bgp_peer_node_sid,omitempty"` - SRv6SIDStructure *srv6.SIDStructure `json:"srv6_sid_structure,omitempty"` -} - -type peerObject struct { - BGPRouterID string `json:"bgp_router_id,omitempty"` -} - -type lsTopologyObject struct { - Key string `json:"_key"` - From string `json:"_from"` - To string `json:"_to"` - Link string `json:"link"` - ProtocolID base.ProtoID `json:"protocol_id"` - DomainID int64 `json:"domain_id"` - MTID uint16 `json:"mt_id"` - AreaID string `json:"area_id"` - Protocol string `json:"protocol"` - LocalLinkID uint32 `json:"local_link_id"` - RemoteLinkID uint32 `json:"remote_link_id"` - LocalLinkIP string `json:"local_link_ip"` - RemoteLinkIP string `json:"remote_link_ip"` - LocalNodeASN uint32 `json:"local_node_asn"` - RemoteNodeASN uint32 `json:"remote_node_asn"` - PeerNodeSID *sr.PeerSID `json:"peer_node_sid,omitempty"` - PeerAdjSID *sr.PeerSID `json:"peer_adj_sid,omitempty"` - PeerSetSID *sr.PeerSID `json:"peer_set_sid,omitempty"` - SRv6BGPPeerNodeSID *srv6.BGPPeerNodeSID `json:"srv6_bgp_peer_node_sid,omitempty"` - SRv6ENDXSID []*srv6.EndXSIDTLV `json:"srv6_endx_sid,omitempty"` - LSAdjacencySID []*sr.AdjacencySIDTLV `json:"ls_adjacency_sid,omitempty"` - UnidirLinkDelay uint32 `json:"unidir_link_delay"` - UnidirLinkDelayMinMax []uint32 `json:"unidir_link_delay_min_max"` - UnidirDelayVariation uint32 `json:"unidir_delay_variation,omitempty"` - UnidirPacketLoss uint32 `json:"unidir_packet_loss,omitempty"` - UnidirResidualBW uint32 `json:"unidir_residual_bw,omitempty"` - UnidirAvailableBW uint32 `json:"unidir_available_bw,omitempty"` - UnidirBWUtilization uint32 `json:"unidir_bw_utilization,omitempty"` - Prefix string `json:"prefix"` - PrefixLen int32 `json:"prefix_len"` - PrefixMetric uint32 `json:"prefix_metric"` - PrefixAttrTLVs *bgpls.PrefixAttrTLVs `json:"prefix_attr_tlvs"` -}