diff --git a/src/go/rucio-dataset-mon-go/configs/setup.go b/src/go/rucio-dataset-mon-go/configs/setup.go deleted file mode 100644 index db18378b..00000000 --- a/src/go/rucio-dataset-mon-go/configs/setup.go +++ /dev/null @@ -1,38 +0,0 @@ -package configs - -import ( - "context" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "log" - "time" -) - -// ConnectDB mongo.Client which is MongoDB connection client -func ConnectDB() *mongo.Client { - client, err := mongo.NewClient(options.Client().ApplyURI(EnvMongoURI())) - if err != nil { - log.Fatal(err) - } - - ctx, _ := context.WithTimeout(context.Background(), time.Duration(EnvConnTimeout())*time.Second) - if err = client.Connect(ctx); err != nil { - log.Fatal(err) - } - - //ping the database - if err := client.Ping(ctx, nil); err != nil { - log.Fatal(err) - } - log.Println("Connected to MongoDB") - return client -} - -// DB MongoDb Client instance(mongo.Client) -var DB = ConnectDB() - -// GetCollection returns Mongo db collection with the given collection name -func GetCollection(client *mongo.Client, collectionName string) *mongo.Collection { - collection := client.Database(EnvMongoDB()).Collection(collectionName) - return collection -} diff --git a/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go b/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go index 730a1ec2..83c5cdf8 100644 --- a/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go +++ b/src/go/rucio-dataset-mon-go/controllers/dataset_controller.go @@ -5,158 +5,109 @@ import ( "encoding/json" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/models" - "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/responses" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/mongo" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/utils" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "io" "log" "net/http" - "strings" "time" ) var ( - // connection instance for the "datasets" collection - datasetCollection = configs.GetCollection(configs.DB, configs.GetEnvVar("COLLECTION_DATASETS")) + // connection instance for the "datasets" datasetsDb + datasetsDb = mongo.GetCollection(mongo.DBClient, configs.GetEnvVar("COLLECTION_DATASETS")) GlobalTotalRecCount int64 ) -// MiddlewareReqHandler handles CORS and HTTP request settings for the context router -func MiddlewareReqHandler() gin.HandlerFunc { - return func(c *gin.Context) { - //c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.Header().Set("Access-Control-Allow-Origin", "*") - c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") - c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") - c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") - if c.Request.Method == "OPTIONS" { - c.AbortWithStatus(204) - return - } - c.Next() - } -} - -// errorResponse returns error response with given msg and error -func errorResponse(c *gin.Context, msg string, err error) { - log.Printf("[ERROR] %s %s", msg, err) - c.JSON(http.StatusInternalServerError, - responses.ErrorResponseStruct{ - Status: http.StatusInternalServerError, - Message: msg, - Data: map[string]string{"data": err.Error()}, - }) -} - -// getRequestBody parse datatable request +// getRequestBody parse datatables request func getRequestBody(c *gin.Context) models.DataTableRequest { // === ~~~~~~ Decode incoming DataTable request json ~~~~~~ === log.SetFlags(log.LstdFlags | log.Lshortfile) body, err := io.ReadAll(c.Request.Body) if err != nil { - errorResponse(c, "Request body read failed", err) + utils.ErrorResponse(c, "Request body read failed", err) } log.Printf("Request Body:\n%#v\n", string(body)) var dataTableRequest models.DataTableRequest if err := json.Unmarshal(body, &dataTableRequest); err != nil { - errorResponse(c, "Unmarshal request body failed", err) + utils.ErrorResponse(c, "Unmarshal request body failed", err) } return dataTableRequest } -// getTotalRecCount total document count in the collection -func getTotalRecCount(ctx context.Context, c *gin.Context) int64 { - if GlobalTotalRecCount == 0 { - countTotal, err := datasetCollection.CountDocuments(ctx, bson.M{}) - if err != nil { - errorResponse(c, "TotalRecCount query failed", err) - } - GlobalTotalRecCount = countTotal - } - log.Printf("[INFO] Total Count %d", GlobalTotalRecCount) - return GlobalTotalRecCount -} - -// getFilteredRecCount filtered document count in the collection -func getFilteredRecCount(ctx context.Context, c *gin.Context, findQuery bson.M) int64 { - filteredRecCount, err := datasetCollection.CountDocuments(ctx, findQuery) - if err != nil { - errorResponse(c, "FilteredRecCount query failed", err) - } - return filteredRecCount -} - -// convertOrderEnumToMongoInt converts DataTable enums ("asc" and "desc") to Mongo sorting integer definitions (1,-1) -func convertOrderEnumToMongoInt(dir string) int { - switch strings.ToLower(dir) { - case "asc": - return 1 - case "desc": - return -1 - default: - return 0 - } -} - -// generateSortOrder creates sort order for the Mongo query by iterating over DataTable json request -func generateSortOrder(dataTableRequest models.DataTableRequest) *options.FindOptions { +// getSortBson creates sort object which support multiple column sort +func getSortBson(dataTableRequest models.DataTableRequest) bson.D { orders := dataTableRequest.Orders columns := dataTableRequest.Columns - sortOpts := options.Find() - sortOpts.Limit = &dataTableRequest.Length - sortOpts.Skip = &dataTableRequest.Start + sortBson := bson.D{} for _, order := range orders { - intSortDirection := convertOrderEnumToMongoInt(order.Dir) + intSortDirection := utils.ConvertOrderEnumToMongoInt(order.Dir) if intSortDirection != 0 { - sortOpts = sortOpts.SetSort( - bson.D{{ - Key: columns[order.Column].Data, // column name - Value: intSortDirection, // column direction as int value - }}, - ) + sortBson = append(sortBson, bson.E{ + Key: columns[order.Column].Data, // column name + Value: intSortDirection, // column direction as int value (0/1) + }) } } - return sortOpts + // Always add unique id column at the end to be able fetch non-unique columns in order + sortBson = append(sortBson, bson.E{Key: "_id", Value: 1}) + return sortBson } -// generateFindQuery creates main search query using regex by default -func generateFindQuery(dataTableRequest models.DataTableRequest) bson.M { +// getSearchBson creates main search query using regex by default +func getSearchBson(dataTableRequest models.DataTableRequest) bson.M { // DataTable main search request struct dtMainSearch := dataTableRequest.Search if dtMainSearch.Value == "" { return bson.M{} } else { log.Printf("[INFO] dtMainSearch.Value is : %s", dtMainSearch.Value) - var findQuery []bson.M - findQuery = append(findQuery, bson.M{"dataset": primitive.Regex{Pattern: dtMainSearch.Value, Options: "im"}}) - // i: case insensitive, m: can use ^ and $. Ref: // https://www.mongodb.com/docs/v5.0/reference/operator/query/regex/ // TODO add individual column search - return bson.M{"$and": findQuery} + return bson.M{"$and": bson.M{"Dataset": primitive.Regex{Pattern: dtMainSearch.Value, Options: "im"}}} } } -// paginateResults get query results efficiently -func paginateResults(ctx context.Context, c *gin.Context, findQuery bson.M, sortOptsOfFind *options.FindOptions) []models.Dataset { - var datasets []models.Dataset - datasetResults, err := datasetCollection.Find(ctx, findQuery, sortOptsOfFind) +// getTotalRecCount total document count in the datasetsDb +func getTotalRecCount(ctx context.Context, c *gin.Context) int64 { + if GlobalTotalRecCount == 0 { + countTotal, err := mongo.GetCount(ctx, datasetsDb, bson.M{}) + if err != nil { + utils.ErrorResponse(c, "TotalRecCount query failed", err) + } + GlobalTotalRecCount = countTotal + } + log.Printf("[INFO] Total Count %d", GlobalTotalRecCount) + return GlobalTotalRecCount +} + +// getFilteredRecCount filtered document count in the datasetsDb +func getFilteredRecCount(ctx context.Context, c *gin.Context, findQuery bson.M) int64 { + filteredRecCount, err := mongo.GetCount(ctx, datasetsDb, findQuery) if err != nil { - errorResponse(c, "datasetCollection.Find query failed", err) + utils.ErrorResponse(c, "FilteredRecCount query failed", err) } + return filteredRecCount +} - // reading from the db in an optimal way - defer func(results *mongo.Cursor, ctx context.Context) { - if err := results.Close(ctx); err != nil { - errorResponse(c, "MongoDB cursor failed", err) - } - }(datasetResults, ctx) +// getQueryResults get query results efficiently +func getQueryResults(ctx context.Context, c *gin.Context, dataTableRequest models.DataTableRequest) []models.Dataset { + var datasets []models.Dataset + searchQuery := getSearchBson(dataTableRequest) + sortQuery := getSortBson(dataTableRequest) + limit := dataTableRequest.Length + skip := dataTableRequest.Start - for datasetResults.Next(ctx) { + datasetsCursor, err := mongo.GetResults(ctx, datasetsDb, searchQuery, sortQuery, skip, limit) + if err != nil { + utils.ErrorResponse(c, "datasetsDb.Find query failed", err) + } + for datasetsCursor.Next(ctx) { var singleDataset models.Dataset - if err = datasetResults.Decode(&singleDataset); err != nil { - errorResponse(c, "datasetResults.Decode failed", err) + if err = datasetsCursor.Decode(&singleDataset); err != nil { + utils.ErrorResponse(c, "datasetResults.Decode failed", err) } datasets = append(datasets, singleDataset) } @@ -171,10 +122,8 @@ func GetDatasets() gin.HandlerFunc { dataTableRequest := getRequestBody(c) totalRecCount := getTotalRecCount(ctx, c) - sortOptsOfFind := generateSortOrder(dataTableRequest) - findQuery := generateFindQuery(dataTableRequest) - filteredRecCount := getFilteredRecCount(ctx, c, findQuery) - datasets := paginateResults(ctx, c, findQuery, sortOptsOfFind) + filteredRecCount := getFilteredRecCount(ctx, c, getSearchBson(dataTableRequest)) + datasets := getQueryResults(ctx, c, dataTableRequest) // Send response in DataTable required format // - Need to return exactly same "Draw" value that DataTable sent in incoming request diff --git a/src/go/rucio-dataset-mon-go/main.go b/src/go/rucio-dataset-mon-go/main.go index 2a5adc75..c4aae4f6 100644 --- a/src/go/rucio-dataset-mon-go/main.go +++ b/src/go/rucio-dataset-mon-go/main.go @@ -3,8 +3,8 @@ package main import ( "flag" "fmt" - "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/controllers" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/mongo" "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/routes" "golang.org/x/sync/errgroup" "log" @@ -40,7 +40,7 @@ func main() { } // connect to database - configs.ConnectDB() + mongo.GetMongoClient() controllers.GitVersion = gitVersion controllers.ServerInfo = info() diff --git a/src/go/rucio-dataset-mon-go/models/dataset_model.go b/src/go/rucio-dataset-mon-go/models/dataset_model.go index df9fee58..481d6f60 100644 --- a/src/go/rucio-dataset-mon-go/models/dataset_model.go +++ b/src/go/rucio-dataset-mon-go/models/dataset_model.go @@ -1,16 +1,14 @@ package models -import "go.mongodb.org/mongo-driver/bson/primitive" - // Dataset struct which includes Rucio and DBS calculated values type Dataset struct { - Id primitive.ObjectID `bson:"_id"` - RseType string `bson:"RSE_TYPE,omitempty" validate:"required"` - Dataset string `bson:"dataset,omitempty" validate:"required"` - LastAccess string `bson:"last_access"` - LastAccessMs int64 `bson:"last_access_ms"` - MaxDatasetSizeInRsesGB float64 `bson:"max_dataset_size_in_rses(GB)"` - MinDatasetSizeInRsesGB float64 `bson:"min_dataset_size_in_rses(GB)"` - SumDatasetSizeInRsesGB float64 `bson:"sum_dataset_size_in_rses(GB)"` - RSEs string `bson:"RSE(s)"` + RseType string `bson:"RseType,omitempty" validate:"required"` + Dataset string `bson:"Dataset,omitempty" validate:"required"` + LastAccess string `bson:"LastAccess"` + LastAccessMs int64 `bson:"LastAccessMs"` + MaxDatasetSizeInRsesGB float64 `bson:"MaxDatasetSizeInRsesGB"` + MinDatasetSizeInRsesGB float64 `bson:"MinDatasetSizeInRsesGB"` + AvgDatasetSizeInRsesGB float64 `bson:"AvgDatasetSizeInRsesGB"` + SumDatasetSizeInRsesGB float64 `bson:"SumDatasetSizeInRsesGB"` + RSEs string `bson:"RSEs"` } diff --git a/src/go/rucio-dataset-mon-go/mongo/mongo.go b/src/go/rucio-dataset-mon-go/mongo/mongo.go new file mode 100644 index 00000000..99de6cb4 --- /dev/null +++ b/src/go/rucio-dataset-mon-go/mongo/mongo.go @@ -0,0 +1,25 @@ +package mongo + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +// GetResults returns cursor of aggregate query results +func GetResults(ctx context.Context, collection *mongo.Collection, match bson.M, sort bson.D, skip int64, limit int64) (*mongo.Cursor, error) { + pipeline := []bson.M{ + {"$match": match}, + {"$sort": sort}, + {"$skip": skip}, + {"$limit": limit}, + } + cursor, err := collection.Aggregate(ctx, pipeline) + return cursor, err +} + +// GetCount returns count of query result +func GetCount(ctx context.Context, collection *mongo.Collection, match bson.M) (int64, error) { + count, err := collection.CountDocuments(ctx, match) + return count, err +} diff --git a/src/go/rucio-dataset-mon-go/mongo/setup.go b/src/go/rucio-dataset-mon-go/mongo/setup.go new file mode 100644 index 00000000..9067a609 --- /dev/null +++ b/src/go/rucio-dataset-mon-go/mongo/setup.go @@ -0,0 +1,36 @@ +package mongo + +import ( + "context" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/configs" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "log" + "time" +) + +// DBClient MongoDb Client instance(mongo.Client) +var DBClient = GetMongoClient() + +// GetMongoClient returns mongo client +func GetMongoClient() *mongo.Client { + var err error + var client *mongo.Client + opts := options.Client() + opts.ApplyURI(configs.EnvMongoURI()) + opts.SetMaxPoolSize(100) + opts.SetConnectTimeout(time.Duration(configs.EnvConnTimeout()) * time.Second) + if client, err = mongo.Connect(context.Background(), opts); err != nil { + log.Fatal(err) + } + if err := client.Ping(context.Background(), nil); err != nil { + log.Fatal(err) + } + return client +} + +// GetCollection returns Mongo db collection with the given collection name +func GetCollection(client *mongo.Client, collectionName string) *mongo.Collection { + collection := client.Database(configs.EnvMongoDB()).Collection(collectionName) + return collection +} diff --git a/src/go/rucio-dataset-mon-go/responses/error_dt_response.go b/src/go/rucio-dataset-mon-go/responses/error_dt_response.go deleted file mode 100644 index fbdaa237..00000000 --- a/src/go/rucio-dataset-mon-go/responses/error_dt_response.go +++ /dev/null @@ -1,8 +0,0 @@ -package responses - -// ErrorResponseStruct custom response struct, used in case of error -type ErrorResponseStruct struct { - Status int `json:"status"` - Message string `json:"message"` - Data map[string]string `json:"data"` -} diff --git a/src/go/rucio-dataset-mon-go/routes/main_router.go b/src/go/rucio-dataset-mon-go/routes/main_router.go index fb235686..6fbb14c8 100644 --- a/src/go/rucio-dataset-mon-go/routes/main_router.go +++ b/src/go/rucio-dataset-mon-go/routes/main_router.go @@ -2,6 +2,7 @@ package routes import ( "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/controllers" + "github.com/dmwm/CMSMonitoring/src/go/rucio-dataset-mon-go/utils" "github.com/gin-gonic/gin" "net/http" ) @@ -12,11 +13,12 @@ func MainRouter() http.Handler { e.Use(gin.Recovery()) // REST - e.Use(controllers.MiddlewareReqHandler()) + e.Use(utils.MiddlewareReqHandler()) e.POST("/api/datasets", controllers.GetDatasets()) + e.Static("/static/img", "./static/img") // Static - e.LoadHTMLGlob("static/*") + e.LoadHTMLGlob("static/index.html") e.GET("/", controllers.GetIndexPage) e.GET("/serverinfo", controllers.GetServerInfo) diff --git a/src/go/rucio-dataset-mon-go/static/img/oc_icon.png b/src/go/rucio-dataset-mon-go/static/img/oc_icon.png new file mode 100644 index 00000000..5ca9c127 Binary files /dev/null and b/src/go/rucio-dataset-mon-go/static/img/oc_icon.png differ diff --git a/src/go/rucio-dataset-mon-go/static/index.html b/src/go/rucio-dataset-mon-go/static/index.html index bd9073ec..df713f22 100644 --- a/src/go/rucio-dataset-mon-go/static/index.html +++ b/src/go/rucio-dataset-mon-go/static/index.html @@ -1,64 +1,166 @@ - - + - - + Go Datatable (Serverside) - + + + + + - Go Datatable (Serverside) + + + + +
+ CMS +

+ RUCIO DATASETS MONITORING +

+
-
-

Go Datatable Serverside

+
- - + + + - - + + + + + + +
Id RseType Dataset LastAccess LastAccessMsMaxDatasetSizeInRsesGB(MaxDatasetSizeInRsesGB MinDatasetSizeInRsesGBAvgDatasetSizeInRsesGB SumDatasetSizeInRsesGB RSEs
RseType DatasetRseSizeLastAccessLastAccessMsMaxDatasetSizeInRsesGBMinDatasetSizeInRsesGBAvgDatasetSizeInRsesGBSumDatasetSizeInRsesGBRSEs
- - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/go/rucio-dataset-mon-go/utils/utils.go b/src/go/rucio-dataset-mon-go/utils/utils.go new file mode 100644 index 00000000..fe986a1e --- /dev/null +++ b/src/go/rucio-dataset-mon-go/utils/utils.go @@ -0,0 +1,54 @@ +package utils + +import ( + "github.com/gin-gonic/gin" + "log" + "net/http" + "strings" +) + +// ErrorResponseStruct custom response struct, used in case of error +type ErrorResponseStruct struct { + Status int `json:"status"` + Message string `json:"message"` + Data map[string]string `json:"data"` +} + +// MiddlewareReqHandler handles CORS and HTTP request settings for the context router +func MiddlewareReqHandler() gin.HandlerFunc { + return func(c *gin.Context) { + //c.Writer.Header().Set("Content-Type", "application/json") + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + c.Next() + } +} + +// ErrorResponse returns error response with given msg and error +func ErrorResponse(c *gin.Context, msg string, err error) { + log.Printf("[ERROR] %s %s", msg, err) + c.JSON(http.StatusInternalServerError, + ErrorResponseStruct{ + Status: http.StatusInternalServerError, + Message: msg, + Data: map[string]string{"data": err.Error()}, + }) +} + +// ConvertOrderEnumToMongoInt converts DataTable enums ("asc" and "desc") to Mongo sorting integer definitions (1,-1) +func ConvertOrderEnumToMongoInt(dir string) int { + switch strings.ToLower(dir) { + case "asc": + return 1 + case "desc": + return -1 + default: + return 0 + } +}