diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..68e24a4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..32cfd57 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,33 @@ +name: goreleaser + +on: + pull_request: + push: + +permissions: + contents: write + +jobs: + goreleaser: + env: + GORELEASER_FLAGS: + runs-on: ubuntu-latest + steps: + - if: ${{ !startsWith(github.ref, 'refs/tags/20') }} + run: echo "GORELEASER_FLAGS=--snapshot" >> $GITHUB_ENV + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean ${{ env.GORELEASER_FLAGS }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0677ae8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +dist +clibana \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..d04a9ec --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,16 @@ +linters: + enable-all: true + disable: + # Dont care: + - exhaustruct + - depguard + - wrapcheck + - tagalign + # Deprecated: + - execinquery + - gomnd + - perfsprint + - gochecknoglobals + +issues: + exclude-rules: [] diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..fc7896b --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,44 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy + # you may remove this if you don't need go generate + - go generate ./... +builds: +- env: + - CGO_ENABLED=0 + - GOEXPERIMENT=rangefunc + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + +archives: +- format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ {{- title .Os }}_ {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "386" }}i386 {{- else }}{{ .Arch }}{{ end }} {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + +# The lines beneath this are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/README.md b/README.md new file mode 100644 index 0000000..da2e031 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Clibana +![GitHub release (with filter)](https://img.shields.io/github/v/release/ivoronin/clibana) +[![Go Report Card](https://goreportcard.com/badge/github.com/ivoronin/clibana)](https://goreportcard.com/report/github.com/ivoronin/clibana) +![GitHub last commit (branch)](https://img.shields.io/github/last-commit/ivoronin/clibana/main) +![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/ivoronin/clibana/main.yml) +![GitHub top language](https://img.shields.io/github/languages/top/ivoronin/clibana) + +## Description + +Clibana is a command-line interface (CLI) tool for OpenSearch that offers Kibana-like log searching and live tailing (`-f` support) capabilities. + +## Features + +- **Log Search**: Execute searches on multiple OpenSearch indices using Lucene query syntax. Clibana can output specified fields or export data in NDJSON format. +- **Live Tailing**: Monitor logs in real-time using the `-f` option, similar to `tail -f`. +- **Cluster Exploration**: List index information and field mappings. +- **AWS and Basic Authentication**: Supports both AWS SigV4 and Basic Authentication methods. + +## Examples + +```bash +clibana -H https://logs.internal -i "k8s.containers.*" search -s now-2h -e now-1h "kubernetes.pod_name:*nginx*" +clibana -H https://logs.internal -i "k8s.containers.*" mappings +clibana -H https://logs.internal -i "k8s.containers.*" indices +``` + +Most options can be set using environment variables. Check `clibana` -h for additional details. + +### AWS Support + +1. Clibana supports the `aws://` scheme to specify an AWS Managed OpenSearch Domain name as a host, which will automatically resolve to its endpoint. Example: `clibana --host aws://logs-internal`. +2. You can use your AWS credentials to authenticate to an AWS Managed OpenSearch Domain. Set the authentication type to `aws`: `clibana -a aws`. \ No newline at end of file diff --git a/aws.go b/aws.go new file mode 100644 index 0000000..695129b --- /dev/null +++ b/aws.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awsopensearch "github.com/aws/aws-sdk-go-v2/service/opensearch" + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/signer/awsv2" +) + +var awsConfig *aws.Config + +func awsInit() { + if awsConfig == nil { + config, err := awsconfig.LoadDefaultConfig(context.TODO()) + if err != nil { + FatalError(fmt.Errorf("failed to load AWS config: %w", err)) + } + + awsConfig = &config + } +} + +func resolveAWSDomainEndpoint(domainName string) string { + awsInit() + + aosClient := awsopensearch.NewFromConfig(*awsConfig) + + domain, err := aosClient.DescribeDomain(context.TODO(), &awsopensearch.DescribeDomainInput{ + DomainName: &domainName, + }) + if err != nil { + FatalError(fmt.Errorf("failed to describe OpenSearch domain: %w", err)) + } + + var endpoint string + + switch { + case domain.DomainStatus.EndpointV2 != nil: + endpoint = *domain.DomainStatus.EndpointV2 + case domain.DomainStatus.Endpoint != nil: + endpoint = *domain.DomainStatus.Endpoint + case domain.DomainStatus.Endpoints != nil: + switch { + case domain.DomainStatus.Endpoints["vpcv2"] != "": + endpoint = domain.DomainStatus.Endpoints["vpcv2"] + case domain.DomainStatus.Endpoints["vpc"] != "": + endpoint = domain.DomainStatus.Endpoints["vpc"] + } + } + + if endpoint == "" { + FatalError(fmt.Errorf("no endpoints found for OpenSearch domain: %s", domainName)) + } + + return "https://" + endpoint +} + +func buildAWSAuthClientConfig() opensearch.Config { + awsInit() + + signer, err := awsv2.NewSigner(*awsConfig) + if err != nil { + FatalError(fmt.Errorf("failed to create AWS V4 signer: %w", err)) + } + + return opensearch.Config{ + Signer: signer, + } +} diff --git a/client.go b/client.go new file mode 100644 index 0000000..5e026e9 --- /dev/null +++ b/client.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" +) + +func buildClientConfig(clibanaConfig ClibanaConfig) opensearch.Config { + var opensearchConfig opensearch.Config + + switch strings.ToLower(clibanaConfig.AuthType) { + case AuthTypeAWS: + opensearchConfig = buildAWSAuthClientConfig() + case AuthTypeBasic: + opensearchConfig = buildBasicAuthClientConfig(clibanaConfig) + default: + FatalError(fmt.Errorf("unsupported authentication type: %s", clibanaConfig.AuthType)) + } + + opensearchConfig.Addresses = []string{clibanaConfig.Host} + opensearchConfig.Transport = &http.Transport{ + ResponseHeaderTimeout: ResponseTimeout * time.Second, + } + + return opensearchConfig +} + +func createClient(clibanaConfig ClibanaConfig) (*opensearch.Client, error) { + if strings.HasPrefix(clibanaConfig.Host, "aws://") { + domainName := strings.TrimPrefix(clibanaConfig.Host, "aws://") + clibanaConfig.Host = resolveAWSDomainEndpoint(domainName) + } + + return opensearch.NewClient(buildClientConfig(clibanaConfig)) +} + +func buildBasicAuthClientConfig(config ClibanaConfig) opensearch.Config { + if config.Username == "" || config.Password == "" { + FatalError(fmt.Errorf("uusername and password must be provided for basic authentication")) + } + + return opensearch.Config{ + Username: config.Username, + Password: config.Password, + } +} + +func doRequest[T any](client *opensearch.Client, request opensearchapi.Request) T { + DebugLogger.Printf("Request: %+v\n", request) + + response, err := request.Do(context.TODO(), client) + if err != nil { + FatalError(fmt.Errorf("request failed: %w", err)) + } + + defer response.Body.Close() + + DebugLogger.Printf("Response: %+v\n", response) + + if response.IsError() { + FatalError(fmt.Errorf("request error: %s", response.String())) + } + + var responseObj T + + if err := json.NewDecoder(response.Body).Decode(&responseObj); err != nil { + FatalError(fmt.Errorf("error decoding response: %w", err)) + } + + return responseObj +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..cced247 --- /dev/null +++ b/config.go @@ -0,0 +1,59 @@ +package main + +import ( + "strings" + + "github.com/alexflint/go-arg" +) + +type SearchConfig struct { + Fields fieldList `arg:"-o,env:CLIBANA_FIELDS" help:"Comma-separated list of fields to output"` + Follow bool `arg:"-f" help:"Enable live tailing of logs"` + Query string `arg:"positional" default:"*" help:"Query string"` + Start string `arg:"-s" default:"now-5m" help:"Start time"` + End string `arg:"-e" default:"now" help:"End time"` +} + +type MappingsConfig struct { + Quiet bool `arg:"-q" help:"Do not show headers"` +} + +type IndicesConfig struct { + Quiet bool `arg:"-q" help:"Do not show headers"` +} + +type ClibanaConfig struct { + Host string `arg:"-H,required,env:CLIBANA_HOST" help:"http[s]://host[:port] or aws://cluster-name"` + Index string `arg:"-i,required,env:CLIBANA_INDEX" help:"Index pattern"` + AuthType string `arg:"-a,env:CLIBANA_AUTH" default:"basic" help:"Authentication type: aws or basic"` + Username string `arg:"-u,env:CLIBANA_USER" help:"Username for basic authentication"` + Password string `arg:"-p,env:CLIBANA_PASSWORD" help:"Password for basic authentication"` + Debug bool `arg:"-d,env:CLIBANA_DEBUG" help:"Enable debug output"` + Search *SearchConfig `arg:"subcommand:search" help:"Search indices matching the index pattern"` + Mappings *MappingsConfig `arg:"subcommand:mappings" help:"Show field mappings in the indices matching the index pattern"` + Indices *IndicesConfig `arg:"subcommand:indices" help:"List indices matching the index pattern"` +} + +func NewClibanaConfig() ClibanaConfig { + var clibanaConfig ClibanaConfig + + arg.MustParse(&clibanaConfig) + + return clibanaConfig +} + +func (ClibanaConfig) Description() string { + return "Clibana - OpenSearch log tailer" +} + +func (ClibanaConfig) Epilogue() string { + return "For more information, see https://github.com/ivoronin/clibana" +} + +type fieldList []string + +func (c *fieldList) UnmarshalText(text []byte) error { //nolint:unparam + *c = strings.Split(string(text), ",") + + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cafc30f --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module github.com/ivoronin/clibana + +go 1.22.4 + +require ( + github.com/alexflint/go-arg v1.5.1 + github.com/aws/aws-sdk-go-v2 v1.30.3 + github.com/aws/aws-sdk-go-v2/config v1.27.27 + github.com/aws/aws-sdk-go-v2/service/opensearch v1.39.2 + github.com/opensearch-project/opensearch-go/v2 v2.3.0 +) + +require ( + github.com/alexflint/go-scalar v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..189bc28 --- /dev/null +++ b/go.sum @@ -0,0 +1,100 @@ +github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y= +github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= +github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw= +github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= +github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= +github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= +github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= +github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/opensearch v1.39.2 h1:px8DLC+DOd2fCLnMm6XlyeLU/9B0dXZWzYXzHSKAzZY= +github.com/aws/aws-sdk-go-v2/service/opensearch v1.39.2/go.mod h1:91AFffUmnw/bumAEE6Sf1yWgW3YdsjexH5c6hePGwSQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= +github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/indices.go b/indices.go new file mode 100644 index 0000000..dfc19db --- /dev/null +++ b/indices.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" +) + +type CatIndicesResponseItem struct { + Index string `json:"index"` + Health string `json:"health"` + Status string `json:"status"` + Pri string `json:"pri"` + Rep string `json:"rep"` + DocsCount string `json:"docs.count"` + DocsDeleted string `json:"docs.deleted"` + StoreSize string `json:"store.size"` + PriStorSize string `json:"pri.store.size"` +} + +type CatIndicesResponse []CatIndicesResponseItem + +func indices(client *opensearch.Client, clibanaConfig ClibanaConfig) { + request := opensearchapi.CatIndicesRequest{ + Format: "json", + Index: []string{clibanaConfig.Index}, + } + + response := doRequest[CatIndicesResponse](client, request) + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + + if !clibanaConfig.Indices.Quiet { + columns := []string{"index", "health", "status", "pri", "rep", "docs.count", "docs.deleted", "store.size", "pri.store"} + writer.Write([]byte(fmt.Sprintf("%s\n", strings.Join(columns, "\t")))) + } + + for _, idx := range response { + writer.Write([]byte(fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + idx.Index, idx.Health, idx.Status, idx.Pri, idx.Rep, idx.DocsCount, idx.DocsDeleted, idx.StoreSize, idx.PriStorSize, + ))) + } + + writer.Flush() +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..ca97a80 --- /dev/null +++ b/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "io" + "log" + "os" +) + +const ( + TailSleep = 5 + ResponseTimeout = 10 + AuthTypeAWS = "aws" + AuthTypeBasic = "basic" + SearchRequestSize = 10000 +) + +var DebugLogger = log.New(io.Discard, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile) + +func FatalError(err error) { + fmt.Fprintf(os.Stderr, "clibana: %v\n", err) + os.Exit(1) +} + +func main() { + clibanaConfig := NewClibanaConfig() + + if clibanaConfig.Debug { + DebugLogger.SetOutput(os.Stderr) + + } + + client, err := createClient(clibanaConfig) + if err != nil { + FatalError(fmt.Errorf("Failed to create OpenSearch client: %w", err)) + } + + DebugLogger.Printf("Configuration: %+v\n", clibanaConfig) + + switch { + case clibanaConfig.Search != nil: + search(client, clibanaConfig) + case clibanaConfig.Mappings != nil: + mappings(client, clibanaConfig) + case clibanaConfig.Indices != nil: + indices(client, clibanaConfig) + default: + FatalError(fmt.Errorf("no subcommand specified")) + } + +} diff --git a/mappings.go b/mappings.go new file mode 100644 index 0000000..2f22ca3 --- /dev/null +++ b/mappings.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "os" + "slices" + "sort" + "text/tabwriter" + + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" +) + +type Field struct { + Name string + Type string +} + +type Mappings struct { + Properties map[string]interface{} `json:"properties"` +} + +type IndicesGetMappingResponseItem struct { + Mappings Mappings `json:"mappings"` +} + +type IndicesGetMappingResponse map[string]IndicesGetMappingResponseItem + +func mappings(client *opensearch.Client, clibanaConfig ClibanaConfig) { + request := opensearchapi.IndicesGetMappingRequest{ + Index: []string{clibanaConfig.Index}, + } + response := doRequest[IndicesGetMappingResponse](client, request) + + var keys []Field + + for _, item := range response { + keys = append(keys, getIndexMapKeysFlattened(item.Mappings.Properties, "")...) + } + + sort.Slice(keys, func(i, j int) bool { + return keys[i].Name < keys[j].Name + }) + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + + if !clibanaConfig.Mappings.Quiet { + writer.Write([]byte("field\ttype\n")) + } + + for _, key := range slices.Compact(keys) { + writer.Write([]byte(fmt.Sprintf("%s\t%s\n", key.Name, key.Type))) + } + + writer.Flush() +} + +func getIndexMapKeysFlattened(props map[string]interface{}, prefix string) []Field { + var keys []Field + + for key, value := range props { + switch v := value.(type) { + case map[string]interface{}: + if _, ok := v["properties"]; ok { + properties := mustAssertType[map[string]interface{}](v["properties"]) + for _, subfield := range getIndexMapKeysFlattened(properties, key+".") { + keys = append(keys, Field{ + Name: prefix + subfield.Name, + Type: subfield.Type, + }) + } + } else { + typ := mustAssertType[string](v["type"]) + keys = append(keys, Field{ + Name: prefix + key, + Type: typ, + }) + } + default: + panic(fmt.Sprintf("unexpected type: %+v", value)) + } + } + + return keys +} diff --git a/search.go b/search.go new file mode 100644 index 0000000..8cb480d --- /dev/null +++ b/search.go @@ -0,0 +1,38 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/opensearch-project/opensearch-go/v2" +) + +func search(client *opensearch.Client, clibanaConfig ClibanaConfig) { + tailer := NewTailer(client, clibanaConfig) + + for hit := range tailer.Tail() { + var output string + + if clibanaConfig.Search.Fields != nil { + var values []string + + for _, field := range clibanaConfig.Search.Fields { + if strValue, ok := getNestedField(hit.Source, field); ok { + values = append(values, strValue) + } + } + + output = strings.Join(values, " ") + } else { + buf, err := json.Marshal(hit.Source) + if err != nil { + FatalError(fmt.Errorf("failed to marshal JSON: %w", err)) + } + + output = string(buf) + } + + fmt.Println(output) //nolint:forbidigo + } +} diff --git a/tailer.go b/tailer.go new file mode 100644 index 0000000..9eaa391 --- /dev/null +++ b/tailer.go @@ -0,0 +1,108 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" +) + +type SearchResponseHitsHit struct { + Sort []interface{} `json:"sort"` + Source map[string]interface{} `json:"_source"` +} + +type SearchResponseHits struct { + Hits []SearchResponseHitsHit `json:"hits"` +} + +type SearchResponse struct { + Hits SearchResponseHits `json:"hits"` +} + +type Tailer struct { + Client *opensearch.Client + ClibanaConfig ClibanaConfig + SearchAfter []interface{} +} + +func NewTailer(client *opensearch.Client, clibanaConfig ClibanaConfig) *Tailer { + return &Tailer{ + Client: client, + ClibanaConfig: clibanaConfig, + } +} + +func (t *Tailer) Tail() func(func(SearchResponseHitsHit) bool) { + size := SearchRequestSize + return func(yield func(SearchResponseHitsHit) bool) { + for { + requestBody := t.buildSearchRequestBody() + request := opensearchapi.SearchRequest{ + Index: []string{t.ClibanaConfig.Index}, + Body: requestBody, + Sort: []string{"@timestamp:asc"}, + Size: &size, + } + + response := doRequest[SearchResponse](t.Client, request) + + for _, hit := range response.Hits.Hits { + t.SearchAfter = hit.Sort + if !yield(hit) { + break + } + } + + if len(response.Hits.Hits) != size { + if t.ClibanaConfig.Search.Follow { + time.Sleep(TailSleep * time.Second) + } else { + break + } + } + } + } +} + +func (t *Tailer) buildSearchRequestBody() *strings.Reader { + query := map[string]interface{}{ + "query": map[string]interface{}{ + "bool": map[string]interface{}{ + "must": []interface{}{ + map[string]interface{}{ + "query_string": map[string]interface{}{ + "query": t.ClibanaConfig.Search.Query, + }, + }, + map[string]interface{}{ + "range": map[string]interface{}{ + "@timestamp": map[string]interface{}{ + "gte": t.ClibanaConfig.Search.Start, + "lte": t.ClibanaConfig.Search.End, + }, + }, + }, + }, + }, + }, + } + + if t.SearchAfter != nil { + query["search_after"] = t.SearchAfter + } + + if len(t.ClibanaConfig.Search.Fields) > 0 { + query["_source"] = append(t.ClibanaConfig.Search.Fields, "@timestamp") + } + + body, err := json.Marshal(query) + if err != nil { + FatalError(fmt.Errorf("failed to marshal search request body to JSON: %w", err)) + } + + return strings.NewReader(string(body)) +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..947aa5b --- /dev/null +++ b/util.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "strings" +) + +func mustAssertType[T any](obj interface{}) T { //nolint:ireturn + var zero T + + if val, ok := obj.(T); ok { + return val + } + + panic(fmt.Sprintf("failed to assert type: %v to %T", obj, zero)) +} + +func getNestedField(source map[string]interface{}, field string) (string, bool) { + var value interface{} = source + + fieldParts := strings.Split(field, ".") + for _, part := range fieldParts { + if nestedMap, ok := value.(map[string]interface{}); ok { + value = nestedMap[part] + } else { + return "", false + } + } + + if strValue, ok := value.(string); ok { + return strValue, true + } + + return "", false +}