Merge pull request #130 from mrceyhun/f-sqoop-refactor

Move sqoop scripts to cms monitoring repo and refactor git workflows

.github/workflows/build-go-tools.yml

@@ -0,0 +1,118 @@
+# Ref: https://github.com/dmwm/dbs2go/blob/master/.github/workflows/build.yml
+name: Build
+
+on:
+  push:
+    tags:
+      - 'go-*.*.*'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.15
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build
+        run: |
+          mkdir cmsmon-tools
+          cd src/go/MONIT
+          go build -o monit monit.go
+          go build -o alert alert.go
+          go build -o annotationManager annotationManager.go
+          go build -o datasources datasources.go
+          go build -o ggus_alerting ggus_alerting.go
+          go build -o ggus_parser ggus_parser.go
+          go build -o intelligence intelligence.go
+          go build -o ssb_alerting ssb_alerting.go
+          go build -o es_exporter es_exporter.go
+          mv monit alert annotationManager datasources ggus_alerting \
+            ggus_parser intelligence ssb_alerting es_exporter ../../../cmsmon-tools
+          cd ../NATS
+          go build -o dbs_vm dbs_vm.go
+          go build -o nats-pub nats-pub.go
+          go build -o nats-sub nats-sub.go
+          mv dbs_vm nats-pub nats-sub ../../../cmsmon-tools
+          cd ../../../
+          tar cfz cmsmon-tools.tar.gz cmsmon-tools
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      - name: Upload binaries
+        id: upload-cmsmon-tools
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./cmsmon-tools.tar.gz
+          asset_name: cmsmon-tools.tar.gz
+          asset_content_type: application/octet-stream
+
+      # --- Build and push docker images ---
+      - name: Get git tag
+        id: get_tag
+        run: echo ::set-output name=tag::${GITHUB_REF/refs\/tags\//}
+
+      - name: Build cmsmon-int image
+        run: |
+          echo Image tag: ${{ steps.get_tag.outputs.tag }}
+          curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/cmsmon-intelligence/Dockerfile
+          sed -i -e "s,ENV CMSMON_TAG=.*,ENV CMSMON_TAG=${{steps.get_tag.outputs.tag}},g" Dockerfile
+          docker build . --tag docker.pkg.github.com/dmwm/cmsmon-int/cmsmon-int
+          docker tag docker.pkg.github.com/dmwm/cmsmon-int/cmsmon-int registry.cern.ch/cmsmonitoring/cmsmon-int
+
+      - name: Login to registry.cern.ch
+        uses: docker/[email protected]
+        with:
+          registry: registry.cern.ch
+          username: ${{ secrets.CERN_LOGIN }}
+          password: ${{ secrets.CERN_TOKEN }}
+
+      - name: Publish cmsmon-int image to registry.cern.ch
+        uses: docker/build-push-action@v1
+        with:
+          username: ${{ secrets.CERN_LOGIN }}
+          password: ${{ secrets.CERN_TOKEN }}
+          registry: registry.cern.ch
+          repository: cmsmonitoring/cmsmon-int
+          tag_with_ref: true
+
+      - name: Build cmsmon-alerts image
+        run: |
+          echo Image tag: ${{ steps.get_tag.outputs.tag }}
+          curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/cmsmon-alerts/Dockerfile
+          sed -i -e "s,ENV CMSMON_TAG=.*,ENV CMSMON_TAG=${{steps.get_tag.outputs.tag}},g" Dockerfile
+          docker build . --tag docker.pkg.github.com/dmwm/cmsmon-alerts/cmsmon-alerts
+          docker tag docker.pkg.github.com/dmwm/cmsmon-alerts/cmsmon-alerts registry.cern.ch/cmsmonitoring/cmsmon-alerts
+
+      - name: Login to registry.cern.ch
+        uses: docker/[email protected]
+        with:
+          registry: registry.cern.ch
+          username: ${{ secrets.CERN_LOGIN }}
+          password: ${{ secrets.CERN_TOKEN }}
+
+      - name: Publish cmsmon-alerts image to registry.cern.ch
+        uses: docker/build-push-action@v1
+        with:
+          username: ${{ secrets.CERN_LOGIN }}
+          password: ${{ secrets.CERN_TOKEN }}
+          registry: registry.cern.ch
+          repository: cmsmonitoring/cmsmon-alerts
+          tag_with_ref: true

.github/workflows/build-sqoop.yml

@@ -0,0 +1,49 @@
+# Ref: https://github.com/dmwm/dbs2go/blob/master/.github/workflows/build.yml
+
+name: Build
+
+on:
+  push:
+    tags:
+      - 'sqoop-*.*.*'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.15
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get git tag
+        id: get_tag
+        run: echo ::set-output name=tag::${GITHUB_REF/refs\/tags\//}
+
+      - name: Build sqoop image
+        run: |
+          echo Image tag: ${{ steps.get_tag.outputs.tag }}
+          curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/sqoop/Dockerfile
+          sed -i -e "s,ENV CMSMON_TAG=.*,ENV CMSMON_TAG=${{steps.get_tag.outputs.tag}},g" Dockerfile
+          docker build . --tag docker.pkg.github.com/dmwm/sqoop/sqoop
+          docker tag docker.pkg.github.com/dmwm/sqoop/sqoop registry.cern.ch/cmsmonitoring/sqoop
+
+      - name: Login to registry.cern.ch
+        uses: docker/[email protected]
+        with:
+          registry: registry.cern.ch
+          username: ${{ secrets.CERN_LOGIN }}
+          password: ${{ secrets.CERN_TOKEN }}
+
+      - name: Publish sqoop image to registry.cern.ch
+        uses: docker/build-push-action@v1
+        with:
+          username: ${{ secrets.CERN_LOGIN }}
+          password: ${{ secrets.CERN_TOKEN }}
+          registry: registry.cern.ch
+          repository: cmsmonitoring/sqoop
+          tag_with_ref: true

.github/workflows/test.yml → .github/workflows/syntaxcheck.yml

@@ -1,21 +1,25 @@
-name: Continuous integration testing
+name: Syntax check
 
-on: [pull_request, push]
+on:
+  push:
+    paths:
+      - '**.json'
+      - '**.schema'
+      - '**.yaml'
+
+  pull_request:
 
 jobs:
-  test:
-
+  syntaxcheck:
     runs-on: ubuntu-latest
-
     steps:
-
       - uses: actions/checkout@v2
-      
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          
+
       - name: check valid json
         run: |
           echo "Checking valid json..."

README.md

@@ -6,3 +6,15 @@ infrastructure.
 
 - Available [here](https://cmsmonit-docs.web.cern.ch/)
 - source [code](https://gitlab.cern.ch/cmsmonitoring/cmsmonit-docs) 
+
+
+### Git Workflows
+
+- On tag `go-*.*.*` 
+    - Builds go executables and release `cmsmon-tools`
+    - Builds `cmsmonitoring/cmsmonit-int` docker image and push to registry.cern.ch
+    - Builds `cmsmonitoring/cmsmonit-alert` docker image and push to registry.cern.ch
+- On tag `sqoop-*.*.*` 
+    - Builds `cmsmonitoring/sqoop` docker image and push to registry.cern.ch
+- Syntaxcheck on specical condition
+    - Check validations of json and yaml files only that kind of files are changed

sqoop/cronjobs.txt

@@ -0,0 +1,36 @@
+#
+# Sqoop jobs taken from AWG and CERN DB IT group to dump CMS databases
+#
+
+# jobmon dump
+#13 04 * * * cd /data/sqoop; ./run.sh ./scripts/cms-jm.sh
+#13 04 * * * cd /data/sqoop; ./run.sh ./scripts/jm-cms-data-pop.sh
+
+# cms-jm-data-popularity dumps
+#23 04 * * * cd /data/sqoop; ./run.sh ./scripts/cmssw-popularity.sh
+
+# cmssw popularity
+#42 04 * * * cd /data/sqoop/cmssw-popularity; ./run.sh ./sqoop-load.sh
+
+# PhEDEX dumps
+22 03 * * * cd /data/sqoop; ./run.sh ./scripts/phedex-blk-replicas-snapshot.sh
+43 03 * * * cd /data/sqoop; ./run.sh ./scripts/phedex-file-catalog.sh
+
+# Rucio dumps
+30 06 * * * cd /data/sqoop; ./run.sh ./scripts/rucio_replicas.sh
+00 01 * * * cd /data/sqoop; ./run.sh ./scripts/rucio_dids.sh
+30 02 * * * cd /data/sqoop; ./run.sh ./scripts/rucio_contents.sh
+
+# DBS dumps
+27 03 * * *   cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-datasets.sh
+32 03 * * *   cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-blocks.sh
+37 03 * * *   cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-files.sh
+03 22 * * MON cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-full-copy-PHYS01.sh
+03 22 * * TUE cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-full-copy-PHYS02.sh
+03 22 * * WED cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-full-copy-PHYS03.sh
+12 22 * * THU cd /data/sqoop; ./run.sh ./scripts/cms-dbs3-full-copy.sh
+
+# ASO dump
+42 03 * * * cd /data/sqoop; ./run.sh ./scripts/cms-aso.sh
+
+40 14 * * * cd /data; /data/sqoop/run.sh /data/monit -query="stats" -token /etc/cmsdb/token -hdfs=/etc/cmsdb/hdfs.json -creds=/etc/cmsdb/cms-es-size.json -verbose 1 -inject 2>&1 1>& monit.log