Merge pull request #3 from clowder-framework/1-make-smm-extractor-clo…

…wder-v2-compatible

1 make smm extractor clowder v2 compatible

.github/workflows/docker.yml

@@ -0,0 +1,174 @@
+name: Docker
+
+# This will run when:
+# - a new release is created, to make sure the right tags of the
+#   docker images are pushed (expects tags to be v1.8.4).
+# - when new code is pushed to main/develop to push the tags
+#   latest and develop
+# - when a pull request is created and updated  to make sure the
+#   Dockerfile is still valid.
+# To be able to push to dockerhub, this execpts the following
+# secrets to be set in the project:
+# - DOCKERHUB_USERNAME : username that can push to the org
+# - DOCKERHUB_PASSWORD : password asscoaited with the username
+on:
+  push:
+    branches:
+      - main
+
+  pull_request:
+
+# Certain actions will only run when this is the main repo.
+env:
+  MAIN_REPO: clowder-framework/smm-extractor
+  DOCKERHUB_ORG: socialmediamacroscope
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: name_entity_recognition_extractor
+            FOLDER: name_entity_recognition_extractor
+            PLATFORM: "linux/amd64,linux/arm64"
+          - name: network_analysis_extractor
+            FOLDER: network_analysis_extractor
+            PLATFORM: "linux/amd64,linux/arm64"
+          - name: preprocessing_extractor
+            FOLDER: preprocessing_extractor
+            PLATFORM: "linux/amd64,linux/arm64"
+          - name: sentiment_analysis_extractor
+            FOLDER: sentiment_analysis_extractor
+            PLATFORM: "linux/amd64,linux/arm64"
+          - name: topic_modeling_extractor
+            FOLDER: topic_modeling_extractor
+            PLATFORM: "linux/amd64,linux/arm64"
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # set environment variables
+      - name: Extractor Version
+        run: |
+          # find out what the BRANCH is, in case of a PR we will use the PR-<number>
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          elif [[ $GITHUB_REF =~ pull ]]; then
+            BRANCH="$(echo $GITHUB_REF | sed 's#refs/pull/\([0-9]*\)/merge#PR-\1#')"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+
+          # should we push to dockerhub, and is there a README
+          DOCKERHUB_PUSH="false"
+          DOCKERHUB_README="false"
+          if [ "$BRANCH" == "main" -a "${{ github.repository }}" == "${{ env.MAIN_REPO }}" ]; then
+            if [ "${{ secrets.SMM_DOCKERHUB_USERNAME }}" != "" -a "${{ secrets.SMM_DOCKERHUB_PASSWORD }}" != "" ]; then
+              DOCKERHUB_PUSH="true"
+              if [ -e "${{ matrix.FOLDER }}/README.md" ]; then
+                DOCKERHUB_README="true"
+              fi
+            fi
+          fi
+
+          # calculate the version and all tags
+          if [ "$BRANCH" == "main" ]; then
+            VERSION="$(awk '/"version":/ { print $2 }' ${{ matrix.FOLDER }}/extractor_info.json | sed 's/^.*"\([0-9\.]*\)".*$/\1/')"
+            tags="latest"
+            oldversion=""
+            tmpversion="${VERSION}"
+            while [ "${oldversion}" != "${tmpversion}" ]; do
+              oldversion="${tmpversion}"
+              tags="${tags} ${tmpversion}"
+              tmpversion=${tmpversion%.*}
+            done
+          else
+            VERSION="test"
+            tags="$BRANCH"
+          fi
+
+          # create a list of all images to be pushed
+          IMAGE=${{ matrix.name }}
+          IMAGES=""
+          for tag in ${tags}; do
+            if [ "$DOCKERHUB_PUSH" == "true" ]; then
+              IMAGES="${IMAGES}${{ env.DOCKERHUB_ORG }}/${IMAGE}:${tag},"
+            fi
+            IMAGES="${IMAGES}ghcr.io/${{ github.repository_owner }}/${IMAGE}:${tag},"
+          done
+          IMAGES="${IMAGES%,*}"
+
+          # save the results in env
+          echo "BRANCH=${BRANCH}"
+          echo "VERSION=${VERSION}"
+          echo "DOCKERHUB_README=${DOCKERHUB_README}"
+          echo "DOCKERHUB_PUSH=${DOCKERHUB_PUSH}"
+          echo "IMAGES=${IMAGES}"
+
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "DOCKERHUB_README=${DOCKERHUB_README}" >> $GITHUB_ENV
+          echo "DOCKERHUB_PUSH=${DOCKERHUB_PUSH}" >> $GITHUB_ENV
+          echo "IMAGES=${IMAGES}" >> $GITHUB_ENV
+
+      # setup docker build
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Inspect Builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      # login to registries
+      - name: Login to DockerHub
+        if: env.DOCKERHUB_PUSH == 'true'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.SMM_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.SMM_DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # build the docker images
+      - name: Build and push ${{ matrix.name }}
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          file: ${{ matrix.FOLDER }}/extractor.dockerfile
+          context: ${{ matrix.FOLDER }}
+          platforms: ${{ matrix.PLATFORM }}
+          cache-from: type=gha,scope=${{ matrix.name }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.name }}
+          tags: ${{ env.IMAGES }}
+          build-args: |
+            VERSION=${{ env.VERSION }}
+            BUILDNUMBER=${{ github.run_number }}
+            GITSHA1=${{ github.sha  }}
+
+      # this will update the README of the dockerhub repo
+      - name: Docker Hub Description
+        if: env.DOCKERHUB_README == 'true'
+        uses: peter-evans/dockerhub-description@v2
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.SMM_DOCKERHUB_USERNAME }}
+          DOCKERHUB_PASSWORD: ${{ secrets.SMM_DOCKERHUB_PASSWORD }}
+          DOCKERHUB_REPOSITORY: ${{ env.DOCKERHUB_ORG }}/${{ matrix.name }}
+          README_FILEPATH: ${{ matrix.FOLDER }}/README.md

command.txt

@@ -0,0 +1,14 @@
+docker build -f extractor.dockerfile -t socialmediamacroscope/name_entity_recognition_extractor:latest .
+docker push socialmediamacroscope/name_entity_recognition_extractor:latest
+
+docker build -f extractor.dockerfile -t socialmediamacroscope/network_analysis_extractor:latest .
+docker push socialmediamacroscope/network_analysis_extractor:latest
+
+docker build -f extractor.dockerfile -t socialmediamacroscope/preprocessing_extractor:latest .
+docker push socialmediamacroscope/preprocessing_extractor:latest
+
+docker build -f extractor.dockerfile -t socialmediamacroscope/sentiment_analysis_extractor:latest .
+docker push socialmediamacroscope/sentiment_analysis_extractor:latest
+
+docker build -f extractor.dockerfile -t socialmediamacroscope/topic_modeling_extractor:latest .
+docker push socialmediamacroscope/topic_modeling_extractor:latest