spark-tests.yaml

# Workflow derived from https://github.com/r-lib/actions/tree/master/examples
# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
on:
  push:
    branches: main
  pull_request:
    branches: main

name: Spark-Tests

jobs:
  Spark-Tests:
    runs-on: ubuntu-latest

    name: ${{ matrix.config.name }}

    strategy:
      fail-fast: false
      matrix:
        config:
          - {spark: '3.3.4', hadoop: '3',   arrow: 'release', livy: '',      dbplyr: '',      name: 'Spark 3.3'}
          - {spark: '3.2.4', hadoop: '3.2', arrow: '',        livy: '',      dbplyr: '',      name: 'Spark 3.2'}
          - {spark: '3.1.3', hadoop: '3.2', arrow: '',        livy: '',      dbplyr: '',      name: 'Spark 3.1'}
          - {spark: '2.4.0', hadoop: '2.7', arrow: '',        livy: '',      dbplyr: '',      name: 'Spark 2.4'}
    env:
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
      R_KEEP_PKG_SOURCE: yes
      SPARK_VERSION:  ${{ matrix.config.spark }}
      HADOOP_VERSION: ${{ matrix.config.hadoop }}
      LIVY_VERSION:   ${{ matrix.config.livy }}
      ARROW_VERSION : ${{ matrix.config.arrow }}
      DBPLYR_VERSION: ${{ matrix.config.dbplyr }}

    steps:
      - uses: actions/checkout@v3

      - uses: r-lib/actions/setup-r@v2
        with:
          r-version: 'release'
          use-public-rspm: true

      - uses: r-lib/actions/setup-r-dependencies@v2
        with:
          extra-packages: |
            any::devtools
            any::sparklyr
            any::arrow

      - name: Cache Spark
        id: cache-spark
        uses: actions/cache@v2
        with:
          path: /home/runner/spark/spark-${{ matrix.config.spark }}-bin-hadoop${{ matrix.config.hadoop }}
          key: sparklyr-spark-${{ matrix.config.spark }}-bin-hadoop${{ matrix.config.hadoop }}-3

      - name: Install Spark (via sparklyr)
        if: steps.cache-spark.outputs.cache-hit != 'true'
        run: |
          sparklyr::spark_install(
            version = Sys.getenv("SPARK_VERSION"),
            hadoop_version = Sys.getenv("HADOOP_VERSION")
          )
        shell: Rscript {0}

      - name: Cache Livy
        if: ${{ matrix.config.livy }} != ''
        id: cache-livy
        uses: actions/cache@v2
        with:
          path: /home/runner/.cache/livy/livy-${{ matrix.config.livy }}
          key: livy-${{ matrix.config.livy }}

      - name: Install Livy (via sparklyr)
        if: ${{ (steps.cache-livy.outputs.cache-hit != 'true') && (matrix.config.livy != '') }}
        run: |
          sparklyr::livy_install(
            version = Sys.getenv("LIVY_VERSION"),
            spark_version = Sys.getenv("SPARK_VERSION")
            )
        shell: Rscript {0}

      - name: Cache Scala
        id: cache-scala
        uses: actions/cache@v2
        with:
          path: /home/runner/scala/
          key: scala-2

      - name: Install Scala (via sparklyr)
        if: steps.cache-scala.outputs.cache-hit != 'true'
        run: |
          sparklyr::download_scalac()
        shell: Rscript {0}

      - name: Switching to Arrow devel if needed
        run: |
          if(Sys.getenv("ARROW_VERSION") == "devel") {
            arrow::install_arrow(nightly = TRUE)
          }
        shell: Rscript {0}

      - name: Installing specified version of dbplyr
        run: |
          if(Sys.getenv("DBPLYR_VERSION") == "dev") {
            devtools::install_github("tidyverse/dbplyr")
          }
        shell: Rscript {0}

      - name: R Session Info
        run: sessionInfo()
        shell: Rscript {0}

      - name: R Environment Variables
        run: Sys.getenv()
        shell: Rscript {0}

      - name: R Installed Packages
        run: |
          m_pkgs <- installed.packages()
          t_pkgs <- as.data.frame(m_pkgs, row.names = FALSE)
          print(t_pkgs[, c("Package", "Version")])
        shell: Rscript {0}

      - name: R Tests
        run: |
          if(Sys.getenv("ARROW_VERSION") != "") {
            library(arrow)
          }
          spark_major <- as.numeric(substr(Sys.getenv("SPARK_VERSION"), 1, 1))
          if(spark_major < 3) Sys.setenv("JAVA_HOME" = Sys.getenv("JAVA_HOME_8_X64"))
          Sys.getenv("JAVA_HOME")
          devtools::load_all()
          devtools::test(reporter = sparklyr_reporter())
        shell: Rscript {0}