build.gradle

/*
 * Copyright 2018 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins {
    id 'java'
    id 'application'
    id 'eclipse'
    id 'idea'
    id 'com.diffplug.spotless' version '6.22.0'
    id 'com.google.cloud.tools.jib' version '3.4.0'
    id 'jacoco'
}

// Licence header enforced by spotless
def javaLicenseHeader = """/*
 * Copyright \$YEAR Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
"""

java {
    sourceCompatibility = JavaVersion.VERSION_17
    targetCompatibility = JavaVersion.VERSION_17
    toolchain {
        languageVersion = JavaLanguageVersion.of(17)
    }
}

application {
    applicationDefaultJvmArgs = ['-Xms4g', '-Xmx16g']
    mainClass = System.getProperty('mainClass', 'com.google.swarm.tokenization.DLPTextToBigQueryStreamingV2')
}

jib {
    from {
        image = 'gcr.io/dataflow-templates-base/java8-template-launcher-base:latest'
    }
    container {
        appRoot = '/template/df-dlp-inspect-template'
        entrypoint = 'INHERIT'
        environment = [DATAFLOW_JAVA_COMMAND_SPEC: '/template/df-dlp-inspect-template/resources/java_command_spec.json']

    }
}
group 'com.google.swarm'
version '0.1.0'


tasks.withType(JavaCompile).configureEach {
    options.encoding = 'UTF-8'
}

repositories {
    mavenCentral()
}

dependencies {
    ext {
        dataflowBeamVersion = '2.46.0'
    }

    implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: dataflowBeamVersion
    implementation(group: 'org.apache.beam', name: 'beam-runners-google-cloud-dataflow-java', version: dataflowBeamVersion) {
        exclude group: 'io.confluent', module: 'kafka-schema-registry-client'
        exclude group: 'io.confluent', module: 'kafka-avro-serializer'
    }
    implementation group: 'org.apache.beam', name: 'beam-runners-direct-java', version: dataflowBeamVersion
    implementation group: 'org.apache.beam', name: 'beam-sdks-java-extensions-ml', version: dataflowBeamVersion
    implementation group: 'org.apache.beam', name: 'beam-sdks-java-io-amazon-web-services', version: dataflowBeamVersion
    implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.13.1'
    implementation group: 'org.apache.parquet', name: 'parquet-hadoop', version: '1.13.1'
    implementation group: 'org.apache.beam', name: 'beam-sdks-java-io-parquet', version: '2.14.0'
    implementation group: 'org.slf4j', name: 'slf4j-jdk14', version: '2.0.6'
    implementation 'com.google.cloud:google-cloud-kms:2.15.0'
    implementation 'com.google.guava:guava:32.1.2-jre'
    implementation group: 'com.google.cloud', name: 'google-cloud-dlp', version: '3.16.0'
    implementation 'com.google.api-client:google-api-client:2.2.0'
    implementation group: 'com.google.apis', name: 'google-api-services-cloudkms', version: 'v1-rev108-1.25.0'
    implementation group: 'org.apache.beam', name: 'beam-sdks-java-io-amazon-web-services', version: dataflowBeamVersion
    implementation "com.google.auto.value:auto-value-annotations:1.10.1"
    implementation 'com.google.cloud:google-cloud-storage:2.20.1'
    implementation group: 'org.apache.orc', name: 'orc-core', version: '1.9.1'
    implementation group: 'org.apache.hadoop', name: 'hadoop-hdfs-client', version: '3.3.6'
    implementation group: 'org.apache.hadoop', name: 'hadoop-common', version: '3.3.6'
    implementation group: 'com.google.cloud.bigdataoss', name: 'gcs-connector', version: 'hadoop3-2.2.17'
    implementation group: 'org.apache.hive', name: 'hive-storage-api', version: '2.8.1'
    implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.10.0'
    annotationProcessor group: 'com.google.auto.value', name: 'auto-value', version: '1.10.1'
    testImplementation group: 'org.apache.beam', name: 'beam-runners-direct-java', version: dataflowBeamVersion
    testImplementation group: 'org.slf4j', name: 'slf4j-jdk14', version: '2.0.6'
    testImplementation group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3'
    testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3'
    testImplementation group: 'junit', name: 'junit', version: '4.13.1'
}

jar {
    manifest {
        attributes('Main-Class': application.mainClass)
    }
}

compileJava {
    options.encoding = 'UTF-8'
    // Add additional jvm arguments
    options.compilerArgs.addAll(['-Xlint:all', '-Xlint:-serial', '-Xlint:-processing'])
}

// default tasks when you run ./gradlew
defaultTasks 'clean', 'assemble', 'jar'
// Google cloud dataflow requires the resource/main directory to exist
tasks.register('resources') {
    def resourcesDir = new File('build/resources/main')
    resourcesDir.mkdirs()
}

run {
    if (project.hasProperty('args')) {
        args project.args.split('\\s')
    }
}

test {
    useJUnit()
    testLogging {
        events "passed", "skipped", "failed"
        showStandardStreams = true
        exceptionFormat "full"
    }
    finalizedBy jacocoTestReport
}

sourceSets {
    test {
        java.srcDir file('src/test')
    }
}

spotless {
    java {
        licenseHeader javaLicenseHeader
        googleJavaFormat()
    }
}
run.mustRunAfter 'resources'
run.mustRunAfter 'resources'
tasks.register('fatJar', Jar) {
    manifest {
        attributes 'Implementation-Title': 'dlp-demo-tool',
                'Implementation-Version': archiveVersion,
                'Main-Class': application.mainClass
    }
    archiveFileName = project.name + '-all'
    from {
        configurations.compile.collect {
            it.isDirectory() ? it : zipTree(it)
        }
    }
    with jar
}

jacocoTestReport {
    dependsOn test
    reports {
        csv.required = true
        html.required = false
        xml.required = true
    }
}