dataproc.yaml.tmpl

configBucket: ${tamr_dataproc_bucket}
gceClusterConfig:
  zoneUri: ${zone}
  subnetworkUri: ${subnetwork_uri}
  internalIpOnly: true
  tags:
  %{ for tag in dataproc_network_tags} - ${tag}
  %{ endfor }
  serviceAccount: ${service_account}
  metadata:
    GCS_CONNECTOR_VERSION: 2.2.0
softwareConfig:
  imageVersion: ${dataproc_image_version}
  properties:
    dataproc:job.history.to-gcs.enabled: true
    dataproc:dataproc.logging.stackdriver.enable: ${stackdriver_logging}
    dataproc:dataproc.logging.stackdriver.job.driver.enable: ${stackdriver_logging}
    dataproc:dataproc.logging.stackdriver.job.yarn.container.enable: ${stackdriver_logging}
    core:fs.gs.outputstream.type: FLUSHABLE_COMPOSITE
    core:fs.gs.outputstream.sync.min.interval.ms: 1500
    core:fs.gs.outputstream.pipe.type: NIO_CHANNEL_PIPE
masterConfig:
  numInstances: 1
  machineTypeUri: ${master_instance_type}
  instanceNames:
    - ephemeral-master
  diskConfig:
    bootDiskType: pd-standard
    bootDiskSizeGb: ${master_disk_size}
workerConfig:
  numInstances: ${worker_num_instances}
  machineTypeUri: ${worker_machine_type}
  diskConfig:
    bootDiskType: pd-ssd
    bootDiskSizeGb: 100
    numLocalSsds: ${worker_num_local_ssds}
secondaryWorkerConfig:
  isPreemptible: true
  numInstances: ${worker_preemptible_num_instances}
  machineTypeUri: ${worker_preemptible_machine_type}
  diskConfig:
    bootDiskType: pd-ssd
    bootDiskSizeGb: 100
    numLocalSsds: ${worker_preemptible_num_local_ssds}