Support running Heron on Amazon Ecs.

heron/config/src/yaml/BUILD

@@ -17,6 +17,11 @@ filegroup(
     srcs = glob(["conf/local/*.yaml"]),
 )
 
+filegroup(
+    name = "conf-ecs-yaml",
+    srcs = glob(["conf/ecs/*.yaml"]),
+)
+
 filegroup(
     name = "conf-aurora-yaml",
     srcs = glob(["conf/aurora/*"]),

heron/config/src/yaml/conf/ecs/README

@@ -0,0 +1,6 @@
+This folder contains sample configs needed for using running heron on AWS Cluster
+Please follow the steps at this google doc for detailed set up and workflow:
+
+https://docs.google.com/document/d/1ecbCuA46cIKPfY0SP0F1dcRlei4DIPz3pZ6ZSZ5zZgc/edit
+
+Then you can run Heorn on AWS !!!

heron/config/src/yaml/conf/ecs/cleanUp.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+aws autoscaling delete-launch-configuration --launch-configuration-name ecs-heron-launch-configuration
+echo "done delete-launch-configuration"
+aws iam remove-role-from-instance-profile --instance-profile-name ecs-heron-instance-profile --role-name ecs-heron-role
+echo "done iam remove-role-from-instance-profile"
+aws iam delete-instance-profile --instance-profile-name ecs-heron-instance-profile
+echo "done iam iam delete-instance-profile"
+aws iam delete-role-policy --role-name ecs-heron-role --policy-name ecs-heron-policy
+echo "done iam delete-role-policy "
+aws iam delete-role --role-name ecs-heron-role
+echo "done iam delete-role "
+aws ec2 delete-key-pair --key-name ecs-heron-keypair
+rm -f ecs-heron-key*.pem
+
+GROUP_ID=$(aws ec2 describe-security-groups --query 'SecurityGroups[?GroupName==`ecs-heron-securitygroup`].GroupId' --output text)
+aws ec2 delete-security-group --group-id "$GROUP_ID"
+echo "done delete security-groups "
+
+aws ecs delete-cluster --cluster ecs-heron-cluster
+echo "done delete cluster "

heron/config/src/yaml/conf/ecs/ecs-heron-policy.json

@@ -0,0 +1,28 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ecs:CreateCluster",
+                "ecs:DeregisterContainerInstance",
+                "ecs:DiscoverPollEndpoint",
+                "ecs:Poll",
+                "ecs:RegisterContainerInstance",
+                "ecs:Submit*",
+                "ecs:ListClusters",
+                "ecs:ListContainerInstances",
+                "ecs:DescribeContainerInstances",
+                "ecs:ListServices",
+                "ecs:DescribeTasks",
+                "ecs:DescribeServices",
+                "ec2:DescribeInstances",
+                "ec2:DescribeTags",
+                "autoscaling:DescribeAutoScalingInstances"
+            ],
+            "Resource": [
+                "*"
+            ]
+        }
+    ]
+}

heron/config/src/yaml/conf/ecs/ecs-heron-role.json

@@ -0,0 +1,8 @@
+{
+    "Version": "2012-10-17",
+    "Statement": {
+	"Effect": "Allow",
+	"Principal": {"Service": "ec2.amazonaws.com"},
+	"Action": "sts:AssumeRole"
+    }
+}

heron/config/src/yaml/conf/ecs/ecs_compose_template.yaml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  CONTAINER_NUMBER:
+    image: ananthgs/onlyheronandubuntu
+    #command: ["sh", "-c", "mkdir /s3; cd /s3 ;aws s3 cp s3://herondockercal/TOPOLOGY_NAME/topology.tar.gz /s3 ;aws s3 cp s3://herondockercal/heron-core-testbuild-ubuntu14.04.tar.gz /s3 ;cd /s3; tar -zxvf topology.tar.gz; tar -zxvf heron-core-testbuild-ubuntu14.04.tar.gz; HERON_EXECUTOR ;"]
+    command: ["sh", "-c", "mkdir /s3; cd /s3 ;aws s3 cp s3://herondockercal/TOPOLOGY_NAME/topology.tar.gz /s3 ;aws s3 cp s3://herondockercal/heron-core.tar.gz /s3 ;cd /s3; tar -zxvf topology.tar.gz; tar -zxvf heron-core.tar.gz; HERON_EXECUTOR ;"]
+    networks:
+      - heron
+    ports:FREEPORTS
+    volumes:
+      - "herondata:/root/.herondata"
+networks:
+  heron:
+    driver: bridge
+volumes:
+  herondata:
+    driver: local

heron/config/src/yaml/conf/ecs/heron_internals.yaml

@@ -0,0 +1,272 @@
+################################################################################
+# Default values for various configs used inside Heron.
+################################################################################
+# All the config associated with time is in the unit of milli-seconds,
+# unless otherwise specified.
+################################################################################
+# All the config associated with data size is in the unit of bytes, unless
+# otherwise specified.
+################################################################################
+
+################################################################################
+# System level configs
+################################################################################
+
+### heron.* configs are general configurations over all componenets
+
+# The relative path to the logging directory
+heron.logging.directory: "log-files"
+
+# The maximum log file size in MB
+heron.logging.maximum.size.mb: 100
+
+# The maximum number of log files
+heron.logging.maximum.files: 5
+
+# The interval in seconds after which to check if the tmaster location has been fetched or not
+heron.check.tmaster.location.interval.sec: 120
+
+# The interval in seconds to prune logging files in C++
+heron.logging.prune.interval.sec: 300
+
+# The interval in seconds to flush log files in C++
+heron.logging.flush.interval.sec: 10
+
+# The threshold level to log error
+heron.logging.err.threshold: 3
+
+# The interval in seconds for different components to export metrics to metrics manager
+heron.metrics.export.interval.sec: 60
+
+# The maximum count of exceptions in one MetricPublisherPublishMessage protobuf
+heron.metrics.max.exceptions.per.message.count: 1024
+
+################################################################################
+# Configs related to Stream Manager, starts with heron.streammgr.*
+################################################################################
+
+# Maximum size in bytes of a packet to be send out from stream manager
+heron.streammgr.packet.maximum.size.bytes: 102400
+
+# The tuple cache (used for batching) can be drained in two ways:
+# (a) Time based
+# (b) size based
+
+# The frequency in ms to drain the tuple cache in stream manager
+heron.streammgr.cache.drain.frequency.ms: 10
+
+# The sized based threshold in MB for draining the tuple cache
+heron.streammgr.cache.drain.size.mb: 100
+
+# For efficient acknowledgements
+heron.streammgr.xormgr.rotatingmap.nbuckets: 3
+
+# The reconnect interval to other stream managers in secs for stream manager client
+heron.streammgr.client.reconnect.interval.sec: 1
+
+# The reconnect interval to tamster in second for stream manager client
+heron.streammgr.client.reconnect.tmaster.interval.sec: 10
+
+# The maximum packet size in MB of stream manager's network options
+heron.streammgr.network.options.maximum.packet.mb: 100
+
+# The interval in seconds to send heartbeat
+heron.streammgr.tmaster.heartbeat.interval.sec: 10
+
+# Maximum batch size in MB to read by stream manager from socket
+heron.streammgr.connection.read.batch.size.mb: 1
+
+# Maximum batch size in MB to write by stream manager to socket
+heron.streammgr.connection.write.batch.size.mb: 1
+
+# Number of times we should wait to see a buffer full while enqueueing data
+# before declaring start of back pressure
+heron.streammgr.network.backpressure.threshold: 3
+
+# High water mark on the num in MB that can be left outstanding on a connection
+heron.streammgr.network.backpressure.highwatermark.mb: 100
+
+# Low water mark on the num in MB that can be left outstanding on a connection
+heron.streammgr.network.backpressure.lowwatermark.mb: 50
+
+################################################################################
+# Configs related to Topology Master, starts with heron.tmaster.*
+################################################################################
+
+# The maximum interval in minutes of metrics to be kept in tmaster
+heron.tmaster.metrics.collector.maximum.interval.min: 180
+
+# The maximum time to retry to establish the tmaster
+heron.tmaster.establish.retry.times: 30
+
+# The interval to retry to establish the tmaster
+heron.tmaster.establish.retry.interval.sec: 1
+
+# Maximum packet size in MB of tmaster's network options to connect to stream managers
+heron.tmaster.network.master.options.maximum.packet.mb: 16
+
+# Maximum packet size in MB of tmaster's network options to connect to scheduler
+heron.tmaster.network.controller.options.maximum.packet.mb: 1
+
+# Maximum packet size in MB of tmaster's network options for stat queries
+heron.tmaster.network.stats.options.maximum.packet.mb: 1
+
+# The interval for tmaster to purge metrics from socket
+heron.tmaster.metrics.collector.purge.interval.sec: 60
+
+# The maximum # of exceptions to be stored in tmetrics collector, to prevent potential OOM
+heron.tmaster.metrics.collector.maximum.exception: 256
+
+# Should the metrics reporter bind on all interfaces
+heron.tmaster.metrics.network.bindallinterfaces: False
+
+# The timeout in seconds for stream mgr, compared with (current time - last heartbeat time)
+heron.tmaster.stmgr.state.timeout.sec: 60
+
+################################################################################
+# Configs related to Topology Master, starts with heron.metricsmgr.*
+################################################################################
+
+# The size of packets to read from socket will be determined by the minimal of:
+# (a) time based
+# (b) size based
+
+# Time based, the maximum batch time in ms for metricsmgr to read from socket
+heron.metricsmgr.network.read.batch.time.ms: 16
+
+# Size based, the maximum batch size in bytes to read from socket
+heron.metricsmgr.network.read.batch.size.bytes: 32768
+
+# The size of packets to write to socket will be determined by the minimum of
+# (a) time based
+# (b) size based
+
+# Time based, the maximum batch time in ms for metricsmgr to write to socket
+heron.metricsmgr.network.write.batch.time.ms: 16
+
+# Size based, the maximum batch size in bytes to write to socket
+heron.metricsmgr.network.write.batch.size.bytes: 32768
+
+# The maximum socket's send buffer size in bytes
+heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 6553600
+
+# The maximum socket's received buffer size in bytes of metricsmgr's network options
+heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 8738000
+
+################################################################################
+# Configs related to Heron Instance, starts with heron.instance.*
+################################################################################
+
+# The queue capacity (num of items) in bolt for buffer packets to read from stream manager
+heron.instance.internal.bolt.read.queue.capacity: 128
+
+# The queue capacity (num of items) in bolt for buffer packets to write to stream manager
+heron.instance.internal.bolt.write.queue.capacity: 128
+
+# The queue capacity (num of items) in spout for buffer packets to read from stream manager
+heron.instance.internal.spout.read.queue.capacity: 1024
+
+# The queue capacity (num of items) in spout for buffer packets to write to stream manager
+heron.instance.internal.spout.write.queue.capacity: 128
+
+# The queue capacity (num of items) for metrics packets to write to metrics manager
+heron.instance.internal.metrics.write.queue.capacity: 128
+
+# The size of packets read from stream manager will be determined by the minimal of
+# (a) time based
+# (b) size based
+
+# Time based, the maximum batch time in ms for instance to read from stream manager per attempt
+heron.instance.network.read.batch.time.ms: 16
+
+# Size based, the maximum batch size in bytes to read from stream manager
+heron.instance.network.read.batch.size.bytes: 32768
+
+# The size of packets written to stream manager will be determined by the minimum of
+# (a) time based
+# (b) size based
+
+# Time based, the maximum batch time in ms for instance to write to stream manager per attempt
+heron.instance.network.write.batch.time.ms: 16
+
+# Size based, the maximum batch size in bytes to write to stream manager
+heron.instance.network.write.batch.size.bytes: 32768
+
+# The maximum socket's send buffer size in bytes
+heron.instance.network.options.socket.send.buffer.size.bytes: 6553600
+
+# The maximum socket's received buffer size in bytes of instance's network options
+heron.instance.network.options.socket.received.buffer.size.bytes: 8738000
+
+# The maximum # of data tuple to batch in a HeronDataTupleSet protobuf
+heron.instance.set.data.tuple.capacity: 1024
+
+# The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf
+heron.instance.set.data.tuple.size.bytes: 8388608
+
+# The maximum # of control tuple to batch in a HeronControlTupleSet protobuf
+heron.instance.set.control.tuple.capacity: 1024
+
+# The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could
+# also break if there are no more ack tuples to process
+heron.instance.ack.batch.time.ms: 128
+
+# The maximum time in ms for an spout instance to emit tuples per attempt
+heron.instance.emit.batch.time.ms: 16
+
+# The maximum batch size in bytes for an spout to emit tuples per attempt
+heron.instance.emit.batch.size.bytes: 32768
+
+# The maximum time in ms for an bolt instance to execute tuples per attempt
+heron.instance.execute.batch.time.ms: 16
+
+# The maximum batch size in bytes for an bolt instance to execute tuples per attempt
+heron.instance.execute.batch.size.bytes: 32768
+
+# The time interval for an instance to check the state change,
+# for example, the interval a spout uses to check whether activate/deactivate is invoked
+heron.instance.state.check.interval.sec: 5
+
+# The time to wait before the instance exits forcibly when uncaught exception happens
+heron.instance.force.exit.timeout.ms: 2000
+
+# Interval in seconds to reconnect to the stream manager, including the request timeout in connecting
+heron.instance.reconnect.streammgr.interval.sec: 5
+heron.instance.reconnect.streammgr.times: 60
+
+# Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting
+heron.instance.reconnect.metricsmgr.interval.sec: 5
+heron.instance.reconnect.metricsmgr.times: 60
+
+# The interval in second for an instance to sample its system metrics, for instance, cpu load.
+heron.instance.metrics.system.sample.interval.sec: 10
+
+heron.instance.slave.fetch.pplan.interval.sec: 1
+
+# For efficient acknowledgement
+heron.instance.acknowledgement.nbuckets: 10
+
+################################################################################
+# For dynamically tuning the available sizes in the interval read & write queues
+# to provide high performance while avoiding GC issues
+################################################################################
+
+# The expected size on read queue in bolt
+heron.instance.tuning.expected.bolt.read.queue.size: 8
+
+# The expected size on write queue in bolt
+heron.instance.tuning.expected.bolt.write.queue.size: 8
+
+# The expected size on read queue in spout
+heron.instance.tuning.expected.spout.read.queue.size: 512
+
+# The exepected size on write queue in spout
+heron.instance.tuning.expected.spout.write.queue.size: 8
+
+# The expected size on metrics write queue
+heron.instance.tuning.expected.metrics.write.queue.size: 8
+
+heron.instance.tuning.current.sample.weight: 0.8
+
+# Interval in ms to tune the size of in & out data queue in instance
+heron.instance.tuning.interval.ms: 100