diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index 6314112be53..0636b2d1fc9 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -895,6 +895,13 @@ def main(): # PEX_ROOT shell environment before forking the processes shell_env = os.environ.copy() shell_env["PEX_ROOT"] = os.path.join(os.path.abspath('.'), ".pex") + # Refer to https://gperftools.github.io/gperftools/heapprofile.html + # for details of settings of gperftools heap profiler + shell_env["HEAPPROFILE"] = "stmgr.hprof" + shell_env["HEAP_PROFILE_ALLOCATION_INTERVAL"] = "0" + shell_env["HEAP_PROFILE_INUSE_INTERVAL"] = "0" + shell_env["HEAPPROFILESIGNAL"] = str(signal.SIGUSR1) + # Instantiate the executor, bind it to signal handlers and launch it executor = HeronExecutor(sys.argv, shell_env) diff --git a/heron/shell/src/python/handlers/__init__.py b/heron/shell/src/python/handlers/__init__.py index 8a414a22b29..83141a70d59 100644 --- a/heron/shell/src/python/handlers/__init__.py +++ b/heron/shell/src/python/handlers/__init__.py @@ -6,7 +6,8 @@ from filestatshandler import FileStatsHandler from jmaphandler import JmapHandler from jstackhandler import JstackHandler +from killexecutorhandler import KillExecutorHandler from memoryhistogramhandler import MemoryHistogramHandler from pmaphandler import PmapHandler from pidhandler import PidHandler -from killexecutorhandler import KillExecutorHandler +from stmgrheapprofhandler import StmgrHeapProfHandler diff --git a/heron/shell/src/python/handlers/stmgrheapprofhandler.py b/heron/shell/src/python/handlers/stmgrheapprofhandler.py new file mode 100644 index 00000000000..581ad4cc747 --- /dev/null +++ b/heron/shell/src/python/handlers/stmgrheapprofhandler.py @@ -0,0 +1,42 @@ +# Copyright 2017 Twitter. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' stmgrheapprofhandler.py ''' +import glob +import os +import signal +import tornado.web + +class StmgrHeapProfHandler(tornado.web.RequestHandler): + """ + Responsible for getting the process ID for an instance. + """ + + # pylint: disable=attribute-defined-outside-init + @tornado.web.asynchronous + def get(self): + ''' get method ''' + self.content_type = 'application/json' + stmgr_pid_files = glob.glob('stmgr*.pid') + try: + pid_file = stmgr_pid_files[0] + with open(pid_file, 'r') as f: + pid = f.read() + os.kill(int(pid), signal.SIGUSR1) + self.write('Performing heap profiling on stream manager...') + self.finish() + except: + self.write("Not stream manager found") + self.set_status(404) + self.finish() diff --git a/heron/shell/src/python/main.py b/heron/shell/src/python/main.py index 401e1f6d84e..633b97cfe51 100644 --- a/heron/shell/src/python/main.py +++ b/heron/shell/src/python/main.py @@ -35,6 +35,7 @@ (r"^/filestats/(.*)", handlers.FileStatsHandler), (r"^/download/(.*)", handlers.DownloadHandler), (r"^/killexecutor", handlers.KillExecutorHandler), + (r"^/stmgrheapprof", handlers.StmgrHeapProfHandler) ] # pylint: disable=dangerous-default-value diff --git a/heron/stmgr/src/cpp/BUILD b/heron/stmgr/src/cpp/BUILD index 9961c8f85c3..30417d2473f 100644 --- a/heron/stmgr/src/cpp/BUILD +++ b/heron/stmgr/src/cpp/BUILD @@ -117,6 +117,7 @@ cc_binary( "server/stmgr-main.cpp", ], copts = [ + "-Ithird_party", "-Iheron", "-Iheron/common/src/cpp", "-Iheron/statemgrs/src/cpp", @@ -135,6 +136,7 @@ cc_binary( "//heron/common/src/cpp/metrics:metrics-cxx", "//heron/statemgrs/src/cpp:statemgrs-cxx", "//third_party/yaml-cpp:yaml-cxx", + "//third_party/gperftools:profiler-cxx", ], linkstatic = 1, ) diff --git a/heron/stmgr/src/cpp/server/stmgr-main.cpp b/heron/stmgr/src/cpp/server/stmgr-main.cpp index cc10a6ca09a..9fb9aa53e66 100644 --- a/heron/stmgr/src/cpp/server/stmgr-main.cpp +++ b/heron/stmgr/src/cpp/server/stmgr-main.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include #include @@ -58,6 +59,8 @@ int main(int argc, char* argv[]) { sp_int32 ckptmgr_port = atoi(argv[13]); sp_string ckptmgr_id = argv[14]; + HeapProfilerStart("stmgr"); + EventLoopImpl ss; // Read heron internals config from local file @@ -85,5 +88,6 @@ int main(int argc, char* argv[]) { ckptmgr_port, ckptmgr_id, high_watermark, low_watermark); mgr.Init(); ss.loop(); + HeapProfilerStop(); return 0; } diff --git a/third_party/gperftools/BUILD b/third_party/gperftools/BUILD index 195b2e2ed3b..5161b76c85b 100644 --- a/third_party/gperftools/BUILD +++ b/third_party/gperftools/BUILD @@ -100,6 +100,7 @@ cc_library( "lib/libprofiler.a", ], hdrs = [ + "include/gperftools/heap-profiler.h", "include/gperftools/profiler.h", ], includes = ["include"],