feat: support oom hook

bin/xprofctl

@@ -37,18 +37,21 @@ const args = yargs
   .command('set_config', '设置 xprofiler 配置',
     yargs => yargs
       .group(['enable_log_uv_handles', 'disable_log_uv_handles', 'log_level',
-        'log_type', 'enable_fatal_error_hook', 'disable_fatal_error_hook'], '配置项:')
+        'log_type', 'enable_fatal_error_hook', 'disable_fatal_error_hook',
+        'enable_oom_hook', 'disable_oom_hook'], '配置项:')
       .describe('enable_log_uv_handles', '开启 libuv 句柄详情采集')
       .describe('disable_log_uv_handles', '关闭 libuv 句柄详情采集')
       .describe('log_level', '日志级别: info, error, debug')
       .describe('log_type', '日志输出未知: 文件, 控制台')
       .describe('enable_fatal_error_hook', '开启 Fatal Error 钩子')
+      .describe('enable_oom_hook', '开启 Out Of Memory 钩子')
       .describe('disable_fatal_error_hook', '关闭 Fatal Error 钩子')
       .boolean('enable_log_uv_handles')
       .boolean('disable_log_uv_handles')
       .choices('log_level', [0, 1, 2])
       .choices('log_type', [0, 1])
       .boolean('enable_fatal_error_hook')
+      .boolean('enable_oom_hook')
       .boolean('disable_fatal_error_hook')
       .hide('v')
       .hide('h'))

bin/xprofctl.json

@@ -21,7 +21,9 @@
       "log_level": "number",
       "log_type": "number",
       "enable_fatal_error_hook": "boolean",
-      "disable_fatal_error_hook": "boolean"
+      "disable_fatal_error_hook": "boolean",
+      "enable_oom_hook": "boolean",
+      "disable_oom_hook": "boolean"
     }
   }
 ]

binding.gyp

@@ -39,6 +39,7 @@
                 "src/commands/report/system_statistics.cc",
                 "src/hooks/set_hooks.cc",
                 "src/hooks/fatal_error.cc",
+                "src/hooks/out_of_memory.cc",
             ],
             "include_dirs": ['<!(node -e "require(\'nan\')")'],
             'cflags_cc!': ['-fno-exceptions'],

index.d.ts

@@ -8,6 +8,7 @@ export interface XprofilerConfig {
   log_format_alinode?: boolean;
   enable_log_uv_handles?: boolean;
   enable_fatal_error_hook?: boolean;
+  enable_oom_hook?: boolean;
   patch_http?: boolean;
   patch_http_timeout?: number;
   check_throw?: boolean;

lib/configure.js

@@ -12,6 +12,7 @@ const defaultConfig = {
   log_level: 1,
   log_type: 0,
   enable_fatal_error_hook: true,
+  enable_oom_hook: false,
   patch_http: true,
   patch_http_timeout: 30, // seconds，
   check_throw: true,

src/commands/simple/config.cc

@@ -29,6 +29,7 @@ COMMAND_CALLBACK(GetXprofilerConfig) {
   data["log_level"] = GetLogLevel();
   data["log_type"] = GetLogType();
   data["enable_fatal_error_hook"] = GetEnableFatalErrorHook();
+  data["enable_oom_hook"] = GetEnableOOMErrorHook();
   data["patch_http"] = GetPatchHttp();
   data["patch_http_timeout"] = GetPatchHttpTimeout();
   data["check_throw"] = GetCheckThrow();
@@ -44,6 +45,7 @@ COMMAND_CALLBACK(SetXprofilerConfig) {
   HANDLE_CONFIG_SETTING(LOG_TYPE, log_type, LogType)
   HANDLE_CONFIG_SETTING(bool, enable_log_uv_handles, EnableLogUvHandles)
   HANDLE_CONFIG_SETTING(bool, enable_fatal_error_hook, EnableFatalErrorHook)
+  HANDLE_CONFIG_SETTING(bool, enable_oom_hook, EnableOOMErrorHook)
 
   if (!setted)
     error(format("not support setting config %s", options.dump().c_str()));

src/configure.cc

@@ -23,6 +23,7 @@ static LOG_TYPE log_type = LOG_TO_FILE;
 static bool enable_log_uv_handles = true;
 static bool log_format_alinode = false;
 static bool enable_fatal_error_hook = true;
+static bool enable_oom_hook = false;
 static bool patch_http = true;
 static uint32_t patch_http_timeout = 30;
 static bool check_throw = true;
@@ -41,6 +42,7 @@ void Configure(const FunctionCallbackInfo<Value> &info) {
   CONVERT_BOOL(enable_log_uv_handles)
   CONVERT_BOOL(log_format_alinode)
   CONVERT_BOOL(enable_fatal_error_hook)
+  CONVERT_BOOL(enable_oom_hook)
   CONVERT_BOOL(patch_http)
   CONVERT_UINT32(patch_http_timeout)
   CONVERT_BOOL(check_throw)
@@ -58,6 +60,7 @@ void GetConfig(const FunctionCallbackInfo<Value> &info) {
   CONFIG_NATIVE_NUMBER(enable_log_uv_handles, Boolean)
   CONFIG_NATIVE_NUMBER(log_format_alinode, Boolean)
   CONFIG_NATIVE_NUMBER(enable_fatal_error_hook, Boolean)
+  CONFIG_NATIVE_NUMBER(enable_oom_hook, Boolean)
   CONFIG_NATIVE_NUMBER(patch_http, Boolean)
   CONFIG_NATIVE_NUMBER(patch_http_timeout, Number)
   CONFIG_NATIVE_NUMBER(check_throw, Boolean)
@@ -73,6 +76,7 @@ DEFINE_GET_SET_FUNCTION(LogType, LOG_TYPE, log_type)
 DEFINE_GET_SET_FUNCTION(FormatAsAlinode, bool, log_format_alinode)
 DEFINE_GET_SET_FUNCTION(EnableLogUvHandles, bool, enable_log_uv_handles)
 DEFINE_GET_SET_FUNCTION(EnableFatalErrorHook, bool, enable_fatal_error_hook)
+DEFINE_GET_SET_FUNCTION(EnableOOMErrorHook, bool, enable_oom_hook)
 DEFINE_GET_SET_FUNCTION(PatchHttp, bool, patch_http)
 DEFINE_GET_SET_FUNCTION(PatchHttpTimeout, uint32_t, patch_http_timeout)
 DEFINE_GET_SET_FUNCTION(CheckThrow, bool, check_throw)

src/configure.h

@@ -62,6 +62,7 @@ DECLARE_GET_SET_FUNCTION(LogType, LOG_TYPE)
 DECLARE_GET_SET_FUNCTION(FormatAsAlinode, bool)
 DECLARE_GET_SET_FUNCTION(EnableLogUvHandles, bool)
 DECLARE_GET_SET_FUNCTION(EnableFatalErrorHook, bool)
+DECLARE_GET_SET_FUNCTION(EnableOOMErrorHook, bool)
 DECLARE_GET_SET_FUNCTION(PatchHttp, bool)
 DECLARE_GET_SET_FUNCTION(PatchHttpTimeout, uint32_t)
 DECLARE_GET_SET_FUNCTION(CheckThrow, bool)

src/hooks/out_of_memory.cc

@@ -0,0 +1,63 @@
+#include "../commands/heapdump/heap_profiler.h"
+#include "../configure.h"
+#include "../library/utils.h"
+#include "../logger.h"
+#include "../platform/platform.h"
+#include "nan.h"
+
+namespace xprofiler {
+using Nan::HandleScope;
+using std::string;
+using std::to_string;
+using v8::Isolate;
+
+static const char module_type[] = "out_of_memory";
+
+static bool oom_flag = false;
+
+Isolate* isolate_ = nullptr;
+
+size_t NearHeapLimitCallback(void* raw_data, size_t current_heap_limit,
+                             size_t initial_heap_limit) {
+  const size_t heapdump_factor = 2;
+  size_t max_limit = (std::numeric_limits<size_t>::max)() / 4;
+  size_t increased_heap =
+      std::min(max_limit, initial_heap_limit * heapdump_factor);
+  Info(module_type,
+       "current_heap_limit is %d, initial_heap_limit is %d, increased_heap is "
+       "%d.",
+       current_heap_limit, initial_heap_limit, increased_heap);
+  return increased_heap;
+}
+
+static void OnOutOfMemoryError(const char* location, bool is_heap_oom) {
+  // avoid endless loop
+  if (oom_flag) {
+    Info(module_type, "heapdump hook before oom has been executed.");
+    return;
+  }
+  oom_flag = true;
+
+#if (NODE_MODULE_VERSION >= 64)
+  Info(module_type, "increase heap limit hook.");
+  HandleScope scope;
+  isolate_->AddNearHeapLimitCallback(NearHeapLimitCallback, nullptr);
+  Sleep(2);
+#endif
+
+  // dump snapshot
+  string filepath = GetLogDir() + GetSep() + "x-oom-" + to_string(GetPid()) +
+                    "-" + ConvertTime("%Y%m%d") + "-" + RandNum() +
+                    ".heapsnapshot";
+  Info(module_type, "heapdump to %s.", filepath.c_str());
+  HeapProfiler::TakeSnapshot(filepath);
+  Info(module_type, "heapsnapshot dumped.");
+  raise(SIGABRT);
+}
+
+void SetOOMErrorHandler() {
+  Isolate* isolate = Isolate::GetCurrent();
+  isolate_ = isolate;
+  isolate->SetOOMErrorHandler(OnOutOfMemoryError);
+}
+}  // namespace xprofiler

src/hooks/out_of_memory.h

@@ -0,0 +1,8 @@
+#ifndef _SRC_HOOKS_OOM_H
+#define _SRC_HOOKS_OOM_H
+
+namespace xprofiler {
+void SetOOMErrorHandler();
+}
+
+#endif

src/hooks/set_hooks.cc

@@ -2,12 +2,18 @@
 
 #include "../configure.h"
 #include "fatal_error.h"
+#include "out_of_memory.h"
 
 namespace xprofiler {
 void SetHooks(const FunctionCallbackInfo<Value> &info) {
   // set fatal error hook
   if (GetEnableFatalErrorHook()) {
     SetFatalErrorHandler();
   }
+
+  // set oom hook
+  if (GetEnableOOMErrorHook()) {
+    SetOOMErrorHandler();
+  }
 }
 }  // namespace xprofiler

test/fixtures/command.test.js

@@ -157,6 +157,7 @@ exports = module.exports = function (logdir) {
         { key: 'data.log_level', rule: /^2$/ },
         { key: 'data.log_type', rule: /^1$/ },
         { key: 'data.enable_fatal_error_hook', rule: { label: 'true', test: value => value === true } },
+        { key: 'data.enable_oom_hook', rule: { label: 'false', test: value => value === false } },
         { key: 'data.patch_http', rule: { label: 'true', test: value => value === true } },
         { key: 'data.patch_http_timeout', rule: /^30$/ },
         { key: 'data.check_throw', rule: { label: 'false', test: value => value === false } },
@@ -166,6 +167,7 @@ exports = module.exports = function (logdir) {
           + '  - check_throw: false\n'
           + '  - enable_fatal_error_hook: true\n'
           + '  - enable_log_uv_handles: true\n'
+          + '  - enable_oom_hook: false\n'
           + `  - log_dir: ${escape(logdir)}\n`
           + '  - log_format_alinode: false\n'
           + '  - log_interval: 60\n'
@@ -178,17 +180,22 @@ exports = module.exports = function (logdir) {
     },
     {
       cmd: 'set_config',
-      options: { enable_log_uv_handles: false, log_level: 2, log_type: 1, enable_fatal_error_hook: false },
+      options: {
+        enable_log_uv_handles: false, log_level: 2, log_type: 1,
+        enable_fatal_error_hook: false, enable_oom_hook: true
+      },
       xctlRules: [
         { key: 'data.enable_log_uv_handles', rule: { label: 'false', test: value => value === false } },
         { key: 'data.log_level', rule: /^2$/ },
         { key: 'data.log_type', rule: /^1$/ },
         { key: 'data.enable_fatal_error_hook', rule: { label: 'false', test: value => value === false } },
+        { key: 'data.enable_oom_hook', rule: { label: 'true', test: value => value === true } },
       ],
       xprofctlRules(data) {
         return [new RegExp(`^X-Profiler 配置\\(pid ${data.pid}\\)成功:\n`
           + '  - enable_fatal_error_hook: false\n'
           + '  - enable_log_uv_handles: false\n'
+          + '  - enable_oom_hook: false\n'
           + '  - log_level: 2\n'
           + '  - log_type: 1')
         ];
@@ -294,6 +301,6 @@ exports = module.exports = function (logdir) {
   ];
 };
 
-exports.profileRule = { diag };
+exports.profileRule = { diag, heapsnapshot };
 
 exports.checkProfile = checkProfile;

test/fixtures/config.test.js

@@ -100,6 +100,12 @@ const configure = {
     envValue: 'NO',
     userValue: false
   },
+  enable_oom_hook: {
+    defaultValue: false,
+    envKey: 'XPROFILER_ENABLE_OOM_HOOK',
+    envValue: 'YES',
+    userValue: true
+  },
   patch_http: {
     defaultValue: true,
     envKey: 'XPROFILER_PATCH_HTTP',

test/fixtures/oom-error.js

@@ -0,0 +1,10 @@
+'use strict';
+
+const xprofiler = require('../../');
+xprofiler();
+
+const array = [];
+setInterval(() => {
+  array.push(new Array(10e6));
+  console.log('now rss:', process.memoryUsage().rss / 1024 / 1024 + ' Mb');
+}, 1);

test/hooks.test.js

@@ -10,15 +10,26 @@ const unlink = promisify(fs.unlink);
 const exists = promisify(fs.exists);
 const readFile = promisify(fs.readFile);
 const utils = require('./fixtures/utils');
-const { profileRule: { diag }, checkProfile } = require('./fixtures/command.test');
+const { profileRule: { diag, heapsnapshot }, checkProfile } = require('./fixtures/command.test');
 
 const logdir = utils.createLogDir('logdir_hooks');
 
-const cases = [{
-  title: 'fatal error hook is valid',
-  subTitle: 'x-fatal-error.diag is created when fatal error occured.',
-  jspath: path.join(__dirname, 'fixtures/fatal-error.js')
-}];
+const cases = [
+  {
+    title: 'fatal error hook is valid',
+    subTitle: 'x-fatal-error.diag is created when fatal error occured.',
+    jspath: path.join(__dirname, 'fixtures/fatal-error.js'),
+    hookFileRegexp: /x-fatal-error-(\d+)-(\d+)-(\d+).diag/,
+    profileRule: diag,
+  },
+  {
+    title: 'oom error hook is valid',
+    subTitle: 'x-oom-error.heapsnapshot is created when oom error occured.',
+    jspath: path.join(__dirname, 'fixtures/oom-error.js'),
+    hookFileRegexp: /x-oom-(\d+)-(\d+)-(\d+).heapsnapshot/,
+    profileRule: heapsnapshot,
+    env: { XPROFILER_ENABLE_OOM_HOOK: 'YES' }
+  }];
 const casesLength = cases.length;
 
 for (const cse of cases) {
@@ -31,13 +42,13 @@ for (const cse of cases) {
           XPROFILER_LOG_DIR: logdir,
           XPROFILER_LOG_LEVEL: 2,
           XPROFILER_LOG_TYPE: 1
-        })
+        }, cse.env)
       });
       await new Promise(resolve => p.on('close', resolve));
       await utils.sleep(2000);
       const files = await readdir(logdir);
       for (const file of files) {
-        if (/x-fatal-error-(\d+)-(\d+)-(\d+).diag/.test(file)) {
+        if (cse.hookFileRegexp.test(file)) {
           hookFile = path.join(logdir, file);
           const fileExists = await exists(hookFile);
           console.log('check hook file exists:', hookFile, fileExists);
@@ -72,7 +83,7 @@ for (const cse of cases) {
       describe(`it has expected structure`, function () {
         const content = fs.readFileSync(hookFile, 'utf8').trim();
         console.log('fatal error report:', content);
-        checkProfile(diag, JSON.parse(content));
+        checkProfile(cse.profileRule, JSON.parse(content));
       });
     });
   });

xprofiler.json

@@ -56,6 +56,14 @@
     ],
     "format": "boolean"
   },
+  {
+    "name": "enable_oom_hook",
+    "env": "XPROFILER_ENABLE_OOM_HOOK",
+    "rules": [
+      "boolean"
+    ],
+    "format": "boolean"
+  },
   {
     "name": "patch_http",
     "env": "XPROFILER_PATCH_HTTP",