From 6348bd17eb325372569168c885786e87caf1823c Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Tue, 6 Mar 2018 00:34:02 +0300 Subject: [PATCH 1/7] add shdict zset --- .cproject | 42 ++ .project | 26 ++ build.sh | 316 ++++++++++++++ src/ngx_http_lua_shdict.c | 878 +++++++++++++++++++++++++++++++++++++- src/ngx_http_lua_shdict.h | 12 + 5 files changed, 1272 insertions(+), 2 deletions(-) create mode 100644 .cproject create mode 100644 .project create mode 100755 build.sh diff --git a/.cproject b/.cproject new file mode 100644 index 0000000000..b800ac47cb --- /dev/null +++ b/.cproject @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000000..40a597d426 --- /dev/null +++ b/.project @@ -0,0 +1,26 @@ + + + lua-nginx-module + + + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..32ac4298b4 --- /dev/null +++ b/build.sh @@ -0,0 +1,316 @@ +#!/bin/bash + +# Copyright, Aleksey Konovkin (alkon2000@mail.ru) +# BSD license type + +download=0 +if [ "$1" == "1" ]; then + download=1 +fi +build_deps=0 + +DIR="$(pwd)" + +VERSION="1.13.6" +PCRE_VERSION="8.39" +LUAJIT_VERSION="2.1.0-beta2" + +SUFFIX="" + +BASE_PREFIX="$DIR/build" +INSTALL_PREFIX="$DIR/install" + +PCRE_PREFIX="$DIR/build/pcre-$PCRE_VERSION" +JIT_PREFIX="$DIR/build/deps/luajit" + +export LUAJIT_INC="$JIT_PREFIX/usr/local/include/luajit-2.1" +export LUAJIT_LIB="$JIT_PREFIX/usr/local/lib" + +export LD_LIBRARY_PATH="$JIT_PREFIX/lib" + +function clean() { + rm -rf install 2>/dev/null + rm -rf $(ls -1d build/* 2>/dev/null | grep -v deps) 2>/dev/null + if [ $download -eq 1 ]; then + rm -rf download 2>/dev/null + fi +} + +if [ "$1" == "clean" ]; then + clean + exit 0 +fi + +function build_luajit() { + echo "Build luajit" + cd LuaJIT-$LUAJIT_VERSION + make -j 8 > /dev/null + r=$? + if [ $r -ne 0 ]; then + exit $r + fi + DESTDIR="$JIT_PREFIX" make install > /dev/null + cd .. +} + +function build_cJSON() { + echo "Build cjson" + cd lua-cjson + PREFIX="$JIT_PREFIX/usr/local" make > /dev/null + r=$? + if [ $r -ne 0 ]; then + exit $r + fi + cd .. +} + +function build_debug() { + cd nginx-$VERSION$SUFFIX + echo "Configuring debug nginx-$VERSION$SUFFIX" + ./configure --prefix="$INSTALL_PREFIX/nginx-$VERSION$SUFFIX" \ + --with-pcre=$PCRE_PREFIX \ + --with-http_stub_status_module \ + --with-stream \ + --with-debug \ + --with-http_auth_request_module \ + --with-cc-opt="-O0" \ + --add-module=../ngx_devel_kit \ + --add-module=../../../lua-nginx-module \ + --add-module=../echo-nginx-module \ + --add-module=../memc-nginx-module > /dev/null 2>/dev/stderr +# --add-module=../drizzle-nginx-module \ + r=$? + if [ $r -ne 0 ]; then + exit $r + fi + + echo "Build debug nginx-$VERSION$SUFFIX" + make -j 8 > /dev/null 2>/dev/stderr + + r=$? + if [ $r -ne 0 ]; then + exit $r + fi + make install > /dev/null + + mv "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/sbin/nginx" "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/sbin/nginx.debug" + cd .. +} + +function build_release() { + cd nginx-$VERSION$SUFFIX + echo "Configuring release nginx-$VERSION$SUFFIX" + ./configure --prefix="$INSTALL_PREFIX/nginx-$VERSION$SUFFIX" \ + --with-pcre=$PCRE_PREFIX \ + --with-http_stub_status_module \ + --with-stream \ + --with-http_auth_request_module \ + --add-module=../ngx_devel_kit \ + --add-module=../../../lua-nginx-module \ + --add-module=../echo-nginx-module \ + --add-module=../memc-nginx-module > /dev/null 2>/dev/stderr +# --add-module=../drizzle-nginx-module \ + + r=$? + if [ $r -ne 0 ]; then + exit $r + fi + + echo "Build release nginx-$VERSION$SUFFIX" + make -j 8 > /dev/null 2>/dev/stderr + + r=$? + if [ $r -ne 0 ]; then + exit $r + fi + make install > /dev/null + cd .. +} + +function download_module() { + if [ -e $DIR/../$2 ]; then + echo "Get $DIR/../$2" + dir=$(pwd) + cd $DIR/.. + tar zcf $dir/$2.tar.gz $(ls -1d $2/* | grep -vE "(install$)|(build$)|(download$)|(.git$)") + cd $dir + else + if [ $download -eq 1 ] || [ ! -e $2.tar.gz ]; then + echo "Download $2 branch=$3" + curl -s -L -O https://github.com/$1/$2/archive/$3.zip + unzip -q $3.zip + mv $2-$3 $2 + tar zcf $2.tar.gz $2 + rm -rf $2 $3.zip + fi + fi +} + +function gitclone() { + git clone $1 > /dev/null 2> /tmp/err + if [ $? -ne 0 ]; then + cat /tmp/err + fi +} + +function download_nginx() { + if [ $download -eq 1 ] || [ ! -e nginx-$VERSION.tar.gz ]; then + echo "Download nginx-$VERSION" + curl -s -L -O http://nginx.org/download/nginx-$VERSION.tar.gz + else + echo "Get nginx-$VERSION.tar.gz" + fi +} + +function download_luajit() { + if [ $download -eq 1 ] || [ ! -e LuaJIT-$LUAJIT_VERSION.tar.gz ]; then + echo "Download LuaJIT-$LUAJIT_VERSION" + curl -s -L -O http://luajit.org/download/LuaJIT-$LUAJIT_VERSION.tar.gz + else + echo "Get LuaJIT-$LUAJIT_VERSION.tar.gz" + fi +} + +function download_pcre() { + if [ $download -eq 1 ] || [ ! -e pcre-$PCRE_VERSION.tar.gz ]; then + echo "Download PCRE-$PCRE_VERSION" + curl -s -L -O http://ftp.cs.stanford.edu/pub/exim/pcre/pcre-$PCRE_VERSION.tar.gz + else + echo "Get pcre-$PCRE_VERSION.tar.gz" + fi +} + +function extract_downloads() { + cd download + + for d in $(ls -1 *.tar.gz) + do + echo "Extracting $d" + tar zxf $d -C ../build --no-overwrite-dir --keep-old-files 2>/dev/null + done + + cd .. +} + +function download() { + mkdir build 2>/dev/null + mkdir build/deps 2>/dev/null + + mkdir download 2>/dev/null + mkdir download/lua_modules 2>/dev/null + + cd download + + download_luajit + download_pcre + download_nginx + + download_module simpl ngx_devel_kit master + download_module openresty lua-cjson master + download_module openresty echo-nginx-module master + download_module openresty drizzle-nginx-module master + download_module openresty memc-nginx-module master + + cd .. +} + +function install_file() { + echo "Install $1" + if [ ! -e "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$2" ]; then + mkdir -p "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$2" + fi + cp -r $3 $1 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$2/" +} + +function install_files() { + for f in $(ls $1) + do + install_file $f $2 $3 + done +} + +function build() { + cd build + + patch -N -p0 < ../lua-cjson-Makefile.patch + + if [ $build_deps -eq 1 ] || [ ! -e deps/luajit ]; then + build_luajit + fi + + build_cJSON + + make clean > /dev/null 2>&1 + build_debug + + make clean > /dev/null 2>&1 + build_release + + install_file "$JIT_PREFIX/usr/local/lib" . + install_file lua-cjson/cjson.so lib/lua/5.1 + + cd .. +} + +clean +download +extract_downloads +build + +function install_resty_module() { + if [ -e $DIR/../$2 ]; then + echo "Get $DIR/../$2" + dir=$(pwd) + cd $DIR/.. + zip -qr $dir/$2.zip $(ls -1d $2/* | grep -vE "(install$)|(build$)|(download$)|(.git$)") + cd $dir + else + if [ $6 -eq 1 ] || [ ! -e $2-$5.zip ] ; then + echo "Download $2 branch=$5" + rm -rf $2-$5 2>/dev/null + curl -s -L -O https://github.com/$1/$2/archive/$5.zip + mv $5.zip $2-$5.zip + else + echo "Get $2-$5" + fi + fi + echo "Install $2/$3" + if [ ! -e "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4" ]; then + mkdir -p "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4" + fi + if [ -e $2-$5.zip ]; then + unzip -q $2-$5.zip + cp -r $2-$5/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" + rm -rf $2-$5 + elif [ -e $2-$5.tar.gz ]; then + tar zxf $2-$5.tar.gz + cp -r $2-$5/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" + rm -rf $2-$5 + elif [ -e $2.zip ]; then + unzip -q $2.zip + cp -r $2/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" + rm -rf $2 + elif [ -e $2.tar.gz ]; then + tar zxf $2.tar.gz + cp -r $2/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" + rm -rf $2 + fi +} + +function make_dir() { + mkdir $INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$1 +} + +cd "$DIR" + +kernel_name=$(uname -s) +kernel_version=$(uname -r) + +cd install + +tar zcvf nginx-$VERSION$SUFFIX-$kernel_name-$kernel_version.tar.gz nginx-$VERSION$SUFFIX +rm -rf nginx-$VERSION$SUFFIX + +cd .. + +exit $r \ No newline at end of file diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 5b48eb4b0d..63a35b1030 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -41,7 +41,11 @@ static int ngx_http_lua_shdict_lpop(lua_State *L); static int ngx_http_lua_shdict_rpop(lua_State *L); static int ngx_http_lua_shdict_pop_helper(lua_State *L, int flags); static int ngx_http_lua_shdict_llen(lua_State *L); - +static int ngx_http_lua_shdict_zadd(lua_State *L); +static int ngx_http_lua_shdict_zrem(lua_State *L); +static int ngx_http_lua_shdict_zgetall(lua_State *L); +static int ngx_http_lua_shdict_zget(lua_State *L); +static int ngx_http_lua_shdict_zcard(lua_State *L); static ngx_inline ngx_shm_zone_t *ngx_http_lua_shdict_get_zone(lua_State *L, int index); @@ -67,6 +71,7 @@ enum { SHDICT_TNUMBER = 3, /* same as LUA_TNUMBER */ SHDICT_TSTRING = 4, /* same as LUA_TSTRING */ SHDICT_TLIST = 5, + SHDICT_TZSET = 6 }; @@ -78,6 +83,14 @@ ngx_http_lua_shdict_get_list_head(ngx_http_lua_shdict_node_t *sd, size_t len) } +static ngx_inline ngx_http_lua_shdict_zset_t * +ngx_http_lua_shdict_get_rbtree(ngx_http_lua_shdict_node_t *sd, size_t len) +{ + return (ngx_http_lua_shdict_zset_t *) ngx_align_ptr(((u_char *) &sd->data + len), + NGX_ALIGNMENT); +} + + ngx_int_t ngx_http_lua_shdict_init_zone(ngx_shm_zone_t *shm_zone, void *data) { @@ -330,7 +343,7 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_createtable(L, 0, lmcf->shdict_zones->nelts /* nrec */); /* ngx.shared */ - lua_createtable(L, 0 /* narr */, 18 /* nrec */); /* shared mt */ + lua_createtable(L, 0 /* narr */, 22 /* nrec */); /* shared mt */ lua_pushcfunction(L, ngx_http_lua_shdict_get); lua_setfield(L, -2, "get"); @@ -374,6 +387,21 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_pushcfunction(L, ngx_http_lua_shdict_llen); lua_setfield(L, -2, "llen"); + lua_pushcfunction(L, ngx_http_lua_shdict_zadd); + lua_setfield(L, -2, "zadd"); + + lua_pushcfunction(L, ngx_http_lua_shdict_zrem); + lua_setfield(L, -2, "zrem"); + + lua_pushcfunction(L, ngx_http_lua_shdict_zget); + lua_setfield(L, -2, "zget"); + + lua_pushcfunction(L, ngx_http_lua_shdict_zgetall); + lua_setfield(L, -2, "zgetall"); + + lua_pushcfunction(L, ngx_http_lua_shdict_zcard); + lua_setfield(L, -2, "zcard"); + lua_pushcfunction(L, ngx_http_lua_shdict_flush_all); lua_setfield(L, -2, "flush_all"); @@ -2167,6 +2195,852 @@ ngx_http_lua_shdict_llen(lua_State *L) } +static int +ngx_http_lua_shdict_zadd(lua_State *L) +{ + int n; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_str_t value; + int value_type; + ngx_rbtree_node_t *node; + ngx_rbtree_node_t *sentinel; + ngx_shm_zone_t *zone; + ngx_str_t zkey; + ngx_http_lua_shdict_zset_t *zset; + ngx_http_lua_shdict_zset_node_t *zset_node; + + n = lua_gettop(L); + + if (n != 3 && n != 4) { + return luaL_error(L, "expecting 3 or 4 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + zkey.data = (u_char *) luaL_checklstring(L, 3, &zkey.len); + + if (zkey.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty zkey"); + return 2; + } + + if (zkey.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "zkey too long"); + return 2; + } + + value.data = NULL; + + if (n == 4) { + value_type = lua_type(L, 4); + + switch (value_type) { + + case SHDICT_TSTRING: + value.data = (u_char *) lua_tolstring(L, 4, &value.len); + break; + + default: + lua_pushnil(L); + lua_pushliteral(L, "bad value type"); + return 2; + } + } + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + /* exists but expired */ + + if (rc == NGX_DONE) { + + if (sd->value_type != SHDICT_TZSET) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict zadd: found old entry and value " + "type not matched, remove it first"); + + ngx_queue_remove(&sd->queue); + + node = (ngx_rbtree_node_t *) + ((u_char *) sd - offsetof(ngx_rbtree_node_t, color)); + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_slab_free_locked(ctx->shpool, node); + + dd("go to init_list"); + goto init_zset; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict zadd: found old entry and value " + "type matched, reusing it"); + + /* free rbtree */ + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + node = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; + + for (; node != sentinel; node = zset->rbtree.root) { + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + ngx_rbtree_delete(&zset->rbtree, node); + if (zset_node->value.data) { + ngx_slab_free_locked(ctx->shpool, zset_node->value.data); + } + ngx_slab_free_locked(ctx->shpool, zset_node); + } + + sd->expires = 0; + sd->value_len = 0; + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + + dd("go to zadd_node"); + goto zadd_node; + } + + /* exists and not expired */ + + if (rc == NGX_OK) { + + if (sd->value_type != SHDICT_TZSET) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a zset"); + return 2; + } + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + + dd("go to zadd_node"); + goto zadd_node; + } + + /* rc == NGX_DECLINED, not found */ + +init_zset: + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict list: creating a new entry"); + + /* NOTICE: we assume the begin point aligned in slab, be careful */ + n = offsetof(ngx_rbtree_node_t, color) + + offsetof(ngx_http_lua_shdict_node_t, data) + + key.len + + sizeof(ngx_http_lua_shdict_zset_t); + + dd("length before aligned: %d", n); + + n = (int) (uintptr_t) ngx_align_ptr(n, NGX_ALIGNMENT); + + dd("length after aligned: %d", n); + + node = ngx_slab_alloc_locked(ctx->shpool, n); + + if (node == NULL) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushboolean(L, 0); + lua_pushliteral(L, "no memory"); + return 2; + } + + sd = (ngx_http_lua_shdict_node_t *) &node->color; + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + node->key = hash; + sd->key_len = (u_short) key.len; + + sd->expires = 0; + + sd->value_len = 0; + + dd("setting value type to %d", (int) SHDICT_TZSET); + + sd->value_type = (uint8_t) SHDICT_TZSET; + + ngx_memcpy(sd->data, key.data, key.len); + + ngx_rbtree_init(&zset->rbtree, &zset->sentinel, + ngx_rbtree_insert_value); + + ngx_rbtree_insert(&ctx->sh->rbtree, node); + + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + +zadd_node: + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict zset: creating a new zset node"); + + /* NOTICE: we assume the begin point aligned in slab, be careful */ + n = offsetof(ngx_rbtree_node_t, color) + + offsetof(ngx_http_lua_shdict_zset_node_t, data) + + zkey.len; + + node = ngx_slab_alloc_locked(ctx->shpool, n); + + if (node == NULL) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushboolean(L, 0); + lua_pushliteral(L, "no memory"); + return 2; + } + + dd("setting zset length to %d", sd->value_len + 1); + + sd->value_len = sd->value_len + 1; + + dd("setting zset node value length to %d", (int) value.len); + + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + if (value.data) { + zset_node->value.len = value.len; + zset_node->value.data = ngx_slab_alloc_locked(ctx->shpool, value.len); + if (zset_node->value.data == NULL) { + ngx_slab_free_locked(ctx->shpool, node); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushboolean(L, 0); + lua_pushliteral(L, "no memory"); + + return 2; + } + ngx_memcpy(zset_node->value.data, value.data, value.len); + } + + node->key = ngx_crc32_short(zkey.data, zkey.len); + zset_node->key_len = (u_short) zkey.len; + + ngx_memcpy(zset_node->data, zkey.data, zkey.len); + + ngx_rbtree_insert(&zset->rbtree, node); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnumber(L, sd->value_len); + return 1; +} + + +static int +ngx_http_lua_shdict_zrem(lua_State *L) +{ + int n; + ngx_str_t name; + ngx_str_t key; + ngx_str_t zkey; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_rbtree_node_t *node; + ngx_rbtree_node_t *sentinel; + ngx_shm_zone_t *zone; + ngx_http_lua_shdict_zset_t *zset; + ngx_http_lua_shdict_zset_node_t *zset_node; + + n = lua_gettop(L); + + if (n != 3) { + return luaL_error(L, "expecting 3 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + name = ctx->name; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + zkey.data = (u_char *) luaL_checklstring(L, 3, &zkey.len); + + if (zkey.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty zkey"); + return 2; + } + + if (zkey.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "zkey too long"); + return 2; + } + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushnil(L); + return 1; + } + + /* rc == NGX_OK */ + + if (sd->value_type != SHDICT_TZSET) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a zset"); + return 2; + } + + if (sd->value_len <= 0) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua zset length found for key %s " + "in shared_dict %s: %lu", key.data, name.data, + (unsigned long) sd->value_len); + } + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + node = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; + + hash = ngx_crc32_short(zkey.data, zkey.len); + + while (node != sentinel) { + + if (hash < node->key) { + node = node->left; + continue; + } + + if (hash > node->key) { + node = node->right; + continue; + } + + /* hash == node->key */ + + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + rc = ngx_memn2cmp(zkey.data, zset_node->data, zkey.len, (size_t) zset_node->key_len); + + if (rc == 0) { + if (zset_node->value.data) { + lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); + ngx_slab_free_locked(ctx->shpool, zset_node->value.data); + } else { + lua_pushlightuserdata(L, NULL); + } + + ngx_rbtree_delete(&zset->rbtree, node); + ngx_slab_free_locked(ctx->shpool, node); + sd->value_len = sd->value_len - 1; + goto ret; + } + + node = (rc < 0) ? node->left : node->right; + } + + lua_pushlightuserdata(L, NULL); + +ret: + + dd("value len: %d", (int) sd->value_len); + + if (sd->value_len == 0) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict zset: empty node after zrem, " + "remove it"); + + ngx_queue_remove(&sd->queue); + + node = (ngx_rbtree_node_t *) + ((u_char *) sd - offsetof(ngx_rbtree_node_t, color)); + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_slab_free_locked(ctx->shpool, node); + + } else { + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return 1; +} + + +static int +ngx_http_lua_shdict_zcard(lua_State *L) +{ + int n; + ngx_str_t name; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_shm_zone_t *zone; + + n = lua_gettop(L); + + if (n != 2) { + return luaL_error(L, "expecting 2 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + name = ctx->name; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushnil(L); + return 1; + } + + /* rc == NGX_OK */ + + if (sd->value_type != SHDICT_TZSET) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a zset"); + return 2; + } + + if (sd->value_len <= 0) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua zgetall length found for key %s " + "in shared_dict %s: %lu", key.data, name.data, + (unsigned long) sd->value_len); + } + + lua_pushinteger(L, sd->value_len); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return 1; +} + + +static int +ngx_http_lua_shdict_zget(lua_State *L) +{ + int n; + ngx_str_t name; + ngx_str_t key; + ngx_str_t zkey; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_rbtree_node_t *node; + ngx_rbtree_node_t *sentinel; + ngx_shm_zone_t *zone; + ngx_http_lua_shdict_zset_t *zset; + ngx_http_lua_shdict_zset_node_t *zset_node; + + n = lua_gettop(L); + + if (n != 3) { + return luaL_error(L, "expecting 3 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + name = ctx->name; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + zkey.data = (u_char *) luaL_checklstring(L, 3, &zkey.len); + + if (zkey.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty zkey"); + return 2; + } + + if (zkey.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "zkey too long"); + return 2; + } + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushnil(L); + return 1; + } + + /* rc == NGX_OK */ + + if (sd->value_type != SHDICT_TZSET) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a zset"); + return 2; + } + + if (sd->value_len <= 0) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua zget length found for key %s " + "in shared_dict %s: %lu", key.data, name.data, + (unsigned long) sd->value_len); + } + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + node = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; + + hash = ngx_crc32_short(zkey.data, zkey.len); + + while (node != sentinel) { + + if (hash < node->key) { + node = node->left; + continue; + } + + if (hash > node->key) { + node = node->right; + continue; + } + + /* hash == node->key */ + + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + rc = ngx_memn2cmp(zkey.data, zset_node->data, zkey.len, (size_t) zset_node->key_len); + + if (rc == 0) { + lua_createtable(L, 2, 0); + + lua_pushlstring(L, (char *) zset_node->data, zset_node->key_len); + lua_rawseti(L, -2, 1); + + if (zset_node->value.data) { + lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); + } else { + lua_pushnil(L); + } + lua_rawseti(L, -2, 2); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return 1; + } + + node = (rc < 0) ? node->left : node->right; + } + + lua_pushnil(L); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return 1; +} + + +static int +ngx_http_lua_shdict_zgetall(lua_State *L) +{ + int n; + ngx_str_t name; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_rbtree_node_t *node; + ngx_rbtree_node_t *sentinel; + ngx_shm_zone_t *zone; + ngx_http_lua_shdict_zset_t *zset; + ngx_http_lua_shdict_zset_node_t *zset_node; + + n = lua_gettop(L); + + if (n != 2) { + return luaL_error(L, "expecting 2 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + name = ctx->name; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushnil(L); + return 1; + } + + /* rc == NGX_OK */ + + if (sd->value_type != SHDICT_TZSET) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a zset"); + return 2; + } + + if (sd->value_len <= 0) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua zgetall length found for key %s " + "in shared_dict %s: %lu", key.data, name.data, + (unsigned long) sd->value_len); + } + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + node = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; + + lua_createtable(L, sd->value_len, 0); + + if (node != sentinel) { + n = 1; + + for (node = ngx_rbtree_min(node, sentinel); + node; + node = ngx_rbtree_next(&zset->rbtree, node)) + { + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + lua_createtable(L, 2, 0); + + lua_pushlstring(L, (char *) zset_node->data, zset_node->key_len); + lua_rawseti(L, -2, 1); + + if (zset_node->value.data) { + lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); + } else { + lua_pushnil(L); + } + lua_rawseti(L, -2, 2); + lua_rawseti(L, -2, n++); + } + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return 1; +} + + ngx_shm_zone_t * ngx_http_lua_find_zone(u_char *name_data, size_t name_len) { diff --git a/src/ngx_http_lua_shdict.h b/src/ngx_http_lua_shdict.h index 90a0099f5d..7d36170862 100644 --- a/src/ngx_http_lua_shdict.h +++ b/src/ngx_http_lua_shdict.h @@ -31,6 +31,18 @@ typedef struct { } ngx_http_lua_shdict_list_node_t; +typedef struct { + u_char color; + ngx_str_t value; + u_short key_len; + u_char data[1]; +} ngx_http_lua_shdict_zset_node_t; + +typedef struct { + ngx_rbtree_t rbtree; + ngx_rbtree_node_t sentinel; +} ngx_http_lua_shdict_zset_t; + typedef struct { ngx_rbtree_t rbtree; ngx_rbtree_node_t sentinel; From 551d98e72d8c47f63618989017712c26eb2fb0ba Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Wed, 7 Mar 2018 00:11:44 +0300 Subject: [PATCH 2/7] shdict zset lexographical key compare --- .cproject | 38 ++- src/ngx_http_lua_shdict.c | 628 +++++++++++++++++++++++++++++++------- src/ngx_http_lua_shdict.h | 2 +- 3 files changed, 551 insertions(+), 117 deletions(-) diff --git a/.cproject b/.cproject index b800ac47cb..d915bd4c52 100644 --- a/.cproject +++ b/.cproject @@ -14,17 +14,35 @@ - + - + - - + + + + + + + + + + - + + @@ -35,8 +53,16 @@ + + + + + + + + + - diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 63a35b1030..5712ff063b 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -41,11 +41,12 @@ static int ngx_http_lua_shdict_lpop(lua_State *L); static int ngx_http_lua_shdict_rpop(lua_State *L); static int ngx_http_lua_shdict_pop_helper(lua_State *L, int flags); static int ngx_http_lua_shdict_llen(lua_State *L); -static int ngx_http_lua_shdict_zadd(lua_State *L); +static int ngx_http_lua_shdict_zset(lua_State *L); static int ngx_http_lua_shdict_zrem(lua_State *L); static int ngx_http_lua_shdict_zgetall(lua_State *L); static int ngx_http_lua_shdict_zget(lua_State *L); static int ngx_http_lua_shdict_zcard(lua_State *L); +static int ngx_http_lua_shdict_zscan(lua_State *L); static ngx_inline ngx_shm_zone_t *ngx_http_lua_shdict_get_zone(lua_State *L, int index); @@ -90,6 +91,111 @@ ngx_http_lua_shdict_get_rbtree(ngx_http_lua_shdict_node_t *sd, size_t len) NGX_ALIGNMENT); } +static ngx_inline void +ngx_http_lua_shdict_push_znode_value(lua_State *L, + ngx_http_lua_shdict_zset_node_t *zset_node) +{ + double num; + u_char c; + + if (zset_node->value.data) { + + switch (zset_node->value_type) { + + case SHDICT_TSTRING: + + lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); + break; + + case SHDICT_TNUMBER: + + ngx_memcpy(&num, zset_node->value.data, sizeof(double)); + + lua_pushnumber(L, num); + break; + + case SHDICT_TBOOLEAN: + + c = *zset_node->value.data; + + lua_pushboolean(L, c ? 1 : 0); + break; + } + } else { + + lua_pushlightuserdata(L, NULL); + } +} + + +void +ngx_http_lua_shdict_zset_insert_value(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, + ngx_rbtree_node_t *sentinel) +{ + ngx_rbtree_node_t **p; + ngx_http_lua_shdict_zset_node_t *sdn, *sdnt; + + for ( ;; ) { + + sdn = (ngx_http_lua_shdict_zset_node_t *) &node->color; + sdnt = (ngx_http_lua_shdict_zset_node_t *) &temp->color; + + p = ngx_strcmp(sdn->data, sdnt->data) < 0 ? &temp->left : &temp->right; + + if (*p == sentinel) { + break; + } + + temp = *p; + } + + *p = node; + node->parent = temp; + node->left = sentinel; + node->right = sentinel; + ngx_rbt_red(node); +} + + +static ngx_inline void +ngx_http_lua_shdict_free_rbtree(ngx_http_lua_shdict_ctx_t *ctx, + ngx_http_lua_shdict_node_t *sd) +{ + ngx_rbtree_node_t *node; + ngx_rbtree_node_t *tmp; + ngx_rbtree_node_t *sentinel; + ngx_http_lua_shdict_zset_t *zset; + ngx_http_lua_shdict_zset_node_t *zset_node; + + zset = ngx_http_lua_shdict_get_rbtree(sd, sd->key_len); + + node = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; + + if (node != sentinel) { + + for (node = ngx_rbtree_min(node, sentinel); + node; + ngx_slab_free_locked(ctx->shpool, tmp)) + { + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + if (zset_node->value.data) { + ngx_slab_free_locked(ctx->shpool, zset_node->value.data); + } + + tmp = node; + + node = ngx_rbtree_next(&zset->rbtree, node); + } + + ngx_memset(zset, 0, sizeof(ngx_http_lua_shdict_zset_t)); + + ngx_rbtree_init(&zset->rbtree, &zset->sentinel, + ngx_http_lua_shdict_zset_insert_value); + } +} + ngx_int_t ngx_http_lua_shdict_init_zone(ngx_shm_zone_t *shm_zone, void *data) @@ -316,6 +422,10 @@ ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n) } } + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); + } + ngx_queue_remove(q); node = (ngx_rbtree_node_t *) @@ -343,7 +453,7 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_createtable(L, 0, lmcf->shdict_zones->nelts /* nrec */); /* ngx.shared */ - lua_createtable(L, 0 /* narr */, 22 /* nrec */); /* shared mt */ + lua_createtable(L, 0 /* narr */, 23 /* nrec */); /* shared mt */ lua_pushcfunction(L, ngx_http_lua_shdict_get); lua_setfield(L, -2, "get"); @@ -387,8 +497,8 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_pushcfunction(L, ngx_http_lua_shdict_llen); lua_setfield(L, -2, "llen"); - lua_pushcfunction(L, ngx_http_lua_shdict_zadd); - lua_setfield(L, -2, "zadd"); + lua_pushcfunction(L, ngx_http_lua_shdict_zset); + lua_setfield(L, -2, "zset"); lua_pushcfunction(L, ngx_http_lua_shdict_zrem); lua_setfield(L, -2, "zrem"); @@ -402,6 +512,9 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_pushcfunction(L, ngx_http_lua_shdict_zcard); lua_setfield(L, -2, "zcard"); + lua_pushcfunction(L, ngx_http_lua_shdict_zscan); + lua_setfield(L, -2, "zscan"); + lua_pushcfunction(L, ngx_http_lua_shdict_flush_all); lua_setfield(L, -2, "flush_all"); @@ -1101,7 +1214,8 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) if (value.data && value.len == (size_t) sd->value_len - && sd->value_type != SHDICT_TLIST) + && sd->value_type != SHDICT_TLIST + && sd->value_type != SHDICT_TZSET) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, @@ -1162,6 +1276,10 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) } } + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); + } + ngx_queue_remove(&sd->queue); node = (ngx_rbtree_node_t *) @@ -2196,7 +2314,7 @@ ngx_http_lua_shdict_llen(lua_State *L) static int -ngx_http_lua_shdict_zadd(lua_State *L) +ngx_http_lua_shdict_zset(lua_State *L) { int n; ngx_str_t key; @@ -2204,19 +2322,26 @@ ngx_http_lua_shdict_zadd(lua_State *L) ngx_int_t rc; ngx_http_lua_shdict_ctx_t *ctx; ngx_http_lua_shdict_node_t *sd; + int value_type = 0; ngx_str_t value; - int value_type; - ngx_rbtree_node_t *node; + ngx_str_t old_value; + double num; + u_char c; + ngx_rbtree_node_t *node = NULL; + ngx_rbtree_node_t *znode = NULL; ngx_rbtree_node_t *sentinel; ngx_shm_zone_t *zone; ngx_str_t zkey; ngx_http_lua_shdict_zset_t *zset; ngx_http_lua_shdict_zset_node_t *zset_node; + lua_Number exptime = 0; + ngx_time_t *tp; + u_char exists = 0; n = lua_gettop(L); - if (n != 3 && n != 4) { - return luaL_error(L, "expecting 3 or 4 arguments, " + if (n < 3 || n > 5) { + return luaL_error(L, "expecting 3, 4 or 5 arguments, " "but only seen %d", n); } @@ -2269,7 +2394,7 @@ ngx_http_lua_shdict_zadd(lua_State *L) value.data = NULL; - if (n == 4) { + if (n > 3) { value_type = lua_type(L, 4); switch (value_type) { @@ -2278,11 +2403,33 @@ ngx_http_lua_shdict_zadd(lua_State *L) value.data = (u_char *) lua_tolstring(L, 4, &value.len); break; + case SHDICT_TNUMBER: + value.len = sizeof(double); + num = lua_tonumber(L, 3); + value.data = (u_char *) # + break; + + case SHDICT_TBOOLEAN: + value.len = sizeof(u_char); + c = lua_toboolean(L, 3) ? 1 : 0; + value.data = &c; + break; + + case SHDICT_TNIL: + break; + default: lua_pushnil(L); lua_pushliteral(L, "bad value type"); return 2; } + + if (n == 5) { + exptime = luaL_checknumber(L, 5); + if (exptime < 0) { + return luaL_error(L, "bad \"exptime\" argument"); + } + } } ngx_shmtx_lock(&ctx->shpool->mutex); @@ -2295,13 +2442,13 @@ ngx_http_lua_shdict_zadd(lua_State *L) dd("shdict lookup returned %d", (int) rc); - /* exists but expired */ - if (rc == NGX_DONE) { + /* exists but expired */ + if (sd->value_type != SHDICT_TZSET) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, - "lua shared dict zadd: found old entry and value " + "lua shared dict zset: found old entry and value " "type not matched, remove it first"); ngx_queue_remove(&sd->queue); @@ -2318,33 +2465,29 @@ ngx_http_lua_shdict_zadd(lua_State *L) } ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, - "lua shared dict zadd: found old entry and value " + "lua shared dict zset: found old entry and value " "type matched, reusing it"); - /* free rbtree */ + ngx_http_lua_shdict_free_rbtree(ctx, sd); zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); - node = zset->rbtree.root; - sentinel = zset->rbtree.sentinel; + if (exptime > 0) { + tp = ngx_timeofday(); + sd->expires = (uint64_t) tp->sec * 1000 + tp->msec + + (uint64_t) (exptime * 1000); - for (; node != sentinel; node = zset->rbtree.root) { - zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; - ngx_rbtree_delete(&zset->rbtree, node); - if (zset_node->value.data) { - ngx_slab_free_locked(ctx->shpool, zset_node->value.data); - } - ngx_slab_free_locked(ctx->shpool, zset_node); + } else { + sd->expires = 0; } - sd->expires = 0; sd->value_len = 0; ngx_queue_remove(&sd->queue); ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); - dd("go to zadd_node"); - goto zadd_node; + dd("go to add_node"); + goto add_node; } /* exists and not expired */ @@ -2364,8 +2507,8 @@ ngx_http_lua_shdict_zadd(lua_State *L) ngx_queue_remove(&sd->queue); ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); - dd("go to zadd_node"); - goto zadd_node; + dd("go to add_node"); + goto add_node; } /* rc == NGX_DECLINED, not found */ @@ -2373,7 +2516,7 @@ ngx_http_lua_shdict_zadd(lua_State *L) init_zset: ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, - "lua shared dict list: creating a new entry"); + "lua shared dict zset: creating a new entry"); /* NOTICE: we assume the begin point aligned in slab, be careful */ n = offsetof(ngx_rbtree_node_t, color) @@ -2404,7 +2547,14 @@ ngx_http_lua_shdict_zadd(lua_State *L) node->key = hash; sd->key_len = (u_short) key.len; - sd->expires = 0; + if (exptime > 0) { + tp = ngx_timeofday(); + sd->expires = (uint64_t) tp->sec * 1000 + tp->msec + + (uint64_t) (exptime * 1000); + + } else { + sd->expires = 0; + } sd->value_len = 0; @@ -2415,65 +2565,140 @@ ngx_http_lua_shdict_zadd(lua_State *L) ngx_memcpy(sd->data, key.data, key.len); ngx_rbtree_init(&zset->rbtree, &zset->sentinel, - ngx_rbtree_insert_value); + ngx_http_lua_shdict_zset_insert_value); ngx_rbtree_insert(&ctx->sh->rbtree, node); ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); -zadd_node: +add_node: - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, - "lua shared dict zset: creating a new zset node"); + /* search first */ - /* NOTICE: we assume the begin point aligned in slab, be careful */ - n = offsetof(ngx_rbtree_node_t, color) - + offsetof(ngx_http_lua_shdict_zset_node_t, data) - + zkey.len; + znode = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; - node = ngx_slab_alloc_locked(ctx->shpool, n); + while (znode != sentinel) { - if (node == NULL) { - ngx_shmtx_unlock(&ctx->shpool->mutex); - lua_pushboolean(L, 0); - lua_pushliteral(L, "no memory"); - return 2; + zset_node = (ngx_http_lua_shdict_zset_node_t *) &znode->color; + + rc = ngx_strcmp(zkey.data, zset_node->data); + + if (rc < 0) { + + znode = znode->left; + continue; + + } else if (rc > 0) { + + znode = znode->right; + continue; + + } + + /* found */ + + exists = 1; + break; } - dd("setting zset length to %d", sd->value_len + 1); + if (exists == 0) { - sd->value_len = sd->value_len + 1; + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict zset: creating a new zset node"); + + /* NOTICE: we assume the begin point aligned in slab, be careful */ + n = offsetof(ngx_rbtree_node_t, color) + + offsetof(ngx_http_lua_shdict_zset_node_t, data) + + zkey.len + 1 /* zero terminated string key */; + + dd("length before aligned: %d", n); + + n = (int) (uintptr_t) ngx_align_ptr(n, NGX_ALIGNMENT); + + dd("length after aligned: %d", n); + + znode = ngx_slab_alloc_locked(ctx->shpool, n); + + if (znode == NULL) { + + if (node != NULL && sd->value_len == 0) { - dd("setting zset node value length to %d", (int) value.len); + ngx_rbtree_delete(&ctx->sh->rbtree, node); - zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + ngx_queue_remove(&sd->queue); + + ngx_slab_free_locked(ctx->shpool, node); + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushboolean(L, 0); + lua_pushliteral(L, "no memory"); + return 2; + } + } + + zset_node = (ngx_http_lua_shdict_zset_node_t *) &znode->color; + + old_value = zset_node->value; if (value.data) { + + dd("setting zset node value length to %d", (int) value.len); + + zset_node->value_type = value_type; zset_node->value.len = value.len; zset_node->value.data = ngx_slab_alloc_locked(ctx->shpool, value.len); + if (zset_node->value.data == NULL) { - ngx_slab_free_locked(ctx->shpool, node); + + if (exists == 0) { + /* not replaced */ + ngx_slab_free_locked(ctx->shpool, znode); + } + + if (node != NULL && sd->value_len == 0) { + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_queue_remove(&sd->queue); + + ngx_slab_free_locked(ctx->shpool, node); + } ngx_shmtx_unlock(&ctx->shpool->mutex); lua_pushboolean(L, 0); lua_pushliteral(L, "no memory"); - return 2; } + ngx_memcpy(zset_node->value.data, value.data, value.len); + } else { + + ngx_str_null(&zset_node->value); + zset_node->value_type = 0; + } + + if (old_value.data) { + ngx_slab_free_locked(ctx->shpool, old_value.data); } - node->key = ngx_crc32_short(zkey.data, zkey.len); - zset_node->key_len = (u_short) zkey.len; + if (exists == 0) { + dd("setting zset length to %d", sd->value_len + 1); + sd->value_len = sd->value_len + 1; + } ngx_memcpy(zset_node->data, zkey.data, zkey.len); + zset_node->data[zkey.len] = 0; - ngx_rbtree_insert(&zset->rbtree, node); + ngx_rbtree_insert(&zset->rbtree, znode); ngx_shmtx_unlock(&ctx->shpool->mutex); lua_pushnumber(L, sd->value_len); + return 1; } @@ -2589,41 +2814,38 @@ ngx_http_lua_shdict_zrem(lua_State *L) node = zset->rbtree.root; sentinel = zset->rbtree.sentinel; - hash = ngx_crc32_short(zkey.data, zkey.len); - while (node != sentinel) { - if (hash < node->key) { + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + rc = ngx_strcmp(zkey.data, zset_node->data); + + if (rc < 0) { + node = node->left; continue; - } - if (hash > node->key) { + } else if (rc > 0) { + node = node->right; continue; + } - /* hash == node->key */ + /* found */ - zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + ngx_http_lua_shdict_push_znode_value(L, zset_node); - rc = ngx_memn2cmp(zkey.data, zset_node->data, zkey.len, (size_t) zset_node->key_len); + if (zset_node->value.data) { + ngx_slab_free_locked(ctx->shpool, zset_node->value.data); + } - if (rc == 0) { - if (zset_node->value.data) { - lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); - ngx_slab_free_locked(ctx->shpool, zset_node->value.data); - } else { - lua_pushlightuserdata(L, NULL); - } + ngx_rbtree_delete(&zset->rbtree, node); + ngx_slab_free_locked(ctx->shpool, node); - ngx_rbtree_delete(&zset->rbtree, node); - ngx_slab_free_locked(ctx->shpool, node); - sd->value_len = sd->value_len - 1; - goto ret; - } + sd->value_len = sd->value_len - 1; - node = (rc < 0) ? node->left : node->right; + goto ret; } lua_pushlightuserdata(L, NULL); @@ -2633,7 +2855,6 @@ ngx_http_lua_shdict_zrem(lua_State *L) dd("value len: %d", (int) sd->value_len); if (sd->value_len == 0) { - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "lua shared dict zset: empty node after zrem, " "remove it"); @@ -2864,45 +3085,33 @@ ngx_http_lua_shdict_zget(lua_State *L) node = zset->rbtree.root; sentinel = zset->rbtree.sentinel; - hash = ngx_crc32_short(zkey.data, zkey.len); - while (node != sentinel) { - if (hash < node->key) { - node = node->left; - continue; - } + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; - if (hash > node->key) { - node = node->right; - continue; - } + rc = ngx_strcmp(zkey.data, zset_node->data); - /* hash == node->key */ + if (rc < 0) { - zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + node = node->left; + continue; - rc = ngx_memn2cmp(zkey.data, zset_node->data, zkey.len, (size_t) zset_node->key_len); + } else if (rc > 0) { - if (rc == 0) { - lua_createtable(L, 2, 0); + node = node->right; + continue; - lua_pushlstring(L, (char *) zset_node->data, zset_node->key_len); - lua_rawseti(L, -2, 1); + } - if (zset_node->value.data) { - lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); - } else { - lua_pushnil(L); - } - lua_rawseti(L, -2, 2); + /* push zkey */ + lua_pushlstring(L, (char *) zkey.data, zkey.len); - ngx_shmtx_unlock(&ctx->shpool->mutex); + /* push zvalue */ + ngx_http_lua_shdict_push_znode_value(L, zset_node); - return 1; - } + ngx_shmtx_unlock(&ctx->shpool->mutex); - node = (rc < 0) ? node->left : node->right; + return 2; } lua_pushnil(L); @@ -3022,15 +3231,14 @@ ngx_http_lua_shdict_zgetall(lua_State *L) lua_createtable(L, 2, 0); - lua_pushlstring(L, (char *) zset_node->data, zset_node->key_len); + /* push zkey */ + lua_pushstring(L, (char *) zset_node->data); lua_rawseti(L, -2, 1); - if (zset_node->value.data) { - lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); - } else { - lua_pushnil(L); - } + /* push zvalue */ + ngx_http_lua_shdict_push_znode_value(L, zset_node); lua_rawseti(L, -2, 2); + lua_rawseti(L, -2, n++); } } @@ -3041,6 +3249,206 @@ ngx_http_lua_shdict_zgetall(lua_State *L) } +static int +ngx_http_lua_shdict_zscan(lua_State *L) +{ + int n; + ngx_str_t name; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_rbtree_node_t *node; + ngx_rbtree_node_t *tmp = NULL; + ngx_rbtree_node_t *sentinel; + ngx_shm_zone_t *zone; + ngx_http_lua_shdict_zset_t *zset; + ngx_http_lua_shdict_zset_node_t *zset_node; + ngx_str_t lbound; + u_char *err_msg; + size_t len; + + n = lua_gettop(L); + + if (n != 3 && n != 4) { + return luaL_error(L, "expecting 3 or 4 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + if (lua_type(L, 3) != LUA_TFUNCTION) { + return luaL_error(L, "bad \"callback\" argument"); + } + + ctx = zone->data; + name = ctx->name; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + if (n == 4) { + lbound.data = (u_char *) lua_tolstring(L, 4, &lbound.len); + } else { + lbound.data = NULL; + } + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushnil(L); + return 1; + } + + /* rc == NGX_OK */ + + if (sd->value_type != SHDICT_TZSET) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a zset"); + return 2; + } + + if (sd->value_len <= 0) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua zgetall length found for key %s " + "in shared_dict %s: %lu", key.data, name.data, + (unsigned long) sd->value_len); + } + + zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + + node = zset->rbtree.root; + sentinel = zset->rbtree.sentinel; + + if (node != sentinel) { + + n = lua_gettop(L); + + if (lbound.data != NULL) { + + while (node != sentinel) { + + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + rc = ngx_strncmp(lbound.data, zset_node->data, lbound.len); + + if (rc <= 0) { + + if (rc == 0) { + tmp = node; + } + + node = node->left; + + continue; + + } else if (rc > 0) { + + node = node->right; + continue; + + } + } + + if (tmp != NULL) { + node = tmp; + } + } else { + + node = ngx_rbtree_min(node, sentinel); + } + + if (node != sentinel) { + + for (; node; node = ngx_rbtree_next(&zset->rbtree, node)) + { + lua_pushvalue(L, 3); + + zset_node = (ngx_http_lua_shdict_zset_node_t *) &node->color; + + /* push zkey */ + lua_pushstring(L, (char *) zset_node->data); + + /* push zvalue */ + ngx_http_lua_shdict_push_znode_value(L, zset_node); + + if (lua_pcall(L, 2, 2, 0) != 0) { + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + /* error occurred when calling user code */ + err_msg = (u_char *) lua_tolstring(L, -1, &len); + + if (err_msg == NULL) { + err_msg = (u_char *) "unknown reason"; + len = sizeof("unknown reason") - 1; + } + + return luaL_error(L, "user callback error " + "shared_dict %s: %s", name.data, err_msg); + } + + if (lua_isboolean(L, -1)) { + + if (lua_toboolean(L, -1)) { + + /* stop on true */ + break; + } + } + + lua_settop(L, n); + } + } + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushboolean(L, 1); + + return 1; +} + + ngx_shm_zone_t * ngx_http_lua_find_zone(u_char *name_data, size_t name_len) { diff --git a/src/ngx_http_lua_shdict.h b/src/ngx_http_lua_shdict.h index 7d36170862..35cc74da45 100644 --- a/src/ngx_http_lua_shdict.h +++ b/src/ngx_http_lua_shdict.h @@ -33,8 +33,8 @@ typedef struct { typedef struct { u_char color; + uint8_t value_type; ngx_str_t value; - u_short key_len; u_char data[1]; } ngx_http_lua_shdict_zset_node_t; From 4c440393cb1735dad30872668b3bf35167285e1e Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Thu, 8 Mar 2018 19:01:20 +0300 Subject: [PATCH 3/7] dev zset --- .gitignore | 1 + build.sh | 38 +----- src/ngx_http_lua_shdict.c | 180 ++++++++++++------------- t/155-shdict-zset.t | 272 ++++++++++++++++++++++++++++++++++++++ tests.sh | 30 +++++ 5 files changed, 388 insertions(+), 133 deletions(-) create mode 100644 t/155-shdict-zset.t create mode 100755 tests.sh diff --git a/.gitignore b/.gitignore index eff337246b..d947cab0a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +install/ build/ work/ tags diff --git a/build.sh b/build.sh index 32ac4298b4..ed32ac3cc0 100755 --- a/build.sh +++ b/build.sh @@ -77,8 +77,7 @@ function build_debug() { --add-module=../ngx_devel_kit \ --add-module=../../../lua-nginx-module \ --add-module=../echo-nginx-module \ - --add-module=../memc-nginx-module > /dev/null 2>/dev/stderr -# --add-module=../drizzle-nginx-module \ + --add-module=../memc-nginx-module >/dev/null 2>&1 r=$? if [ $r -ne 0 ]; then exit $r @@ -93,37 +92,6 @@ function build_debug() { fi make install > /dev/null - mv "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/sbin/nginx" "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/sbin/nginx.debug" - cd .. -} - -function build_release() { - cd nginx-$VERSION$SUFFIX - echo "Configuring release nginx-$VERSION$SUFFIX" - ./configure --prefix="$INSTALL_PREFIX/nginx-$VERSION$SUFFIX" \ - --with-pcre=$PCRE_PREFIX \ - --with-http_stub_status_module \ - --with-stream \ - --with-http_auth_request_module \ - --add-module=../ngx_devel_kit \ - --add-module=../../../lua-nginx-module \ - --add-module=../echo-nginx-module \ - --add-module=../memc-nginx-module > /dev/null 2>/dev/stderr -# --add-module=../drizzle-nginx-module \ - - r=$? - if [ $r -ne 0 ]; then - exit $r - fi - - echo "Build release nginx-$VERSION$SUFFIX" - make -j 8 > /dev/null 2>/dev/stderr - - r=$? - if [ $r -ne 0 ]; then - exit $r - fi - make install > /dev/null cd .. } @@ -208,7 +176,6 @@ function download() { download_module simpl ngx_devel_kit master download_module openresty lua-cjson master download_module openresty echo-nginx-module master - download_module openresty drizzle-nginx-module master download_module openresty memc-nginx-module master cd .. @@ -243,9 +210,6 @@ function build() { make clean > /dev/null 2>&1 build_debug - make clean > /dev/null 2>&1 - build_release - install_file "$JIT_PREFIX/usr/local/lib" . install_file lua-cjson/cjson.so lib/lua/5.1 diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 5712ff063b..5837e7b640 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -83,6 +83,27 @@ ngx_http_lua_shdict_get_list_head(ngx_http_lua_shdict_node_t *sd, size_t len) NGX_ALIGNMENT); } +static ngx_inline void +ngx_http_lua_shdict_free_list(ngx_http_lua_shdict_ctx_t *ctx, + ngx_http_lua_shdict_node_t *sd) +{ + ngx_queue_t *queue, *q; + u_char *p; + + queue = ngx_http_lua_shdict_get_list_head(sd, sd->key_len); + + for (q = ngx_queue_head(queue); + q != ngx_queue_sentinel(queue); + q = ngx_queue_next(q)) + { + p = (u_char *) ngx_queue_data(q, + ngx_http_lua_shdict_list_node_t, + queue); + + ngx_slab_free_locked(ctx->shpool, p); + } +} + static ngx_inline ngx_http_lua_shdict_zset_t * ngx_http_lua_shdict_get_rbtree(ngx_http_lua_shdict_node_t *sd, size_t len) @@ -188,11 +209,6 @@ ngx_http_lua_shdict_free_rbtree(ngx_http_lua_shdict_ctx_t *ctx, node = ngx_rbtree_next(&zset->rbtree, node); } - - ngx_memset(zset, 0, sizeof(ngx_http_lua_shdict_zset_t)); - - ngx_rbtree_init(&zset->rbtree, &zset->sentinel, - ngx_http_lua_shdict_zset_insert_value); } } @@ -369,12 +385,11 @@ ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n) { ngx_time_t *tp; uint64_t now; - ngx_queue_t *q, *list_queue, *lq; + ngx_queue_t *q; int64_t ms; ngx_rbtree_node_t *node; ngx_http_lua_shdict_node_t *sd; int freed = 0; - ngx_http_lua_shdict_list_node_t *lnode; tp = ngx_timeofday(); @@ -409,17 +424,7 @@ ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n) } if (sd->value_type == SHDICT_TLIST) { - list_queue = ngx_http_lua_shdict_get_list_head(sd, sd->key_len); - - for (lq = ngx_queue_head(list_queue); - lq != ngx_queue_sentinel(list_queue); - lq = ngx_queue_next(lq)) - { - lnode = ngx_queue_data(lq, ngx_http_lua_shdict_list_node_t, - queue); - - ngx_slab_free_locked(ctx->shpool, lnode); - } + ngx_http_lua_shdict_free_list(ctx, sd); } if (sd->value_type == SHDICT_TZSET) { @@ -719,6 +724,14 @@ ngx_http_lua_shdict_get_helper(lua_State *L, int get_stale) lua_pushliteral(L, "value is a list"); return 2; + case SHDICT_TZSET: + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value is a zset"); + return 2; + default: ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -819,7 +832,7 @@ ngx_http_lua_shdict_flush_all(lua_State *L) static int ngx_http_lua_shdict_flush_expired(lua_State *L) { - ngx_queue_t *q, *prev, *list_queue, *lq; + ngx_queue_t *q, *prev; ngx_http_lua_shdict_node_t *sd; ngx_http_lua_shdict_ctx_t *ctx; ngx_shm_zone_t *zone; @@ -829,7 +842,6 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) ngx_rbtree_node_t *node; uint64_t now; int n; - ngx_http_lua_shdict_list_node_t *lnode; n = lua_gettop(L); @@ -872,17 +884,11 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) if (sd->expires != 0 && sd->expires <= now) { if (sd->value_type == SHDICT_TLIST) { - list_queue = ngx_http_lua_shdict_get_list_head(sd, sd->key_len); - - for (lq = ngx_queue_head(list_queue); - lq != ngx_queue_sentinel(list_queue); - lq = ngx_queue_next(lq)) - { - lnode = ngx_queue_data(lq, ngx_http_lua_shdict_list_node_t, - queue); + ngx_http_lua_shdict_free_list(ctx, sd); + } - ngx_slab_free_locked(ctx->shpool, lnode); - } + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); } ngx_queue_remove(q); @@ -1065,7 +1071,6 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) /* indicates whether to foricibly override other * valid entries */ int32_t user_flags = 0; - ngx_queue_t *queue, *q; n = lua_gettop(L); @@ -1262,18 +1267,7 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) remove: if (sd->value_type == SHDICT_TLIST) { - queue = ngx_http_lua_shdict_get_list_head(sd, key.len); - - for (q = ngx_queue_head(queue); - q != ngx_queue_sentinel(queue); - q = ngx_queue_next(q)) - { - p = (u_char *) ngx_queue_data(q, - ngx_http_lua_shdict_list_node_t, - queue); - - ngx_slab_free_locked(ctx->shpool, p); - } + ngx_http_lua_shdict_free_list(ctx, sd); } if (sd->value_type == SHDICT_TZSET) { @@ -1410,7 +1404,6 @@ ngx_http_lua_shdict_incr(lua_State *L) /* indicates whether to foricibly override other * valid entries */ int forcible = 0; - ngx_queue_t *queue, *q; n = lua_gettop(L); @@ -1488,7 +1481,8 @@ ngx_http_lua_shdict_incr(lua_State *L) /* found an expired item */ if ((size_t) sd->value_len == sizeof(double) - && sd->value_type != SHDICT_TLIST) + && sd->value_type != SHDICT_TLIST + && sd->value_type != SHDICT_TZSET) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "lua shared dict incr: found old entry and " @@ -1544,18 +1538,11 @@ ngx_http_lua_shdict_incr(lua_State *L) "NOT matched, removing it first"); if (sd->value_type == SHDICT_TLIST) { - queue = ngx_http_lua_shdict_get_list_head(sd, key.len); - - for (q = ngx_queue_head(queue); - q != ngx_queue_sentinel(queue); - q = ngx_queue_next(q)) - { - p = (u_char *) ngx_queue_data(q, - ngx_http_lua_shdict_list_node_t, - queue); + ngx_http_lua_shdict_free_list(ctx, sd); + } - ngx_slab_free_locked(ctx->shpool, p); - } + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); } ngx_queue_remove(&sd->queue); @@ -1773,8 +1760,8 @@ ngx_http_lua_shdict_push_helper(lua_State *L, int flags) double num; ngx_rbtree_node_t *node; ngx_shm_zone_t *zone; - ngx_queue_t *queue, *q; ngx_http_lua_shdict_list_node_t *lnode; + ngx_queue_t *queue; n = lua_gettop(L); @@ -1857,6 +1844,10 @@ ngx_http_lua_shdict_push_helper(lua_State *L, int flags) "lua shared dict push: found old entry and value " "type not matched, remove it first"); + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); + } + ngx_queue_remove(&sd->queue); node = (ngx_rbtree_node_t *) @@ -1878,16 +1869,9 @@ ngx_http_lua_shdict_push_helper(lua_State *L, int flags) /* free list nodes */ - queue = ngx_http_lua_shdict_get_list_head(sd, key.len); + ngx_http_lua_shdict_free_list(ctx, sd); - for (q = ngx_queue_head(queue); - q != ngx_queue_sentinel(queue); - q = ngx_queue_next(q)) - { - /* TODO: reuse matched size list node */ - lnode = ngx_queue_data(q, ngx_http_lua_shdict_list_node_t, queue); - ngx_slab_free_locked(ctx->shpool, lnode); - } + queue = ngx_http_lua_shdict_get_list_head(sd, key.len); ngx_queue_init(queue); @@ -2405,13 +2389,13 @@ ngx_http_lua_shdict_zset(lua_State *L) case SHDICT_TNUMBER: value.len = sizeof(double); - num = lua_tonumber(L, 3); + num = lua_tonumber(L, 4); value.data = (u_char *) # break; case SHDICT_TBOOLEAN: value.len = sizeof(u_char); - c = lua_toboolean(L, 3) ? 1 : 0; + c = lua_toboolean(L, 4) ? 1 : 0; value.data = &c; break; @@ -2451,6 +2435,10 @@ ngx_http_lua_shdict_zset(lua_State *L) "lua shared dict zset: found old entry and value " "type not matched, remove it first"); + if (sd->value_type == SHDICT_TLIST) { + ngx_http_lua_shdict_free_list(ctx, sd); + } + ngx_queue_remove(&sd->queue); node = (ngx_rbtree_node_t *) @@ -2460,7 +2448,7 @@ ngx_http_lua_shdict_zset(lua_State *L) ngx_slab_free_locked(ctx->shpool, node); - dd("go to init_list"); + dd("go to init_zset"); goto init_zset; } @@ -2468,10 +2456,15 @@ ngx_http_lua_shdict_zset(lua_State *L) "lua shared dict zset: found old entry and value " "type matched, reusing it"); + /* free rbtree */ + ngx_http_lua_shdict_free_rbtree(ctx, sd); zset = ngx_http_lua_shdict_get_rbtree(sd, key.len); + ngx_rbtree_init(&zset->rbtree, &zset->sentinel, + ngx_http_lua_shdict_zset_insert_value); + if (exptime > 0) { tp = ngx_timeofday(); sd->expires = (uint64_t) tp->sec * 1000 + tp->msec @@ -2618,7 +2611,7 @@ ngx_http_lua_shdict_zset(lua_State *L) dd("length after aligned: %d", n); - znode = ngx_slab_alloc_locked(ctx->shpool, n); + znode = ngx_slab_calloc_locked(ctx->shpool, n); if (znode == NULL) { @@ -2649,7 +2642,8 @@ ngx_http_lua_shdict_zset(lua_State *L) zset_node->value_type = value_type; zset_node->value.len = value.len; - zset_node->value.data = ngx_slab_alloc_locked(ctx->shpool, value.len); + zset_node->value.data = ngx_slab_alloc_locked(ctx->shpool, + (uintptr_t) ngx_align_ptr(value.len, NGX_ALIGNMENT)); if (zset_node->value.data == NULL) { @@ -3502,7 +3496,6 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, uint32_t hash; ngx_int_t rc; ngx_time_t *tp; - ngx_queue_t *queue, *q; ngx_rbtree_node_t *node; ngx_http_lua_shdict_ctx_t *ctx; ngx_http_lua_shdict_node_t *sd; @@ -3606,7 +3599,8 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, if (str_value_buf && str_value_len == (size_t) sd->value_len - && sd->value_type != SHDICT_TLIST) + && sd->value_type != SHDICT_TLIST + && sd->value_type != SHDICT_TZSET) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, @@ -3650,18 +3644,11 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, remove: if (sd->value_type == SHDICT_TLIST) { - queue = ngx_http_lua_shdict_get_list_head(sd, key_len); - - for (q = ngx_queue_head(queue); - q != ngx_queue_sentinel(queue); - q = ngx_queue_next(q)) - { - p = (u_char *) ngx_queue_data(q, - ngx_http_lua_shdict_list_node_t, - queue); + ngx_http_lua_shdict_free_list(ctx, sd); + } - ngx_slab_free_locked(ctx->shpool, p); - } + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); } ngx_queue_remove(&sd->queue); @@ -3875,6 +3862,13 @@ ngx_http_lua_ffi_shdict_get(ngx_shm_zone_t *zone, u_char *key, *err = "value is a list"; return NGX_ERROR; + case SHDICT_TZSET: + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + *err = "value is a zset"; + return NGX_ERROR; + default: ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -3915,7 +3909,6 @@ ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key, double num; ngx_rbtree_node_t *node; u_char *p; - ngx_queue_t *queue, *q; if (zone == NULL) { return NGX_ERROR; @@ -3953,7 +3946,8 @@ ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key, /* found an expired item */ if ((size_t) sd->value_len == sizeof(double) - && sd->value_type != SHDICT_TLIST) + && sd->value_type != SHDICT_TLIST + && sd->value_type != SHDICT_TZSET) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "lua shared dict incr: found old entry and " @@ -4006,17 +4000,11 @@ ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key, "NOT matched, removing it first"); if (sd->value_type == SHDICT_TLIST) { - queue = ngx_http_lua_shdict_get_list_head(sd, key_len); - - for (q = ngx_queue_head(queue); - q != ngx_queue_sentinel(queue); - q = ngx_queue_next(q)) - { - p = (u_char *) ngx_queue_data(q, ngx_http_lua_shdict_list_node_t, - queue); + ngx_http_lua_shdict_free_list(ctx, sd); + } - ngx_slab_free_locked(ctx->shpool, p); - } + if (sd->value_type == SHDICT_TZSET) { + ngx_http_lua_shdict_free_rbtree(ctx, sd); } ngx_queue_remove(&sd->queue); diff --git a/t/155-shdict-zset.t b/t/155-shdict-zset.t new file mode 100644 index 0000000000..f220f70524 --- /dev/null +++ b/t/155-shdict-zset.t @@ -0,0 +1,272 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use Test::Nginx::Socket::Lua; + +#worker_connections(1014); +#master_process_enabled(1); +#log_level('warn'); + +repeat_each(2); + +plan tests => repeat_each() * (blocks() * 3 + 0); + +#no_diff(); +no_long_string(); +#master_on(); +#workers(2); + +run_tests(); + +__DATA__ + +=== TEST 1: zset & zget & zrem +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:zset("foo", "bar", "hello") + if len then + ngx.say("zset ", len) + else + ngx.say("szet err: ", err) + end + + local len, err = dogs:zset("foo", "foo", 999) + if len then + ngx.say("zset ", len) + else + ngx.say("szet err: ", err) + end + + local zkey, val = dogs:zget("foo", "bar") + ngx.say(zkey, " ", val) + + local zkey, val = dogs:zget("foo", "foo") + ngx.say(zkey, " ", val) + + local val, err = dogs:zrem("foo", "bar") + if val then + ngx.say(val) + else + ngx.say("zrem err: ", err) + end + + local val, err = dogs:zrem("foo", "foo") + if val then + ngx.say(val) + else + ngx.say("zrem err: ", err) + end + + dogs:delete("foo") + } + } +--- request +GET /test +--- response_body +zset 1 +zset 2 +bar hello +foo 999 +hello +999 +--- no_error_log +[error] + + +=== TEST 2: exptime +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:zset("foo", "bar", "hello", 1) + if len then + ngx.say("zset ", len) + else + ngx.say("zset err: ", err) + end + + local zkey, val = dogs:zget("foo", "bar") + ngx.say(zkey, " ", val) + + ngx.sleep(2) + + local zkey, val = dogs:zget("foo", "bar") + ngx.say(zkey) + + local len, err = dogs:zset("foo", "bar", "hello2") + if len then + ngx.say("zset ", len) + else + ngx.say("zset err: ", err) + end + + local zkey, val = dogs:zget("foo", "bar") + ngx.say(zkey, " ", val) + + local val, err = dogs:zrem("foo", "bar") + if val then + ngx.say(val) + else + ngx.say("zrem err: ", err) + end + + dogs:delete("foo") + } + } +--- request +GET /test +--- response_body +zset 1 +bar hello +nil +zset 1 +bar hello2 +hello2 +--- no_error_log +[error] + + +=== TEST 3: zset & zgetall +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local vals = { + { "a", 1 }, { "b", 2 }, { "c", 3 }, { "d", 4 }, { "e", 5 } + } + + for _,v in ipairs(vals) do + local len, err = dogs:zset("foo", unpack(v)) + if not len then + ngx.say("zset err: ", err) + end + end + + ngx.say(dogs:zcard("foo")) + + local v = dogs:zgetall("foo") + for _,i in ipairs(v) do + ngx.say(unpack(i)) + end + + for _,i in pairs(vals) do + local zkey = unpack(i) + ngx.print(dogs:zrem("foo", zkey)) + end + ngx.say() + ngx.say(dogs:zcard("foo")) + + dogs:delete("foo") + } + } +--- request +GET /test +--- response_body +5 +a1 +b2 +c3 +d4 +e5 +12345 +nil +--- no_error_log +[error] + + +=== TEST 4: zset & zscan +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local vals = { + { "a", 1 }, { "b", 2 }, { "c", 3 }, { "d", 4 }, { "e", 5 } + } + + for _,v in ipairs(vals) do + local len, err = dogs:zset("foo", unpack(v)) + if not len then + ngx.say("zset err: ", err) + end + end + + ngx.say(dogs:zcard("foo")) + + dogs:zscan("foo", function(k,v) + ngx.say(k, v) + end) + + dogs:delete("foo") + } + } +--- request +GET /test +--- response_body +5 +a1 +b2 +c3 +d4 +e5 +--- no_error_log +[error] + + +=== TEST 5: zset & zscan (range) +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local vals = { + { "a", 1 }, + { "aa", 11 }, + { "b", 2 }, + { "bb", 22 }, + { "aaa", 111 }, + { "aab", 112 }, + { "x", 0 } + } + + for _,v in ipairs(vals) do + local len, err = dogs:zset("foo", unpack(v)) + if not len then + ngx.say("zset err: ", err) + end + end + + ngx.say(dogs:zcard("foo")) + + dogs:zscan("foo", function(k,v) + if k:sub(1,2) ~= "aa" then + return true + end + ngx.say(k, v) + end, "aa") + + dogs:delete("foo") + } + } +--- request +GET /test +--- response_body +7 +aa11 +aaa111 +aab112 +--- no_error_log +[error] diff --git a/tests.sh b/tests.sh new file mode 100755 index 0000000000..5b8186f07b --- /dev/null +++ b/tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +DIR=$(pwd) +nginx_fname=$(ls -1 $DIR/install/*.tar.gz) + +[ -d install/tmp ] || mkdir install/tmp +tar zxf $nginx_fname -C install/tmp + +folder="$(ls -1 $DIR/install/tmp | grep nginx)" + +export PATH=$DIR/install/tmp/$folder/sbin:$PATH +export LD_LIBRARY_PATH=$DIR/install/tmp/$folder/lib + +export LUA_CPATH=$DIR/install/tmp/$folder/lib/lua/5.1/cjson.so + +ret=0 + +for t in $(ls t/*.t) +do + echo "Tests : "$t + prove $t + if [ $? -ne 0 ]; then + ret=$? + fi +done + +rm -rf t/servroot +rm -rf install/tmp + +exit $ret \ No newline at end of file From c79371bbab73ded5f852273ac90c75c0f12c8eba Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Thu, 8 Mar 2018 20:51:46 +0300 Subject: [PATCH 4/7] prepare for merge --- .cproject | 68 ------------- .gitignore | 1 - .project | 26 ----- build.sh | 280 ----------------------------------------------------- tests.sh | 30 ------ 5 files changed, 405 deletions(-) delete mode 100644 .cproject delete mode 100644 .project delete mode 100755 build.sh delete mode 100755 tests.sh diff --git a/.cproject b/.cproject deleted file mode 100644 index d915bd4c52..0000000000 --- a/.cproject +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.gitignore b/.gitignore index d947cab0a1..eff337246b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -install/ build/ work/ tags diff --git a/.project b/.project deleted file mode 100644 index 40a597d426..0000000000 --- a/.project +++ /dev/null @@ -1,26 +0,0 @@ - - - lua-nginx-module - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/build.sh b/build.sh deleted file mode 100755 index ed32ac3cc0..0000000000 --- a/build.sh +++ /dev/null @@ -1,280 +0,0 @@ -#!/bin/bash - -# Copyright, Aleksey Konovkin (alkon2000@mail.ru) -# BSD license type - -download=0 -if [ "$1" == "1" ]; then - download=1 -fi -build_deps=0 - -DIR="$(pwd)" - -VERSION="1.13.6" -PCRE_VERSION="8.39" -LUAJIT_VERSION="2.1.0-beta2" - -SUFFIX="" - -BASE_PREFIX="$DIR/build" -INSTALL_PREFIX="$DIR/install" - -PCRE_PREFIX="$DIR/build/pcre-$PCRE_VERSION" -JIT_PREFIX="$DIR/build/deps/luajit" - -export LUAJIT_INC="$JIT_PREFIX/usr/local/include/luajit-2.1" -export LUAJIT_LIB="$JIT_PREFIX/usr/local/lib" - -export LD_LIBRARY_PATH="$JIT_PREFIX/lib" - -function clean() { - rm -rf install 2>/dev/null - rm -rf $(ls -1d build/* 2>/dev/null | grep -v deps) 2>/dev/null - if [ $download -eq 1 ]; then - rm -rf download 2>/dev/null - fi -} - -if [ "$1" == "clean" ]; then - clean - exit 0 -fi - -function build_luajit() { - echo "Build luajit" - cd LuaJIT-$LUAJIT_VERSION - make -j 8 > /dev/null - r=$? - if [ $r -ne 0 ]; then - exit $r - fi - DESTDIR="$JIT_PREFIX" make install > /dev/null - cd .. -} - -function build_cJSON() { - echo "Build cjson" - cd lua-cjson - PREFIX="$JIT_PREFIX/usr/local" make > /dev/null - r=$? - if [ $r -ne 0 ]; then - exit $r - fi - cd .. -} - -function build_debug() { - cd nginx-$VERSION$SUFFIX - echo "Configuring debug nginx-$VERSION$SUFFIX" - ./configure --prefix="$INSTALL_PREFIX/nginx-$VERSION$SUFFIX" \ - --with-pcre=$PCRE_PREFIX \ - --with-http_stub_status_module \ - --with-stream \ - --with-debug \ - --with-http_auth_request_module \ - --with-cc-opt="-O0" \ - --add-module=../ngx_devel_kit \ - --add-module=../../../lua-nginx-module \ - --add-module=../echo-nginx-module \ - --add-module=../memc-nginx-module >/dev/null 2>&1 - r=$? - if [ $r -ne 0 ]; then - exit $r - fi - - echo "Build debug nginx-$VERSION$SUFFIX" - make -j 8 > /dev/null 2>/dev/stderr - - r=$? - if [ $r -ne 0 ]; then - exit $r - fi - make install > /dev/null - - cd .. -} - -function download_module() { - if [ -e $DIR/../$2 ]; then - echo "Get $DIR/../$2" - dir=$(pwd) - cd $DIR/.. - tar zcf $dir/$2.tar.gz $(ls -1d $2/* | grep -vE "(install$)|(build$)|(download$)|(.git$)") - cd $dir - else - if [ $download -eq 1 ] || [ ! -e $2.tar.gz ]; then - echo "Download $2 branch=$3" - curl -s -L -O https://github.com/$1/$2/archive/$3.zip - unzip -q $3.zip - mv $2-$3 $2 - tar zcf $2.tar.gz $2 - rm -rf $2 $3.zip - fi - fi -} - -function gitclone() { - git clone $1 > /dev/null 2> /tmp/err - if [ $? -ne 0 ]; then - cat /tmp/err - fi -} - -function download_nginx() { - if [ $download -eq 1 ] || [ ! -e nginx-$VERSION.tar.gz ]; then - echo "Download nginx-$VERSION" - curl -s -L -O http://nginx.org/download/nginx-$VERSION.tar.gz - else - echo "Get nginx-$VERSION.tar.gz" - fi -} - -function download_luajit() { - if [ $download -eq 1 ] || [ ! -e LuaJIT-$LUAJIT_VERSION.tar.gz ]; then - echo "Download LuaJIT-$LUAJIT_VERSION" - curl -s -L -O http://luajit.org/download/LuaJIT-$LUAJIT_VERSION.tar.gz - else - echo "Get LuaJIT-$LUAJIT_VERSION.tar.gz" - fi -} - -function download_pcre() { - if [ $download -eq 1 ] || [ ! -e pcre-$PCRE_VERSION.tar.gz ]; then - echo "Download PCRE-$PCRE_VERSION" - curl -s -L -O http://ftp.cs.stanford.edu/pub/exim/pcre/pcre-$PCRE_VERSION.tar.gz - else - echo "Get pcre-$PCRE_VERSION.tar.gz" - fi -} - -function extract_downloads() { - cd download - - for d in $(ls -1 *.tar.gz) - do - echo "Extracting $d" - tar zxf $d -C ../build --no-overwrite-dir --keep-old-files 2>/dev/null - done - - cd .. -} - -function download() { - mkdir build 2>/dev/null - mkdir build/deps 2>/dev/null - - mkdir download 2>/dev/null - mkdir download/lua_modules 2>/dev/null - - cd download - - download_luajit - download_pcre - download_nginx - - download_module simpl ngx_devel_kit master - download_module openresty lua-cjson master - download_module openresty echo-nginx-module master - download_module openresty memc-nginx-module master - - cd .. -} - -function install_file() { - echo "Install $1" - if [ ! -e "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$2" ]; then - mkdir -p "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$2" - fi - cp -r $3 $1 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$2/" -} - -function install_files() { - for f in $(ls $1) - do - install_file $f $2 $3 - done -} - -function build() { - cd build - - patch -N -p0 < ../lua-cjson-Makefile.patch - - if [ $build_deps -eq 1 ] || [ ! -e deps/luajit ]; then - build_luajit - fi - - build_cJSON - - make clean > /dev/null 2>&1 - build_debug - - install_file "$JIT_PREFIX/usr/local/lib" . - install_file lua-cjson/cjson.so lib/lua/5.1 - - cd .. -} - -clean -download -extract_downloads -build - -function install_resty_module() { - if [ -e $DIR/../$2 ]; then - echo "Get $DIR/../$2" - dir=$(pwd) - cd $DIR/.. - zip -qr $dir/$2.zip $(ls -1d $2/* | grep -vE "(install$)|(build$)|(download$)|(.git$)") - cd $dir - else - if [ $6 -eq 1 ] || [ ! -e $2-$5.zip ] ; then - echo "Download $2 branch=$5" - rm -rf $2-$5 2>/dev/null - curl -s -L -O https://github.com/$1/$2/archive/$5.zip - mv $5.zip $2-$5.zip - else - echo "Get $2-$5" - fi - fi - echo "Install $2/$3" - if [ ! -e "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4" ]; then - mkdir -p "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4" - fi - if [ -e $2-$5.zip ]; then - unzip -q $2-$5.zip - cp -r $2-$5/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" - rm -rf $2-$5 - elif [ -e $2-$5.tar.gz ]; then - tar zxf $2-$5.tar.gz - cp -r $2-$5/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" - rm -rf $2-$5 - elif [ -e $2.zip ]; then - unzip -q $2.zip - cp -r $2/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" - rm -rf $2 - elif [ -e $2.tar.gz ]; then - tar zxf $2.tar.gz - cp -r $2/$3 "$INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$4/" - rm -rf $2 - fi -} - -function make_dir() { - mkdir $INSTALL_PREFIX/nginx-$VERSION$SUFFIX/$1 -} - -cd "$DIR" - -kernel_name=$(uname -s) -kernel_version=$(uname -r) - -cd install - -tar zcvf nginx-$VERSION$SUFFIX-$kernel_name-$kernel_version.tar.gz nginx-$VERSION$SUFFIX -rm -rf nginx-$VERSION$SUFFIX - -cd .. - -exit $r \ No newline at end of file diff --git a/tests.sh b/tests.sh deleted file mode 100755 index 5b8186f07b..0000000000 --- a/tests.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -DIR=$(pwd) -nginx_fname=$(ls -1 $DIR/install/*.tar.gz) - -[ -d install/tmp ] || mkdir install/tmp -tar zxf $nginx_fname -C install/tmp - -folder="$(ls -1 $DIR/install/tmp | grep nginx)" - -export PATH=$DIR/install/tmp/$folder/sbin:$PATH -export LD_LIBRARY_PATH=$DIR/install/tmp/$folder/lib - -export LUA_CPATH=$DIR/install/tmp/$folder/lib/lua/5.1/cjson.so - -ret=0 - -for t in $(ls t/*.t) -do - echo "Tests : "$t - prove $t - if [ $? -ne 0 ]; then - ret=$? - fi -done - -rm -rf t/servroot -rm -rf install/tmp - -exit $ret \ No newline at end of file From d09d58e86092f02e88ab867dcffa68c454f32096 Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Thu, 8 Mar 2018 20:59:38 +0300 Subject: [PATCH 5/7] code style --- src/ngx_http_lua_shdict.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 5837e7b640..7e27872944 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -108,8 +108,8 @@ ngx_http_lua_shdict_free_list(ngx_http_lua_shdict_ctx_t *ctx, static ngx_inline ngx_http_lua_shdict_zset_t * ngx_http_lua_shdict_get_rbtree(ngx_http_lua_shdict_node_t *sd, size_t len) { - return (ngx_http_lua_shdict_zset_t *) ngx_align_ptr(((u_char *) &sd->data + len), - NGX_ALIGNMENT); + return (ngx_http_lua_shdict_zset_t *) + ngx_align_ptr(((u_char *) &sd->data + len), NGX_ALIGNMENT); } static ngx_inline void @@ -125,7 +125,8 @@ ngx_http_lua_shdict_push_znode_value(lua_State *L, case SHDICT_TSTRING: - lua_pushlstring(L, (char *) zset_node->value.data, zset_node->value.len); + lua_pushlstring(L, (char *) zset_node->value.data, + zset_node->value.len); break; case SHDICT_TNUMBER: @@ -150,8 +151,8 @@ ngx_http_lua_shdict_push_znode_value(lua_State *L, void -ngx_http_lua_shdict_zset_insert_value(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, - ngx_rbtree_node_t *sentinel) +ngx_http_lua_shdict_zset_insert_value(ngx_rbtree_node_t *temp, + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) { ngx_rbtree_node_t **p; ngx_http_lua_shdict_zset_node_t *sdn, *sdnt; From 4c392c60110bbf0e6e51f7b569c6868afd334964 Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Thu, 8 Mar 2018 21:22:15 +0300 Subject: [PATCH 6/7] fix shdict memtatable size test --- t/062-count.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/062-count.t b/t/062-count.t index a69c33edb9..511119b82d 100644 --- a/t/062-count.t +++ b/t/062-count.t @@ -283,7 +283,7 @@ n = 5 --- request GET /test --- response_body -n = 18 +n = 24 --- no_error_log [error] From 68b37588770a39cf0a873d47ecd9ce31aaa4540f Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Thu, 8 Mar 2018 21:27:30 +0300 Subject: [PATCH 7/7] fix shdict ngx.shared package size --- src/ngx_http_lua_shdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 7e27872944..da551aa5c9 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -459,7 +459,7 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_createtable(L, 0, lmcf->shdict_zones->nelts /* nrec */); /* ngx.shared */ - lua_createtable(L, 0 /* narr */, 23 /* nrec */); /* shared mt */ + lua_createtable(L, 0 /* narr */, 24 /* nrec */); /* shared mt */ lua_pushcfunction(L, ngx_http_lua_shdict_get); lua_setfield(L, -2, "get");