diff --git a/README.markdown b/README.markdown index 7902550d5e..922536739a 100644 --- a/README.markdown +++ b/README.markdown @@ -3261,6 +3261,8 @@ Nginx API for Lua * [ngx.shared.DICT.get_stale](#ngxshareddictget_stale) * [ngx.shared.DICT.set](#ngxshareddictset) * [ngx.shared.DICT.safe_set](#ngxshareddictsafe_set) +* [ngx.shared.DICT.set_when](#ngxshareddictset_when) +* [ngx.shared.DICT.safe_set_when](#ngxshareddictsafe_set_when) * [ngx.shared.DICT.add](#ngxshareddictadd) * [ngx.shared.DICT.safe_add](#ngxshareddictsafe_add) * [ngx.shared.DICT.replace](#ngxshareddictreplace) @@ -6400,6 +6402,8 @@ The resulting object `dict` has the following methods: * [get_stale](#ngxshareddictget_stale) * [set](#ngxshareddictset) * [safe_set](#ngxshareddictsafe_set) +* [set_when](#ngxshareddictset_when) +* [safe_set_when](#ngxshareddictsafe_set_when) * [add](#ngxshareddictadd) * [safe_add](#ngxshareddictsafe_add) * [replace](#ngxshareddictreplace) @@ -6590,6 +6594,42 @@ See also [ngx.shared.DICT](#ngxshareddict). [Back to TOC](#nginx-api-for-lua) +ngx.shared.DICT.set_when +------------------------ +**syntax:** *success, err, forcible = ngx.shared.DICT:set(key, old_value, value, exptime?, flags?)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** + +Just like the [set](#ngxshareddictset) method, but only stores the key-value pair into the dictionary [ngx.shared.DICT](#ngxshareddict) if the value for key is the same as `old_value`. + +If the `key` argument does *not* exist in the dictionary (or expired already), the `success` return value will be `true`. + +If the value for the key is *not* the same as `old_value`, the `success` return value will be `false` and the `err` value will be `"already modified"`. + +This feature was first introduced in the `v0.10.16rc1` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + +ngx.shared.DICT.safe_set_when +----------------------------- +**syntax:** *ok, err = ngx.shared.DICT:safe_set_when(key, old_value, value, exptime?, flags?)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** + +Similar to the [set_when](#ngxshareddictset_when) method, but never overrides the (least recently used) unexpired items in the store when running out of storage in the shared memory zone. In this case, it will immediately return `nil` and the string "no memory". + +If the `key` argument does *not* exist in the dictionary (or expired already), the `success` return value will be `true`. + +If the value for the key is *not* the same as `old_value`, the `success` return value will be `false` and the `err` value will be `"already modified"`. + +This feature was first introduced in the `v0.10.16rc1` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + ngx.shared.DICT.add ------------------- diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 1993b05a5d..1617f599d4 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -5393,6 +5393,8 @@ The resulting object dict has the following methods: * [[#ngx.shared.DICT.get_stale|get_stale]] * [[#ngx.shared.DICT.set|set]] * [[#ngx.shared.DICT.safe_set|safe_set]] +* [[#ngx.shared.DICT.set_when|set_when]] +* [[#ngx.shared.DICT.safe_set_when|safe_set_when]] * [[#ngx.shared.DICT.add|add]] * [[#ngx.shared.DICT.safe_add|safe_add]] * [[#ngx.shared.DICT.replace|replace]] @@ -5563,6 +5565,36 @@ This feature was first introduced in the v0.7.18 release. See also [[#ngx.shared.DICT|ngx.shared.DICT]]. +== ngx.shared.DICT.set_when == +'''syntax:''' ''success, err, forcible = ngx.shared.DICT:set(key, old_value, value, exptime?, flags?)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' + +Just like the [[#ngx.shared.DICT.set|set]] method, but only stores the key-value pair into the dictionary [[#ngx.shared.DICT|ngx.shared.DICT]] if the value for key is the same as old_value. + +If the key argument does ''not'' exist in the dictionary (or expired already), the success return value will be true. + +If the value for the key is ''not'' the same as old_value, the success return value will be false and the err value will be "already modified". + +This feature was first introduced in the v0.10.16rc1 release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + +== ngx.shared.DICT.safe_set_when == +'''syntax:''' ''ok, err = ngx.shared.DICT:safe_set_when(key, old_value, value, exptime?, flags?)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' + +Similar to the [[#ngx.shared.DICT.set_when|set_when]] method, but never overrides the (least recently used) unexpired items in the store when running out of storage in the shared memory zone. In this case, it will immediately return nil and the string "no memory". + +If the key argument does ''not'' exist in the dictionary (or expired already), the success return value will be true. + +If the value for the key is ''not'' the same as old_value, the success return value will be false and the err value will be "already modified". + +This feature was first introduced in the v0.10.16rc1 release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + == ngx.shared.DICT.add == '''syntax:''' ''success, err, forcible = ngx.shared.DICT:add(key, value, exptime?, flags?)'' diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 2fd2a94c13..7f5b75a037 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -1569,6 +1569,309 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, } +int +ngx_http_lua_ffi_shdict_store_when(ngx_shm_zone_t *zone, int op, u_char *key, + size_t key_len, int old_value_type, u_char *old_str_value_buf, + size_t old_str_value_len, double old_num_value, int value_type, + u_char *str_value_buf, size_t str_value_len, double num_value, + long exptime, int user_flags, char **errmsg, int *forcible) +{ + int i, n; + u_char old_c, c, *p; + uint32_t hash; + ngx_int_t rc; + ngx_time_t *tp; + ngx_queue_t *queue, *q; + ngx_rbtree_node_t *node; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + + dd("exptime: %ld", exptime); + + ctx = zone->data; + + *forcible = 0; + + hash = ngx_crc32_short(key, key_len); + + switch (old_value_type) { + + case SHDICT_TSTRING: + /* do nothing */ + break; + + case SHDICT_TNUMBER: + dd("num value: %lf", old_num_value); + old_str_value_buf = (u_char *) &old_num_value; + old_str_value_len = sizeof(double); + break; + + case SHDICT_TBOOLEAN: + old_c = old_num_value ? 1 : 0; + old_str_value_buf = &old_c; + old_str_value_len = sizeof(u_char); + break; + + case LUA_TNIL: + old_str_value_buf = NULL; + old_str_value_len = 0; + break; + + default: + *errmsg = "unsupported old_value type"; + return NGX_ERROR; + } + + switch (value_type) { + + case SHDICT_TSTRING: + /* do nothing */ + break; + + case SHDICT_TNUMBER: + dd("num value: %lf", num_value); + str_value_buf = (u_char *) &num_value; + str_value_len = sizeof(double); + break; + + case SHDICT_TBOOLEAN: + c = num_value ? 1 : 0; + str_value_buf = &c; + str_value_len = sizeof(u_char); + break; + + case LUA_TNIL: + if (op & (NGX_HTTP_LUA_SHDICT_ADD|NGX_HTTP_LUA_SHDICT_REPLACE)) { + *errmsg = "attempt to add or replace nil values"; + return NGX_ERROR; + } + + str_value_buf = NULL; + str_value_len = 0; + break; + + default: + *errmsg = "unsupported value type"; + return NGX_ERROR; + } + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key, key_len, &sd); + + dd("lookup returns %d", (int) rc); + + if (op & NGX_HTTP_LUA_SHDICT_REPLACE) { + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + *errmsg = "not found"; + return NGX_DECLINED; + } + + /* rc == NGX_OK */ + + goto replace; + } + + if (op & NGX_HTTP_LUA_SHDICT_ADD) { + + if (rc == NGX_OK) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + *errmsg = "exists"; + return NGX_DECLINED; + } + + if (rc == NGX_DONE) { + /* exists but expired */ + + dd("go to replace"); + goto replace; + } + + /* rc == NGX_DECLINED */ + + dd("go to insert"); + goto insert; + } + + if (rc == NGX_OK || rc == NGX_DONE) { + + if (sd->value_type != old_value_type + || (size_t) sd->value_len != old_str_value_len + || memcmp(sd->data + sd->key_len, old_str_value_buf, + sd->value_len) != 0) + { + ngx_shmtx_unlock(&ctx->shpool->mutex); + *errmsg = "already modified"; + return NGX_ERROR; + } + + if (value_type == LUA_TNIL) { + goto remove; + } + +replace: + + if (str_value_buf + && str_value_len == (size_t) sd->value_len + && sd->value_type != SHDICT_TLIST) + { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict set: found old entry and value " + "size matched, reusing it"); + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + + sd->key_len = (u_short) key_len; + + if (exptime > 0) { + tp = ngx_timeofday(); + sd->expires = (uint64_t) tp->sec * 1000 + tp->msec + + (uint64_t) exptime; + + } else { + sd->expires = 0; + } + + sd->user_flags = user_flags; + + sd->value_len = (uint32_t) str_value_len; + + dd("setting value type to %d", value_type); + + sd->value_type = (uint8_t) value_type; + + p = ngx_copy(sd->data, key, key_len); + ngx_memcpy(p, str_value_buf, str_value_len); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return NGX_OK; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict set: found old entry but value size " + "NOT matched, removing it first"); + +remove: + + if (sd->value_type == SHDICT_TLIST) { + queue = ngx_http_lua_shdict_get_list_head(sd, key_len); + + for (q = ngx_queue_head(queue); + q != ngx_queue_sentinel(queue); + q = ngx_queue_next(q)) + { + p = (u_char *) ngx_queue_data(q, + ngx_http_lua_shdict_list_node_t, + queue); + + ngx_slab_free_locked(ctx->shpool, p); + } + } + + ngx_queue_remove(&sd->queue); + + node = (ngx_rbtree_node_t *) + ((u_char *) sd - offsetof(ngx_rbtree_node_t, color)); + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_slab_free_locked(ctx->shpool, node); + + } + +insert: + + /* rc == NGX_DECLINED or value size unmatch */ + + if (str_value_buf == NULL) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + return NGX_OK; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict set: creating a new entry"); + + n = offsetof(ngx_rbtree_node_t, color) + + offsetof(ngx_http_lua_shdict_node_t, data) + + key_len + + str_value_len; + + node = ngx_slab_alloc_locked(ctx->shpool, n); + + if (node == NULL) { + + if (op & NGX_HTTP_LUA_SHDICT_SAFE_STORE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + *errmsg = "no memory"; + return NGX_ERROR; + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict set: overriding non-expired items " + "due to memory shortage for entry \"%*s\"", key_len, + key); + + for (i = 0; i < 30; i++) { + if (ngx_http_lua_shdict_expire(ctx, 0) == 0) { + break; + } + + *forcible = 1; + + node = ngx_slab_alloc_locked(ctx->shpool, n); + if (node != NULL) { + goto allocated; + } + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + *errmsg = "no memory"; + return NGX_ERROR; + } + +allocated: + + sd = (ngx_http_lua_shdict_node_t *) &node->color; + + node->key = hash; + sd->key_len = (u_short) key_len; + + if (exptime > 0) { + tp = ngx_timeofday(); + sd->expires = (uint64_t) tp->sec * 1000 + tp->msec + + (uint64_t) exptime; + + } else { + sd->expires = 0; + } + + sd->user_flags = user_flags; + sd->value_len = (uint32_t) str_value_len; + dd("setting value type to %d", value_type); + sd->value_type = (uint8_t) value_type; + + p = ngx_copy(sd->data, key, key_len); + ngx_memcpy(p, str_value_buf, str_value_len); + + ngx_rbtree_insert(&ctx->sh->rbtree, node); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return NGX_OK; +} + + int ngx_http_lua_ffi_shdict_get(ngx_shm_zone_t *zone, u_char *key, size_t key_len, int *value_type, u_char **str_value_buf, diff --git a/t/062-count.t b/t/062-count.t index b481763abf..c3153a75d3 100644 --- a/t/062-count.t +++ b/t/062-count.t @@ -283,7 +283,7 @@ n = 5 --- request GET /test --- response_body -n = 22 +n = 24 --- no_error_log [error] diff --git a/t/162-shdict-set-when.t b/t/162-shdict-set-when.t new file mode 100644 index 0000000000..b9701baed9 --- /dev/null +++ b/t/162-shdict-set-when.t @@ -0,0 +1,139 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use Test::Nginx::Socket::Lua; + +#worker_connections(1014); +#master_process_enabled(1); +#log_level('warn'); + +#repeat_each(2); + +plan tests => repeat_each() * (blocks() * 3); + +#no_diff(); +no_long_string(); +#master_on(); +#workers(2); + +run_tests(); + +__DATA__ + +=== TEST 1: set_when success +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua ' + local dogs = ngx.shared.dogs + dogs:set("foo", 32) + dogs:set_when("foo", 32, 33) + local val = dogs:get("foo") + ngx.say(val, " ", type(val)) + '; + } +--- request +GET /test +--- response_body +33 number +--- no_error_log +[error] + + + +=== TEST 2: set_when fail +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua ' + local dogs = ngx.shared.dogs + dogs:set("foo", 32) + local ok, err, forcible = dogs:set_when("foo", 32, 33) + ngx.say(ok, " ", err, " ", forcible) + local ok, err, forcible = dogs:set_when("foo", 32, 34) + ngx.say(ok, " ", err, " ", forcible) + local val = dogs:get("foo") + ngx.say(val, " ", type(val)) + '; + } +--- request +GET /test +--- response_body +true nil false +false already modified false +33 number +--- no_error_log +[error] + + + +=== TEST 3: set_when success for expired value +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua ' + local dogs = ngx.shared.dogs + dogs:set("foo", 32, 0.01) + ngx.sleep(0.02) + local ok, err, forcible = dogs:set_when("foo", 32, 33) + ngx.say(ok, " ", err, " ", forcible) + local val = dogs:get("foo") + ngx.say(val, " ", type(val)) + '; + } +--- request +GET /test +--- response_body +true nil false +33 number +--- no_error_log +[error] + + + +=== TEST 4: set_when success for unmatched expired value +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua ' + local dogs = ngx.shared.dogs + dogs:set("foo", 32, 0.01) + ngx.sleep(0.02) + local ok, err, forcible = dogs:set_when("foo", 31, 33) + ngx.say(ok, " ", err, " ", forcible) + local val = dogs:get("foo") + ngx.say(val, " ", type(val)) + '; + } +--- request +GET /test +--- response_body +true nil false +33 number +--- no_error_log +[error] + + + +=== TEST 5: set_when success when old_value did not exist +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua ' + local dogs = ngx.shared.dogs + local ok, err, forcible = dogs:set_when("foo", 32, 33) + ngx.say(ok, " ", err, " ", forcible) + local val = dogs:get("foo") + ngx.say(val, " ", type(val)) + '; + } +--- request +GET /test +--- response_body +true nil false +33 number +--- no_error_log +[error]