From 22aab1ce94f5725f1c8d0fa3a062fd2e12957967 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Tue, 5 Oct 2021 11:31:18 +0300 Subject: [PATCH 1/5] Redis Functions - Move code to make review process easier. This commit is only move code around without changing it. The reason behind this is to make review process easier by allowing the reviewer to simply ignore all code movements. changes: 1. rename scripting.c to eval.c 2. introduce and new file, script_lua.c, and move parts of Lua functionality to this new file. script_lua.c will eventually contains the shared code between legacy lua and lua engine. This commit does not compiled on purpose. Its only purpose is to move code and rename files. --- src/Makefile | 2 +- src/{scripting.c => eval.c} | 1420 +--------------------------------- src/script_lua.c | 1423 +++++++++++++++++++++++++++++++++++ 3 files changed, 1449 insertions(+), 1396 deletions(-) rename src/{scripting.c => eval.c} (55%) create mode 100644 src/script_lua.c diff --git a/src/Makefile b/src/Makefile index 34b5c3566..469e8eb54 100644 --- a/src/Makefile +++ b/src/Makefile @@ -309,7 +309,7 @@ endif REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX) -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX) REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX) diff --git a/src/scripting.c b/src/eval.c similarity index 55% rename from src/scripting.c rename to src/eval.c index f27840956..6e6aeac7c 100644 --- a/src/scripting.c +++ b/src/eval.c @@ -33,6 +33,7 @@ #include "cluster.h" #include "monotonic.h" #include "resp_parser.h" +#include "script_lua.h" #include #include @@ -40,23 +41,6 @@ #include #include -static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_BulkString(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_NullBulkString(void *ctx, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_NullArray(void *ctx, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_Status(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_Error(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_Array(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); -static void redisProtocolToLuaType_Map(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); -static void redisProtocolToLuaType_Set(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); -static void redisProtocolToLuaType_Null(void *ctx, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_Bool(void *ctx, int val, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_BigNumber(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_VerbatimString(void *ctx, const char *format, const char *str, size_t len, const char *proto, size_t proto_len); -static void redisProtocolToLuaType_Attribute(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); -int redis_math_random (lua_State *L); -int redis_math_randomseed (lua_State *L); void ldbInit(void); void ldbDisable(client *c); void ldbEnable(client *c); @@ -115,1035 +99,6 @@ void sha1hex(char *digest, char *script, size_t len) { digest[40] = '\0'; } -/* --------------------------------------------------------------------------- - * Redis reply to Lua type conversion functions. - * ------------------------------------------------------------------------- */ - -/* Take a Redis reply in the Redis protocol format and convert it into a - * Lua type. Thanks to this function, and the introduction of not connected - * clients, it is trivial to implement the redis() lua function. - * - * Basically we take the arguments, execute the Redis command in the context - * of a non connected client, then take the generated reply and convert it - * into a suitable Lua type. With this trick the scripting feature does not - * need the introduction of a full Redis internals API. The script - * is like a normal client that bypasses all the slow I/O paths. - * - * Note: in this function we do not do any sanity check as the reply is - * generated by Redis directly. This allows us to go faster. - * - * Errors are returned as a table with a single 'err' field set to the - * error string. - */ - -static const ReplyParserCallbacks DefaultLuaTypeParserCallbacks = { - .null_array_callback = redisProtocolToLuaType_NullArray, - .bulk_string_callback = redisProtocolToLuaType_BulkString, - .null_bulk_string_callback = redisProtocolToLuaType_NullBulkString, - .error_callback = redisProtocolToLuaType_Error, - .simple_str_callback = redisProtocolToLuaType_Status, - .long_callback = redisProtocolToLuaType_Int, - .array_callback = redisProtocolToLuaType_Array, - .set_callback = redisProtocolToLuaType_Set, - .map_callback = redisProtocolToLuaType_Map, - .bool_callback = redisProtocolToLuaType_Bool, - .double_callback = redisProtocolToLuaType_Double, - .null_callback = redisProtocolToLuaType_Null, - .big_number_callback = redisProtocolToLuaType_BigNumber, - .verbatim_string_callback = redisProtocolToLuaType_VerbatimString, - .attribute_callback = redisProtocolToLuaType_Attribute, - .error = NULL, -}; - -void redisProtocolToLuaType(lua_State *lua, char* reply) { - ReplyParser parser = {.curr_location = reply, .callbacks = DefaultLuaTypeParserCallbacks}; - - parseReply(&parser, lua); -} - -static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushnumber(lua,(lua_Number)val); -} - -static void redisProtocolToLuaType_NullBulkString(void *ctx, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushboolean(lua,0); -} - -static void redisProtocolToLuaType_NullArray(void *ctx, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - lua_State *lua = ctx; - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushboolean(lua,0); -} - - -static void redisProtocolToLuaType_BulkString(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushlstring(lua,str,len); -} - -static void redisProtocolToLuaType_Status(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 3)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua,"ok"); - lua_pushlstring(lua,str,len); - lua_settable(lua,-3); -} - -static void redisProtocolToLuaType_Error(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 3)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua,"err"); - lua_pushlstring(lua,str,len); - lua_settable(lua,-3); -} - -static void redisProtocolToLuaType_Map(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { - UNUSED(proto); - lua_State *lua = ctx; - if (lua) { - if (!lua_checkstack(lua, 3)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua, "map"); - lua_newtable(lua); - } - for (size_t j = 0; j < len; j++) { - parseReply(parser,lua); - parseReply(parser,lua); - if (lua) lua_settable(lua,-3); - } - if (lua) lua_settable(lua,-3); -} - -static void redisProtocolToLuaType_Set(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { - UNUSED(proto); - - lua_State *lua = ctx; - if (lua) { - if (!lua_checkstack(lua, 3)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua, "set"); - lua_newtable(lua); - } - for (size_t j = 0; j < len; j++) { - parseReply(parser,lua); - if (lua) { - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. - * Notice that here we need to check the stack again because the recursive - * call to redisProtocolToLuaType might have use the room allocated in the stack*/ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushboolean(lua,1); - lua_settable(lua,-3); - } - } - if (lua) lua_settable(lua,-3); -} - -static void redisProtocolToLuaType_Array(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { - UNUSED(proto); - - lua_State *lua = ctx; - if (lua){ - if (!lua_checkstack(lua, 2)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - } - for (size_t j = 0; j < len; j++) { - if (lua) lua_pushnumber(lua,j+1); - parseReply(parser,lua); - if (lua) lua_settable(lua,-3); - } -} - -static void redisProtocolToLuaType_Attribute(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { - UNUSED(proto); - - /* Parse the attribute reply. - * Currently, we do not expose the attribute to the Lua script so - * we just need to continue parsing and ignore it (the NULL ensures that the - * reply will be ignored). */ - for (size_t j = 0; j < len; j++) { - parseReply(parser,NULL); - parseReply(parser,NULL); - } - - /* Parse the reply itself. */ - parseReply(parser,ctx); -} - -static void redisProtocolToLuaType_VerbatimString(void *ctx, const char *format, const char *str, size_t len, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 5)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua,"verbatim_string"); - lua_newtable(lua); - lua_pushstring(lua,"string"); - lua_pushlstring(lua,str,len); - lua_settable(lua,-3); - lua_pushstring(lua,"format"); - lua_pushlstring(lua,format,3); - lua_settable(lua,-3); - lua_settable(lua,-3); -} - -static void redisProtocolToLuaType_BigNumber(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 3)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua,"big_number"); - lua_pushlstring(lua,str,len); - lua_settable(lua,-3); -} - -static void redisProtocolToLuaType_Null(void *ctx, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushnil(lua); -} - -static void redisProtocolToLuaType_Bool(void *ctx, int val, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 1)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_pushboolean(lua,val); -} - -static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto, size_t proto_len) { - UNUSED(proto); - UNUSED(proto_len); - if (!ctx) { - return; - } - - lua_State *lua = ctx; - if (!lua_checkstack(lua, 3)) { - /* Increase the Lua stack if needed, to make sure there is enough room - * to push elements to the stack. On failure, exit with panic. */ - serverPanic("lua stack limit reach when parsing redis.call reply"); - } - lua_newtable(lua); - lua_pushstring(lua,"double"); - lua_pushnumber(lua,d); - lua_settable(lua,-3); -} - -/* This function is used in order to push an error on the Lua stack in the - * format used by redis.pcall to return errors, which is a lua table - * with a single "err" field set to the error string. Note that this - * table is never a valid reply by proper commands, since the returned - * tables are otherwise always indexed by integers, never by strings. */ -void luaPushError(lua_State *lua, char *error) { - lua_Debug dbg; - - /* If debugging is active and in step mode, log errors resulting from - * Redis commands. */ - if (ldb.active && ldb.step) { - ldbLog(sdscatprintf(sdsempty()," %s",error)); - } - - lua_newtable(lua); - lua_pushstring(lua,"err"); - - /* Attempt to figure out where this function was called, if possible */ - if(lua_getstack(lua, 1, &dbg) && lua_getinfo(lua, "nSl", &dbg)) { - sds msg = sdscatprintf(sdsempty(), "%s: %d: %s", - dbg.source, dbg.currentline, error); - lua_pushstring(lua, msg); - sdsfree(msg); - } else { - lua_pushstring(lua, error); - } - lua_settable(lua,-3); -} - -/* In case the error set into the Lua stack by luaPushError() was generated - * by the non-error-trapping version of redis.pcall(), which is redis.call(), - * this function will raise the Lua error so that the execution of the - * script will be halted. */ -int luaRaiseError(lua_State *lua) { - lua_pushstring(lua,"err"); - lua_gettable(lua,-2); - return lua_error(lua); -} - -/* Sort the array currently in the stack. We do this to make the output - * of commands like KEYS or SMEMBERS something deterministic when called - * from Lua (to play well with AOf/replication). - * - * The array is sorted using table.sort itself, and assuming all the - * list elements are strings. */ -void luaSortArray(lua_State *lua) { - /* Initial Stack: array */ - lua_getglobal(lua,"table"); - lua_pushstring(lua,"sort"); - lua_gettable(lua,-2); /* Stack: array, table, table.sort */ - lua_pushvalue(lua,-3); /* Stack: array, table, table.sort, array */ - if (lua_pcall(lua,1,0,0)) { - /* Stack: array, table, error */ - - /* We are not interested in the error, we assume that the problem is - * that there are 'false' elements inside the array, so we try - * again with a slower function but able to handle this case, that - * is: table.sort(table, __redis__compare_helper) */ - lua_pop(lua,1); /* Stack: array, table */ - lua_pushstring(lua,"sort"); /* Stack: array, table, sort */ - lua_gettable(lua,-2); /* Stack: array, table, table.sort */ - lua_pushvalue(lua,-3); /* Stack: array, table, table.sort, array */ - lua_getglobal(lua,"__redis__compare_helper"); - /* Stack: array, table, table.sort, array, __redis__compare_helper */ - lua_call(lua,2,0); - } - /* Stack: array (sorted), table */ - lua_pop(lua,1); /* Stack: array (sorted) */ -} - -/* --------------------------------------------------------------------------- - * Lua reply to Redis reply conversion functions. - * ------------------------------------------------------------------------- */ - -/* Reply to client 'c' converting the top element in the Lua stack to a - * Redis reply. As a side effect the element is consumed from the stack. */ -void luaReplyToRedisReply(client *c, lua_State *lua) { - int t = lua_type(lua,-1); - - if (!lua_checkstack(lua, 4)) { - /* Increase the Lua stack if needed to make sure there is enough room - * to push 4 elements to the stack. On failure, return error. - * Notice that we need, in the worst case, 4 elements because returning a map might - * require push 4 elements to the Lua stack.*/ - addReplyErrorFormat(c, "reached lua stack limit"); - lua_pop(lua,1); /* pop the element from the stack */ - return; - } - - switch(t) { - case LUA_TSTRING: - addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); - break; - case LUA_TBOOLEAN: - if (server.lua_client->resp == 2) - addReply(c,lua_toboolean(lua,-1) ? shared.cone : - shared.null[c->resp]); - else - addReplyBool(c,lua_toboolean(lua,-1)); - break; - case LUA_TNUMBER: - addReplyLongLong(c,(long long)lua_tonumber(lua,-1)); - break; - case LUA_TTABLE: - /* We need to check if it is an array, an error, or a status reply. - * Error are returned as a single element table with 'err' field. - * Status replies are returned as single element table with 'ok' - * field. */ - - /* Handle error reply. */ - /* we took care of the stack size on function start */ - lua_pushstring(lua,"err"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TSTRING) { - addReplyErrorFormat(c,"-%s",lua_tostring(lua,-1)); - lua_pop(lua,2); - return; - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle status reply. */ - lua_pushstring(lua,"ok"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TSTRING) { - sds ok = sdsnew(lua_tostring(lua,-1)); - sdsmapchars(ok,"\r\n"," ",2); - addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); - sdsfree(ok); - lua_pop(lua,2); - return; - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle double reply. */ - lua_pushstring(lua,"double"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TNUMBER) { - addReplyDouble(c,lua_tonumber(lua,-1)); - lua_pop(lua,2); - return; - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle big number reply. */ - lua_pushstring(lua,"big_number"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TSTRING) { - sds big_num = sdsnewlen(lua_tostring(lua,-1), lua_strlen(lua,-1)); - sdsmapchars(big_num,"\r\n"," ",2); - addReplyBigNum(c,big_num,sdslen(big_num)); - sdsfree(big_num); - lua_pop(lua,2); - return; - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle verbatim reply. */ - lua_pushstring(lua,"verbatim_string"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TTABLE) { - lua_pushstring(lua,"format"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TSTRING){ - char* format = (char*)lua_tostring(lua,-1); - lua_pushstring(lua,"string"); - lua_gettable(lua,-3); - t = lua_type(lua,-1); - if (t == LUA_TSTRING){ - size_t len; - char* str = (char*)lua_tolstring(lua,-1,&len); - addReplyVerbatim(c, str, len, format); - lua_pop(lua,4); - return; - } - lua_pop(lua,1); - } - lua_pop(lua,1); - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle map reply. */ - lua_pushstring(lua,"map"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TTABLE) { - int maplen = 0; - void *replylen = addReplyDeferredLen(c); - /* we took care of the stack size on function start */ - lua_pushnil(lua); /* Use nil to start iteration. */ - while (lua_next(lua,-2)) { - /* Stack now: table, key, value */ - lua_pushvalue(lua,-2); /* Dup key before consuming. */ - luaReplyToRedisReply(c, lua); /* Return key. */ - luaReplyToRedisReply(c, lua); /* Return value. */ - /* Stack now: table, key. */ - maplen++; - } - setDeferredMapLen(c,replylen,maplen); - lua_pop(lua,2); - return; - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle set reply. */ - lua_pushstring(lua,"set"); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TTABLE) { - int setlen = 0; - void *replylen = addReplyDeferredLen(c); - /* we took care of the stack size on function start */ - lua_pushnil(lua); /* Use nil to start iteration. */ - while (lua_next(lua,-2)) { - /* Stack now: table, key, true */ - lua_pop(lua,1); /* Discard the boolean value. */ - lua_pushvalue(lua,-1); /* Dup key before consuming. */ - luaReplyToRedisReply(c, lua); /* Return key. */ - /* Stack now: table, key. */ - setlen++; - } - setDeferredSetLen(c,replylen,setlen); - lua_pop(lua,2); - return; - } - lua_pop(lua,1); /* Discard field name pushed before. */ - - /* Handle the array reply. */ - void *replylen = addReplyDeferredLen(c); - int j = 1, mbulklen = 0; - while(1) { - /* we took care of the stack size on function start */ - lua_pushnumber(lua,j++); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TNIL) { - lua_pop(lua,1); - break; - } - luaReplyToRedisReply(c, lua); - mbulklen++; - } - setDeferredArrayLen(c,replylen,mbulklen); - break; - default: - addReplyNull(c); - } - lua_pop(lua,1); -} - -/* --------------------------------------------------------------------------- - * Lua redis.* functions implementations. - * ------------------------------------------------------------------------- */ - -#define LUA_CMD_OBJCACHE_SIZE 32 -#define LUA_CMD_OBJCACHE_MAX_LEN 64 -int luaRedisGenericCommand(lua_State *lua, int raise_error) { - int j, argc = lua_gettop(lua); - struct redisCommand *cmd; - client *c = server.lua_client; - sds reply; - - /* Cached across calls. */ - static robj **argv = NULL; - static int argv_size = 0; - static robj *cached_objects[LUA_CMD_OBJCACHE_SIZE]; - static size_t cached_objects_len[LUA_CMD_OBJCACHE_SIZE]; - static int inuse = 0; /* Recursive calls detection. */ - - /* By using Lua debug hooks it is possible to trigger a recursive call - * to luaRedisGenericCommand(), which normally should never happen. - * To make this function reentrant is futile and makes it slower, but - * we should at least detect such a misuse, and abort. */ - if (inuse) { - char *recursion_warning = - "luaRedisGenericCommand() recursive call detected. " - "Are you doing funny stuff with Lua debug hooks?"; - serverLog(LL_WARNING,"%s",recursion_warning); - luaPushError(lua,recursion_warning); - return 1; - } - inuse++; - - /* Require at least one argument */ - if (argc == 0) { - luaPushError(lua, - "Please specify at least one argument for redis.call()"); - inuse--; - return raise_error ? luaRaiseError(lua) : 1; - } - - /* Build the arguments vector */ - if (argv_size < argc) { - argv = zrealloc(argv,sizeof(robj*)*argc); - argv_size = argc; - } - - for (j = 0; j < argc; j++) { - char *obj_s; - size_t obj_len; - char dbuf[64]; - - if (lua_type(lua,j+1) == LUA_TNUMBER) { - /* We can't use lua_tolstring() for number -> string conversion - * since Lua uses a format specifier that loses precision. */ - lua_Number num = lua_tonumber(lua,j+1); - - obj_len = snprintf(dbuf,sizeof(dbuf),"%.17g",(double)num); - obj_s = dbuf; - } else { - obj_s = (char*)lua_tolstring(lua,j+1,&obj_len); - if (obj_s == NULL) break; /* Not a string. */ - } - - /* Try to use a cached object. */ - if (j < LUA_CMD_OBJCACHE_SIZE && cached_objects[j] && - cached_objects_len[j] >= obj_len) - { - sds s = cached_objects[j]->ptr; - argv[j] = cached_objects[j]; - cached_objects[j] = NULL; - memcpy(s,obj_s,obj_len+1); - sdssetlen(s, obj_len); - } else { - argv[j] = createStringObject(obj_s, obj_len); - } - } - - /* Check if one of the arguments passed by the Lua script - * is not a string or an integer (lua_isstring() return true for - * integers as well). */ - if (j != argc) { - j--; - while (j >= 0) { - decrRefCount(argv[j]); - j--; - } - luaPushError(lua, - "Lua redis() command arguments must be strings or integers"); - inuse--; - return raise_error ? luaRaiseError(lua) : 1; - } - - /* Pop all arguments from the stack, we do not need them anymore - * and this way we guaranty we will have room on the stack for the result. */ - lua_pop(lua, argc); - - /* Setup our fake client for command execution */ - c->argv = argv; - c->argc = argc; - c->user = server.lua_caller->user; - - /* Process module hooks */ - moduleCallCommandFilters(c); - argv = c->argv; - argc = c->argc; - - /* Log the command if debugging is active. */ - if (ldb.active && ldb.step) { - sds cmdlog = sdsnew(""); - for (j = 0; j < c->argc; j++) { - if (j == 10) { - cmdlog = sdscatprintf(cmdlog," ... (%d more)", - c->argc-j-1); - break; - } else { - cmdlog = sdscatlen(cmdlog," ",1); - cmdlog = sdscatsds(cmdlog,c->argv[j]->ptr); - } - } - ldbLog(cmdlog); - } - - /* Command lookup */ - cmd = lookupCommand(argv,argc); - if (!cmd || ((cmd->arity > 0 && cmd->arity != argc) || - (argc < -cmd->arity))) - { - if (cmd) - luaPushError(lua, - "Wrong number of args calling Redis command From Lua script"); - else - luaPushError(lua,"Unknown Redis command called from Lua script"); - goto cleanup; - } - c->cmd = c->lastcmd = cmd; - - /* There are commands that are not allowed inside scripts. */ - if (!server.lua_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { - luaPushError(lua, "This Redis command is not allowed from scripts"); - goto cleanup; - } - - /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ - if ((server.lua_caller->cmd->proc == evalRoCommand || - server.lua_caller->cmd->proc == evalShaRoCommand) && - (cmd->flags & CMD_WRITE)) - { - luaPushError(lua, "Write commands are not allowed from read-only scripts"); - goto cleanup; - } - - /* Check the ACLs. */ - int acl_errpos; - int acl_retval = ACLCheckAllPerm(c,&acl_errpos); - if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,ACL_LOG_CTX_LUA,acl_errpos,NULL,NULL); - switch (acl_retval) { - case ACL_DENIED_CMD: - luaPushError(lua, "The user executing the script can't run this " - "command or subcommand"); - break; - case ACL_DENIED_KEY: - luaPushError(lua, "The user executing the script can't access " - "at least one of the keys mentioned in the " - "command arguments"); - break; - case ACL_DENIED_CHANNEL: - luaPushError(lua, "The user executing the script can't publish " - "to the channel mentioned in the command"); - break; - default: - luaPushError(lua, "The user executing the script is lacking the " - "permissions for the command"); - break; - } - goto cleanup; - } - - /* Write commands are forbidden against read-only slaves, or if a - * command marked as non-deterministic was already called in the context - * of this script. */ - if (cmd->flags & CMD_WRITE) { - int deny_write_type = writeCommandsDeniedByDiskError(); - if (server.lua_random_dirty && !server.lua_replicate_commands) { - luaPushError(lua, - "Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); - goto cleanup; - } else if (server.masterhost && server.repl_slave_ro && - server.lua_caller->id != CLIENT_ID_AOF && - !(server.lua_caller->flags & CLIENT_MASTER)) - { - luaPushError(lua, shared.roslaveerr->ptr); - goto cleanup; - } else if (deny_write_type != DISK_ERROR_TYPE_NONE) { - if (deny_write_type == DISK_ERROR_TYPE_RDB) { - luaPushError(lua, shared.bgsaveerr->ptr); - } else { - sds aof_write_err = sdscatfmt(sdsempty(), - "-MISCONF Errors writing to the AOF file: %s\r\n", - strerror(server.aof_last_write_errno)); - luaPushError(lua, aof_write_err); - sdsfree(aof_write_err); - } - goto cleanup; - } - } - - /* If we reached the memory limit configured via maxmemory, commands that - * could enlarge the memory usage are not allowed, but only if this is the - * first write in the context of this script, otherwise we can't stop - * in the middle. */ - if (server.maxmemory && /* Maxmemory is actually enabled. */ - server.lua_caller->id != CLIENT_ID_AOF && /* Don't care about mem if loading from AOF. */ - !server.masterhost && /* Slave must execute the script. */ - server.lua_write_dirty == 0 && /* Script had no side effects so far. */ - server.lua_oom && /* Detected OOM when script start. */ - (cmd->flags & CMD_DENYOOM)) - { - luaPushError(lua, shared.oomerr->ptr); - goto cleanup; - } - - if (cmd->flags & CMD_RANDOM) server.lua_random_dirty = 1; - if (cmd->flags & CMD_WRITE) server.lua_write_dirty = 1; - - /* If this is a Redis Cluster node, we need to make sure Lua is not - * trying to access non-local keys, with the exception of commands - * received from our master or when loading the AOF back in memory. */ - if (server.cluster_enabled && server.lua_caller->id != CLIENT_ID_AOF && - !(server.lua_caller->flags & CLIENT_MASTER)) - { - int error_code; - /* Duplicate relevant flags in the lua client. */ - c->flags &= ~(CLIENT_READONLY|CLIENT_ASKING); - c->flags |= server.lua_caller->flags & (CLIENT_READONLY|CLIENT_ASKING); - if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != - server.cluster->myself) - { - if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { - luaPushError(lua, - "Lua script attempted to execute a write command while the " - "cluster is down and readonly"); - } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { - luaPushError(lua, - "Lua script attempted to execute a command while the " - "cluster is down"); - } else { - luaPushError(lua, - "Lua script attempted to access a non local key in a " - "cluster node"); - } - - goto cleanup; - } - } - - /* If we are using single commands replication, we need to wrap what - * we propagate into a MULTI/EXEC block, so that it will be atomic like - * a Lua script in the context of AOF and slaves. */ - if (server.lua_replicate_commands && - !server.lua_multi_emitted && - !(server.lua_caller->flags & CLIENT_MULTI) && - server.lua_write_dirty && - server.lua_repl != PROPAGATE_NONE) - { - execCommandPropagateMulti(server.lua_caller->db->id); - server.lua_multi_emitted = 1; - /* Now we are in the MULTI context, the lua_client should be - * flag as CLIENT_MULTI. */ - c->flags |= CLIENT_MULTI; - } - - /* Run the command */ - int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; - if (server.lua_replicate_commands) { - /* Set flags according to redis.set_repl() settings. */ - if (server.lua_repl & PROPAGATE_AOF) - call_flags |= CMD_CALL_PROPAGATE_AOF; - if (server.lua_repl & PROPAGATE_REPL) - call_flags |= CMD_CALL_PROPAGATE_REPL; - } - call(c,call_flags); - serverAssert((c->flags & CLIENT_BLOCKED) == 0); - - /* Convert the result of the Redis command into a suitable Lua type. - * The first thing we need is to create a single string from the client - * output buffers. */ - if (listLength(c->reply) == 0 && (size_t)c->bufpos < c->buf_usable_size) { - /* This is a fast path for the common case of a reply inside the - * client static buffer. Don't create an SDS string but just use - * the client buffer directly. */ - c->buf[c->bufpos] = '\0'; - reply = c->buf; - c->bufpos = 0; - } else { - reply = sdsnewlen(c->buf,c->bufpos); - c->bufpos = 0; - while(listLength(c->reply)) { - clientReplyBlock *o = listNodeValue(listFirst(c->reply)); - - reply = sdscatlen(reply,o->buf,o->used); - listDelNode(c->reply,listFirst(c->reply)); - } - } - if (raise_error && reply[0] != '-') raise_error = 0; - redisProtocolToLuaType(lua,reply); - - /* If the debugger is active, log the reply from Redis. */ - if (ldb.active && ldb.step) - ldbLogRedisReply(reply); - - /* Sort the output array if needed, assuming it is a non-null multi bulk - * reply as expected. */ - if ((cmd->flags & CMD_SORT_FOR_SCRIPT) && - (server.lua_replicate_commands == 0) && - (reply[0] == '*' && reply[1] != '-')) { - luaSortArray(lua); - } - if (reply != c->buf) sdsfree(reply); - c->reply_bytes = 0; - -cleanup: - /* Clean up. Command code may have changed argv/argc so we use the - * argv/argc of the client instead of the local variables. */ - for (j = 0; j < c->argc; j++) { - robj *o = c->argv[j]; - - /* Try to cache the object in the cached_objects array. - * The object must be small, SDS-encoded, and with refcount = 1 - * (we must be the only owner) for us to cache it. */ - if (j < LUA_CMD_OBJCACHE_SIZE && - o->refcount == 1 && - (o->encoding == OBJ_ENCODING_RAW || - o->encoding == OBJ_ENCODING_EMBSTR) && - sdslen(o->ptr) <= LUA_CMD_OBJCACHE_MAX_LEN) - { - sds s = o->ptr; - if (cached_objects[j]) decrRefCount(cached_objects[j]); - cached_objects[j] = o; - cached_objects_len[j] = sdsalloc(s); - } else { - decrRefCount(o); - } - } - - if (c->argv != argv) { - zfree(c->argv); - argv = NULL; - argv_size = 0; - } - - c->user = NULL; - - if (raise_error) { - /* If we are here we should have an error in the stack, in the - * form of a table with an "err" field. Extract the string to - * return the plain error. */ - inuse--; - return luaRaiseError(lua); - } - inuse--; - return 1; -} - -/* redis.call() */ -int luaRedisCallCommand(lua_State *lua) { - return luaRedisGenericCommand(lua,1); -} - -/* redis.pcall() */ -int luaRedisPCallCommand(lua_State *lua) { - return luaRedisGenericCommand(lua,0); -} - -/* This adds redis.sha1hex(string) to Lua scripts using the same hashing - * function used for sha1ing lua scripts. */ -int luaRedisSha1hexCommand(lua_State *lua) { - int argc = lua_gettop(lua); - char digest[41]; - size_t len; - char *s; - - if (argc != 1) { - lua_pushstring(lua, "wrong number of arguments"); - return lua_error(lua); - } - - s = (char*)lua_tolstring(lua,1,&len); - sha1hex(digest,s,len); - lua_pushstring(lua,digest); - return 1; -} - -/* Returns a table with a single field 'field' set to the string value - * passed as argument. This helper function is handy when returning - * a Redis Protocol error or status reply from Lua: - * - * return redis.error_reply("ERR Some Error") - * return redis.status_reply("ERR Some Error") - */ -int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { - if (lua_gettop(lua) != 1 || lua_type(lua,-1) != LUA_TSTRING) { - luaPushError(lua, "wrong number or type of arguments"); - return 1; - } - - lua_newtable(lua); - lua_pushstring(lua, field); - lua_pushvalue(lua, -3); - lua_settable(lua, -3); - return 1; -} - -/* redis.error_reply() */ -int luaRedisErrorReplyCommand(lua_State *lua) { - return luaRedisReturnSingleFieldTable(lua,"err"); -} - -/* redis.status_reply() */ -int luaRedisStatusReplyCommand(lua_State *lua) { - return luaRedisReturnSingleFieldTable(lua,"ok"); -} - -/* redis.replicate_commands() - * - * Turn on single commands replication if the script never called - * a write command so far, and returns true. Otherwise if the script - * already started to write, returns false and stick to whole scripts - * replication, which is our default. */ -int luaRedisReplicateCommandsCommand(lua_State *lua) { - if (server.lua_write_dirty) { - lua_pushboolean(lua,0); - } else { - server.lua_replicate_commands = 1; - /* When we switch to single commands replication, we can provide - * different math.random() sequences at every call, which is what - * the user normally expects. */ - redisSrand48(rand()); - lua_pushboolean(lua,1); - } - return 1; -} - /* redis.breakpoint() * * Allows to stop execution during a debugging session from within @@ -1176,165 +131,24 @@ int luaRedisDebugCommand(lua_State *lua) { return 0; } -/* redis.set_repl() +/* redis.replicate_commands() * - * Set the propagation of write commands executed in the context of the - * script to on/off for AOF and slaves. */ -int luaRedisSetReplCommand(lua_State *lua) { - int argc = lua_gettop(lua); - int flags; - - if (server.lua_replicate_commands == 0) { - lua_pushstring(lua, "You can set the replication behavior only after turning on single commands replication with redis.replicate_commands()."); - return lua_error(lua); - } else if (argc != 1) { - lua_pushstring(lua, "redis.set_repl() requires two arguments."); - return lua_error(lua); + * Turn on single commands replication if the script never called + * a write command so far, and returns true. Otherwise if the script + * already started to write, returns false and stick to whole scripts + * replication, which is our default. */ +int luaRedisReplicateCommandsCommand(lua_State *lua) { + if (server.lua_write_dirty) { + lua_pushboolean(lua,0); + } else { + server.lua_replicate_commands = 1; + /* When we switch to single commands replication, we can provide + * different math.random() sequences at every call, which is what + * the user normally expects. */ + redisSrand48(rand()); + lua_pushboolean(lua,1); } - - flags = lua_tonumber(lua,-1); - if ((flags & ~(PROPAGATE_AOF|PROPAGATE_REPL)) != 0) { - lua_pushstring(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); - return lua_error(lua); - } - server.lua_repl = flags; - return 0; -} - -/* redis.log() */ -int luaLogCommand(lua_State *lua) { - int j, argc = lua_gettop(lua); - int level; - sds log; - - if (argc < 2) { - lua_pushstring(lua, "redis.log() requires two arguments or more."); - return lua_error(lua); - } else if (!lua_isnumber(lua,-argc)) { - lua_pushstring(lua, "First argument must be a number (log level)."); - return lua_error(lua); - } - level = lua_tonumber(lua,-argc); - if (level < LL_DEBUG || level > LL_WARNING) { - lua_pushstring(lua, "Invalid debug level."); - return lua_error(lua); - } - if (level < server.verbosity) return 0; - - /* Glue together all the arguments */ - log = sdsempty(); - for (j = 1; j < argc; j++) { - size_t len; - char *s; - - s = (char*)lua_tolstring(lua,(-argc)+j,&len); - if (s) { - if (j != 1) log = sdscatlen(log," ",1); - log = sdscatlen(log,s,len); - } - } - serverLogRaw(level,log); - sdsfree(log); - return 0; -} - -/* redis.setresp() */ -int luaSetResp(lua_State *lua) { - int argc = lua_gettop(lua); - - if (argc != 1) { - lua_pushstring(lua, "redis.setresp() requires one argument."); - return lua_error(lua); - } - - int resp = lua_tonumber(lua,-argc); - if (resp != 2 && resp != 3) { - lua_pushstring(lua, "RESP version must be 2 or 3."); - return lua_error(lua); - } - - server.lua_client->resp = resp; - return 0; -} - -/* --------------------------------------------------------------------------- - * Lua engine initialization and reset. - * ------------------------------------------------------------------------- */ - -void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) { - lua_pushcfunction(lua, luafunc); - lua_pushstring(lua, libname); - lua_call(lua, 1, 0); -} - -LUALIB_API int (luaopen_cjson) (lua_State *L); -LUALIB_API int (luaopen_struct) (lua_State *L); -LUALIB_API int (luaopen_cmsgpack) (lua_State *L); -LUALIB_API int (luaopen_bit) (lua_State *L); - -void luaLoadLibraries(lua_State *lua) { - luaLoadLib(lua, "", luaopen_base); - luaLoadLib(lua, LUA_TABLIBNAME, luaopen_table); - luaLoadLib(lua, LUA_STRLIBNAME, luaopen_string); - luaLoadLib(lua, LUA_MATHLIBNAME, luaopen_math); - luaLoadLib(lua, LUA_DBLIBNAME, luaopen_debug); - luaLoadLib(lua, "cjson", luaopen_cjson); - luaLoadLib(lua, "struct", luaopen_struct); - luaLoadLib(lua, "cmsgpack", luaopen_cmsgpack); - luaLoadLib(lua, "bit", luaopen_bit); - -#if 0 /* Stuff that we don't load currently, for sandboxing concerns. */ - luaLoadLib(lua, LUA_LOADLIBNAME, luaopen_package); - luaLoadLib(lua, LUA_OSLIBNAME, luaopen_os); -#endif -} - -/* Remove a functions that we don't want to expose to the Redis scripting - * environment. */ -void luaRemoveUnsupportedFunctions(lua_State *lua) { - lua_pushnil(lua); - lua_setglobal(lua,"loadfile"); - lua_pushnil(lua); - lua_setglobal(lua,"dofile"); -} - -/* This function installs metamethods in the global table _G that prevent - * the creation of globals accidentally. - * - * It should be the last to be called in the scripting engine initialization - * sequence, because it may interact with creation of globals. */ -void scriptingEnableGlobalsProtection(lua_State *lua) { - char *s[32]; - sds code = sdsempty(); - int j = 0; - - /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html. - * Modified to be adapted to Redis. */ - s[j++]="local dbg=debug\n"; - s[j++]="local mt = {}\n"; - s[j++]="setmetatable(_G, mt)\n"; - s[j++]="mt.__newindex = function (t, n, v)\n"; - s[j++]=" if dbg.getinfo(2) then\n"; - s[j++]=" local w = dbg.getinfo(2, \"S\").what\n"; - s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; - s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"; - s[j++]=" end\n"; - s[j++]=" end\n"; - s[j++]=" rawset(t, n, v)\n"; - s[j++]="end\n"; - s[j++]="mt.__index = function (t, n)\n"; - s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n"; - s[j++]=" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n"; - s[j++]=" end\n"; - s[j++]=" return rawget(t, n)\n"; - s[j++]="end\n"; - s[j++]="debug = nil\n"; - s[j++]=NULL; - - for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j])); - luaL_loadbuffer(lua,code,sdslen(code),"@enable_strict_lua"); - lua_pcall(lua,0,0,0); - sdsfree(code); + return 1; } /* Initialize the scripting environment. @@ -1359,96 +173,16 @@ void scriptingInit(int setup) { ldbInit(); } - luaLoadLibraries(lua); - luaRemoveUnsupportedFunctions(lua); - /* Initialize a dictionary we use to map SHAs to scripts. * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType); server.lua_scripts_mem = 0; - /* Register the redis commands table and fields */ - lua_newtable(lua); + luaEngineRegisterRedisAPI(lua); - /* redis.call */ - lua_pushstring(lua,"call"); - lua_pushcfunction(lua,luaRedisCallCommand); - lua_settable(lua,-3); - - /* redis.pcall */ - lua_pushstring(lua,"pcall"); - lua_pushcfunction(lua,luaRedisPCallCommand); - lua_settable(lua,-3); - - /* redis.log and log levels. */ - lua_pushstring(lua,"log"); - lua_pushcfunction(lua,luaLogCommand); - lua_settable(lua,-3); - - /* redis.setresp */ - lua_pushstring(lua,"setresp"); - lua_pushcfunction(lua,luaSetResp); - lua_settable(lua,-3); - - lua_pushstring(lua,"LOG_DEBUG"); - lua_pushnumber(lua,LL_DEBUG); - lua_settable(lua,-3); - - lua_pushstring(lua,"LOG_VERBOSE"); - lua_pushnumber(lua,LL_VERBOSE); - lua_settable(lua,-3); - - lua_pushstring(lua,"LOG_NOTICE"); - lua_pushnumber(lua,LL_NOTICE); - lua_settable(lua,-3); - - lua_pushstring(lua,"LOG_WARNING"); - lua_pushnumber(lua,LL_WARNING); - lua_settable(lua,-3); - - /* redis.sha1hex */ - lua_pushstring(lua, "sha1hex"); - lua_pushcfunction(lua, luaRedisSha1hexCommand); - lua_settable(lua, -3); - - /* redis.error_reply and redis.status_reply */ - lua_pushstring(lua, "error_reply"); - lua_pushcfunction(lua, luaRedisErrorReplyCommand); - lua_settable(lua, -3); - lua_pushstring(lua, "status_reply"); - lua_pushcfunction(lua, luaRedisStatusReplyCommand); - lua_settable(lua, -3); - - /* redis.replicate_commands */ - lua_pushstring(lua, "replicate_commands"); - lua_pushcfunction(lua, luaRedisReplicateCommandsCommand); - lua_settable(lua, -3); - - /* redis.set_repl and associated flags. */ - lua_pushstring(lua,"set_repl"); - lua_pushcfunction(lua,luaRedisSetReplCommand); - lua_settable(lua,-3); - - lua_pushstring(lua,"REPL_NONE"); - lua_pushnumber(lua,PROPAGATE_NONE); - lua_settable(lua,-3); - - lua_pushstring(lua,"REPL_AOF"); - lua_pushnumber(lua,PROPAGATE_AOF); - lua_settable(lua,-3); - - lua_pushstring(lua,"REPL_SLAVE"); - lua_pushnumber(lua,PROPAGATE_REPL); - lua_settable(lua,-3); - - lua_pushstring(lua,"REPL_REPLICA"); - lua_pushnumber(lua,PROPAGATE_REPL); - lua_settable(lua,-3); - - lua_pushstring(lua,"REPL_ALL"); - lua_pushnumber(lua,PROPAGATE_AOF|PROPAGATE_REPL); - lua_settable(lua,-3); + /* register debug commands */ + lua_getglobal(lua,"redis"); /* redis.breakpoint */ lua_pushstring(lua,"breakpoint"); @@ -1460,22 +194,13 @@ void scriptingInit(int setup) { lua_pushcfunction(lua,luaRedisDebugCommand); lua_settable(lua,-3); - /* Finally set the table as 'redis' global var. */ + /* redis.replicate_commands */ + lua_pushstring(lua, "replicate_commands"); + lua_pushcfunction(lua, luaRedisReplicateCommandsCommand); + lua_settable(lua, -3); + lua_setglobal(lua,"redis"); - /* Replace math.random and math.randomseed with our implementations. */ - lua_getglobal(lua,"math"); - - lua_pushstring(lua,"random"); - lua_pushcfunction(lua,redis_math_random); - lua_settable(lua,-3); - - lua_pushstring(lua,"randomseed"); - lua_pushcfunction(lua,redis_math_randomseed); - lua_settable(lua,-3); - - lua_setglobal(lua,"math"); - /* Add a helper function that we use to sort the multi bulk output of non * deterministic commands, when containing 'false' elements. */ { @@ -1545,62 +270,6 @@ void scriptingReset(int async) { scriptingInit(0); } -/* Set an array of Redis String Objects as a Lua array (table) stored into a - * global variable. */ -void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec) { - int j; - - lua_newtable(lua); - for (j = 0; j < elec; j++) { - lua_pushlstring(lua,(char*)elev[j]->ptr,sdslen(elev[j]->ptr)); - lua_rawseti(lua,-2,j+1); - } - lua_setglobal(lua,var); -} - -/* --------------------------------------------------------------------------- - * Redis provided math.random - * ------------------------------------------------------------------------- */ - -/* We replace math.random() with our implementation that is not affected - * by specific libc random() implementations and will output the same sequence - * (for the same seed) in every arch. */ - -/* The following implementation is the one shipped with Lua itself but with - * rand() replaced by redisLrand48(). */ -int redis_math_random (lua_State *L) { - /* the `%' avoids the (rare) case of r==1, and is needed also because on - some systems (SunOS!) `rand()' may return a value larger than RAND_MAX */ - lua_Number r = (lua_Number)(redisLrand48()%REDIS_LRAND48_MAX) / - (lua_Number)REDIS_LRAND48_MAX; - switch (lua_gettop(L)) { /* check number of arguments */ - case 0: { /* no arguments */ - lua_pushnumber(L, r); /* Number between 0 and 1 */ - break; - } - case 1: { /* only upper limit */ - int u = luaL_checkint(L, 1); - luaL_argcheck(L, 1<=u, 1, "interval is empty"); - lua_pushnumber(L, floor(r*u)+1); /* int between 1 and `u' */ - break; - } - case 2: { /* lower and upper limits */ - int l = luaL_checkint(L, 1); - int u = luaL_checkint(L, 2); - luaL_argcheck(L, l<=u, 2, "interval is empty"); - lua_pushnumber(L, floor(r*(u-l+1))+l); /* int between `l' and `u' */ - break; - } - default: return luaL_error(L, "wrong number of arguments"); - } - return 1; -} - -int redis_math_randomseed (lua_State *L) { - redisSrand48(luaL_checkint(L, 1)); - return 0; -} - /* --------------------------------------------------------------------------- * EVAL and SCRIPT commands implementation * ------------------------------------------------------------------------- */ @@ -1676,45 +345,6 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { return sha; } -/* This is the Lua script "count" hook that we use to detect scripts timeout. */ -void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { - long long elapsed = elapsedMs(server.lua_time_start); - UNUSED(ar); - UNUSED(lua); - - /* Set the timeout condition if not already set and the maximum - * execution time was reached. */ - if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { - serverLog(LL_WARNING, - "Lua slow script detected: still in execution after %lld milliseconds. " - "You can try killing the script using the SCRIPT KILL command. " - "Script SHA1 is: %s", - elapsed, server.lua_cur_script); - server.lua_timedout = 1; - blockingOperationStarts(); - /* Once the script timeouts we reenter the event loop to permit others - * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason - * we need to mask the client executing the script from the event loop. - * If we don't do that the client may disconnect and could no longer be - * here when the EVAL command will return. */ - protectClient(server.lua_caller); - } - if (server.lua_timedout) processEventsWhileBlocked(); - if (server.lua_kill) { - serverLog(LL_WARNING,"Lua script killed by user with SCRIPT KILL."); - - /* - * Set the hook to invoke all the time so the user -         * will not be able to catch the error with pcall and invoke -         * pcall again which will prevent the script from ever been killed - */ - lua_sethook(lua, luaMaskCountHook, LUA_MASKLINE, 0); - - lua_pushstring(lua,"Script killed by user with SCRIPT KILL..."); - lua_error(lua); - } -} - void prepareLuaClient(void) { /* Select the right DB in the context of the Lua client */ selectDb(server.lua_client,server.lua_caller->db->id); diff --git a/src/script_lua.c b/src/script_lua.c new file mode 100644 index 000000000..0685a4bb3 --- /dev/null +++ b/src/script_lua.c @@ -0,0 +1,1423 @@ +/* + * Copyright (c) 2009-2021, Redis Labs Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "script_lua.h" + +#include "server.h" +#include "sha1.h" +#include "rand.h" +#include "cluster.h" +#include "monotonic.h" +#include "resp_parser.h" +#include +#include +#include +#include +#include "functions.h" + +int redis_math_random (lua_State *L); +int redis_math_randomseed (lua_State *L); +static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_BulkString(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_NullBulkString(void *ctx, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_NullArray(void *ctx, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_Status(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_Error(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_Array(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); +static void redisProtocolToLuaType_Map(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); +static void redisProtocolToLuaType_Set(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); +static void redisProtocolToLuaType_Null(void *ctx, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_Bool(void *ctx, int val, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_BigNumber(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_VerbatimString(void *ctx, const char *format, const char *str, size_t len, const char *proto, size_t proto_len); +static void redisProtocolToLuaType_Attribute(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); + +/* --------------------------------------------------------------------------- + * Redis reply to Lua type conversion functions. + * ------------------------------------------------------------------------- */ + +/* Take a Redis reply in the Redis protocol format and convert it into a + * Lua type. Thanks to this function, and the introduction of not connected + * clients, it is trivial to implement the redis() lua function. + * + * Basically we take the arguments, execute the Redis command in the context + * of a non connected client, then take the generated reply and convert it + * into a suitable Lua type. With this trick the scripting feature does not + * need the introduction of a full Redis internals API. The script + * is like a normal client that bypasses all the slow I/O paths. + * + * Note: in this function we do not do any sanity check as the reply is + * generated by Redis directly. This allows us to go faster. + * + * Errors are returned as a table with a single 'err' field set to the + * error string. + */ + +static const ReplyParserCallbacks DefaultLuaTypeParserCallbacks = { + .null_array_callback = redisProtocolToLuaType_NullArray, + .bulk_string_callback = redisProtocolToLuaType_BulkString, + .null_bulk_string_callback = redisProtocolToLuaType_NullBulkString, + .error_callback = redisProtocolToLuaType_Error, + .simple_str_callback = redisProtocolToLuaType_Status, + .long_callback = redisProtocolToLuaType_Int, + .array_callback = redisProtocolToLuaType_Array, + .set_callback = redisProtocolToLuaType_Set, + .map_callback = redisProtocolToLuaType_Map, + .bool_callback = redisProtocolToLuaType_Bool, + .double_callback = redisProtocolToLuaType_Double, + .null_callback = redisProtocolToLuaType_Null, + .big_number_callback = redisProtocolToLuaType_BigNumber, + .verbatim_string_callback = redisProtocolToLuaType_VerbatimString, + .attribute_callback = redisProtocolToLuaType_Attribute, + .error = NULL, +}; + +void redisProtocolToLuaType(lua_State *lua, char* reply) { + ReplyParser parser = {.curr_location = reply, .callbacks = DefaultLuaTypeParserCallbacks}; + + parseReply(&parser, lua); +} + +static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushnumber(lua,(lua_Number)val); +} + +static void redisProtocolToLuaType_NullBulkString(void *ctx, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushboolean(lua,0); +} + +static void redisProtocolToLuaType_NullArray(void *ctx, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + lua_State *lua = ctx; + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushboolean(lua,0); +} + + +static void redisProtocolToLuaType_BulkString(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushlstring(lua,str,len); +} + +static void redisProtocolToLuaType_Status(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 3)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua,"ok"); + lua_pushlstring(lua,str,len); + lua_settable(lua,-3); +} + +static void redisProtocolToLuaType_Error(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 3)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua,"err"); + lua_pushlstring(lua,str,len); + lua_settable(lua,-3); +} + +static void redisProtocolToLuaType_Map(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { + UNUSED(proto); + lua_State *lua = ctx; + if (lua) { + if (!lua_checkstack(lua, 3)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua, "map"); + lua_newtable(lua); + } + for (size_t j = 0; j < len; j++) { + parseReply(parser,lua); + parseReply(parser,lua); + if (lua) lua_settable(lua,-3); + } + if (lua) lua_settable(lua,-3); +} + +static void redisProtocolToLuaType_Set(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { + UNUSED(proto); + + lua_State *lua = ctx; + if (lua) { + if (!lua_checkstack(lua, 3)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua, "set"); + lua_newtable(lua); + } + for (size_t j = 0; j < len; j++) { + parseReply(parser,lua); + if (lua) { + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. + * Notice that here we need to check the stack again because the recursive + * call to redisProtocolToLuaType might have use the room allocated in the stack*/ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushboolean(lua,1); + lua_settable(lua,-3); + } + } + if (lua) lua_settable(lua,-3); +} + +static void redisProtocolToLuaType_Array(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { + UNUSED(proto); + + lua_State *lua = ctx; + if (lua){ + if (!lua_checkstack(lua, 2)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + } + for (size_t j = 0; j < len; j++) { + if (lua) lua_pushnumber(lua,j+1); + parseReply(parser,lua); + if (lua) lua_settable(lua,-3); + } +} + +static void redisProtocolToLuaType_Attribute(struct ReplyParser *parser, void *ctx, size_t len, const char *proto) { + UNUSED(proto); + + /* Parse the attribute reply. + * Currently, we do not expose the attribute to the Lua script so + * we just need to continue parsing and ignore it (the NULL ensures that the + * reply will be ignored). */ + for (size_t j = 0; j < len; j++) { + parseReply(parser,NULL); + parseReply(parser,NULL); + } + + /* Parse the reply itself. */ + parseReply(parser,ctx); +} + +static void redisProtocolToLuaType_VerbatimString(void *ctx, const char *format, const char *str, size_t len, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 5)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua,"verbatim_string"); + lua_newtable(lua); + lua_pushstring(lua,"string"); + lua_pushlstring(lua,str,len); + lua_settable(lua,-3); + lua_pushstring(lua,"format"); + lua_pushlstring(lua,format,3); + lua_settable(lua,-3); + lua_settable(lua,-3); +} + +static void redisProtocolToLuaType_BigNumber(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 3)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua,"big_number"); + lua_pushlstring(lua,str,len); + lua_settable(lua,-3); +} + +static void redisProtocolToLuaType_Null(void *ctx, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushnil(lua); +} + +static void redisProtocolToLuaType_Bool(void *ctx, int val, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 1)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_pushboolean(lua,val); +} + +static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto, size_t proto_len) { + UNUSED(proto); + UNUSED(proto_len); + if (!ctx) { + return; + } + + lua_State *lua = ctx; + if (!lua_checkstack(lua, 3)) { + /* Increase the Lua stack if needed, to make sure there is enough room + * to push elements to the stack. On failure, exit with panic. */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + lua_newtable(lua); + lua_pushstring(lua,"double"); + lua_pushnumber(lua,d); + lua_settable(lua,-3); +} + +/* This function is used in order to push an error on the Lua stack in the + * format used by redis.pcall to return errors, which is a lua table + * with a single "err" field set to the error string. Note that this + * table is never a valid reply by proper commands, since the returned + * tables are otherwise always indexed by integers, never by strings. */ +void luaPushError(lua_State *lua, char *error) { + lua_Debug dbg; + + /* If debugging is active and in step mode, log errors resulting from + * Redis commands. */ + if (ldb.active && ldb.step) { + ldbLog(sdscatprintf(sdsempty()," %s",error)); + } + + lua_newtable(lua); + lua_pushstring(lua,"err"); + + /* Attempt to figure out where this function was called, if possible */ + if(lua_getstack(lua, 1, &dbg) && lua_getinfo(lua, "nSl", &dbg)) { + sds msg = sdscatprintf(sdsempty(), "%s: %d: %s", + dbg.source, dbg.currentline, error); + lua_pushstring(lua, msg); + sdsfree(msg); + } else { + lua_pushstring(lua, error); + } + lua_settable(lua,-3); +} + +/* In case the error set into the Lua stack by luaPushError() was generated + * by the non-error-trapping version of redis.pcall(), which is redis.call(), + * this function will raise the Lua error so that the execution of the + * script will be halted. */ +int luaRaiseError(lua_State *lua) { + lua_pushstring(lua,"err"); + lua_gettable(lua,-2); + return lua_error(lua); +} + +/* Sort the array currently in the stack. We do this to make the output + * of commands like KEYS or SMEMBERS something deterministic when called + * from Lua (to play well with AOf/replication). + * + * The array is sorted using table.sort itself, and assuming all the + * list elements are strings. */ +void luaSortArray(lua_State *lua) { + /* Initial Stack: array */ + lua_getglobal(lua,"table"); + lua_pushstring(lua,"sort"); + lua_gettable(lua,-2); /* Stack: array, table, table.sort */ + lua_pushvalue(lua,-3); /* Stack: array, table, table.sort, array */ + if (lua_pcall(lua,1,0,0)) { + /* Stack: array, table, error */ + + /* We are not interested in the error, we assume that the problem is + * that there are 'false' elements inside the array, so we try + * again with a slower function but able to handle this case, that + * is: table.sort(table, __redis__compare_helper) */ + lua_pop(lua,1); /* Stack: array, table */ + lua_pushstring(lua,"sort"); /* Stack: array, table, sort */ + lua_gettable(lua,-2); /* Stack: array, table, table.sort */ + lua_pushvalue(lua,-3); /* Stack: array, table, table.sort, array */ + lua_getglobal(lua,"__redis__compare_helper"); + /* Stack: array, table, table.sort, array, __redis__compare_helper */ + lua_call(lua,2,0); + } + /* Stack: array (sorted), table */ + lua_pop(lua,1); /* Stack: array (sorted) */ +} + +/* --------------------------------------------------------------------------- + * Lua reply to Redis reply conversion functions. + * ------------------------------------------------------------------------- */ + +/* Reply to client 'c' converting the top element in the Lua stack to a + * Redis reply. As a side effect the element is consumed from the stack. */ +void luaReplyToRedisReply(client *c, lua_State *lua) { + int t = lua_type(lua,-1); + + if (!lua_checkstack(lua, 4)) { + /* Increase the Lua stack if needed to make sure there is enough room + * to push 4 elements to the stack. On failure, return error. + * Notice that we need, in the worst case, 4 elements because returning a map might + * require push 4 elements to the Lua stack.*/ + addReplyErrorFormat(c, "reached lua stack limit"); + lua_pop(lua,1); /* pop the element from the stack */ + return; + } + + switch(t) { + case LUA_TSTRING: + addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); + break; + case LUA_TBOOLEAN: + if (server.lua_client->resp == 2) + addReply(c,lua_toboolean(lua,-1) ? shared.cone : + shared.null[c->resp]); + else + addReplyBool(c,lua_toboolean(lua,-1)); + break; + case LUA_TNUMBER: + addReplyLongLong(c,(long long)lua_tonumber(lua,-1)); + break; + case LUA_TTABLE: + /* We need to check if it is an array, an error, or a status reply. + * Error are returned as a single element table with 'err' field. + * Status replies are returned as single element table with 'ok' + * field. */ + + /* Handle error reply. */ + /* we took care of the stack size on function start */ + lua_pushstring(lua,"err"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TSTRING) { + addReplyErrorFormat(c,"-%s",lua_tostring(lua,-1)); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle status reply. */ + lua_pushstring(lua,"ok"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TSTRING) { + sds ok = sdsnew(lua_tostring(lua,-1)); + sdsmapchars(ok,"\r\n"," ",2); + addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); + sdsfree(ok); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle double reply. */ + lua_pushstring(lua,"double"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNUMBER) { + addReplyDouble(c,lua_tonumber(lua,-1)); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle big number reply. */ + lua_pushstring(lua,"big_number"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TSTRING) { + sds big_num = sdsnewlen(lua_tostring(lua,-1), lua_strlen(lua,-1)); + sdsmapchars(big_num,"\r\n"," ",2); + addReplyBigNum(c,big_num,sdslen(big_num)); + sdsfree(big_num); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle verbatim reply. */ + lua_pushstring(lua,"verbatim_string"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + lua_pushstring(lua,"format"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TSTRING){ + char* format = (char*)lua_tostring(lua,-1); + lua_pushstring(lua,"string"); + lua_gettable(lua,-3); + t = lua_type(lua,-1); + if (t == LUA_TSTRING){ + size_t len; + char* str = (char*)lua_tolstring(lua,-1,&len); + addReplyVerbatim(c, str, len, format); + lua_pop(lua,4); + return; + } + lua_pop(lua,1); + } + lua_pop(lua,1); + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle map reply. */ + lua_pushstring(lua,"map"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int maplen = 0; + void *replylen = addReplyDeferredLen(c); + /* we took care of the stack size on function start */ + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, value */ + lua_pushvalue(lua,-2); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + luaReplyToRedisReply(c, lua); /* Return value. */ + /* Stack now: table, key. */ + maplen++; + } + setDeferredMapLen(c,replylen,maplen); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle set reply. */ + lua_pushstring(lua,"set"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int setlen = 0; + void *replylen = addReplyDeferredLen(c); + /* we took care of the stack size on function start */ + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, true */ + lua_pop(lua,1); /* Discard the boolean value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + setlen++; + } + setDeferredSetLen(c,replylen,setlen); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle the array reply. */ + void *replylen = addReplyDeferredLen(c); + int j = 1, mbulklen = 0; + while(1) { + /* we took care of the stack size on function start */ + lua_pushnumber(lua,j++); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNIL) { + lua_pop(lua,1); + break; + } + luaReplyToRedisReply(c, lua); + mbulklen++; + } + setDeferredArrayLen(c,replylen,mbulklen); + break; + default: + addReplyNull(c); + } + lua_pop(lua,1); +} + +/* --------------------------------------------------------------------------- + * Lua redis.* functions implementations. + * ------------------------------------------------------------------------- */ + +#define LUA_CMD_OBJCACHE_SIZE 32 +#define LUA_CMD_OBJCACHE_MAX_LEN 64 +int luaRedisGenericCommand(lua_State *lua, int raise_error) { + int j, argc = lua_gettop(lua); + struct redisCommand *cmd; + client *c = server.lua_client; + sds reply; + + /* Cached across calls. */ + static robj **argv = NULL; + static int argv_size = 0; + static robj *cached_objects[LUA_CMD_OBJCACHE_SIZE]; + static size_t cached_objects_len[LUA_CMD_OBJCACHE_SIZE]; + static int inuse = 0; /* Recursive calls detection. */ + + /* By using Lua debug hooks it is possible to trigger a recursive call + * to luaRedisGenericCommand(), which normally should never happen. + * To make this function reentrant is futile and makes it slower, but + * we should at least detect such a misuse, and abort. */ + if (inuse) { + char *recursion_warning = + "luaRedisGenericCommand() recursive call detected. " + "Are you doing funny stuff with Lua debug hooks?"; + serverLog(LL_WARNING,"%s",recursion_warning); + luaPushError(lua,recursion_warning); + return 1; + } + inuse++; + + /* Require at least one argument */ + if (argc == 0) { + luaPushError(lua, + "Please specify at least one argument for redis.call()"); + inuse--; + return raise_error ? luaRaiseError(lua) : 1; + } + + /* Build the arguments vector */ + if (argv_size < argc) { + argv = zrealloc(argv,sizeof(robj*)*argc); + argv_size = argc; + } + + for (j = 0; j < argc; j++) { + char *obj_s; + size_t obj_len; + char dbuf[64]; + + if (lua_type(lua,j+1) == LUA_TNUMBER) { + /* We can't use lua_tolstring() for number -> string conversion + * since Lua uses a format specifier that loses precision. */ + lua_Number num = lua_tonumber(lua,j+1); + + obj_len = snprintf(dbuf,sizeof(dbuf),"%.17g",(double)num); + obj_s = dbuf; + } else { + obj_s = (char*)lua_tolstring(lua,j+1,&obj_len); + if (obj_s == NULL) break; /* Not a string. */ + } + + /* Try to use a cached object. */ + if (j < LUA_CMD_OBJCACHE_SIZE && cached_objects[j] && + cached_objects_len[j] >= obj_len) + { + sds s = cached_objects[j]->ptr; + argv[j] = cached_objects[j]; + cached_objects[j] = NULL; + memcpy(s,obj_s,obj_len+1); + sdssetlen(s, obj_len); + } else { + argv[j] = createStringObject(obj_s, obj_len); + } + } + + /* Check if one of the arguments passed by the Lua script + * is not a string or an integer (lua_isstring() return true for + * integers as well). */ + if (j != argc) { + j--; + while (j >= 0) { + decrRefCount(argv[j]); + j--; + } + luaPushError(lua, + "Lua redis() command arguments must be strings or integers"); + inuse--; + return raise_error ? luaRaiseError(lua) : 1; + } + + /* Pop all arguments from the stack, we do not need them anymore + * and this way we guaranty we will have room on the stack for the result. */ + lua_pop(lua, argc); + + /* Setup our fake client for command execution */ + c->argv = argv; + c->argc = argc; + c->user = server.lua_caller->user; + + /* Process module hooks */ + moduleCallCommandFilters(c); + argv = c->argv; + argc = c->argc; + + /* Log the command if debugging is active. */ + if (ldb.active && ldb.step) { + sds cmdlog = sdsnew(""); + for (j = 0; j < c->argc; j++) { + if (j == 10) { + cmdlog = sdscatprintf(cmdlog," ... (%d more)", + c->argc-j-1); + break; + } else { + cmdlog = sdscatlen(cmdlog," ",1); + cmdlog = sdscatsds(cmdlog,c->argv[j]->ptr); + } + } + ldbLog(cmdlog); + } + + /* Command lookup */ + cmd = lookupCommand(argv[0]->ptr); + if (!cmd || ((cmd->arity > 0 && cmd->arity != argc) || + (argc < -cmd->arity))) + { + if (cmd) + luaPushError(lua, + "Wrong number of args calling Redis command From Lua script"); + else + luaPushError(lua,"Unknown Redis command called from Lua script"); + goto cleanup; + } + c->cmd = c->lastcmd = cmd; + + /* There are commands that are not allowed inside scripts. */ + if (!server.lua_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { + luaPushError(lua, "This Redis command is not allowed from scripts"); + goto cleanup; + } + + /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ + if ((server.lua_caller->cmd->proc == evalRoCommand || + server.lua_caller->cmd->proc == evalShaRoCommand) && + (cmd->flags & CMD_WRITE)) + { + luaPushError(lua, "Write commands are not allowed from read-only scripts"); + goto cleanup; + } + + /* Check the ACLs. */ + int acl_errpos; + int acl_retval = ACLCheckAllPerm(c,&acl_errpos); + if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,ACL_LOG_CTX_LUA,acl_errpos,NULL,NULL); + switch (acl_retval) { + case ACL_DENIED_CMD: + luaPushError(lua, "The user executing the script can't run this " + "command or subcommand"); + break; + case ACL_DENIED_KEY: + luaPushError(lua, "The user executing the script can't access " + "at least one of the keys mentioned in the " + "command arguments"); + break; + case ACL_DENIED_CHANNEL: + luaPushError(lua, "The user executing the script can't publish " + "to the channel mentioned in the command"); + break; + default: + luaPushError(lua, "The user executing the script is lacking the " + "permissions for the command"); + break; + } + goto cleanup; + } + + /* Write commands are forbidden against read-only slaves, or if a + * command marked as non-deterministic was already called in the context + * of this script. */ + if (cmd->flags & CMD_WRITE) { + int deny_write_type = writeCommandsDeniedByDiskError(); + if (server.lua_random_dirty && !server.lua_replicate_commands) { + luaPushError(lua, + "Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); + goto cleanup; + } else if (server.masterhost && server.repl_slave_ro && + server.lua_caller->id != CLIENT_ID_AOF && + !(server.lua_caller->flags & CLIENT_MASTER)) + { + luaPushError(lua, shared.roslaveerr->ptr); + goto cleanup; + } else if (deny_write_type != DISK_ERROR_TYPE_NONE) { + if (deny_write_type == DISK_ERROR_TYPE_RDB) { + luaPushError(lua, shared.bgsaveerr->ptr); + } else { + sds aof_write_err = sdscatfmt(sdsempty(), + "-MISCONF Errors writing to the AOF file: %s\r\n", + strerror(server.aof_last_write_errno)); + luaPushError(lua, aof_write_err); + sdsfree(aof_write_err); + } + goto cleanup; + } + } + + /* If we reached the memory limit configured via maxmemory, commands that + * could enlarge the memory usage are not allowed, but only if this is the + * first write in the context of this script, otherwise we can't stop + * in the middle. */ + if (server.maxmemory && /* Maxmemory is actually enabled. */ + server.lua_caller->id != CLIENT_ID_AOF && /* Don't care about mem if loading from AOF. */ + !server.masterhost && /* Slave must execute the script. */ + server.lua_write_dirty == 0 && /* Script had no side effects so far. */ + server.lua_oom && /* Detected OOM when script start. */ + (cmd->flags & CMD_DENYOOM)) + { + luaPushError(lua, shared.oomerr->ptr); + goto cleanup; + } + + if (cmd->flags & CMD_RANDOM) server.lua_random_dirty = 1; + if (cmd->flags & CMD_WRITE) server.lua_write_dirty = 1; + + /* If this is a Redis Cluster node, we need to make sure Lua is not + * trying to access non-local keys, with the exception of commands + * received from our master or when loading the AOF back in memory. */ + if (server.cluster_enabled && server.lua_caller->id != CLIENT_ID_AOF && + !(server.lua_caller->flags & CLIENT_MASTER)) + { + int error_code; + /* Duplicate relevant flags in the lua client. */ + c->flags &= ~(CLIENT_READONLY|CLIENT_ASKING); + c->flags |= server.lua_caller->flags & (CLIENT_READONLY|CLIENT_ASKING); + if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != + server.cluster->myself) + { + if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + luaPushError(lua, + "Lua script attempted to execute a write command while the " + "cluster is down and readonly"); + } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { + luaPushError(lua, + "Lua script attempted to execute a command while the " + "cluster is down"); + } else { + luaPushError(lua, + "Lua script attempted to access a non local key in a " + "cluster node"); + } + + goto cleanup; + } + } + + /* If we are using single commands replication, we need to wrap what + * we propagate into a MULTI/EXEC block, so that it will be atomic like + * a Lua script in the context of AOF and slaves. */ + if (server.lua_replicate_commands && + !server.lua_multi_emitted && + !(server.lua_caller->flags & CLIENT_MULTI) && + server.lua_write_dirty && + server.lua_repl != PROPAGATE_NONE) + { + execCommandPropagateMulti(server.lua_caller->db->id); + server.lua_multi_emitted = 1; + /* Now we are in the MULTI context, the lua_client should be + * flag as CLIENT_MULTI. */ + c->flags |= CLIENT_MULTI; + } + + /* Run the command */ + int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; + if (server.lua_replicate_commands) { + /* Set flags according to redis.set_repl() settings. */ + if (server.lua_repl & PROPAGATE_AOF) + call_flags |= CMD_CALL_PROPAGATE_AOF; + if (server.lua_repl & PROPAGATE_REPL) + call_flags |= CMD_CALL_PROPAGATE_REPL; + } + call(c,call_flags); + serverAssert((c->flags & CLIENT_BLOCKED) == 0); + + /* Convert the result of the Redis command into a suitable Lua type. + * The first thing we need is to create a single string from the client + * output buffers. */ + if (listLength(c->reply) == 0 && (size_t)c->bufpos < c->buf_usable_size) { + /* This is a fast path for the common case of a reply inside the + * client static buffer. Don't create an SDS string but just use + * the client buffer directly. */ + c->buf[c->bufpos] = '\0'; + reply = c->buf; + c->bufpos = 0; + } else { + reply = sdsnewlen(c->buf,c->bufpos); + c->bufpos = 0; + while(listLength(c->reply)) { + clientReplyBlock *o = listNodeValue(listFirst(c->reply)); + + reply = sdscatlen(reply,o->buf,o->used); + listDelNode(c->reply,listFirst(c->reply)); + } + } + if (raise_error && reply[0] != '-') raise_error = 0; + redisProtocolToLuaType(lua,reply); + + /* If the debugger is active, log the reply from Redis. */ + if (ldb.active && ldb.step) + ldbLogRedisReply(reply); + + /* Sort the output array if needed, assuming it is a non-null multi bulk + * reply as expected. */ + if ((cmd->flags & CMD_SORT_FOR_SCRIPT) && + (server.lua_replicate_commands == 0) && + (reply[0] == '*' && reply[1] != '-')) { + luaSortArray(lua); + } + if (reply != c->buf) sdsfree(reply); + c->reply_bytes = 0; + +cleanup: + /* Clean up. Command code may have changed argv/argc so we use the + * argv/argc of the client instead of the local variables. */ + for (j = 0; j < c->argc; j++) { + robj *o = c->argv[j]; + + /* Try to cache the object in the cached_objects array. + * The object must be small, SDS-encoded, and with refcount = 1 + * (we must be the only owner) for us to cache it. */ + if (j < LUA_CMD_OBJCACHE_SIZE && + o->refcount == 1 && + (o->encoding == OBJ_ENCODING_RAW || + o->encoding == OBJ_ENCODING_EMBSTR) && + sdslen(o->ptr) <= LUA_CMD_OBJCACHE_MAX_LEN) + { + sds s = o->ptr; + if (cached_objects[j]) decrRefCount(cached_objects[j]); + cached_objects[j] = o; + cached_objects_len[j] = sdsalloc(s); + } else { + decrRefCount(o); + } + } + + if (c->argv != argv) { + zfree(c->argv); + argv = NULL; + argv_size = 0; + } + + c->user = NULL; + + if (raise_error) { + /* If we are here we should have an error in the stack, in the + * form of a table with an "err" field. Extract the string to + * return the plain error. */ + inuse--; + return luaRaiseError(lua); + } + inuse--; + return 1; +} + +/* redis.call() */ +int luaRedisCallCommand(lua_State *lua) { + return luaRedisGenericCommand(lua,1); +} + +/* redis.pcall() */ +int luaRedisPCallCommand(lua_State *lua) { + return luaRedisGenericCommand(lua,0); +} + +/* This adds redis.sha1hex(string) to Lua scripts using the same hashing + * function used for sha1ing lua scripts. */ +int luaRedisSha1hexCommand(lua_State *lua) { + int argc = lua_gettop(lua); + char digest[41]; + size_t len; + char *s; + + if (argc != 1) { + lua_pushstring(lua, "wrong number of arguments"); + return lua_error(lua); + } + + s = (char*)lua_tolstring(lua,1,&len); + sha1hex(digest,s,len); + lua_pushstring(lua,digest); + return 1; +} + +/* Returns a table with a single field 'field' set to the string value + * passed as argument. This helper function is handy when returning + * a Redis Protocol error or status reply from Lua: + * + * return redis.error_reply("ERR Some Error") + * return redis.status_reply("ERR Some Error") + */ +int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { + if (lua_gettop(lua) != 1 || lua_type(lua,-1) != LUA_TSTRING) { + luaPushError(lua, "wrong number or type of arguments"); + return 1; + } + + lua_newtable(lua); + lua_pushstring(lua, field); + lua_pushvalue(lua, -3); + lua_settable(lua, -3); + return 1; +} + +/* redis.error_reply() */ +int luaRedisErrorReplyCommand(lua_State *lua) { + return luaRedisReturnSingleFieldTable(lua,"err"); +} + +/* redis.status_reply() */ +int luaRedisStatusReplyCommand(lua_State *lua) { + return luaRedisReturnSingleFieldTable(lua,"ok"); +} + +/* redis.set_repl() + * + * Set the propagation of write commands executed in the context of the + * script to on/off for AOF and slaves. */ +int luaRedisSetReplCommand(lua_State *lua) { + int argc = lua_gettop(lua); + int flags; + + if (server.lua_replicate_commands == 0) { + lua_pushstring(lua, "You can set the replication behavior only after turning on single commands replication with redis.replicate_commands()."); + return lua_error(lua); + } else if (argc != 1) { + lua_pushstring(lua, "redis.set_repl() requires two arguments."); + return lua_error(lua); + } + + flags = lua_tonumber(lua,-1); + if ((flags & ~(PROPAGATE_AOF|PROPAGATE_REPL)) != 0) { + lua_pushstring(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); + return lua_error(lua); + } + server.lua_repl = flags; + return 0; +} + +/* redis.log() */ +int luaLogCommand(lua_State *lua) { + int j, argc = lua_gettop(lua); + int level; + sds log; + + if (argc < 2) { + lua_pushstring(lua, "redis.log() requires two arguments or more."); + return lua_error(lua); + } else if (!lua_isnumber(lua,-argc)) { + lua_pushstring(lua, "First argument must be a number (log level)."); + return lua_error(lua); + } + level = lua_tonumber(lua,-argc); + if (level < LL_DEBUG || level > LL_WARNING) { + lua_pushstring(lua, "Invalid debug level."); + return lua_error(lua); + } + if (level < server.verbosity) return 0; + + /* Glue together all the arguments */ + log = sdsempty(); + for (j = 1; j < argc; j++) { + size_t len; + char *s; + + s = (char*)lua_tolstring(lua,(-argc)+j,&len); + if (s) { + if (j != 1) log = sdscatlen(log," ",1); + log = sdscatlen(log,s,len); + } + } + serverLogRaw(level,log); + sdsfree(log); + return 0; +} + +/* redis.setresp() */ +int luaSetResp(lua_State *lua) { + int argc = lua_gettop(lua); + + if (argc != 1) { + lua_pushstring(lua, "redis.setresp() requires one argument."); + return lua_error(lua); + } + + int resp = lua_tonumber(lua,-argc); + if (resp != 2 && resp != 3) { + lua_pushstring(lua, "RESP version must be 2 or 3."); + return lua_error(lua); + } + + server.lua_client->resp = resp; + return 0; +} + +/* --------------------------------------------------------------------------- + * Lua engine initialization and reset. + * ------------------------------------------------------------------------- */ + +void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) { + lua_pushcfunction(lua, luafunc); + lua_pushstring(lua, libname); + lua_call(lua, 1, 0); +} + +LUALIB_API int (luaopen_cjson) (lua_State *L); +LUALIB_API int (luaopen_struct) (lua_State *L); +LUALIB_API int (luaopen_cmsgpack) (lua_State *L); +LUALIB_API int (luaopen_bit) (lua_State *L); + +void luaLoadLibraries(lua_State *lua) { + luaLoadLib(lua, "", luaopen_base); + luaLoadLib(lua, LUA_TABLIBNAME, luaopen_table); + luaLoadLib(lua, LUA_STRLIBNAME, luaopen_string); + luaLoadLib(lua, LUA_MATHLIBNAME, luaopen_math); + luaLoadLib(lua, LUA_DBLIBNAME, luaopen_debug); + luaLoadLib(lua, "cjson", luaopen_cjson); + luaLoadLib(lua, "struct", luaopen_struct); + luaLoadLib(lua, "cmsgpack", luaopen_cmsgpack); + luaLoadLib(lua, "bit", luaopen_bit); + +#if 0 /* Stuff that we don't load currently, for sandboxing concerns. */ + luaLoadLib(lua, LUA_LOADLIBNAME, luaopen_package); + luaLoadLib(lua, LUA_OSLIBNAME, luaopen_os); +#endif +} + +/* Remove a functions that we don't want to expose to the Redis scripting + * environment. */ +void luaRemoveUnsupportedFunctions(lua_State *lua) { + lua_pushnil(lua); + lua_setglobal(lua,"loadfile"); + lua_pushnil(lua); + lua_setglobal(lua,"dofile"); +} + +/* This function installs metamethods in the global table _G that prevent + * the creation of globals accidentally. + * + * It should be the last to be called in the scripting engine initialization + * sequence, because it may interact with creation of globals. */ +void scriptingEnableGlobalsProtection(lua_State *lua) { + char *s[32]; + sds code = sdsempty(); + int j = 0; + + /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html. + * Modified to be adapted to Redis. */ + s[j++]="local dbg=debug\n"; + s[j++]="local mt = {}\n"; + s[j++]="setmetatable(_G, mt)\n"; + s[j++]="mt.__newindex = function (t, n, v)\n"; + s[j++]=" if dbg.getinfo(2) then\n"; + s[j++]=" local w = dbg.getinfo(2, \"S\").what\n"; + s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; + s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"; + s[j++]=" end\n"; + s[j++]=" end\n"; + s[j++]=" rawset(t, n, v)\n"; + s[j++]="end\n"; + s[j++]="mt.__index = function (t, n)\n"; + s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n"; + s[j++]=" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n"; + s[j++]=" end\n"; + s[j++]=" return rawget(t, n)\n"; + s[j++]="end\n"; + s[j++]="debug = nil\n"; + s[j++]=NULL; + + for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j])); + luaL_loadbuffer(lua,code,sdslen(code),"@enable_strict_lua"); + lua_pcall(lua,0,0,0); + sdsfree(code); +} + +void luaEngineRegisterRedisAPI(lua_State* lua) { + luaLoadLibraries(lua); + luaRemoveUnsupportedFunctions(lua); + + /* Register the redis commands table and fields */ + lua_newtable(lua); + + /* redis.call */ + lua_pushstring(lua,"call"); + lua_pushcfunction(lua,luaRedisCallCommand); + lua_settable(lua,-3); + + /* redis.pcall */ + lua_pushstring(lua,"pcall"); + lua_pushcfunction(lua,luaRedisPCallCommand); + lua_settable(lua,-3); + + /* redis.log and log levels. */ + lua_pushstring(lua,"log"); + lua_pushcfunction(lua,luaLogCommand); + lua_settable(lua,-3); + + /* redis.setresp */ + lua_pushstring(lua,"setresp"); + lua_pushcfunction(lua,luaSetResp); + lua_settable(lua,-3); + + lua_pushstring(lua,"LOG_DEBUG"); + lua_pushnumber(lua,LL_DEBUG); + lua_settable(lua,-3); + + lua_pushstring(lua,"LOG_VERBOSE"); + lua_pushnumber(lua,LL_VERBOSE); + lua_settable(lua,-3); + + lua_pushstring(lua,"LOG_NOTICE"); + lua_pushnumber(lua,LL_NOTICE); + lua_settable(lua,-3); + + lua_pushstring(lua,"LOG_WARNING"); + lua_pushnumber(lua,LL_WARNING); + lua_settable(lua,-3); + + /* redis.sha1hex */ + lua_pushstring(lua, "sha1hex"); + lua_pushcfunction(lua, luaRedisSha1hexCommand); + lua_settable(lua, -3); + + /* redis.error_reply and redis.status_reply */ + lua_pushstring(lua, "error_reply"); + lua_pushcfunction(lua, luaRedisErrorReplyCommand); + lua_settable(lua, -3); + lua_pushstring(lua, "status_reply"); + lua_pushcfunction(lua, luaRedisStatusReplyCommand); + lua_settable(lua, -3); + + /* redis.set_repl and associated flags. */ + lua_pushstring(lua,"set_repl"); + lua_pushcfunction(lua,luaRedisSetReplCommand); + lua_settable(lua,-3); + + lua_pushstring(lua,"REPL_NONE"); + lua_pushnumber(lua,PROPAGATE_NONE); + lua_settable(lua,-3); + + lua_pushstring(lua,"REPL_AOF"); + lua_pushnumber(lua,PROPAGATE_AOF); + lua_settable(lua,-3); + + lua_pushstring(lua,"REPL_SLAVE"); + lua_pushnumber(lua,PROPAGATE_REPL); + lua_settable(lua,-3); + + lua_pushstring(lua,"REPL_REPLICA"); + lua_pushnumber(lua,PROPAGATE_REPL); + lua_settable(lua,-3); + + lua_pushstring(lua,"REPL_ALL"); + lua_pushnumber(lua,PROPAGATE_AOF|PROPAGATE_REPL); + + lua_settable(lua,-3); + /* Finally set the table as 'redis' global var. */ + lua_setglobal(lua,"redis"); + + /* Replace math.random and math.randomseed with our implementations. */ + lua_getglobal(lua,"math"); + + lua_pushstring(lua,"random"); + lua_pushcfunction(lua,redis_math_random); + lua_settable(lua,-3); + + lua_pushstring(lua,"randomseed"); + lua_pushcfunction(lua,redis_math_randomseed); + lua_settable(lua,-3); + + lua_setglobal(lua,"math"); +} + +/* Set an array of Redis String Objects as a Lua array (table) stored into a + * global variable. */ +void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec) { + int j; + + lua_newtable(lua); + for (j = 0; j < elec; j++) { + lua_pushlstring(lua,(char*)elev[j]->ptr,sdslen(elev[j]->ptr)); + lua_rawseti(lua,-2,j+1); + } + lua_setglobal(lua,var); +} + +/* --------------------------------------------------------------------------- + * Redis provided math.random + * ------------------------------------------------------------------------- */ + +/* We replace math.random() with our implementation that is not affected + * by specific libc random() implementations and will output the same sequence + * (for the same seed) in every arch. */ + +/* The following implementation is the one shipped with Lua itself but with + * rand() replaced by redisLrand48(). */ +int redis_math_random (lua_State *L) { + /* the `%' avoids the (rare) case of r==1, and is needed also because on + some systems (SunOS!) `rand()' may return a value larger than RAND_MAX */ + lua_Number r = (lua_Number)(redisLrand48()%REDIS_LRAND48_MAX) / + (lua_Number)REDIS_LRAND48_MAX; + switch (lua_gettop(L)) { /* check number of arguments */ + case 0: { /* no arguments */ + lua_pushnumber(L, r); /* Number between 0 and 1 */ + break; + } + case 1: { /* only upper limit */ + int u = luaL_checkint(L, 1); + luaL_argcheck(L, 1<=u, 1, "interval is empty"); + lua_pushnumber(L, floor(r*u)+1); /* int between 1 and `u' */ + break; + } + case 2: { /* lower and upper limits */ + int l = luaL_checkint(L, 1); + int u = luaL_checkint(L, 2); + luaL_argcheck(L, l<=u, 2, "interval is empty"); + lua_pushnumber(L, floor(r*(u-l+1))+l); /* int between `l' and `u' */ + break; + } + default: return luaL_error(L, "wrong number of arguments"); + } + return 1; +} + +int redis_math_randomseed (lua_State *L) { + redisSrand48(luaL_checkint(L, 1)); + return 0; +} + +/* This is the Lua script "count" hook that we use to detect scripts timeout. */ +void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { + long long elapsed = elapsedMs(server.lua_time_start); + UNUSED(ar); + UNUSED(lua); + + /* Set the timeout condition if not already set and the maximum + * execution time was reached. */ + if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { + serverLog(LL_WARNING, + "Lua slow script detected: still in execution after %lld milliseconds. " + "You can try killing the script using the SCRIPT KILL command. " + "Script SHA1 is: %s", + elapsed, server.lua_cur_script); + server.lua_timedout = 1; + blockingOperationStarts(); + /* Once the script timeouts we reenter the event loop to permit others + * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason + * we need to mask the client executing the script from the event loop. + * If we don't do that the client may disconnect and could no longer be + * here when the EVAL command will return. */ + protectClient(server.lua_caller); + } + if (server.lua_timedout) processEventsWhileBlocked(); + if (server.lua_kill) { + serverLog(LL_WARNING,"Lua script killed by user with SCRIPT KILL."); + + /* + * Set the hook to invoke all the time so the user +         * will not be able to catch the error with pcall and invoke +         * pcall again which will prevent the script from ever been killed + */ + lua_sethook(lua, luaMaskCountHook, LUA_MASKLINE, 0); + + lua_pushstring(lua,"Script killed by user with SCRIPT KILL..."); + lua_error(lua); + } +} From e0cd580aefe13e49df802fec5135e4f22d46e758 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Tue, 5 Oct 2021 17:03:12 +0300 Subject: [PATCH 2/5] Redis Functions - Move Lua related variable into luaCtx struct The following variable was renamed: 1. lua_caller -> script_caller 2. lua_time_limit -> script_time_limit 3. lua_timedout -> script_timedout 4. lua_oom -> script_oom 5. lua_disable_deny_script -> script_disable_deny_script 6. in_eval -> in_script The following variables was moved to lctx under eval.c 1. lua 2. lua_client 3. lua_cur_script 4. lua_scripts 5. lua_scripts_mem 6. lua_replicate_commands 7. lua_write_dirty 8. lua_random_dirty 9. lua_multi_emitted 10. lua_repl 11. lua_kill 12. lua_time_start 13. lua_time_snapshot This commit is in a low risk of introducing any issues and it is just moving varibales around and not changing any logic. --- src/acl.c | 2 +- src/config.c | 2 +- src/db.c | 4 +- src/debug.c | 2 +- src/defrag.c | 2 +- src/eval.c | 171 +++++++++++++++++++++++++++-------------------- src/evict.c | 2 +- src/module.c | 8 +-- src/networking.c | 2 +- src/object.c | 6 +- src/rdb.c | 6 +- src/script_lua.c | 82 +++++++++++------------ src/script_lua.h | 65 ++++++++++++++++++ src/server.c | 30 ++++----- src/server.h | 39 +++++------ 15 files changed, 253 insertions(+), 170 deletions(-) create mode 100644 src/script_lua.h diff --git a/src/acl.c b/src/acl.c index 5d1598b39..9c23cffa8 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1897,7 +1897,7 @@ void addACLLogEntry(client *c, int reason, int context, int argpos, sds username } client *realclient = c; - if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; + if (realclient->flags & CLIENT_LUA) realclient = server.script_caller; le->cinfo = catClientInfoString(sdsempty(),realclient); le->context = context; diff --git a/src/config.c b/src/config.c index 2b2ff737d..939a5342c 100644 --- a/src/config.c +++ b/src/config.c @@ -2650,7 +2650,7 @@ standardConfig configs[] = { createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), /* Long Long configs */ - createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ + createLongLongConfig("script-time-limit", "lua-time-limit", MODIFIABLE_CONFIG, 0, LONG_MAX, server.script_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), diff --git a/src/db.c b/src/db.c index b98310cf3..749f5535b 100644 --- a/src/db.c +++ b/src/db.c @@ -1501,8 +1501,8 @@ int keyIsExpired(redisDb *db, robj *key) { * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. * See issue #1525 on Github for more information. */ - if (server.lua_caller) { - now = server.lua_time_snapshot; + if (server.script_caller) { + now = evalTimeSnapshot(); } /* If we are in the middle of a command execution, we still want to use * a reference time that does not change: in that case we just use the diff --git a/src/debug.c b/src/debug.c index 8776de38f..d4f3f5dd2 100644 --- a/src/debug.c +++ b/src/debug.c @@ -919,7 +919,7 @@ NULL addReplyStatus(c,"Apparently Redis did not crash: test passed"); } else if (!strcasecmp(c->argv[1]->ptr,"set-disable-deny-scripts") && c->argc == 3) { - server.lua_disable_deny_script = atoi(c->argv[2]->ptr);; + server.script_disable_deny_script = atoi(c->argv[2]->ptr);; addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"config-rewrite-force-all") && c->argc == 2) { diff --git a/src/defrag.c b/src/defrag.c index 734174c3a..ffd6eaba6 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -939,7 +939,7 @@ long defragOtherGlobals() { /* there are many more pointers to defrag (e.g. client argv, output / aof buffers, etc. * but we assume most of these are short lived, we only need to defrag allocations * that remain static for a long time */ - defragged += activeDefragSdsDict(server.lua_scripts, DEFRAG_SDS_DICT_VAL_IS_STROB); + defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_IS_STROB); defragged += activeDefragSdsListAndDict(server.repl_scriptcache_fifo, server.repl_scriptcache_dict, DEFRAG_SDS_DICT_NO_VAL); defragged += moduleDefragGlobals(); return defragged; diff --git a/src/eval.c b/src/eval.c index 6e6aeac7c..5c4c419b5 100644 --- a/src/eval.c +++ b/src/eval.c @@ -50,6 +50,9 @@ void ldbLog(sds entry); void ldbLogRedisReply(char *reply); sds ldbCatStackValue(sds s, lua_State *lua, int idx); +/* Lua context */ +luaCtx lctx; + /* Debugger shared state is stored inside this global structure. */ #define LDB_BREAKPOINTS_MAX 64 /* Max number of breakpoints. */ #define LDB_MAX_LEN_DEFAULT 256 /* Default len limit for replies / var dumps. */ @@ -138,10 +141,10 @@ int luaRedisDebugCommand(lua_State *lua) { * already started to write, returns false and stick to whole scripts * replication, which is our default. */ int luaRedisReplicateCommandsCommand(lua_State *lua) { - if (server.lua_write_dirty) { + if (lctx.lua_write_dirty) { lua_pushboolean(lua,0); } else { - server.lua_replicate_commands = 1; + lctx.lua_replicate_commands = 1; /* When we switch to single commands replication, we can provide * different math.random() sequences at every call, which is what * the user normally expects. */ @@ -165,19 +168,19 @@ void scriptingInit(int setup) { lua_State *lua = lua_open(); if (setup) { - server.lua_client = NULL; - server.lua_caller = NULL; - server.lua_cur_script = NULL; - server.lua_timedout = 0; - server.lua_disable_deny_script = 0; + lctx.lua_client = NULL; + server.script_caller = NULL; + lctx.lua_cur_script = NULL; + server.script_timedout = 0; + server.script_disable_deny_script = 0; ldbInit(); } /* Initialize a dictionary we use to map SHAs to scripts. * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ - server.lua_scripts = dictCreate(&shaScriptObjectDictType); - server.lua_scripts_mem = 0; + lctx.lua_scripts = dictCreate(&shaScriptObjectDictType); + lctx.lua_scripts_mem = 0; luaEngineRegisterRedisAPI(lua); @@ -238,12 +241,12 @@ void scriptingInit(int setup) { * inside the Lua interpreter. * Note: there is no need to create it again when this function is called * by scriptingReset(). */ - if (server.lua_client == NULL) { - server.lua_client = createClient(NULL); - server.lua_client->flags |= CLIENT_LUA; + if (lctx.lua_client == NULL) { + lctx.lua_client = createClient(NULL); + lctx.lua_client->flags |= CLIENT_LUA; /* We do not want to allow blocking commands inside Lua */ - server.lua_client->flags |= CLIENT_DENY_BLOCKING; + lctx.lua_client->flags |= CLIENT_DENY_BLOCKING; } /* Lua beginners often don't use "local", this is likely to introduce @@ -251,18 +254,18 @@ void scriptingInit(int setup) { * to global variables. */ scriptingEnableGlobalsProtection(lua); - server.lua = lua; + lctx.lua = lua; } /* Release resources related to Lua scripting. * This function is used in order to reset the scripting environment. */ void scriptingRelease(int async) { if (async) - freeLuaScriptsAsync(server.lua_scripts); + freeLuaScriptsAsync(lctx.lua_scripts); else - dictRelease(server.lua_scripts); - server.lua_scripts_mem = 0; - lua_close(server.lua); + dictRelease(lctx.lua_scripts); + lctx.lua_scripts_mem = 0; + lua_close(lctx.lua); } void scriptingReset(int async) { @@ -291,7 +294,7 @@ void scriptingReset(int async) { * * If 'c' is not NULL, on error the client is informed with an appropriate * error describing the nature of the problem and the Lua interpreter error. */ -sds luaCreateFunction(client *c, lua_State *lua, robj *body) { +sds luaCreateFunction(client *c, robj *body) { char funcname[43]; dictEntry *de; @@ -300,7 +303,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { sha1hex(funcname+2,body->ptr,sdslen(body->ptr)); sds sha = sdsnewlen(funcname+2,40); - if ((de = dictFind(server.lua_scripts,sha)) != NULL) { + if ((de = dictFind(lctx.lua_scripts,sha)) != NULL) { sdsfree(sha); return dictGetKey(de); } @@ -312,25 +315,25 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { funcdef = sdscatlen(funcdef,body->ptr,sdslen(body->ptr)); funcdef = sdscatlen(funcdef,"\nend",4); - if (luaL_loadbuffer(lua,funcdef,sdslen(funcdef),"@user_script")) { + if (luaL_loadbuffer(lctx.lua,funcdef,sdslen(funcdef),"@user_script")) { if (c != NULL) { addReplyErrorFormat(c, "Error compiling script (new function): %s\n", - lua_tostring(lua,-1)); + lua_tostring(lctx.lua,-1)); } - lua_pop(lua,1); + lua_pop(lctx.lua,1); sdsfree(sha); sdsfree(funcdef); return NULL; } sdsfree(funcdef); - if (lua_pcall(lua,0,0,0)) { + if (lua_pcall(lctx.lua,0,0,0)) { if (c != NULL) { addReplyErrorFormat(c,"Error running script (new function): %s\n", - lua_tostring(lua,-1)); + lua_tostring(lctx.lua,-1)); } - lua_pop(lua,1); + lua_pop(lctx.lua,1); sdsfree(sha); return NULL; } @@ -338,31 +341,31 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { /* We also save a SHA1 -> Original script map in a dictionary * so that we can replicate / write in the AOF all the * EVALSHA commands as EVAL using the original script. */ - int retval = dictAdd(server.lua_scripts,sha,body); - serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); - server.lua_scripts_mem += sdsZmallocSize(sha) + getStringObjectSdsUsedMemory(body); + int retval = dictAdd(lctx.lua_scripts,sha,body); + serverAssertWithInfo(c ? c : lctx.lua_client,NULL,retval == DICT_OK); + lctx.lua_scripts_mem += sdsZmallocSize(sha) + getStringObjectSdsUsedMemory(body); incrRefCount(body); return sha; } void prepareLuaClient(void) { /* Select the right DB in the context of the Lua client */ - selectDb(server.lua_client,server.lua_caller->db->id); - server.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */ + selectDb(lctx.lua_client,server.script_caller->db->id); + lctx.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */ /* If we are in MULTI context, flag Lua client as CLIENT_MULTI. */ - if (server.lua_caller->flags & CLIENT_MULTI) { - server.lua_client->flags |= CLIENT_MULTI; + if (server.script_caller->flags & CLIENT_MULTI) { + lctx.lua_client->flags |= CLIENT_MULTI; } } void resetLuaClient(void) { /* After the script done, remove the MULTI state. */ - server.lua_client->flags &= ~CLIENT_MULTI; + lctx.lua_client->flags &= ~CLIENT_MULTI; } void evalGenericCommand(client *c, int evalsha) { - lua_State *lua = server.lua; + lua_State *lua = lctx.lua; char funcname[43]; long long numkeys; long long initial_server_dirty = server.dirty; @@ -380,11 +383,11 @@ void evalGenericCommand(client *c, int evalsha) { * * Thanks to this flag we'll raise an error every time a write command * is called after a random command was used. */ - server.lua_random_dirty = 0; - server.lua_write_dirty = 0; - server.lua_replicate_commands = server.lua_always_replicate_commands; - server.lua_multi_emitted = 0; - server.lua_repl = PROPAGATE_AOF|PROPAGATE_REPL; + lctx.lua_random_dirty = 0; + lctx.lua_write_dirty = 0; + lctx.lua_replicate_commands = server.lua_always_replicate_commands; + lctx.lua_multi_emitted = 0; + lctx.lua_repl = PROPAGATE_AOF|PROPAGATE_REPL; /* Get the number of arguments that are keys */ if (getLongLongFromObjectOrReply(c,c->argv[2],&numkeys,NULL) != C_OK) @@ -433,7 +436,7 @@ void evalGenericCommand(client *c, int evalsha) { addReplyErrorObject(c, shared.noscripterr); return; } - if (luaCreateFunction(c,lua,c->argv[1]) == NULL) { + if (luaCreateFunction(c,c->argv[1]) == NULL) { lua_pop(lua,1); /* remove the error handler from the stack. */ /* The error is sent to the client by luaCreateFunction() * itself when it returns NULL. */ @@ -456,17 +459,17 @@ void evalGenericCommand(client *c, int evalsha) { * * If we are debugging, we set instead a "line" hook so that the * debugger is call-back at every line executed by the script. */ - server.in_eval = 1; - server.lua_caller = c; - server.lua_cur_script = funcname + 2; - server.lua_time_start = getMonotonicUs(); - server.lua_time_snapshot = mstime(); - server.lua_kill = 0; - if (server.lua_time_limit > 0 && ldb.active == 0) { + server.in_script = 1; + server.script_caller = c; + lctx.lua_cur_script = funcname + 2; + lctx.lua_time_start = getMonotonicUs(); + lctx.lua_time_snapshot = mstime(); + lctx.lua_kill = 0; + if (server.script_time_limit > 0 && ldb.active == 0) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); delhook = 1; } else if (ldb.active) { - lua_sethook(server.lua,luaLdbLineHook,LUA_MASKLINE|LUA_MASKCOUNT,100000); + lua_sethook(lctx.lua,luaLdbLineHook,LUA_MASKLINE|LUA_MASKCOUNT,100000); delhook = 1; } @@ -481,8 +484,8 @@ void evalGenericCommand(client *c, int evalsha) { /* Perform some cleanup that we need to do both on error and success. */ if (delhook) lua_sethook(lua,NULL,0,0); /* Disable hook */ - if (server.lua_timedout) { - server.lua_timedout = 0; + if (server.script_timedout) { + server.script_timedout = 0; blockingOperationEnds(); /* Restore the client that was protected when the script timeout * was detected. */ @@ -490,9 +493,9 @@ void evalGenericCommand(client *c, int evalsha) { if (server.masterhost && server.master) queueClientForReprocessing(server.master); } - server.in_eval = 0; - server.lua_caller = NULL; - server.lua_cur_script = NULL; + server.in_script = 0; + server.script_caller = NULL; + lctx.lua_cur_script = NULL; /* Call the Lua garbage collector from time to time to avoid a * full cycle performed by Lua, which adds too latency. @@ -524,9 +527,9 @@ void evalGenericCommand(client *c, int evalsha) { /* If we are using single commands replication, emit EXEC if there * was at least a write. */ - if (server.lua_replicate_commands) { + if (lctx.lua_replicate_commands) { preventCommandPropagation(c); - if (server.lua_multi_emitted) { + if (lctx.lua_multi_emitted) { execCommandPropagateExec(c->db->id); } } @@ -541,12 +544,12 @@ void evalGenericCommand(client *c, int evalsha) { * For replication, every time a new slave attaches to the master, we need to * flush our cache of scripts that can be replicated as EVALSHA, while * for AOF we need to do so every time we rewrite the AOF file. */ - if (evalsha && !server.lua_replicate_commands) { + if (evalsha && !lctx.lua_replicate_commands) { if (!replicationScriptCacheExists(c->argv[1]->ptr)) { /* This script is not in our script cache, replicate it as * EVAL, then add it into the script cache, as from now on * slaves and AOF know about it. */ - robj *script = dictFetchValue(server.lua_scripts,c->argv[1]->ptr); + robj *script = dictFetchValue(lctx.lua_scripts,c->argv[1]->ptr); replicationScriptCacheAdd(c->argv[1]->ptr); serverAssertWithInfo(c,NULL,script != NULL); @@ -648,25 +651,25 @@ NULL addReplyArrayLen(c, c->argc-2); for (j = 2; j < c->argc; j++) { - if (dictFind(server.lua_scripts,c->argv[j]->ptr)) + if (dictFind(lctx.lua_scripts,c->argv[j]->ptr)) addReply(c,shared.cone); else addReply(c,shared.czero); } } else if (c->argc == 3 && !strcasecmp(c->argv[1]->ptr,"load")) { - sds sha = luaCreateFunction(c,server.lua,c->argv[2]); + sds sha = luaCreateFunction(c,c->argv[2]); if (sha == NULL) return; /* The error was sent by luaCreateFunction(). */ addReplyBulkCBuffer(c,sha,40); forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF); } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { - if (server.lua_caller == NULL) { + if (server.script_caller == NULL) { addReplyError(c,"-NOTBUSY No scripts in execution right now."); - } else if (server.lua_caller->flags & CLIENT_MASTER) { + } else if (server.script_caller->flags & CLIENT_MASTER) { addReplyError(c,"-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed."); - } else if (server.lua_write_dirty) { + } else if (lctx.lua_write_dirty) { addReplyError(c,"-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command."); } else { - server.lua_kill = 1; + lctx.lua_kill = 1; addReply(c,shared.ok); } } else if (c->argc == 3 && !strcasecmp(c->argv[1]->ptr,"debug")) { @@ -693,6 +696,26 @@ NULL } } +unsigned long evalMemory() { + return lua_gc(lctx.lua, LUA_GCCOUNT, 0) * 1024LL; +} + +dict* evalScriptsDict() { + return lctx.lua_scripts; +} + +unsigned long evalScriptsMemory() { + return lctx.lua_scripts_mem + + dictSize(lctx.lua_scripts) * sizeof(dictEntry) + + dictSlots(lctx.lua_scripts) * sizeof(dictEntry*); +} + +/* Returns the time when the script invocation started */ +mstime_t evalTimeSnapshot() { + return lctx.lua_time_snapshot; +} + + /* --------------------------------------------------------------------------- * LDB: Redis Lua debugging facilities * ------------------------------------------------------------------------- */ @@ -717,6 +740,10 @@ void ldbFlushLog(list *log) { listDelNode(log,ln); } +int ldbIsEnabled(){ + return ldb.active && ldb.step; +} + /* Enable debug mode of Lua scripts for this client. */ void ldbEnable(client *c) { c->flags |= CLIENT_LUA_DEBUG; @@ -1447,7 +1474,7 @@ void ldbEval(lua_State *lua, sds *argv, int argc) { * implementation, with ldb.step enabled, so as a side effect the Redis command * and its reply are logged. */ void ldbRedis(lua_State *lua, sds *argv, int argc) { - int j, saved_rc = server.lua_replicate_commands; + int j, saved_rc = lctx.lua_replicate_commands; if (!lua_checkstack(lua, argc + 1)) { /* Increase the Lua stack if needed to make sure there is enough room @@ -1466,10 +1493,10 @@ void ldbRedis(lua_State *lua, sds *argv, int argc) { for (j = 1; j < argc; j++) lua_pushlstring(lua,argv[j],sdslen(argv[j])); ldb.step = 1; /* Force redis.call() to log. */ - server.lua_replicate_commands = 1; + lctx.lua_replicate_commands = 1; lua_pcall(lua,argc-1,1,0); /* Stack: redis, result */ ldb.step = 0; /* Disable logging. */ - server.lua_replicate_commands = saved_rc; + lctx.lua_replicate_commands = saved_rc; lua_pop(lua,2); /* Discard the result and clean the stack. */ } @@ -1657,9 +1684,9 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { /* Check if a timeout occurred. */ if (ar->event == LUA_HOOKCOUNT && ldb.step == 0 && bp == 0) { - mstime_t elapsed = elapsedMs(server.lua_time_start); - mstime_t timelimit = server.lua_time_limit ? - server.lua_time_limit : 5000; + mstime_t elapsed = elapsedMs(server.script_time_limit); + mstime_t timelimit = server.script_time_limit ? + server.script_time_limit : 5000; if (elapsed >= timelimit) { timeout = 1; ldb.step = 1; @@ -1687,7 +1714,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { lua_pushstring(lua, "timeout during Lua debugging with client closing connection"); lua_error(lua); } - server.lua_time_start = getMonotonicUs(); - server.lua_time_snapshot = mstime(); + lctx.lua_time_start = getMonotonicUs(); + lctx.lua_time_snapshot = mstime(); } } diff --git a/src/evict.c b/src/evict.c index 4186378a2..2d87546cb 100644 --- a/src/evict.c +++ b/src/evict.c @@ -472,7 +472,7 @@ static int evictionTimeProc( static int isSafeToPerformEvictions(void) { /* - There must be no script in timeout condition. * - Nor we are loading data right now. */ - if (server.lua_timedout || server.loading) return 0; + if (server.script_timedout || server.loading) return 0; /* By default replicas should ignore maxmemory * and just be masters exact copies. */ diff --git a/src/module.c b/src/module.c index 7c1cc054b..bc7599b75 100644 --- a/src/module.c +++ b/src/module.c @@ -629,7 +629,7 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { /* If this command is executed from with Lua or MULTI/EXEC we do not * need to propagate EXEC */ - if (server.in_eval || server.in_exec) return; + if (server.in_script || server.in_exec) return; /* Handle the replication of the final EXEC, since whatever a command * emits is always wrapped around MULTI/EXEC. */ @@ -2333,7 +2333,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* Skip this if client explicitly wrap the command with MULTI, or if * the module command was called by a script. */ - if (server.in_eval || server.in_exec) return; + if (server.in_script || server.in_exec) return; /* If we already emitted MULTI return ASAP. */ if (server.propagate_in_transaction) return; /* If this is a thread safe context, we do not want to wrap commands @@ -2709,7 +2709,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { } } - if (server.in_eval) + if (server.in_script) flags |= REDISMODULE_CTX_FLAGS_LUA; if (server.in_exec) @@ -6215,7 +6215,7 @@ void unblockClientFromModule(client *c) { */ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { client *c = ctx->client; - int islua = server.in_eval; + int islua = server.in_script; int ismulti = server.in_exec; c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); diff --git a/src/networking.c b/src/networking.c index 1f1a2e169..fb4a487a3 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2199,7 +2199,7 @@ int processInputBuffer(client *c) { * condition on the slave. We want just to accumulate the replication * stream (instead of replying -BUSY like we do with other clients) and * later resume the processing. */ - if (server.lua_timedout && c->flags & CLIENT_MASTER) break; + if (server.script_timedout && c->flags & CLIENT_MASTER) break; /* CLIENT_CLOSE_AFTER_REPLY closes the connection once the reply is * written to the client. Make sure to not let the reply grow after diff --git a/src/object.c b/src/object.c index 5831f196d..dab0648a8 100644 --- a/src/object.c +++ b/src/object.c @@ -1203,9 +1203,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total+=mem; - mem = server.lua_scripts_mem; - mem += dictSize(server.lua_scripts) * sizeof(dictEntry) + - dictSlots(server.lua_scripts) * sizeof(dictEntry*); + mem = evalScriptsMemory(); mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) + dictSlots(server.repl_scriptcache_dict) * sizeof(dictEntry*); if (listLength(server.repl_scriptcache_fifo) > 0) { @@ -1325,7 +1323,7 @@ sds getMemoryDoctorReport(void) { } /* Too many scripts are cached? */ - if (dictSize(server.lua_scripts) > 1000) { + if (dictSize(evalScriptsDict()) > 1000) { many_scripts = 1; num_reports++; } diff --git a/src/rdb.c b/src/rdb.c index 4a65f2cc9..dcbc83785 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1303,8 +1303,8 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { * the script cache as well: on successful PSYNC after a restart, we need * to be able to process any EVALSHA inside the replication backlog the * master will send us. */ - if (rsi && dictSize(server.lua_scripts)) { - di = dictGetIterator(server.lua_scripts); + if (rsi && dictSize(evalScriptsDict())) { + di = dictGetIterator(evalScriptsDict()); while((de = dictNext(di)) != NULL) { robj *body = dictGetVal(de); if (rdbSaveAuxField(rdb,"lua",3,body->ptr,sdslen(body->ptr)) == -1) @@ -2808,7 +2808,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi, redisDb *dbarray) { if (rsi) rsi->repl_offset = strtoll(auxval->ptr,NULL,10); } else if (!strcasecmp(auxkey->ptr,"lua")) { /* Load the script back in memory. */ - if (luaCreateFunction(NULL,server.lua,auxval) == NULL) { + if (luaCreateFunction(NULL, auxval) == NULL) { rdbReportCorruptRDB( "Can't load Lua script from RDB file! " "BODY: %s", (char*)auxval->ptr); diff --git a/src/script_lua.c b/src/script_lua.c index 0685a4bb3..bee5d6c93 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -399,7 +399,7 @@ void luaPushError(lua_State *lua, char *error) { /* If debugging is active and in step mode, log errors resulting from * Redis commands. */ - if (ldb.active && ldb.step) { + if (ldbIsEnabled()) { ldbLog(sdscatprintf(sdsempty()," %s",error)); } @@ -483,7 +483,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); break; case LUA_TBOOLEAN: - if (server.lua_client->resp == 2) + if (lctx.lua_client->resp == 2) addReply(c,lua_toboolean(lua,-1) ? shared.cone : shared.null[c->resp]); else @@ -653,7 +653,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { int luaRedisGenericCommand(lua_State *lua, int raise_error) { int j, argc = lua_gettop(lua); struct redisCommand *cmd; - client *c = server.lua_client; + client *c = lctx.lua_client; sds reply; /* Cached across calls. */ @@ -744,7 +744,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { /* Setup our fake client for command execution */ c->argv = argv; c->argc = argc; - c->user = server.lua_caller->user; + c->user = server.script_caller->user; /* Process module hooks */ moduleCallCommandFilters(c); @@ -752,7 +752,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { argc = c->argc; /* Log the command if debugging is active. */ - if (ldb.active && ldb.step) { + if (ldbIsEnabled()) { sds cmdlog = sdsnew(""); for (j = 0; j < c->argc; j++) { if (j == 10) { @@ -782,14 +782,14 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { c->cmd = c->lastcmd = cmd; /* There are commands that are not allowed inside scripts. */ - if (!server.lua_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { + if (!server.script_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { luaPushError(lua, "This Redis command is not allowed from scripts"); goto cleanup; } /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ - if ((server.lua_caller->cmd->proc == evalRoCommand || - server.lua_caller->cmd->proc == evalShaRoCommand) && + if ((server.script_caller->cmd->proc == evalRoCommand || + server.script_caller->cmd->proc == evalShaRoCommand) && (cmd->flags & CMD_WRITE)) { luaPushError(lua, "Write commands are not allowed from read-only scripts"); @@ -828,13 +828,13 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * of this script. */ if (cmd->flags & CMD_WRITE) { int deny_write_type = writeCommandsDeniedByDiskError(); - if (server.lua_random_dirty && !server.lua_replicate_commands) { + if (lctx.lua_random_dirty && !lctx.lua_replicate_commands) { luaPushError(lua, "Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); goto cleanup; } else if (server.masterhost && server.repl_slave_ro && - server.lua_caller->id != CLIENT_ID_AOF && - !(server.lua_caller->flags & CLIENT_MASTER)) + server.script_caller->id != CLIENT_ID_AOF && + !(server.script_caller->flags & CLIENT_MASTER)) { luaPushError(lua, shared.roslaveerr->ptr); goto cleanup; @@ -857,29 +857,29 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * first write in the context of this script, otherwise we can't stop * in the middle. */ if (server.maxmemory && /* Maxmemory is actually enabled. */ - server.lua_caller->id != CLIENT_ID_AOF && /* Don't care about mem if loading from AOF. */ + server.script_caller->id != CLIENT_ID_AOF && /* Don't care about mem if loading from AOF. */ !server.masterhost && /* Slave must execute the script. */ - server.lua_write_dirty == 0 && /* Script had no side effects so far. */ - server.lua_oom && /* Detected OOM when script start. */ + lctx.lua_write_dirty == 0 && /* Script had no side effects so far. */ + server.script_oom && /* Detected OOM when script start. */ (cmd->flags & CMD_DENYOOM)) { luaPushError(lua, shared.oomerr->ptr); goto cleanup; } - if (cmd->flags & CMD_RANDOM) server.lua_random_dirty = 1; - if (cmd->flags & CMD_WRITE) server.lua_write_dirty = 1; + if (cmd->flags & CMD_RANDOM) lctx.lua_random_dirty = 1; + if (cmd->flags & CMD_WRITE) lctx.lua_write_dirty = 1; /* If this is a Redis Cluster node, we need to make sure Lua is not * trying to access non-local keys, with the exception of commands * received from our master or when loading the AOF back in memory. */ - if (server.cluster_enabled && server.lua_caller->id != CLIENT_ID_AOF && - !(server.lua_caller->flags & CLIENT_MASTER)) + if (server.cluster_enabled && server.script_caller->id != CLIENT_ID_AOF && + !(server.script_caller->flags & CLIENT_MASTER)) { int error_code; /* Duplicate relevant flags in the lua client. */ c->flags &= ~(CLIENT_READONLY|CLIENT_ASKING); - c->flags |= server.lua_caller->flags & (CLIENT_READONLY|CLIENT_ASKING); + c->flags |= server.script_caller->flags & (CLIENT_READONLY|CLIENT_ASKING); if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != server.cluster->myself) { @@ -904,14 +904,14 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { /* If we are using single commands replication, we need to wrap what * we propagate into a MULTI/EXEC block, so that it will be atomic like * a Lua script in the context of AOF and slaves. */ - if (server.lua_replicate_commands && - !server.lua_multi_emitted && - !(server.lua_caller->flags & CLIENT_MULTI) && - server.lua_write_dirty && - server.lua_repl != PROPAGATE_NONE) + if (lctx.lua_replicate_commands && + !lctx.lua_multi_emitted && + !(server.script_caller->flags & CLIENT_MULTI) && + lctx.lua_write_dirty && + lctx.lua_repl != PROPAGATE_NONE) { - execCommandPropagateMulti(server.lua_caller->db->id); - server.lua_multi_emitted = 1; + execCommandPropagateMulti(server.script_caller->db->id); + lctx.lua_multi_emitted = 1; /* Now we are in the MULTI context, the lua_client should be * flag as CLIENT_MULTI. */ c->flags |= CLIENT_MULTI; @@ -919,11 +919,11 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { /* Run the command */ int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; - if (server.lua_replicate_commands) { + if (lctx.lua_replicate_commands) { /* Set flags according to redis.set_repl() settings. */ - if (server.lua_repl & PROPAGATE_AOF) + if (lctx.lua_repl & PROPAGATE_AOF) call_flags |= CMD_CALL_PROPAGATE_AOF; - if (server.lua_repl & PROPAGATE_REPL) + if (lctx.lua_repl & PROPAGATE_REPL) call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); @@ -953,13 +953,13 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { redisProtocolToLuaType(lua,reply); /* If the debugger is active, log the reply from Redis. */ - if (ldb.active && ldb.step) + if (ldbIsEnabled()) ldbLogRedisReply(reply); /* Sort the output array if needed, assuming it is a non-null multi bulk * reply as expected. */ if ((cmd->flags & CMD_SORT_FOR_SCRIPT) && - (server.lua_replicate_commands == 0) && + (lctx.lua_replicate_commands == 0) && (reply[0] == '*' && reply[1] != '-')) { luaSortArray(lua); } @@ -1076,7 +1076,7 @@ int luaRedisSetReplCommand(lua_State *lua) { int argc = lua_gettop(lua); int flags; - if (server.lua_replicate_commands == 0) { + if (lctx.lua_replicate_commands == 0) { lua_pushstring(lua, "You can set the replication behavior only after turning on single commands replication with redis.replicate_commands()."); return lua_error(lua); } else if (argc != 1) { @@ -1089,7 +1089,7 @@ int luaRedisSetReplCommand(lua_State *lua) { lua_pushstring(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); return lua_error(lua); } - server.lua_repl = flags; + lctx.lua_repl = flags; return 0; } @@ -1145,7 +1145,7 @@ int luaSetResp(lua_State *lua) { return lua_error(lua); } - server.lua_client->resp = resp; + lctx.lua_client->resp = resp; return 0; } @@ -1385,29 +1385,29 @@ int redis_math_randomseed (lua_State *L) { /* This is the Lua script "count" hook that we use to detect scripts timeout. */ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { - long long elapsed = elapsedMs(server.lua_time_start); + long long elapsed = elapsedMs(lctx.lua_time_start); UNUSED(ar); UNUSED(lua); /* Set the timeout condition if not already set and the maximum * execution time was reached. */ - if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { + if (elapsed >= server.script_time_limit && server.script_timedout == 0) { serverLog(LL_WARNING, "Lua slow script detected: still in execution after %lld milliseconds. " "You can try killing the script using the SCRIPT KILL command. " "Script SHA1 is: %s", - elapsed, server.lua_cur_script); - server.lua_timedout = 1; + elapsed, lctx.lua_cur_script); + server.script_timedout = 1; blockingOperationStarts(); /* Once the script timeouts we reenter the event loop to permit others * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason * we need to mask the client executing the script from the event loop. * If we don't do that the client may disconnect and could no longer be * here when the EVAL command will return. */ - protectClient(server.lua_caller); + protectClient(server.script_caller); } - if (server.lua_timedout) processEventsWhileBlocked(); - if (server.lua_kill) { + if (server.script_timedout) processEventsWhileBlocked(); + if (lctx.lua_kill) { serverLog(LL_WARNING,"Lua script killed by user with SCRIPT KILL."); /* diff --git a/src/script_lua.h b/src/script_lua.h new file mode 100644 index 000000000..f9cf6f18a --- /dev/null +++ b/src/script_lua.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2009-2021, Redis Labs Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SCRIPT_LUA_H_ +#define __SCRIPT_LUA_H_ + +#include "server.h" +#include "script.h" +#include +#include +#include + +typedef struct luaCtx { + lua_State *lua; /* The Lua interpreter. We use just one for all clients */ + client *lua_client; /* The "fake client" to query Redis from Lua */ + char *lua_cur_script; /* SHA1 of the script currently running, or NULL */ + dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ + int lua_replicate_commands; /* True if we are doing single commands repl. */ + int lua_write_dirty; + int lua_random_dirty; + int lua_multi_emitted; + int lua_repl; + int lua_kill; + monotime lua_time_start; /* monotonic timer to detect timed-out script */ + mstime_t lua_time_snapshot; /* Snapshot of mstime when script is started */ +} luaCtx; + +extern luaCtx lctx; + +void luaEngineRegisterRedisAPI(lua_State* lua); +void scriptingEnableGlobalsProtection(lua_State *lua); +void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec); +void luaMaskCountHook(lua_State *lua, lua_Debug *ar); +void luaReplyToRedisReply(client *c, lua_State *lua); + + + +#endif /* __SCRIPT_LUA_H_ */ diff --git a/src/server.c b/src/server.c index 05ab6b93a..8d682c207 100644 --- a/src/server.c +++ b/src/server.c @@ -2994,7 +2994,7 @@ void cronUpdateMemoryStats() { /* LUA memory isn't part of zmalloc_used, but it is part of the process RSS, * so we must deduct it in order to be able to calculate correct * "allocator fragmentation" ratio */ - size_t lua_memory = lua_gc(server.lua,LUA_GCCOUNT,0)*1024LL; + size_t lua_memory = evalMemory(); server.cron_malloc_stats.allocator_resident = server.cron_malloc_stats.process_rss - lua_memory; } if (!server.cron_malloc_stats.allocator_active) @@ -4247,7 +4247,7 @@ void initServer(void) { server.pubsub_channels = dictCreate(&keylistDictType); server.pubsub_patterns = dictCreate(&keylistDictType); server.cronloops = 0; - server.in_eval = 0; + server.in_script = 0; server.in_exec = 0; server.propagate_in_transaction = 0; server.client_pause_in_transaction = 0; @@ -4937,11 +4937,11 @@ void call(client *c, int flags) { /* If the caller is Lua, we want to force the EVAL caller to propagate * the script if the command flag or client flag are forcing the * propagation. */ - if (c->flags & CLIENT_LUA && server.lua_caller) { + if (c->flags & CLIENT_LUA && server.script_caller) { if (c->flags & CLIENT_FORCE_REPL) - server.lua_caller->flags |= CLIENT_FORCE_REPL; + server.script_caller->flags |= CLIENT_FORCE_REPL; if (c->flags & CLIENT_FORCE_AOF) - server.lua_caller->flags |= CLIENT_FORCE_AOF; + server.script_caller->flags |= CLIENT_FORCE_AOF; } /* Note: the code below uses the real command that was executed @@ -5070,8 +5070,8 @@ void call(client *c, int flags) { /* If the client has keys tracking enabled for client side caching, * make sure to remember the keys it fetched via this command. */ if (c->cmd->flags & CMD_READONLY) { - client *caller = (c->flags & CLIENT_LUA && server.lua_caller) ? - server.lua_caller : c; + client *caller = (c->flags & CLIENT_LUA && server.script_caller) ? + server.script_caller : c; if (caller->flags & CLIENT_TRACKING && !(caller->flags & CLIENT_TRACKING_BCAST)) { @@ -5172,14 +5172,14 @@ void populateCommandMovableKeys(struct redisCommand *cmd) { * other operations can be performed by the caller. Otherwise * if C_ERR is returned the client was destroyed (i.e. after QUIT). */ int processCommand(client *c) { - if (!server.lua_timedout) { + if (!server.script_timedout) { /* Both EXEC and EVAL call call() directly so there should be * no way in_exec or in_eval or propagate_in_transaction is 1. * That is unless lua_timedout, in which case client may run * some commands. */ serverAssert(!server.propagate_in_transaction); serverAssert(!server.in_exec); - serverAssert(!server.in_eval); + serverAssert(!server.in_script); } moduleCallCommandFilters(c); @@ -5274,7 +5274,7 @@ int processCommand(client *c) { if (server.cluster_enabled && !(c->flags & CLIENT_MASTER) && !(c->flags & CLIENT_LUA && - server.lua_caller->flags & CLIENT_MASTER) && + server.script_caller->flags & CLIENT_MASTER) && !(!c->cmd->movablekeys && c->cmd->key_specs_num == 0 && c->cmd->proc != execCommand)) { @@ -5309,7 +5309,7 @@ int processCommand(client *c) { * the event loop since there is a busy Lua script running in timeout * condition, to avoid mixing the propagation of scripts with the * propagation of DELs due to eviction. */ - if (server.maxmemory && !server.lua_timedout) { + if (server.maxmemory && !server.script_timedout) { int out_of_memory = (performEvictions() == EVICT_FAIL); /* performEvictions may evict keys, so we need flush pending tracking @@ -5345,7 +5345,7 @@ int processCommand(client *c) { * until first write within script, memory used by lua stack and * arguments might interfere. */ if (c->cmd->proc == evalCommand || c->cmd->proc == evalShaCommand) { - server.lua_oom = out_of_memory; + server.script_oom = out_of_memory; } } @@ -5432,7 +5432,7 @@ int processCommand(client *c) { * the MULTI plus a few initial commands refused, then the timeout * condition resolves, and the bottom-half of the transaction gets * executed, see Github PR #7022. */ - if (server.lua_timedout && + if (server.script_timedout && c->cmd->proc != authCommand && c->cmd->proc != helloCommand && c->cmd->proc != replconfCommand && @@ -6286,7 +6286,7 @@ sds genRedisInfoString(const char *section) { size_t zmalloc_used = zmalloc_used_memory(); size_t total_system_mem = server.system_memory_size; const char *evict_policy = evictPolicyToString(); - long long memory_lua = server.lua ? (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024 : 0; + long long memory_lua = evalMemory(); struct redisMemOverhead *mh = getMemoryOverheadData(); /* Peak memory is updated from time to time by serverCron() so it @@ -6369,7 +6369,7 @@ sds genRedisInfoString(const char *section) { used_memory_lua_hmem, (long long) mh->lua_caches, used_memory_scripts_hmem, - dictSize(server.lua_scripts), + dictSize(evalScriptsDict()), server.maxmemory, maxmemory_hmem, evict_policy, diff --git a/src/server.h b/src/server.h index c06b23631..959b0e2d1 100644 --- a/src/server.h +++ b/src/server.h @@ -1334,7 +1334,7 @@ struct redisServer { int sentinel_mode; /* True if this instance is a Sentinel. */ size_t initial_memory_usage; /* Bytes used after initialization. */ int always_show_logo; /* Show logo even for non-stdout logging. */ - int in_eval; /* Are we inside EVAL? */ + int in_script; /* Are we inside EVAL? */ int in_exec; /* Are we inside EXEC? */ int propagate_in_transaction; /* Make sure we don't propagate nested MULTI/EXEC */ char *ignore_warnings; /* Config: warnings that should be ignored. */ @@ -1719,28 +1719,13 @@ struct redisServer { is down? */ int cluster_config_file_lock_fd; /* cluster config fd, will be flock */ /* Scripting */ - lua_State *lua; /* The Lua interpreter. We use just one for all clients */ - client *lua_client; /* The "fake client" to query Redis from Lua */ - client *lua_caller; /* The client running EVAL right now, or NULL */ - char* lua_cur_script; /* SHA1 of the script currently running, or NULL */ - dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ - unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ - mstime_t lua_time_limit; /* Script timeout in milliseconds */ - monotime lua_time_start; /* monotonic timer to detect timed-out script */ - mstime_t lua_time_snapshot; /* Snapshot of mstime when script is started */ - int lua_write_dirty; /* True if a write command was called during the - execution of the current script. */ - int lua_random_dirty; /* True if a random command was called during the - execution of the current script. */ - int lua_replicate_commands; /* True if we are doing single commands repl. */ - int lua_multi_emitted;/* True if we already propagated MULTI. */ - int lua_repl; /* Script replication flags for redis.set_repl(). */ - int lua_timedout; /* True if we reached the time limit for script - execution. */ - int lua_kill; /* Kill the script if true. */ + client *script_caller; /* The client running script right now, or NULL */ + mstime_t script_time_limit; /* Script timeout in milliseconds */ + int script_timedout; /* True if we reached the time limit for script + execution. */ int lua_always_replicate_commands; /* Default replication type. */ - int lua_oom; /* OOM detected when script start? */ - int lua_disable_deny_script; /* Allow running commands marked "no-script" inside a script. */ + int script_oom; /* OOM detected when script start */ + int script_disable_deny_script; /* Allow running commands marked "no-script" inside a script. */ /* Lazy free */ int lazyfree_lazy_eviction; int lazyfree_lazy_expire; @@ -2722,8 +2707,16 @@ void scriptingInit(int setup); int ldbRemoveChild(pid_t pid); void ldbKillForkedSessions(void); int ldbPendingChildren(void); -sds luaCreateFunction(client *c, lua_State *lua, robj *body); +sds luaCreateFunction(client *c, robj *body); void freeLuaScriptsAsync(dict *lua_scripts); +int ldbIsEnabled(); +void ldbLog(sds entry); +void ldbLogRedisReply(char *reply); +void sha1hex(char *digest, char *script, size_t len); +unsigned long evalMemory(); +dict* evalScriptsDict(); +unsigned long evalScriptsMemory(); +mstime_t evalTimeSnapshot(); /* Blocked clients */ void processUnblockedClients(void); From fc731bc67f8ecd07e83aa138b03a073028f9f3f2 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Tue, 5 Oct 2021 19:37:03 +0300 Subject: [PATCH 3/5] Redis Functions - Introduce script unit. Script unit is a new unit located on script.c. Its purpose is to provides an API for functions (and eval) to interact with Redis. Interaction includes mostly executing commands, but also functionalities like calling Redis back on long scripts or check if the script was killed. The interaction is done using a scriptRunCtx object that need to be created by the user and initialized using scriptPrepareForRun. Detailed list of functionalities expose by the unit: 1. Calling commands (including all the validation checks such as acl, cluster, read only run, ...) 2. Set Resp 3. Set Replication method (AOF/REPLICATION/NONE) 4. Call Redis back to on long running scripts to allow Redis reply to clients and perform script kill The commit introduce the new unit and uses it on eval commands to interact with Redis. --- src/Makefile | 2 +- src/acl.c | 2 +- src/db.c | 3 +- src/eval.c | 108 ++++------ src/evict.c | 3 +- src/networking.c | 7 +- src/replication.c | 2 +- src/script.c | 432 +++++++++++++++++++++++++++++++++++++++ src/script.h | 96 +++++++++ src/script_lua.c | 312 ++++++++-------------------- src/script_lua.h | 46 +++-- src/server.c | 15 +- src/server.h | 4 +- src/sort.c | 2 +- src/t_stream.c | 2 +- tests/unit/scripting.tcl | 2 +- 16 files changed, 698 insertions(+), 340 deletions(-) create mode 100644 src/script.c create mode 100644 src/script.h diff --git a/src/Makefile b/src/Makefile index 469e8eb54..076305b58 100644 --- a/src/Makefile +++ b/src/Makefile @@ -309,7 +309,7 @@ endif REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX) -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX) REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX) diff --git a/src/acl.c b/src/acl.c index 9c23cffa8..b9efd5401 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1897,7 +1897,7 @@ void addACLLogEntry(client *c, int reason, int context, int argpos, sds username } client *realclient = c; - if (realclient->flags & CLIENT_LUA) realclient = server.script_caller; + if (realclient->flags & CLIENT_SCRIPT) realclient = server.script_caller; le->cinfo = catClientInfoString(sdsempty(),realclient); le->context = context; diff --git a/src/db.c b/src/db.c index 749f5535b..807653767 100644 --- a/src/db.c +++ b/src/db.c @@ -31,6 +31,7 @@ #include "cluster.h" #include "atomicvar.h" #include "latency.h" +#include "script.h" #include #include @@ -88,7 +89,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { * commands is to make writable replicas behave consistently. It * shall not be used in readonly commands. Modules are accepted so * that we don't break old modules. */ - client *c = server.in_eval ? server.lua_client : server.current_client; + client *c = server.in_script ? scriptGetClient() : server.current_client; serverAssert(!c || !c->cmd || (c->cmd->flags & (CMD_WRITE|CMD_MODULE))); } if (expireIfNeeded(db, key, force_delete_expired)) { diff --git a/src/eval.c b/src/eval.c index 5c4c419b5..7d5c40a53 100644 --- a/src/eval.c +++ b/src/eval.c @@ -51,7 +51,14 @@ void ldbLogRedisReply(char *reply); sds ldbCatStackValue(sds s, lua_State *lua, int idx); /* Lua context */ -luaCtx lctx; +struct luaCtx { + lua_State *lua; /* The Lua interpreter. We use just one for all clients */ + client *lua_client; /* The "fake client" to query Redis from Lua */ + char *lua_cur_script; /* SHA1 of the script currently running, or NULL */ + dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ + int lua_replicate_commands; /* True if we are doing single commands repl. */ +} lctx; /* Debugger shared state is stored inside this global structure. */ #define LDB_BREAKPOINTS_MAX 64 /* Max number of breakpoints. */ @@ -141,10 +148,12 @@ int luaRedisDebugCommand(lua_State *lua) { * already started to write, returns false and stick to whole scripts * replication, which is our default. */ int luaRedisReplicateCommandsCommand(lua_State *lua) { - if (lctx.lua_write_dirty) { + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + if (rctx->flags & SCRIPT_WRITE_DIRTY) { lua_pushboolean(lua,0); } else { lctx.lua_replicate_commands = 1; + rctx->flags &= ~SCRIPT_EVAL_REPLICATION; /* When we switch to single commands replication, we can provide * different math.random() sequences at every call, which is what * the user normally expects. */ @@ -171,7 +180,6 @@ void scriptingInit(int setup) { lctx.lua_client = NULL; server.script_caller = NULL; lctx.lua_cur_script = NULL; - server.script_timedout = 0; server.script_disable_deny_script = 0; ldbInit(); } @@ -182,7 +190,7 @@ void scriptingInit(int setup) { lctx.lua_scripts = dictCreate(&shaScriptObjectDictType); lctx.lua_scripts_mem = 0; - luaEngineRegisterRedisAPI(lua); + luaRegisterRedisAPI(lua); /* register debug commands */ lua_getglobal(lua,"redis"); @@ -243,7 +251,7 @@ void scriptingInit(int setup) { * by scriptingReset(). */ if (lctx.lua_client == NULL) { lctx.lua_client = createClient(NULL); - lctx.lua_client->flags |= CLIENT_LUA; + lctx.lua_client->flags |= CLIENT_SCRIPT; /* We do not want to allow blocking commands inside Lua */ lctx.lua_client->flags |= CLIENT_DENY_BLOCKING; @@ -252,7 +260,7 @@ void scriptingInit(int setup) { /* Lua beginners often don't use "local", this is likely to introduce * subtle bugs in their code. To prevent problems we protect accesses * to global variables. */ - scriptingEnableGlobalsProtection(lua); + luaEnableGlobalsProtection(lua); lctx.lua = lua; } @@ -375,19 +383,7 @@ void evalGenericCommand(client *c, int evalsha) { * every call so that our PRNG is not affected by external state. */ redisSrand48(0); - /* We set this flag to zero to remember that so far no random command - * was called. This way we can allow the user to call commands like - * SRANDMEMBER or RANDOMKEY from Lua scripts as far as no write command - * is called (otherwise the replication and AOF would end with non - * deterministic sequences). - * - * Thanks to this flag we'll raise an error every time a write command - * is called after a random command was used. */ - lctx.lua_random_dirty = 0; - lctx.lua_write_dirty = 0; lctx.lua_replicate_commands = server.lua_always_replicate_commands; - lctx.lua_multi_emitted = 0; - lctx.lua_repl = PROPAGATE_AOF|PROPAGATE_REPL; /* Get the number of arguments that are keys */ if (getLongLongFromObjectOrReply(c,c->argv[2],&numkeys,NULL) != C_OK) @@ -452,19 +448,24 @@ void evalGenericCommand(client *c, int evalsha) { luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys); luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys); - /* Set a hook in order to be able to stop the script execution if it - * is running for too much time. - * We set the hook only if the time limit is enabled as the hook will - * make the Lua script execution slower. - * - * If we are debugging, we set instead a "line" hook so that the - * debugger is call-back at every line executed by the script. */ - server.in_script = 1; - server.script_caller = c; lctx.lua_cur_script = funcname + 2; - lctx.lua_time_start = getMonotonicUs(); - lctx.lua_time_snapshot = mstime(); - lctx.lua_kill = 0; + + scriptRunCtx rctx; + scriptPrepareForRun(&rctx, lctx.lua_client, c, lctx.lua_cur_script); + + /* We must set it before we set the Lua hook, theoretically the + * Lua hook might be called wheneven we run any Lua instruction + * such as 'luaSetGlobalArray' and we want the rctx to be available + * each time the Lua hook is invoked. */ + luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, &rctx); + + if (!lctx.lua_replicate_commands) rctx.flags |= SCRIPT_EVAL_REPLICATION; + /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ + if ((server.script_caller->cmd->proc == evalRoCommand || + server.script_caller->cmd->proc == evalShaRoCommand)) { + rctx.flags |= SCRIPT_READ_ONLY; + } + if (server.script_time_limit > 0 && ldb.active == 0) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); delhook = 1; @@ -473,29 +474,17 @@ void evalGenericCommand(client *c, int evalsha) { delhook = 1; } - prepareLuaClient(); - /* At this point whether this script was never seen before or if it was * already defined, we can call it. We have zero arguments and expect * a single return value. */ err = lua_pcall(lua,0,1,-2); - resetLuaClient(); + scriptResetRun(&rctx); /* Perform some cleanup that we need to do both on error and success. */ if (delhook) lua_sethook(lua,NULL,0,0); /* Disable hook */ - if (server.script_timedout) { - server.script_timedout = 0; - blockingOperationEnds(); - /* Restore the client that was protected when the script timeout - * was detected. */ - unprotectClient(c); - if (server.masterhost && server.master) - queueClientForReprocessing(server.master); - } - server.in_script = 0; - server.script_caller = NULL; lctx.lua_cur_script = NULL; + luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, NULL); /* Call the Lua garbage collector from time to time to avoid a * full cycle performed by Lua, which adds too latency. @@ -521,19 +510,10 @@ void evalGenericCommand(client *c, int evalsha) { } else { /* On success convert the Lua return value into Redis protocol, and * send it to * the client. */ - luaReplyToRedisReply(c,lua); /* Convert and consume the reply. */ + luaReplyToRedisReply(c,rctx.c,lua); /* Convert and consume the reply. */ lua_pop(lua,1); /* Remove the error handler. */ } - /* If we are using single commands replication, emit EXEC if there - * was at least a write. */ - if (lctx.lua_replicate_commands) { - preventCommandPropagation(c); - if (lctx.lua_multi_emitted) { - execCommandPropagateExec(c->db->id); - } - } - /* EVALSHA should be propagated to Slave and AOF file as full EVAL, unless * we are sure that the script was already in the context of all the * attached slaves *and* the current AOF file if enabled. @@ -662,16 +642,7 @@ NULL addReplyBulkCBuffer(c,sha,40); forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF); } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { - if (server.script_caller == NULL) { - addReplyError(c,"-NOTBUSY No scripts in execution right now."); - } else if (server.script_caller->flags & CLIENT_MASTER) { - addReplyError(c,"-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed."); - } else if (lctx.lua_write_dirty) { - addReplyError(c,"-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command."); - } else { - lctx.lua_kill = 1; - addReply(c,shared.ok); - } + scriptKill(c); } else if (c->argc == 3 && !strcasecmp(c->argv[1]->ptr,"debug")) { if (clientHasPendingReplies(c)) { addReplyError(c,"SCRIPT DEBUG must be called outside a pipeline"); @@ -712,7 +683,7 @@ unsigned long evalScriptsMemory() { /* Returns the time when the script invocation started */ mstime_t evalTimeSnapshot() { - return lctx.lua_time_snapshot; + return scriptTimeSnapshot(); } @@ -1672,6 +1643,7 @@ ldbLog(sdsnew(" next line of code.")); /* This is the core of our Lua debugger, called each time Lua is about * to start executing a new line. */ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); lua_getstack(lua,0,ar); lua_getinfo(lua,"Sl",ar); ldb.currentline = ar->currentline; @@ -1684,7 +1656,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { /* Check if a timeout occurred. */ if (ar->event == LUA_HOOKCOUNT && ldb.step == 0 && bp == 0) { - mstime_t elapsed = elapsedMs(server.script_time_limit); + mstime_t elapsed = elapsedMs(rctx->start_time); mstime_t timelimit = server.script_time_limit ? server.script_time_limit : 5000; if (elapsed >= timelimit) { @@ -1714,7 +1686,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { lua_pushstring(lua, "timeout during Lua debugging with client closing connection"); lua_error(lua); } - lctx.lua_time_start = getMonotonicUs(); - lctx.lua_time_snapshot = mstime(); + rctx->start_time = getMonotonicUs(); + rctx->snapshot_time = mstime(); } } diff --git a/src/evict.c b/src/evict.c index 2d87546cb..a10c2d20e 100644 --- a/src/evict.c +++ b/src/evict.c @@ -33,6 +33,7 @@ #include "server.h" #include "bio.h" #include "atomicvar.h" +#include "script.h" #include /* ---------------------------------------------------------------------------- @@ -472,7 +473,7 @@ static int evictionTimeProc( static int isSafeToPerformEvictions(void) { /* - There must be no script in timeout condition. * - Nor we are loading data right now. */ - if (server.script_timedout || server.loading) return 0; + if (scriptIsTimedout() || server.loading) return 0; /* By default replicas should ignore maxmemory * and just be masters exact copies. */ diff --git a/src/networking.c b/src/networking.c index fb4a487a3..4275d62fb 100644 --- a/src/networking.c +++ b/src/networking.c @@ -30,6 +30,7 @@ #include "server.h" #include "atomicvar.h" #include "cluster.h" +#include "script.h" #include #include #include @@ -260,7 +261,7 @@ void clientInstallWriteHandler(client *c) { int prepareClientToWrite(client *c) { /* If it's the Lua client we always return ok without installing any * handler since there is no socket at all. */ - if (c->flags & (CLIENT_LUA|CLIENT_MODULE)) return C_OK; + if (c->flags & (CLIENT_SCRIPT|CLIENT_MODULE)) return C_OK; /* If CLIENT_CLOSE_ASAP flag is set, we need not write anything. */ if (c->flags & CLIENT_CLOSE_ASAP) return C_ERR; @@ -1491,7 +1492,7 @@ void freeClientAsync(client *c) { * may access the list while Redis uses I/O threads. All the other accesses * are in the context of the main thread while the other threads are * idle. */ - if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return; + if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_SCRIPT) return; c->flags |= CLIENT_CLOSE_ASAP; if (server.io_threads_num == 1) { /* no need to bother with locking if there's just one thread (the main thread) */ @@ -2199,7 +2200,7 @@ int processInputBuffer(client *c) { * condition on the slave. We want just to accumulate the replication * stream (instead of replying -BUSY like we do with other clients) and * later resume the processing. */ - if (server.script_timedout && c->flags & CLIENT_MASTER) break; + if (scriptIsTimedout() && c->flags & CLIENT_MASTER) break; /* CLIENT_CLOSE_AFTER_REPLY closes the connection once the reply is * written to the client. Make sure to not let the reply grow after diff --git a/src/replication.c b/src/replication.c index 1a4aa4c2d..cfd7f1f37 100644 --- a/src/replication.c +++ b/src/replication.c @@ -546,7 +546,7 @@ void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv, gettimeofday(&tv,NULL); cmdrepr = sdscatprintf(cmdrepr,"%ld.%06ld ",(long)tv.tv_sec,(long)tv.tv_usec); - if (c->flags & CLIENT_LUA) { + if (c->flags & CLIENT_SCRIPT) { cmdrepr = sdscatprintf(cmdrepr,"[%d lua] ",dictid); } else if (c->flags & CLIENT_UNIX_SOCKET) { cmdrepr = sdscatprintf(cmdrepr,"[%d unix:%s] ",dictid,server.unixsocket); diff --git a/src/script.c b/src/script.c new file mode 100644 index 000000000..0d6014508 --- /dev/null +++ b/src/script.c @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2009-2021, Redis Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "script.h" +#include "cluster.h" + +/* On script invocation, holding the current run context */ +static scriptRunCtx *curr_run_ctx = NULL; + +static void exitScriptTimedoutMode(scriptRunCtx *run_ctx) { + serverAssert(run_ctx == curr_run_ctx); + serverAssert(scriptIsTimedout()); + run_ctx->flags &= ~SCRIPT_TIMEDOUT; + blockingOperationEnds(); + /* if we are a replica and we have an active master, set it for continue processing */ + if (server.masterhost && server.master) queueClientForReprocessing(server.master); +} + +static void enterScriptTimedoutMode(scriptRunCtx *run_ctx) { + serverAssert(run_ctx == curr_run_ctx); + serverAssert(!scriptIsTimedout()); + /* Mark script as timedout */ + run_ctx->flags |= SCRIPT_TIMEDOUT; + blockingOperationStarts(); +} + +int scriptIsTimedout() { + return scriptIsRunning() && (curr_run_ctx->flags & SCRIPT_TIMEDOUT); +} + +client* scriptGetClient() { + serverAssert(scriptIsRunning()); + return curr_run_ctx->c; +} + +/* interrupt function for scripts, should be call + * from time to time to reply some special command (like ping) + * and also check if the run should be terminated. */ +int scriptInterrupt(scriptRunCtx *run_ctx) { + if (run_ctx->flags & SCRIPT_TIMEDOUT) { + /* script already timedout + we just need to precess some events and return */ + processEventsWhileBlocked(); + return (run_ctx->flags & SCRIPT_KILLED) ? SCRIPT_KILL : SCRIPT_CONTINUE; + } + + long long elapsed = elapsedMs(run_ctx->start_time); + if (elapsed < server.script_time_limit) { + return SCRIPT_CONTINUE; + } + + serverLog(LL_WARNING, + "Slow script detected: still in execution after %lld milliseconds. " + "You can try killing the script using the SCRIPT KILL command.", + elapsed); + + enterScriptTimedoutMode(run_ctx); + /* Once the script timeouts we reenter the event loop to permit others + * some commands execution. For this reason + * we need to mask the client executing the script from the event loop. + * If we don't do that the client may disconnect and could no longer be + * here when the EVAL command will return. */ + protectClient(run_ctx->original_client); + + processEventsWhileBlocked(); + + return (run_ctx->flags & SCRIPT_KILLED) ? SCRIPT_KILL : SCRIPT_CONTINUE; +} + +/* Prepare the given run ctx for execution */ +void scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *caller, const char *funcname) { + serverAssert(!curr_run_ctx); + /* set the curr_run_ctx so we can use it to kill the script if needed */ + curr_run_ctx = run_ctx; + + run_ctx->c = engine_client; + run_ctx->original_client = caller; + run_ctx->funcname = funcname; + + client *script_client = run_ctx->c; + client *curr_client = run_ctx->original_client; + server.script_caller = curr_client; + + /* Select the right DB in the context of the Lua client */ + selectDb(script_client, curr_client->db->id); + script_client->resp = 2; /* Default is RESP2, scripts can change it. */ + + /* If we are in MULTI context, flag Lua client as CLIENT_MULTI. */ + if (curr_client->flags & CLIENT_MULTI) { + script_client->flags |= CLIENT_MULTI; + } + + server.in_script = 1; + + run_ctx->start_time = getMonotonicUs(); + run_ctx->snapshot_time = mstime(); + + run_ctx->flags = 0; + run_ctx->repl_flags = PROPAGATE_AOF | PROPAGATE_REPL; +} + +/* Reset the given run ctx after execution */ +void scriptResetRun(scriptRunCtx *run_ctx) { + serverAssert(curr_run_ctx); + + /* After the script done, remove the MULTI state. */ + run_ctx->c->flags &= ~CLIENT_MULTI; + + server.in_script = 0; + server.script_caller = NULL; + + if (scriptIsTimedout()) { + exitScriptTimedoutMode(run_ctx); + /* Restore the client that was protected when the script timeout + * was detected. */ + unprotectClient(run_ctx->original_client); + } + + if (!(run_ctx->flags & SCRIPT_EVAL_REPLICATION)) { + preventCommandPropagation(run_ctx->original_client); + if (run_ctx->flags & SCRIPT_MULTI_EMMITED) { + execCommandPropagateExec(run_ctx->original_client->db->id); + } + } + + /* unset curr_run_ctx so we will know there is no running script */ + curr_run_ctx = NULL; +} + +/* return true if a script is currently running */ +int scriptIsRunning() { + return curr_run_ctx != NULL; +} + +/* Kill the current running script */ +void scriptKill(client *c) { + if (!curr_run_ctx) { + addReplyError(c, "-NOTBUSY No scripts in execution right now."); + return; + } + if (curr_run_ctx->original_client->flags & CLIENT_MASTER) { + addReplyError(c, + "-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed."); + } + if (curr_run_ctx->flags & SCRIPT_WRITE_DIRTY) { + addReplyError(c, + "-UNKILLABLE Sorry the script already executed write " + "commands against the dataset. You can either wait the " + "script termination or kill the server in a hard way " + "using the SHUTDOWN NOSAVE command."); + return; + } + curr_run_ctx->flags |= SCRIPT_KILLED; + addReply(c, shared.ok); +} + +static int scriptVerifyCommandArity(struct redisCommand *cmd, int argc, sds *err) { + if (!cmd || ((cmd->arity > 0 && cmd->arity != argc) || (argc < cmd->arity))) { + if (cmd) + *err = sdsnew("Wrong number of args calling Redis command from script"); + else + *err = sdsnew("Unknown Redis command called from script"); + return C_ERR; + } + return C_OK; +} + +static int scriptVerifyACL(client *c, sds *err) { + /* Check the ACLs. */ + int acl_errpos; + int acl_retval = ACLCheckAllPerm(c, &acl_errpos); + if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,ACL_LOG_CTX_LUA,acl_errpos,NULL,NULL); + switch (acl_retval) { + case ACL_DENIED_CMD: + *err = sdsnew("The user executing the script can't run this " + "command or subcommand"); + break; + case ACL_DENIED_KEY: + *err = sdsnew("The user executing the script can't access " + "at least one of the keys mentioned in the " + "command arguments"); + break; + case ACL_DENIED_CHANNEL: + *err = sdsnew("The user executing the script can't publish " + "to the channel mentioned in the command"); + break; + default: + *err = sdsnew("The user executing the script is lacking the " + "permissions for the command"); + break; + } + return C_ERR; + } + return C_OK; +} + +static int scriptVerifyWriteCommandAllow(scriptRunCtx *run_ctx, char **err) { + if (!(run_ctx->c->cmd->flags & CMD_WRITE)) { + return C_OK; + } + + if (run_ctx->flags & SCRIPT_READ_ONLY) { + /* We know its a write command, on a read only run we do not allow it. */ + *err = sdsnew("Write commands are not allowed from read-only scripts."); + return C_ERR; + } + + if ((run_ctx->flags & SCRIPT_RANDOM_DIRTY) && (run_ctx->flags & SCRIPT_EVAL_REPLICATION)) { + *err = sdsnew("Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); + return C_ERR; + } + + /* Write commands are forbidden against read-only slaves, or if a + * command marked as non-deterministic was already called in the context + * of this script. */ + int deny_write_type = writeCommandsDeniedByDiskError(); + + if (server.masterhost && server.repl_slave_ro && run_ctx->original_client->flags != CLIENT_ID_AOF + && !(run_ctx->original_client->flags & CLIENT_MASTER)) + { + *err = sdsdup(shared.roslaveerr->ptr); + return C_ERR; + } + + if (deny_write_type != DISK_ERROR_TYPE_NONE) { + if (deny_write_type == DISK_ERROR_TYPE_RDB) { + *err = sdsdup(shared.bgsaveerr->ptr); + } else { + *err = sdsempty(); + *err = sdscatfmt(*err, + "MISCONF Errors writing to the AOF file: %s\r\n", + strerror(server.aof_last_write_errno)); + } + return C_ERR; + } + + return C_OK; +} + +static int scriptVerifyOOM(scriptRunCtx *run_ctx, char **err) { + /* If we reached the memory limit configured via maxmemory, commands that + * could enlarge the memory usage are not allowed, but only if this is the + * first write in the context of this script, otherwise we can't stop + * in the middle. */ + + if (server.maxmemory && /* Maxmemory is actually enabled. */ + run_ctx->original_client->id != CLIENT_ID_AOF && /* Don't care about mem if loading from AOF. */ + !server.masterhost && /* Slave must execute the script. */ + !(run_ctx->flags & SCRIPT_WRITE_DIRTY) && /* Script had no side effects so far. */ + server.script_oom && /* Detected OOM when script start. */ + (run_ctx->c->cmd->flags & CMD_DENYOOM)) + { + *err = sdsdup(shared.oomerr->ptr); + return C_ERR; + } + + return C_OK; +} + +static int scriptVerifyClusterState(client *c, client *original_c, sds *err) { + if (!server.cluster_enabled || original_c->id == CLIENT_ID_AOF || (original_c->flags & CLIENT_MASTER)) { + return C_OK; + } + /* If this is a Redis Cluster node, we need to make sure the script is not + * trying to access non-local keys, with the exception of commands + * received from our master or when loading the AOF back in memory. */ + int error_code; + /* Duplicate relevant flags in the script client. */ + c->flags &= ~(CLIENT_READONLY | CLIENT_ASKING); + c->flags |= original_c->flags & (CLIENT_READONLY | CLIENT_ASKING); + if (getNodeByQuery(c, c->cmd, c->argv, c->argc, NULL, &error_code) != server.cluster->myself) { + if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + *err = sdsnew( + "Script attempted to execute a write command while the " + "cluster is down and readonly"); + } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { + *err = sdsnew("Script attempted to execute a command while the " + "cluster is down"); + } else { + *err = sdsnew("Script attempted to access a non local key in a " + "cluster node"); + } + return C_ERR; + } + return C_OK; +} + +static void scriptEmitMultiIfNeeded(scriptRunCtx *run_ctx) { + /* If we are using single commands replication, we need to wrap what + * we propagate into a MULTI/EXEC block, so that it will be atomic like + * a Lua script in the context of AOF and slaves. */ + client *c = run_ctx->c; + if (!(run_ctx->flags & SCRIPT_EVAL_REPLICATION) + && !(run_ctx->flags & SCRIPT_MULTI_EMMITED) + && !(run_ctx->original_client->flags & CLIENT_MULTI) + && (run_ctx->flags & SCRIPT_WRITE_DIRTY) + && ((run_ctx->repl_flags & PROPAGATE_AOF) + || (run_ctx->repl_flags & PROPAGATE_REPL))) + { + execCommandPropagateMulti(run_ctx->original_client->db->id); + run_ctx->flags |= SCRIPT_MULTI_EMMITED; + /* Now we are in the MULTI context, the lua_client should be + * flag as CLIENT_MULTI. */ + c->flags |= CLIENT_MULTI; + } +} + +/* set RESP for a given run_ctx */ +int scriptSetResp(scriptRunCtx *run_ctx, int resp) { + if (resp != 2 && resp != 3) { + return C_ERR; + } + + run_ctx->c->resp = resp; + return C_OK; +} + +/* set Repl for a given run_ctx + * either: PROPAGATE_AOF | PROPAGATE_REPL*/ +int scriptSetRepl(scriptRunCtx *run_ctx, int repl) { + if ((repl & ~(PROPAGATE_AOF | PROPAGATE_REPL)) != 0) { + return C_ERR; + } + run_ctx->repl_flags = repl; + return C_OK; +} + +/* Call a Redis command. + * The reply is written to the run_ctx client and it is + * up to the engine to take and parse. + * The err out variable is set only if error occurs and describe the error. + * If err is set on reply is written to the run_ctx client. */ +void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { + client *c = run_ctx->c; + + /* Setup our fake client for command execution */ + c->argv = argv; + c->argc = argc; + c->user = run_ctx->original_client->user; + + /* Process module hooks */ + moduleCallCommandFilters(c); + argv = c->argv; + argc = c->argc; + + struct redisCommand *cmd = lookupCommand(argv, argc); + if (scriptVerifyCommandArity(cmd, argc, err) != C_OK) { + return; + } + + c->cmd = c->lastcmd = cmd; + + /* There are commands that are not allowed inside scripts. */ + if (!server.script_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { + *err = sdsnew("This Redis command is not allowed from script"); + return; + } + + if (scriptVerifyACL(c, err) != C_OK) { + return; + } + + if (scriptVerifyWriteCommandAllow(run_ctx, err) != C_OK) { + return; + } + + if (scriptVerifyOOM(run_ctx, err) != C_OK) { + return; + } + + if (cmd->flags & CMD_WRITE) { + /* signify that we already change the data in this execution */ + run_ctx->flags |= SCRIPT_WRITE_DIRTY; + } + + if (cmd->flags & CMD_RANDOM) { + /* signify that we already perform a random command in this execution */ + run_ctx->flags |= SCRIPT_RANDOM_DIRTY; + } + + if (scriptVerifyClusterState(c, run_ctx->original_client, err) != C_OK) { + return; + } + + scriptEmitMultiIfNeeded(run_ctx); + + int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; + if (!(run_ctx->flags & SCRIPT_EVAL_REPLICATION)) { + if (run_ctx->repl_flags & PROPAGATE_AOF) { + call_flags |= CMD_CALL_PROPAGATE_AOF; + } + if (run_ctx->repl_flags & PROPAGATE_REPL) { + call_flags |= CMD_CALL_PROPAGATE_REPL; + } + } + call(c, call_flags); + serverAssert((c->flags & CLIENT_BLOCKED) == 0); +} + +/* Returns the time when the script invocation started */ +mstime_t scriptTimeSnapshot() { + serverAssert(!curr_run_ctx); + return curr_run_ctx->snapshot_time; +} diff --git a/src/script.h b/src/script.h new file mode 100644 index 000000000..4d5e92966 --- /dev/null +++ b/src/script.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2009-2021, Redis Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SCRIPT_H_ +#define __SCRIPT_H_ + +/* + * Script.c unit provides an API for functions and eval + * to interact with Redis. Interaction includes mostly + * executing commands, but also functionalities like calling + * Redis back on long scripts or check if the script was killed. + * + * The interaction is done using a scriptRunCtx object that + * need to be created by the user and initialized using scriptPrepareForRun. + * + * Detailed list of functionalities expose by the unit: + * 1. Calling commands (including all the validation checks such as + * acl, cluster, read only run, ...) + * 2. Set Resp + * 3. Set Replication method (AOF/REPLICATION/NONE) + * 4. Call Redis back to on long running scripts to allow Redis reply + * to clients and perform script kill + */ + +/* + * scriptInterrupt function will return one of those value, + * + * - SCRIPT_KILL - kill the current running script. + * - SCRIPT_CONTINUE - keep running the current script. + */ +#define SCRIPT_KILL 1 +#define SCRIPT_CONTINUE 2 + +/* runCtx flags */ +#define SCRIPT_WRITE_DIRTY (1ULL<<0) /* indicate that the current script already performed a write command */ +#define SCRIPT_RANDOM_DIRTY (1ULL<<1) /* indicate that the current script already performed a random reply command. + Thanks to this flag we'll raise an error every time a write command + is called after a random command and prevent none deterministic + replication or AOF. */ +#define SCRIPT_MULTI_EMMITED (1ULL<<2) /* indicate that we already wrote a multi command to replication/aof */ +#define SCRIPT_TIMEDOUT (1ULL<<3) /* indicate that the current script timedout */ +#define SCRIPT_KILLED (1ULL<<4) /* indicate that the current script was marked to be killed */ +#define SCRIPT_READ_ONLY (1ULL<<5) /* indicate that the current script should only perform read commands */ +#define SCRIPT_EVAL_REPLICATION (1ULL<<6) /* mode for eval, indicate that we replicate the + script invocation and not the effects */ +typedef struct scriptRunCtx scriptRunCtx; + +struct scriptRunCtx { + const char *funcname; + client *c; + client *original_client; + int flags; + int repl_flags; + monotime start_time; + mstime_t snapshot_time; +}; + +void scriptPrepareForRun(scriptRunCtx *r_ctx, client *engine_client, client *caller, const char *funcname); +void scriptResetRun(scriptRunCtx *r_ctx); +int scriptSetResp(scriptRunCtx *r_ctx, int resp); +int scriptSetRepl(scriptRunCtx *r_ctx, int repl); +void scriptCall(scriptRunCtx *r_ctx, robj **argv, int argc, sds *err); +int scriptInterrupt(scriptRunCtx *r_ctx); +void scriptKill(client *c); +int scriptIsRunning(); +int scriptIsTimedout(); +client* scriptGetClient(); +mstime_t scriptTimeSnapshot(); + +#endif /* __SCRIPT_H_ */ diff --git a/src/script_lua.c b/src/script_lua.c index bee5d6c93..c7206095c 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2021, Redis Labs Ltd. + * Copyright (c) 2009-2021, Redis Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,10 +39,9 @@ #include #include #include -#include "functions.h" -int redis_math_random (lua_State *L); -int redis_math_randomseed (lua_State *L); +static int redis_math_random (lua_State *L); +static int redis_math_randomseed (lua_State *L); static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len); static void redisProtocolToLuaType_BulkString(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); static void redisProtocolToLuaType_NullBulkString(void *ctx, const char *proto, size_t proto_len); @@ -59,6 +58,39 @@ static void redisProtocolToLuaType_BigNumber(void *ctx, const char *str, size_t static void redisProtocolToLuaType_VerbatimString(void *ctx, const char *format, const char *str, size_t len, const char *proto, size_t proto_len); static void redisProtocolToLuaType_Attribute(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); +/* + * Save the give pointer on Lua registry, used to save the Lua context and + * function context so we can retrieve them from lua_State. + */ +void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr) { + lua_pushstring(lua, name); + if (ptr) { + lua_pushlightuserdata(lua, ptr); + } else { + lua_pushnil(lua); + } + lua_settable(lua, LUA_REGISTRYINDEX); +} + +/* + * Get a saved pointer from registry + */ +void* luaGetFromRegistry(lua_State* lua, const char* name) { + lua_pushstring(lua, name); + lua_gettable(lua, LUA_REGISTRYINDEX); + + /* must be light user data */ + serverAssert(lua_islightuserdata(lua, -1)); + + void* ptr = (void*) lua_topointer(lua, -1); + serverAssert(ptr); + + /* pops the value */ + lua_pop(lua, 1); + + return ptr; +} + /* --------------------------------------------------------------------------- * Redis reply to Lua type conversion functions. * ------------------------------------------------------------------------- */ @@ -99,7 +131,7 @@ static const ReplyParserCallbacks DefaultLuaTypeParserCallbacks = { .error = NULL, }; -void redisProtocolToLuaType(lua_State *lua, char* reply) { +static void redisProtocolToLuaType(lua_State *lua, char* reply) { ReplyParser parser = {.curr_location = reply, .callbacks = DefaultLuaTypeParserCallbacks}; parseReply(&parser, lua); @@ -394,7 +426,7 @@ static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto * with a single "err" field set to the error string. Note that this * table is never a valid reply by proper commands, since the returned * tables are otherwise always indexed by integers, never by strings. */ -void luaPushError(lua_State *lua, char *error) { +static void luaPushError(lua_State *lua, char *error) { lua_Debug dbg; /* If debugging is active and in step mode, log errors resulting from @@ -422,7 +454,7 @@ void luaPushError(lua_State *lua, char *error) { * by the non-error-trapping version of redis.pcall(), which is redis.call(), * this function will raise the Lua error so that the execution of the * script will be halted. */ -int luaRaiseError(lua_State *lua) { +static int luaRaiseError(lua_State *lua) { lua_pushstring(lua,"err"); lua_gettable(lua,-2); return lua_error(lua); @@ -434,7 +466,7 @@ int luaRaiseError(lua_State *lua) { * * The array is sorted using table.sort itself, and assuming all the * list elements are strings. */ -void luaSortArray(lua_State *lua) { +static void luaSortArray(lua_State *lua) { /* Initial Stack: array */ lua_getglobal(lua,"table"); lua_pushstring(lua,"sort"); @@ -465,7 +497,7 @@ void luaSortArray(lua_State *lua) { /* Reply to client 'c' converting the top element in the Lua stack to a * Redis reply. As a side effect the element is consumed from the stack. */ -void luaReplyToRedisReply(client *c, lua_State *lua) { +void luaReplyToRedisReply(client *c, client* script_client, lua_State *lua) { int t = lua_type(lua,-1); if (!lua_checkstack(lua, 4)) { @@ -483,7 +515,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); break; case LUA_TBOOLEAN: - if (lctx.lua_client->resp == 2) + if (script_client->resp == 2) addReply(c,lua_toboolean(lua,-1) ? shared.cone : shared.null[c->resp]); else @@ -587,8 +619,8 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { while (lua_next(lua,-2)) { /* Stack now: table, key, value */ lua_pushvalue(lua,-2); /* Dup key before consuming. */ - luaReplyToRedisReply(c, lua); /* Return key. */ - luaReplyToRedisReply(c, lua); /* Return value. */ + luaReplyToRedisReply(c, script_client, lua); /* Return key. */ + luaReplyToRedisReply(c, script_client, lua); /* Return value. */ /* Stack now: table, key. */ maplen++; } @@ -611,7 +643,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { /* Stack now: table, key, true */ lua_pop(lua,1); /* Discard the boolean value. */ lua_pushvalue(lua,-1); /* Dup key before consuming. */ - luaReplyToRedisReply(c, lua); /* Return key. */ + luaReplyToRedisReply(c, script_client, lua); /* Return key. */ /* Stack now: table, key. */ setlen++; } @@ -633,7 +665,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { lua_pop(lua,1); break; } - luaReplyToRedisReply(c, lua); + luaReplyToRedisReply(c, script_client, lua); mbulklen++; } setDeferredArrayLen(c,replylen,mbulklen); @@ -650,10 +682,11 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { #define LUA_CMD_OBJCACHE_SIZE 32 #define LUA_CMD_OBJCACHE_MAX_LEN 64 -int luaRedisGenericCommand(lua_State *lua, int raise_error) { +static int luaRedisGenericCommand(lua_State *lua, int raise_error) { int j, argc = lua_gettop(lua); - struct redisCommand *cmd; - client *c = lctx.lua_client; + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + sds err = NULL; + client* c = rctx->c; sds reply; /* Cached across calls. */ @@ -741,16 +774,6 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * and this way we guaranty we will have room on the stack for the result. */ lua_pop(lua, argc); - /* Setup our fake client for command execution */ - c->argv = argv; - c->argc = argc; - c->user = server.script_caller->user; - - /* Process module hooks */ - moduleCallCommandFilters(c); - argv = c->argv; - argc = c->argc; - /* Log the command if debugging is active. */ if (ldbIsEnabled()) { sds cmdlog = sdsnew(""); @@ -767,167 +790,13 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { ldbLog(cmdlog); } - /* Command lookup */ - cmd = lookupCommand(argv[0]->ptr); - if (!cmd || ((cmd->arity > 0 && cmd->arity != argc) || - (argc < -cmd->arity))) - { - if (cmd) - luaPushError(lua, - "Wrong number of args calling Redis command From Lua script"); - else - luaPushError(lua,"Unknown Redis command called from Lua script"); + + scriptCall(rctx, argv, argc, &err); + if (err) { + luaPushError(lua, err); + sdsfree(err); goto cleanup; } - c->cmd = c->lastcmd = cmd; - - /* There are commands that are not allowed inside scripts. */ - if (!server.script_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { - luaPushError(lua, "This Redis command is not allowed from scripts"); - goto cleanup; - } - - /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ - if ((server.script_caller->cmd->proc == evalRoCommand || - server.script_caller->cmd->proc == evalShaRoCommand) && - (cmd->flags & CMD_WRITE)) - { - luaPushError(lua, "Write commands are not allowed from read-only scripts"); - goto cleanup; - } - - /* Check the ACLs. */ - int acl_errpos; - int acl_retval = ACLCheckAllPerm(c,&acl_errpos); - if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,ACL_LOG_CTX_LUA,acl_errpos,NULL,NULL); - switch (acl_retval) { - case ACL_DENIED_CMD: - luaPushError(lua, "The user executing the script can't run this " - "command or subcommand"); - break; - case ACL_DENIED_KEY: - luaPushError(lua, "The user executing the script can't access " - "at least one of the keys mentioned in the " - "command arguments"); - break; - case ACL_DENIED_CHANNEL: - luaPushError(lua, "The user executing the script can't publish " - "to the channel mentioned in the command"); - break; - default: - luaPushError(lua, "The user executing the script is lacking the " - "permissions for the command"); - break; - } - goto cleanup; - } - - /* Write commands are forbidden against read-only slaves, or if a - * command marked as non-deterministic was already called in the context - * of this script. */ - if (cmd->flags & CMD_WRITE) { - int deny_write_type = writeCommandsDeniedByDiskError(); - if (lctx.lua_random_dirty && !lctx.lua_replicate_commands) { - luaPushError(lua, - "Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); - goto cleanup; - } else if (server.masterhost && server.repl_slave_ro && - server.script_caller->id != CLIENT_ID_AOF && - !(server.script_caller->flags & CLIENT_MASTER)) - { - luaPushError(lua, shared.roslaveerr->ptr); - goto cleanup; - } else if (deny_write_type != DISK_ERROR_TYPE_NONE) { - if (deny_write_type == DISK_ERROR_TYPE_RDB) { - luaPushError(lua, shared.bgsaveerr->ptr); - } else { - sds aof_write_err = sdscatfmt(sdsempty(), - "-MISCONF Errors writing to the AOF file: %s\r\n", - strerror(server.aof_last_write_errno)); - luaPushError(lua, aof_write_err); - sdsfree(aof_write_err); - } - goto cleanup; - } - } - - /* If we reached the memory limit configured via maxmemory, commands that - * could enlarge the memory usage are not allowed, but only if this is the - * first write in the context of this script, otherwise we can't stop - * in the middle. */ - if (server.maxmemory && /* Maxmemory is actually enabled. */ - server.script_caller->id != CLIENT_ID_AOF && /* Don't care about mem if loading from AOF. */ - !server.masterhost && /* Slave must execute the script. */ - lctx.lua_write_dirty == 0 && /* Script had no side effects so far. */ - server.script_oom && /* Detected OOM when script start. */ - (cmd->flags & CMD_DENYOOM)) - { - luaPushError(lua, shared.oomerr->ptr); - goto cleanup; - } - - if (cmd->flags & CMD_RANDOM) lctx.lua_random_dirty = 1; - if (cmd->flags & CMD_WRITE) lctx.lua_write_dirty = 1; - - /* If this is a Redis Cluster node, we need to make sure Lua is not - * trying to access non-local keys, with the exception of commands - * received from our master or when loading the AOF back in memory. */ - if (server.cluster_enabled && server.script_caller->id != CLIENT_ID_AOF && - !(server.script_caller->flags & CLIENT_MASTER)) - { - int error_code; - /* Duplicate relevant flags in the lua client. */ - c->flags &= ~(CLIENT_READONLY|CLIENT_ASKING); - c->flags |= server.script_caller->flags & (CLIENT_READONLY|CLIENT_ASKING); - if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != - server.cluster->myself) - { - if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { - luaPushError(lua, - "Lua script attempted to execute a write command while the " - "cluster is down and readonly"); - } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { - luaPushError(lua, - "Lua script attempted to execute a command while the " - "cluster is down"); - } else { - luaPushError(lua, - "Lua script attempted to access a non local key in a " - "cluster node"); - } - - goto cleanup; - } - } - - /* If we are using single commands replication, we need to wrap what - * we propagate into a MULTI/EXEC block, so that it will be atomic like - * a Lua script in the context of AOF and slaves. */ - if (lctx.lua_replicate_commands && - !lctx.lua_multi_emitted && - !(server.script_caller->flags & CLIENT_MULTI) && - lctx.lua_write_dirty && - lctx.lua_repl != PROPAGATE_NONE) - { - execCommandPropagateMulti(server.script_caller->db->id); - lctx.lua_multi_emitted = 1; - /* Now we are in the MULTI context, the lua_client should be - * flag as CLIENT_MULTI. */ - c->flags |= CLIENT_MULTI; - } - - /* Run the command */ - int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; - if (lctx.lua_replicate_commands) { - /* Set flags according to redis.set_repl() settings. */ - if (lctx.lua_repl & PROPAGATE_AOF) - call_flags |= CMD_CALL_PROPAGATE_AOF; - if (lctx.lua_repl & PROPAGATE_REPL) - call_flags |= CMD_CALL_PROPAGATE_REPL; - } - call(c,call_flags); - serverAssert((c->flags & CLIENT_BLOCKED) == 0); /* Convert the result of the Redis command into a suitable Lua type. * The first thing we need is to create a single string from the client @@ -958,8 +827,8 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { /* Sort the output array if needed, assuming it is a non-null multi bulk * reply as expected. */ - if ((cmd->flags & CMD_SORT_FOR_SCRIPT) && - (lctx.lua_replicate_commands == 0) && + if ((c->cmd->flags & CMD_SORT_FOR_SCRIPT) && + (rctx->flags & SCRIPT_EVAL_REPLICATION) && (reply[0] == '*' && reply[1] != '-')) { luaSortArray(lua); } @@ -1010,18 +879,18 @@ cleanup: } /* redis.call() */ -int luaRedisCallCommand(lua_State *lua) { +static int luaRedisCallCommand(lua_State *lua) { return luaRedisGenericCommand(lua,1); } /* redis.pcall() */ -int luaRedisPCallCommand(lua_State *lua) { +static int luaRedisPCallCommand(lua_State *lua) { return luaRedisGenericCommand(lua,0); } /* This adds redis.sha1hex(string) to Lua scripts using the same hashing * function used for sha1ing lua scripts. */ -int luaRedisSha1hexCommand(lua_State *lua) { +static int luaRedisSha1hexCommand(lua_State *lua) { int argc = lua_gettop(lua); char digest[41]; size_t len; @@ -1045,7 +914,7 @@ int luaRedisSha1hexCommand(lua_State *lua) { * return redis.error_reply("ERR Some Error") * return redis.status_reply("ERR Some Error") */ -int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { +static int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { if (lua_gettop(lua) != 1 || lua_type(lua,-1) != LUA_TSTRING) { luaPushError(lua, "wrong number or type of arguments"); return 1; @@ -1059,12 +928,12 @@ int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { } /* redis.error_reply() */ -int luaRedisErrorReplyCommand(lua_State *lua) { +static int luaRedisErrorReplyCommand(lua_State *lua) { return luaRedisReturnSingleFieldTable(lua,"err"); } /* redis.status_reply() */ -int luaRedisStatusReplyCommand(lua_State *lua) { +static int luaRedisStatusReplyCommand(lua_State *lua) { return luaRedisReturnSingleFieldTable(lua,"ok"); } @@ -1072,11 +941,13 @@ int luaRedisStatusReplyCommand(lua_State *lua) { * * Set the propagation of write commands executed in the context of the * script to on/off for AOF and slaves. */ -int luaRedisSetReplCommand(lua_State *lua) { +static int luaRedisSetReplCommand(lua_State *lua) { int argc = lua_gettop(lua); int flags; - if (lctx.lua_replicate_commands == 0) { + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + + if (rctx->flags & SCRIPT_EVAL_REPLICATION) { lua_pushstring(lua, "You can set the replication behavior only after turning on single commands replication with redis.replicate_commands()."); return lua_error(lua); } else if (argc != 1) { @@ -1089,12 +960,13 @@ int luaRedisSetReplCommand(lua_State *lua) { lua_pushstring(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); return lua_error(lua); } - lctx.lua_repl = flags; + + scriptSetRepl(rctx, flags); return 0; } /* redis.log() */ -int luaLogCommand(lua_State *lua) { +static int luaLogCommand(lua_State *lua) { int j, argc = lua_gettop(lua); int level; sds log; @@ -1131,7 +1003,7 @@ int luaLogCommand(lua_State *lua) { } /* redis.setresp() */ -int luaSetResp(lua_State *lua) { +static int luaSetResp(lua_State *lua) { int argc = lua_gettop(lua); if (argc != 1) { @@ -1144,8 +1016,8 @@ int luaSetResp(lua_State *lua) { lua_pushstring(lua, "RESP version must be 2 or 3."); return lua_error(lua); } - - lctx.lua_client->resp = resp; + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + scriptSetResp(rctx, resp); return 0; } @@ -1153,7 +1025,7 @@ int luaSetResp(lua_State *lua) { * Lua engine initialization and reset. * ------------------------------------------------------------------------- */ -void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) { +static void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) { lua_pushcfunction(lua, luafunc); lua_pushstring(lua, libname); lua_call(lua, 1, 0); @@ -1164,7 +1036,7 @@ LUALIB_API int (luaopen_struct) (lua_State *L); LUALIB_API int (luaopen_cmsgpack) (lua_State *L); LUALIB_API int (luaopen_bit) (lua_State *L); -void luaLoadLibraries(lua_State *lua) { +static void luaLoadLibraries(lua_State *lua) { luaLoadLib(lua, "", luaopen_base); luaLoadLib(lua, LUA_TABLIBNAME, luaopen_table); luaLoadLib(lua, LUA_STRLIBNAME, luaopen_string); @@ -1183,7 +1055,7 @@ void luaLoadLibraries(lua_State *lua) { /* Remove a functions that we don't want to expose to the Redis scripting * environment. */ -void luaRemoveUnsupportedFunctions(lua_State *lua) { +static void luaRemoveUnsupportedFunctions(lua_State *lua) { lua_pushnil(lua); lua_setglobal(lua,"loadfile"); lua_pushnil(lua); @@ -1195,7 +1067,7 @@ void luaRemoveUnsupportedFunctions(lua_State *lua) { * * It should be the last to be called in the scripting engine initialization * sequence, because it may interact with creation of globals. */ -void scriptingEnableGlobalsProtection(lua_State *lua) { +void luaEnableGlobalsProtection(lua_State *lua) { char *s[32]; sds code = sdsempty(); int j = 0; @@ -1229,7 +1101,7 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { sdsfree(code); } -void luaEngineRegisterRedisAPI(lua_State* lua) { +void luaRegisterRedisAPI(lua_State* lua) { luaLoadLibraries(lua); luaRemoveUnsupportedFunctions(lua); @@ -1350,7 +1222,7 @@ void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec) { /* The following implementation is the one shipped with Lua itself but with * rand() replaced by redisLrand48(). */ -int redis_math_random (lua_State *L) { +static int redis_math_random (lua_State *L) { /* the `%' avoids the (rare) case of r==1, and is needed also because on some systems (SunOS!) `rand()' may return a value larger than RAND_MAX */ lua_Number r = (lua_Number)(redisLrand48()%REDIS_LRAND48_MAX) / @@ -1378,36 +1250,16 @@ int redis_math_random (lua_State *L) { return 1; } -int redis_math_randomseed (lua_State *L) { +static int redis_math_randomseed (lua_State *L) { redisSrand48(luaL_checkint(L, 1)); return 0; } /* This is the Lua script "count" hook that we use to detect scripts timeout. */ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { - long long elapsed = elapsedMs(lctx.lua_time_start); UNUSED(ar); - UNUSED(lua); - - /* Set the timeout condition if not already set and the maximum - * execution time was reached. */ - if (elapsed >= server.script_time_limit && server.script_timedout == 0) { - serverLog(LL_WARNING, - "Lua slow script detected: still in execution after %lld milliseconds. " - "You can try killing the script using the SCRIPT KILL command. " - "Script SHA1 is: %s", - elapsed, lctx.lua_cur_script); - server.script_timedout = 1; - blockingOperationStarts(); - /* Once the script timeouts we reenter the event loop to permit others - * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason - * we need to mask the client executing the script from the event loop. - * If we don't do that the client may disconnect and could no longer be - * here when the EVAL command will return. */ - protectClient(server.script_caller); - } - if (server.script_timedout) processEventsWhileBlocked(); - if (lctx.lua_kill) { + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + if (scriptInterrupt(rctx) == SCRIPT_KILL) { serverLog(LL_WARNING,"Lua script killed by user with SCRIPT KILL."); /* diff --git a/src/script_lua.h b/src/script_lua.h index f9cf6f18a..5ae9225bc 100644 --- a/src/script_lua.h +++ b/src/script_lua.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2021, Redis Labs Ltd. + * Copyright (c) 2009-2021, Redis Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,35 +30,39 @@ #ifndef __SCRIPT_LUA_H_ #define __SCRIPT_LUA_H_ +/* + * script_lua.c unit provides shared functionality between + * eval.c and function_lua.c. Functionality provided: + * + * * Execute Lua code, assuming that the code is located on + * the top of the Lua stack. In addition, parsing the execution + * result and convert it to the resp and reply ot the client. + * + * * Run Redis commands from within the Lua code (Including + * parsing the reply and create a Lua object out of it). + * + * * Register Redis API to the Lua interpreter. Only shared + * API are registered (API that is only relevant on eval.c + * (like debugging) are registered on eval.c). + * + * Uses script.c for interaction back with Redis. + */ + #include "server.h" #include "script.h" #include #include #include -typedef struct luaCtx { - lua_State *lua; /* The Lua interpreter. We use just one for all clients */ - client *lua_client; /* The "fake client" to query Redis from Lua */ - char *lua_cur_script; /* SHA1 of the script currently running, or NULL */ - dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ - unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ - int lua_replicate_commands; /* True if we are doing single commands repl. */ - int lua_write_dirty; - int lua_random_dirty; - int lua_multi_emitted; - int lua_repl; - int lua_kill; - monotime lua_time_start; /* monotonic timer to detect timed-out script */ - mstime_t lua_time_snapshot; /* Snapshot of mstime when script is started */ -} luaCtx; +#define REGISTRY_RUN_CTX_NAME "__RUN_CTX__" -extern luaCtx lctx; - -void luaEngineRegisterRedisAPI(lua_State* lua); -void scriptingEnableGlobalsProtection(lua_State *lua); +void luaRegisterRedisAPI(lua_State* lua); +void luaEnableGlobalsProtection(lua_State *lua); void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec); void luaMaskCountHook(lua_State *lua, lua_Debug *ar); -void luaReplyToRedisReply(client *c, lua_State *lua); +void luaReplyToRedisReply(client *c, client* script_client, lua_State *lua); +void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr); +void* luaGetFromRegistry(lua_State* lua, const char* name); diff --git a/src/server.c b/src/server.c index 8d682c207..1fbf3fa3a 100644 --- a/src/server.c +++ b/src/server.c @@ -35,6 +35,7 @@ #include "latency.h" #include "atomicvar.h" #include "mt19937-64.h" +#include "script.h" #include #include @@ -4931,13 +4932,13 @@ void call(client *c, int flags) { /* When EVAL is called loading the AOF we don't want commands called * from Lua to go into the slowlog or to populate statistics. */ - if (server.loading && c->flags & CLIENT_LUA) + if (server.loading && c->flags & CLIENT_SCRIPT) flags &= ~(CMD_CALL_SLOWLOG | CMD_CALL_STATS); /* If the caller is Lua, we want to force the EVAL caller to propagate * the script if the command flag or client flag are forcing the * propagation. */ - if (c->flags & CLIENT_LUA && server.script_caller) { + if (c->flags & CLIENT_SCRIPT && server.script_caller) { if (c->flags & CLIENT_FORCE_REPL) server.script_caller->flags |= CLIENT_FORCE_REPL; if (c->flags & CLIENT_FORCE_AOF) @@ -5070,7 +5071,7 @@ void call(client *c, int flags) { /* If the client has keys tracking enabled for client side caching, * make sure to remember the keys it fetched via this command. */ if (c->cmd->flags & CMD_READONLY) { - client *caller = (c->flags & CLIENT_LUA && server.script_caller) ? + client *caller = (c->flags & CLIENT_SCRIPT && server.script_caller) ? server.script_caller : c; if (caller->flags & CLIENT_TRACKING && !(caller->flags & CLIENT_TRACKING_BCAST)) @@ -5172,7 +5173,7 @@ void populateCommandMovableKeys(struct redisCommand *cmd) { * other operations can be performed by the caller. Otherwise * if C_ERR is returned the client was destroyed (i.e. after QUIT). */ int processCommand(client *c) { - if (!server.script_timedout) { + if (!scriptIsTimedout()) { /* Both EXEC and EVAL call call() directly so there should be * no way in_exec or in_eval or propagate_in_transaction is 1. * That is unless lua_timedout, in which case client may run @@ -5273,7 +5274,7 @@ int processCommand(client *c) { * 2) The command has no key arguments. */ if (server.cluster_enabled && !(c->flags & CLIENT_MASTER) && - !(c->flags & CLIENT_LUA && + !(c->flags & CLIENT_SCRIPT && server.script_caller->flags & CLIENT_MASTER) && !(!c->cmd->movablekeys && c->cmd->key_specs_num == 0 && c->cmd->proc != execCommand)) @@ -5309,7 +5310,7 @@ int processCommand(client *c) { * the event loop since there is a busy Lua script running in timeout * condition, to avoid mixing the propagation of scripts with the * propagation of DELs due to eviction. */ - if (server.maxmemory && !server.script_timedout) { + if (server.maxmemory && !scriptIsTimedout()) { int out_of_memory = (performEvictions() == EVICT_FAIL); /* performEvictions may evict keys, so we need flush pending tracking @@ -5432,7 +5433,7 @@ int processCommand(client *c) { * the MULTI plus a few initial commands refused, then the timeout * condition resolves, and the bottom-half of the transaction gets * executed, see Github PR #7022. */ - if (server.script_timedout && + if (scriptIsTimedout() && c->cmd->proc != authCommand && c->cmd->proc != helloCommand && c->cmd->proc != replconfCommand && diff --git a/src/server.h b/src/server.h index 959b0e2d1..2b1ef1ce7 100644 --- a/src/server.h +++ b/src/server.h @@ -260,7 +260,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CLIENT_CLOSE_AFTER_REPLY (1<<6) /* Close after writing entire reply. */ #define CLIENT_UNBLOCKED (1<<7) /* This client was unblocked and is stored in server.unblocked_clients */ -#define CLIENT_LUA (1<<8) /* This is a non connected client used by Lua */ +#define CLIENT_SCRIPT (1<<8) /* This is a non connected client used by Lua */ #define CLIENT_ASKING (1<<9) /* Client issued the ASKING command */ #define CLIENT_CLOSE_ASAP (1<<10)/* Close this client ASAP */ #define CLIENT_UNIX_SOCKET (1<<11) /* Client connected via Unix domain socket */ @@ -1721,8 +1721,6 @@ struct redisServer { /* Scripting */ client *script_caller; /* The client running script right now, or NULL */ mstime_t script_time_limit; /* Script timeout in milliseconds */ - int script_timedout; /* True if we reached the time limit for script - execution. */ int lua_always_replicate_commands; /* Default replication type. */ int script_oom; /* OOM detected when script start */ int script_disable_deny_script; /* Allow running commands marked "no-script" inside a script. */ diff --git a/src/sort.c b/src/sort.c index fe0d55fea..153d6ba79 100644 --- a/src/sort.c +++ b/src/sort.c @@ -294,7 +294,7 @@ void sortCommandGeneric(client *c, int readonly) { * scripting and replication. */ if (dontsort && sortval->type == OBJ_SET && - (storekey || c->flags & CLIENT_LUA)) + (storekey || c->flags & CLIENT_SCRIPT)) { /* Force ALPHA sorting */ dontsort = 0; diff --git a/src/t_stream.c b/src/t_stream.c index af32bc18f..14dd2bd02 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1998,7 +1998,7 @@ void xreadCommand(client *c) { int moreargs = c->argc-i-1; char *o = c->argv[i]->ptr; if (!strcasecmp(o,"BLOCK") && moreargs) { - if (c->flags & CLIENT_LUA) { + if (c->flags & CLIENT_SCRIPT) { /* * Although the CLIENT_DENY_BLOCKING flag should protect from blocking the client * on Lua/MULTI/RM_Call we want special treatment for Lua to keep backward compatibility. diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 68c819aa0..f62f68970 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -984,7 +984,7 @@ start_server {tags {"scripting needs:debug external:skip"}} { r write $cmd r flush set ret [r read] - assert_match {*Unknown Redis command called from Lua script*} $ret + assert_match {*Unknown Redis command called from*} $ret # make sure the server is still ok reconnect assert_equal [r ping] {PONG} From f21dc38a6ed3851a5e6501199e803ff0b93795cf Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Wed, 6 Oct 2021 09:36:37 +0300 Subject: [PATCH 4/5] Redis Functions - Moved invoke Lua code functionality to script_lua.c The functionality was moved to script_lua.c under callFunction function. Its purpose is to call the Lua function already located on the top of the Lua stack. Used the new function on eval.c to invoke Lua code. The function will also be used to invoke Lua code on the Lua engine. --- src/eval.c | 62 ++----------------------------------------- src/script_lua.c | 69 +++++++++++++++++++++++++++++++++++++++++++++--- src/script_lua.h | 5 +--- src/server.h | 1 + 4 files changed, 70 insertions(+), 67 deletions(-) diff --git a/src/eval.c b/src/eval.c index 7d5c40a53..f8fe8bf37 100644 --- a/src/eval.c +++ b/src/eval.c @@ -45,9 +45,6 @@ void ldbInit(void); void ldbDisable(client *c); void ldbEnable(client *c); void evalGenericCommandWithDebugging(client *c, int evalsha); -void luaLdbLineHook(lua_State *lua, lua_Debug *ar); -void ldbLog(sds entry); -void ldbLogRedisReply(char *reply); sds ldbCatStackValue(sds s, lua_State *lua, int idx); /* Lua context */ @@ -377,7 +374,6 @@ void evalGenericCommand(client *c, int evalsha) { char funcname[43]; long long numkeys; long long initial_server_dirty = server.dirty; - int delhook = 0, err; /* When we replicate whole scripts, we want the same PRNG sequence at * every call so that our PRNG is not affected by external state. */ @@ -443,22 +439,10 @@ void evalGenericCommand(client *c, int evalsha) { serverAssert(!lua_isnil(lua,-1)); } - /* Populate the argv and keys table accordingly to the arguments that - * EVAL received. */ - luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys); - luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys); - lctx.lua_cur_script = funcname + 2; scriptRunCtx rctx; scriptPrepareForRun(&rctx, lctx.lua_client, c, lctx.lua_cur_script); - - /* We must set it before we set the Lua hook, theoretically the - * Lua hook might be called wheneven we run any Lua instruction - * such as 'luaSetGlobalArray' and we want the rctx to be available - * each time the Lua hook is invoked. */ - luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, &rctx); - if (!lctx.lua_replicate_commands) rctx.flags |= SCRIPT_EVAL_REPLICATION; /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ if ((server.script_caller->cmd->proc == evalRoCommand || @@ -466,53 +450,11 @@ void evalGenericCommand(client *c, int evalsha) { rctx.flags |= SCRIPT_READ_ONLY; } - if (server.script_time_limit > 0 && ldb.active == 0) { - lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); - delhook = 1; - } else if (ldb.active) { - lua_sethook(lctx.lua,luaLdbLineHook,LUA_MASKLINE|LUA_MASKCOUNT,100000); - delhook = 1; - } - - /* At this point whether this script was never seen before or if it was - * already defined, we can call it. We have zero arguments and expect - * a single return value. */ - err = lua_pcall(lua,0,1,-2); - + luaCallFunction(&rctx, lua, c->argv+3, numkeys, c->argv+3+numkeys, c->argc-3-numkeys, ldb.active); + lua_pop(lua,1); /* Remove the error handler. */ scriptResetRun(&rctx); - /* Perform some cleanup that we need to do both on error and success. */ - if (delhook) lua_sethook(lua,NULL,0,0); /* Disable hook */ lctx.lua_cur_script = NULL; - luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, NULL); - - /* Call the Lua garbage collector from time to time to avoid a - * full cycle performed by Lua, which adds too latency. - * - * The call is performed every LUA_GC_CYCLE_PERIOD executed commands - * (and for LUA_GC_CYCLE_PERIOD collection steps) because calling it - * for every command uses too much CPU. */ - #define LUA_GC_CYCLE_PERIOD 50 - { - static long gc_count = 0; - - gc_count++; - if (gc_count == LUA_GC_CYCLE_PERIOD) { - lua_gc(lua,LUA_GCSTEP,LUA_GC_CYCLE_PERIOD); - gc_count = 0; - } - } - - if (err) { - addReplyErrorFormat(c,"Error running script (call to %s): %s\n", - funcname, lua_tostring(lua,-1)); - lua_pop(lua,2); /* Consume the Lua reply and remove error handler. */ - } else { - /* On success convert the Lua return value into Redis protocol, and - * send it to * the client. */ - luaReplyToRedisReply(c,rctx.c,lua); /* Convert and consume the reply. */ - lua_pop(lua,1); /* Remove the error handler. */ - } /* EVALSHA should be propagated to Slave and AOF file as full EVAL, unless * we are sure that the script was already in the context of all the diff --git a/src/script_lua.c b/src/script_lua.c index c7206095c..fb3fa397b 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -57,6 +57,7 @@ static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto static void redisProtocolToLuaType_BigNumber(void *ctx, const char *str, size_t len, const char *proto, size_t proto_len); static void redisProtocolToLuaType_VerbatimString(void *ctx, const char *format, const char *str, size_t len, const char *proto, size_t proto_len); static void redisProtocolToLuaType_Attribute(struct ReplyParser *parser, void *ctx, size_t len, const char *proto); +static void luaReplyToRedisReply(client *c, client* script_client, lua_State *lua); /* * Save the give pointer on Lua registry, used to save the Lua context and @@ -497,7 +498,7 @@ static void luaSortArray(lua_State *lua) { /* Reply to client 'c' converting the top element in the Lua stack to a * Redis reply. As a side effect the element is consumed from the stack. */ -void luaReplyToRedisReply(client *c, client* script_client, lua_State *lua) { +static void luaReplyToRedisReply(client *c, client* script_client, lua_State *lua) { int t = lua_type(lua,-1); if (!lua_checkstack(lua, 4)) { @@ -1201,7 +1202,7 @@ void luaRegisterRedisAPI(lua_State* lua) { /* Set an array of Redis String Objects as a Lua array (table) stored into a * global variable. */ -void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec) { +static void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec) { int j; lua_newtable(lua); @@ -1256,7 +1257,7 @@ static int redis_math_randomseed (lua_State *L) { } /* This is the Lua script "count" hook that we use to detect scripts timeout. */ -void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { +static void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { UNUSED(ar); scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); if (scriptInterrupt(rctx) == SCRIPT_KILL) { @@ -1273,3 +1274,65 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { lua_error(lua); } } + +void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled) { + client* c = run_ctx->original_client; + int delhook = 0; + + /* We must set it before we set the Lua hook, theoretically the + * Lua hook might be called wheneven we run any Lua instruction + * such as 'luaSetGlobalArray' and we want the run_ctx to be available + * each time the Lua hook is invoked. */ + luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, run_ctx); + + if (server.script_time_limit > 0 && !debug_enabled) { + lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); + delhook = 1; + } else if (debug_enabled) { + lua_sethook(lua,luaLdbLineHook,LUA_MASKLINE|LUA_MASKCOUNT,100000); + delhook = 1; + } + + /* Populate the argv and keys table accordingly to the arguments that + * EVAL received. */ + luaSetGlobalArray(lua,"KEYS",keys,nkeys); + luaSetGlobalArray(lua,"ARGV",args,nargs); + + /* At this point whether this script was never seen before or if it was + * already defined, we can call it. We have zero arguments and expect + * a single return value. */ + int err = lua_pcall(lua,0,1,-2); + + /* Call the Lua garbage collector from time to time to avoid a + * full cycle performed by Lua, which adds too latency. + * + * The call is performed every LUA_GC_CYCLE_PERIOD executed commands + * (and for LUA_GC_CYCLE_PERIOD collection steps) because calling it + * for every command uses too much CPU. */ + #define LUA_GC_CYCLE_PERIOD 50 + { + static long gc_count = 0; + + gc_count++; + if (gc_count == LUA_GC_CYCLE_PERIOD) { + lua_gc(lua,LUA_GCSTEP,LUA_GC_CYCLE_PERIOD); + gc_count = 0; + } + } + + if (err) { + addReplyErrorFormat(c,"Error running script (call to %s): %s\n", + run_ctx->funcname, lua_tostring(lua,-1)); + lua_pop(lua,1); /* Consume the Lua reply and remove error handler. */ + } else { + /* On success convert the Lua return value into Redis protocol, and + * send it to * the client. */ + luaReplyToRedisReply(c, run_ctx->c, lua); /* Convert and consume the reply. */ + } + + /* Perform some cleanup that we need to do both on error and success. */ + if (delhook) lua_sethook(lua,NULL,0,0); /* Disable hook */ + + /* remove run_ctx from registry, its only applicable for the current script. */ + luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, NULL); +} diff --git a/src/script_lua.h b/src/script_lua.h index 5ae9225bc..c18ad40bd 100644 --- a/src/script_lua.h +++ b/src/script_lua.h @@ -58,12 +58,9 @@ void luaRegisterRedisAPI(lua_State* lua); void luaEnableGlobalsProtection(lua_State *lua); -void luaSetGlobalArray(lua_State *lua, char *var, robj **elev, int elec); -void luaMaskCountHook(lua_State *lua, lua_Debug *ar); -void luaReplyToRedisReply(client *c, client* script_client, lua_State *lua); void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr); void* luaGetFromRegistry(lua_State* lua, const char* name); - +void luaCallFunction(scriptRunCtx* r_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled); #endif /* __SCRIPT_LUA_H_ */ diff --git a/src/server.h b/src/server.h index 2b1ef1ce7..3511ff04b 100644 --- a/src/server.h +++ b/src/server.h @@ -2706,6 +2706,7 @@ int ldbRemoveChild(pid_t pid); void ldbKillForkedSessions(void); int ldbPendingChildren(void); sds luaCreateFunction(client *c, robj *body); +void luaLdbLineHook(lua_State *lua, lua_Debug *ar); void freeLuaScriptsAsync(dict *lua_scripts); int ldbIsEnabled(); void ldbLog(sds entry); From cbd463175f8b52d594fd4e6b953fa58a5db053c3 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Thu, 7 Oct 2021 14:41:26 +0300 Subject: [PATCH 5/5] Redis Functions - Added redis function unit and Lua engine Redis function unit is located inside functions.c and contains Redis Function implementation: 1. FUNCTION commands: * FUNCTION CREATE * FCALL * FCALL_RO * FUNCTION DELETE * FUNCTION KILL * FUNCTION INFO 2. Register engine In addition, this commit introduce the first engine that uses the Redis Function capabilities, the Lua engine. --- README.md | 11 +- src/Makefile | 2 +- src/aof.c | 4 +- src/db.c | 5 + src/eval.c | 8 +- src/function_lua.c | 183 ++++++++++ src/functions.c | 538 ++++++++++++++++++++++++++++++ src/functions.h | 126 +++++++ src/object.c | 8 +- src/rdb.c | 104 +++++- src/rdb.h | 4 +- src/replication.c | 14 +- src/script.c | 38 ++- src/script.h | 7 +- src/script_lua.c | 16 +- src/script_lua.h | 3 +- src/server.c | 94 +++++- src/server.h | 18 +- tests/integration/replication.tcl | 19 ++ tests/unit/functions.tcl | 280 ++++++++++++++++ tests/unit/scripting.tcl | 308 +++++++++++------ 21 files changed, 1647 insertions(+), 143 deletions(-) create mode 100644 src/function_lua.c create mode 100644 src/functions.c create mode 100644 src/functions.h create mode 100644 tests/unit/functions.tcl diff --git a/README.md b/README.md index 308bbcb98..f84ba2504 100644 --- a/README.md +++ b/README.md @@ -443,6 +443,16 @@ This file also implements both the `SYNC` and `PSYNC` commands that are used in order to perform the first synchronization between masters and replicas, or to continue the replication after a disconnection. +Script +--- +The script unit is compose of 3 units +* `script.c` - integration of scripts with Redis (commands execution, set replication/resp, ..) +* `script_lua.c` - responsible to execute Lua code, uses script.c to interact with Redis from within the Lua code. +* `function_lua.c` - contains the Lua engine implementation, uses script_lua.c to execute the Lua code. +* `functions.c` - Contains Redis Functions implementation (FUNCTION command), uses functions_lua.c if the function it wants to invoke needs the Lua engine. +* `eval.c` - Contains the `eval` implementation using `script_lua.c` to invoke the Lua code. + + Other C files --- @@ -451,7 +461,6 @@ Other C files * `sds.c` is the Redis string library, check https://github.com/antirez/sds for more information. * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. * `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally. -* `scripting.c` implements Lua scripting. It is completely self-contained and isolated from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API. * `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][4]. [4]: https://redis.io/topics/cluster-spec diff --git a/src/Makefile b/src/Makefile index 076305b58..1bc6d86ed 100644 --- a/src/Makefile +++ b/src/Makefile @@ -309,7 +309,7 @@ endif REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX) -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX) REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX) diff --git a/src/aof.c b/src/aof.c index 4b9900ab2..8f609edc6 100644 --- a/src/aof.c +++ b/src/aof.c @@ -30,6 +30,7 @@ #include "server.h" #include "bio.h" #include "rio.h" +#include "functions.h" #include #include @@ -754,7 +755,8 @@ int loadAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); - if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL,server.db) != C_OK) { + + if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL) != C_OK) { serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted"); goto readerr; } else { diff --git a/src/db.c b/src/db.c index 807653767..d0c8b5903 100644 --- a/src/db.c +++ b/src/db.c @@ -1744,6 +1744,11 @@ int evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult * return genericGetKeys(0, 2, 3, 1, argv, argc, result); } +int functionGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + UNUSED(cmd); + return genericGetKeys(0, 2, 3, 1, argv, argc, result); +} + int lmpopGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { UNUSED(cmd); return genericGetKeys(0, 1, 2, 1, argv, argc, result); diff --git a/src/eval.c b/src/eval.c index f8fe8bf37..977cad721 100644 --- a/src/eval.c +++ b/src/eval.c @@ -257,7 +257,7 @@ void scriptingInit(int setup) { /* Lua beginners often don't use "local", this is likely to introduce * subtle bugs in their code. To prevent problems we protect accesses * to global variables. */ - luaEnableGlobalsProtection(lua); + luaEnableGlobalsProtection(lua, 1); lctx.lua = lua; } @@ -443,6 +443,8 @@ void evalGenericCommand(client *c, int evalsha) { scriptRunCtx rctx; scriptPrepareForRun(&rctx, lctx.lua_client, c, lctx.lua_cur_script); + rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as legacy so we + will get legacy error messages and logs */ if (!lctx.lua_replicate_commands) rctx.flags |= SCRIPT_EVAL_REPLICATION; /* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */ if ((server.script_caller->cmd->proc == evalRoCommand || @@ -584,7 +586,7 @@ NULL addReplyBulkCBuffer(c,sha,40); forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF); } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { - scriptKill(c); + scriptKill(c, 1); } else if (c->argc == 3 && !strcasecmp(c->argv[1]->ptr,"debug")) { if (clientHasPendingReplies(c)) { addReplyError(c,"SCRIPT DEBUG must be called outside a pipeline"); @@ -610,7 +612,7 @@ NULL } unsigned long evalMemory() { - return lua_gc(lctx.lua, LUA_GCCOUNT, 0) * 1024LL; + return luaMemory(lctx.lua); } dict* evalScriptsDict() { diff --git a/src/function_lua.c b/src/function_lua.c new file mode 100644 index 000000000..864ced809 --- /dev/null +++ b/src/function_lua.c @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2021, Redis Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * function_lua.c unit provides the Lua engine functionality. + * Including registering the engine and implementing the engine + * callbacks: + * * Create a function from blob (usually text) + * * Invoke a function + * * Free function memory + * * Get memory usage + * + * Uses script_lua.c to run the Lua code. + */ + +#include "functions.h" +#include "script_lua.h" +#include +#include +#include + +#define LUA_ENGINE_NAME "LUA" +#define REGISTRY_ENGINE_CTX_NAME "__ENGINE_CTX__" +#define REGISTRY_ERROR_HANDLER_NAME "__ERROR_HANDLER__" + +/* Lua engine ctx */ +typedef struct luaEngineCtx { + lua_State *lua; +} luaEngineCtx; + +/* Lua function ctx */ +typedef struct luaFunctionCtx { + /* Special ID that allows getting the Lua function object from the Lua registry */ + int lua_function_ref; +} luaFunctionCtx; + +/* + * Compile a given blob and save it on the registry. + * Return a function ctx with Lua ref that allows to later retrieve the + * function from the registry. + * + * Return NULL on compilation error and set the error to the err variable + */ +static void* luaEngineCreate(void *engine_ctx, sds blob, sds *err) { + luaEngineCtx *lua_engine_ctx = engine_ctx; + lua_State *lua = lua_engine_ctx->lua; + if (luaL_loadbuffer(lua, blob, sdslen(blob), "@user_function")) { + *err = sdsempty(); + *err = sdscatprintf(*err, "Error compiling function: %s", + lua_tostring(lua, -1)); + lua_pop(lua, 1); + return NULL; + } + + serverAssert(lua_isfunction(lua, -1)); + + int lua_function_ref = luaL_ref(lua, LUA_REGISTRYINDEX); + + luaFunctionCtx *f_ctx = zmalloc(sizeof(*f_ctx)); + *f_ctx = (luaFunctionCtx ) { .lua_function_ref = lua_function_ref, }; + + return f_ctx; +} + +/* + * Invole the give function with the given keys and args + */ +static void luaEngineCall(scriptRunCtx *run_ctx, + void *engine_ctx, + void *compiled_function, + robj **keys, + size_t nkeys, + robj **args, + size_t nargs) +{ + luaEngineCtx *lua_engine_ctx = engine_ctx; + lua_State *lua = lua_engine_ctx->lua; + luaFunctionCtx *f_ctx = compiled_function; + + /* Push error handler */ + lua_pushstring(lua, REGISTRY_ERROR_HANDLER_NAME); + lua_gettable(lua, LUA_REGISTRYINDEX); + + lua_rawgeti(lua, LUA_REGISTRYINDEX, f_ctx->lua_function_ref); + + serverAssert(lua_isfunction(lua, -1)); + + luaCallFunction(run_ctx, lua, keys, nkeys, args, nargs, 0); + lua_pop(lua, 1); /* Pop error handler */ +} + +static size_t luaEngineGetUsedMemoy(void *engine_ctx) { + luaEngineCtx *lua_engine_ctx = engine_ctx; + return luaMemory(lua_engine_ctx->lua); +} + +static size_t luaEngineFunctionMemoryOverhead(void *compiled_function) { + return zmalloc_size(compiled_function); +} + +static size_t luaEngineMemoryOverhead(void *engine_ctx) { + luaEngineCtx *lua_engine_ctx = engine_ctx; + return zmalloc_size(lua_engine_ctx); +} + +static void luaEngineFreeFunction(void *engine_ctx, void *compiled_function) { + luaEngineCtx *lua_engine_ctx = engine_ctx; + lua_State *lua = lua_engine_ctx->lua; + luaFunctionCtx *f_ctx = compiled_function; + lua_unref(lua, f_ctx->lua_function_ref); + zfree(f_ctx); +} + +/* Initialize Lua engine, should be called once on start. */ +int luaEngineInitEngine() { + luaEngineCtx *lua_engine_ctx = zmalloc(sizeof(*lua_engine_ctx)); + lua_engine_ctx->lua = lua_open(); + + luaRegisterRedisAPI(lua_engine_ctx->lua); + + /* Save error handler to registry */ + lua_pushstring(lua_engine_ctx->lua, REGISTRY_ERROR_HANDLER_NAME); + char *errh_func = "local dbg = debug\n" + "local error_handler = function (err)\n" + " local i = dbg.getinfo(2,'nSl')\n" + " if i and i.what == 'C' then\n" + " i = dbg.getinfo(3,'nSl')\n" + " end\n" + " if i then\n" + " return i.source .. ':' .. i.currentline .. ': ' .. err\n" + " else\n" + " return err\n" + " end\n" + "end\n" + "return error_handler"; + luaL_loadbuffer(lua_engine_ctx->lua, errh_func, strlen(errh_func), "@err_handler_def"); + lua_pcall(lua_engine_ctx->lua,0,1,0); + lua_settable(lua_engine_ctx->lua, LUA_REGISTRYINDEX); + + /* save the engine_ctx on the registry so we can get it from the Lua interpreter */ + luaSaveOnRegistry(lua_engine_ctx->lua, REGISTRY_ENGINE_CTX_NAME, lua_engine_ctx); + + luaEnableGlobalsProtection(lua_engine_ctx->lua, 0); + + + engine *lua_engine = zmalloc(sizeof(*lua_engine)); + *lua_engine = (engine) { + .engine_ctx = lua_engine_ctx, + .create = luaEngineCreate, + .call = luaEngineCall, + .get_used_memory = luaEngineGetUsedMemoy, + .get_function_memory_overhead = luaEngineFunctionMemoryOverhead, + .get_engine_memory_overhead = luaEngineMemoryOverhead, + .free_function = luaEngineFreeFunction, + }; + return functionsRegisterEngine(LUA_ENGINE_NAME, lua_engine); +} diff --git a/src/functions.c b/src/functions.c new file mode 100644 index 000000000..b0d7a2305 --- /dev/null +++ b/src/functions.c @@ -0,0 +1,538 @@ +/* + * Copyright (c) 2021, Redis Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "functions.h" +#include "sds.h" +#include "dict.h" +#include "adlist.h" +#include "atomicvar.h" + +static size_t engine_cache_memory = 0; + +/* Forward declaration */ +static void engineFunctionDispose(dict *d, void *obj); + +struct functionsCtx { + dict *functions; /* Function name -> Function object that can be used to run the function */ + size_t cache_memory /* Overhead memory (structs, dictionaries, ..) used by all the functions */; +}; + +dictType engineDictType = { + dictSdsCaseHash, /* hash function */ + dictSdsDup, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCaseCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + NULL, /* val destructor */ + NULL /* allow to expand */ +}; + +dictType functionDictType = { + dictSdsHash, /* hash function */ + dictSdsDup, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + engineFunctionDispose,/* val destructor */ + NULL /* allow to expand */ +}; + +/* Dictionary of engines */ +static dict *engines = NULL; + +/* Functions Ctx. + * Contains the dictionary that map a function name to + * function object and the cache memory used by all the functions */ +static functionsCtx *functions_ctx = NULL; + +static size_t functionMallocSize(functionInfo *fi) { + return zmalloc_size(fi) + sdsZmallocSize(fi->name) + + (fi->desc ? sdsZmallocSize(fi->desc) : 0) + + sdsZmallocSize(fi->code) + + fi->ei->engine->get_function_memory_overhead(fi->function); +} + +/* Dispose function memory */ +static void engineFunctionDispose(dict *d, void *obj) { + UNUSED(d); + functionInfo *fi = obj; + sdsfree(fi->code); + sdsfree(fi->name); + if (fi->desc) { + sdsfree(fi->desc); + } + engine *engine = fi->ei->engine; + engine->free_function(engine->engine_ctx, fi->function); + zfree(fi); +} + +/* Free function memory and detele it from the functions dictionary */ +static void engineFunctionFree(functionInfo *fi, functionsCtx *functions) { + functions->cache_memory -= functionMallocSize(fi); + + dictDelete(functions->functions, fi->name); +} + +/* Clear all the functions from the given functions ctx */ +void functionsCtxClear(functionsCtx *functions_ctx) { + dictEmpty(functions_ctx->functions, NULL); + functions_ctx->cache_memory = 0; +} + +/* Free the given functions ctx */ +void functionsCtxFree(functionsCtx *functions_ctx) { + functionsCtxClear(functions_ctx); + dictRelease(functions_ctx->functions); + zfree(functions_ctx); +} + +/* Swap the current functions ctx with the given one. + * Free the old functions ctx. */ +void functionsCtxSwapWithCurrent(functionsCtx *new_functions_ctx) { + functionsCtxFree(functions_ctx); + functions_ctx = new_functions_ctx; +} + +/* return the current functions ctx */ +functionsCtx* functionsCtxGetCurrent() { + return functions_ctx; +} + +/* Create a new functions ctx */ +functionsCtx* functionsCtxCreate() { + functionsCtx *ret = zmalloc(sizeof(functionsCtx)); + ret->functions = dictCreate(&functionDictType); + ret->cache_memory = 0; + return ret; +} + +/* + * Register a function info to functions dictionary + * 1. Set the function client + * 2. Add function to functions dictionary + * 3. update cache memory + */ +static void engineFunctionRegister(functionInfo *fi, functionsCtx *functions) { + int res = dictAdd(functions->functions, fi->name, fi); + serverAssert(res == DICT_OK); + + functions->cache_memory += functionMallocSize(fi); +} + +/* + * Creating a function info object and register it. + * Return the created object + */ +static functionInfo* engineFunctionCreate(sds name, void *function, engineInfo *ei, + sds desc, sds code, functionsCtx *functions) +{ + + functionInfo *fi = zmalloc(sizeof(*fi)); + *fi = (functionInfo ) { + .name = sdsdup(name), + .function = function, + .ei = ei, + .code = sdsdup(code), + .desc = desc ? sdsdup(desc) : NULL, + }; + + engineFunctionRegister(fi, functions); + + return fi; +} + +/* Register an engine, should be called once by the engine on startup and give the following: + * + * - engine_name - name of the engine to register + * - engine_ctx - the engine ctx that should be used by Redis to interact with the engine */ +int functionsRegisterEngine(const char *engine_name, engine *engine) { + sds engine_name_sds = sdsnew(engine_name); + if (dictFetchValue(engines, engine_name_sds)) { + serverLog(LL_WARNING, "Same engine was registered twice"); + sdsfree(engine_name_sds); + return C_ERR; + } + + client *c = createClient(NULL); + c->flags |= (CLIENT_DENY_BLOCKING | CLIENT_SCRIPT); + engineInfo *ei = zmalloc(sizeof(*ei)); + *ei = (engineInfo ) { .name = engine_name_sds, .engine = engine, .c = c,}; + + dictAdd(engines, engine_name_sds, ei); + + engine_cache_memory += zmalloc_size(ei) + sdsZmallocSize(ei->name) + + zmalloc_size(engine) + + engine->get_engine_memory_overhead(engine->engine_ctx); + + return C_OK; +} + +/* + * FUNCTION STATS + */ +void functionsStatsCommand(client *c) { + if (scriptIsRunning() && scriptIsEval()) { + addReplyErrorObject(c, shared.slowevalerr); + return; + } + + addReplyMapLen(c, 2); + + addReplyBulkCString(c, "running_script"); + if (!scriptIsRunning()) { + addReplyNull(c); + } else { + addReplyMapLen(c, 3); + addReplyBulkCString(c, "name"); + addReplyBulkCString(c, scriptCurrFunction()); + addReplyBulkCString(c, "command"); + client *script_client = scriptGetCaller(); + addReplyArrayLen(c, script_client->argc); + for (int i = 0 ; i < script_client->argc ; ++i) { + addReplyBulkCBuffer(c, script_client->argv[i]->ptr, sdslen(script_client->argv[i]->ptr)); + } + addReplyBulkCString(c, "duration_ms"); + addReplyLongLong(c, scriptRunDuration()); + } + + addReplyBulkCString(c, "engines"); + addReplyArrayLen(c, dictSize(engines)); + dictIterator *iter = dictGetIterator(engines); + dictEntry *entry = NULL; + while ((entry = dictNext(iter))) { + engineInfo *ei = dictGetVal(entry); + addReplyBulkCString(c, ei->name); + } + dictReleaseIterator(iter); +} + +/* + * FUNCTION LIST + */ +void functionsListCommand(client *c) { + /* general information on all the functions */ + addReplyArrayLen(c, dictSize(functions_ctx->functions)); + dictIterator *iter = dictGetIterator(functions_ctx->functions); + dictEntry *entry = NULL; + while ((entry = dictNext(iter))) { + functionInfo *fi = dictGetVal(entry); + addReplyMapLen(c, 3); + addReplyBulkCString(c, "name"); + addReplyBulkCBuffer(c, fi->name, sdslen(fi->name)); + addReplyBulkCString(c, "engine"); + addReplyBulkCBuffer(c, fi->ei->name, sdslen(fi->ei->name)); + addReplyBulkCString(c, "description"); + if (fi->desc) { + addReplyBulkCBuffer(c, fi->desc, sdslen(fi->desc)); + } else { + addReplyNull(c); + } + } + dictReleaseIterator(iter); +} + +/* + * FUNCTION INFO [WITHCODE] + */ +void functionsInfoCommand(client *c) { + if (c->argc > 4) { + addReplyErrorFormat(c,"wrong number of arguments for '%s' command or subcommand", c->cmd->name); + return; + } + /* dedicated information on specific function */ + robj *function_name = c->argv[2]; + int with_code = 0; + if (c->argc == 4) { + robj *with_code_arg = c->argv[3]; + if (!strcasecmp(with_code_arg->ptr, "withcode")) { + with_code = 1; + } + } + + functionInfo *fi = dictFetchValue(functions_ctx->functions, function_name->ptr); + if (!fi) { + addReplyError(c, "Function does not exists"); + return; + } + addReplyMapLen(c, with_code? 4 : 3); + addReplyBulkCString(c, "name"); + addReplyBulkCBuffer(c, fi->name, sdslen(fi->name)); + addReplyBulkCString(c, "engine"); + addReplyBulkCBuffer(c, fi->ei->name, sdslen(fi->ei->name)); + addReplyBulkCString(c, "description"); + if (fi->desc) { + addReplyBulkCBuffer(c, fi->desc, sdslen(fi->desc)); + } else { + addReplyNull(c); + } + if (with_code) { + addReplyBulkCString(c, "code"); + addReplyBulkCBuffer(c, fi->code, sdslen(fi->code)); + } +} + +/* + * FUNCTION DELETE + */ +void functionsDeleteCommand(client *c) { + if (server.masterhost && server.repl_slave_ro && !(c->flags & CLIENT_MASTER)) { + addReplyError(c, "Can not delete a function on a read only replica"); + return; + } + + robj *function_name = c->argv[2]; + functionInfo *fi = dictFetchValue(functions_ctx->functions, function_name->ptr); + if (!fi) { + addReplyError(c, "Function not found"); + return; + } + + engineFunctionFree(fi, functions_ctx); + forceCommandPropagation(c, PROPAGATE_REPL | PROPAGATE_AOF); + addReply(c, shared.ok); +} + +void functionsKillCommand(client *c) { + scriptKill(c, 0); +} + +static void fcallCommandGeneric(client *c, int ro) { + robj *function_name = c->argv[1]; + functionInfo *fi = dictFetchValue(functions_ctx->functions, function_name->ptr); + if (!fi) { + addReplyError(c, "Function not found"); + return; + } + engine *engine = fi->ei->engine; + + long long numkeys; + /* Get the number of arguments that are keys */ + if (getLongLongFromObject(c->argv[2], &numkeys) != C_OK) { + addReplyError(c, "Bad number of keys provided"); + return; + } + if (numkeys > (c->argc - 3)) { + addReplyError(c, "Number of keys can't be greater than number of args"); + return; + } else if (numkeys < 0) { + addReplyError(c, "Number of keys can't be negative"); + return; + } + + scriptRunCtx run_ctx; + + scriptPrepareForRun(&run_ctx, fi->ei->c, c, fi->name); + if (ro) { + run_ctx.flags |= SCRIPT_READ_ONLY; + } + engine->call(&run_ctx, engine->engine_ctx, fi->function, c->argv + 3, numkeys, + c->argv + 3 + numkeys, c->argc - 3 - numkeys); + scriptResetRun(&run_ctx); +} + +/* + * FCALL nkeys + */ +void fcallCommand(client *c) { + fcallCommandGeneric(c, 0); +} + +/* + * FCALL_RO nkeys + */ +void fcallCommandReadOnly(client *c) { + fcallCommandGeneric(c, 1); +} + +void functionsHelpCommand(client *c) { + const char *help[] = { +"CREATE [REPLACE] [DESC ] ", +" Create a new function with the given function name and code.", +"DELETE ", +" Delete the given function.", +"INFO [WITHCODE]", +" For each function, print the following information about the function:", +" * Function name", +" * The engine used to run the function", +" * Function description", +" * Function code (only if WITHCODE is given)", +"LIST", +" Return general information on all the functions:", +" * Function name", +" * The engine used to run the function", +" * Function description", +"STATS", +" Return information about the current function running:", +" * Function name", +" * Command used to run the function", +" * Duration in MS that the function is running", +" If not function is running, return nil", +" In addition, returns a list of available engines.", +"KILL", +" Kill the current running function.", +NULL }; + addReplyHelp(c, help); +} + +/* Compile and save the given function, return C_OK on success and C_ERR on failure. + * In case on failure the err out param is set with relevant error message */ +int functionsCreateWithFunctionCtx(sds function_name,sds engine_name, sds desc, sds code, + int replace, sds* err, functionsCtx *functions) { + engineInfo *ei = dictFetchValue(engines, engine_name); + if (!ei) { + *err = sdsnew("Engine not found"); + return C_ERR; + } + engine *engine = ei->engine; + + functionInfo *fi = dictFetchValue(functions->functions, function_name); + if (fi && !replace) { + *err = sdsnew("Function already exists"); + return C_ERR; + } + + void *function = engine->create(engine->engine_ctx, code, err); + if (*err) { + return C_ERR; + } + + if (fi) { + /* free the already existing function as we are going to replace it */ + engineFunctionFree(fi, functions); + } + + engineFunctionCreate(function_name, function, ei, desc, code, functions); + + return C_OK; +} + +/* + * FUNCTION CREATE + * [REPLACE] [DESC ] + * + * ENGINE NAME - name of the engine to use the run the function + * FUNCTION NAME - name to use to invoke the function + * REPLACE - optional, replace existing function + * DESCRIPTION - optional, function description + * FUNCTION CODE - function code to pass to the engine + */ +void functionsCreateCommand(client *c) { + + if (server.masterhost && server.repl_slave_ro && !(c->flags & CLIENT_MASTER)) { + addReplyError(c, "Can not create a function on a read only replica"); + return; + } + + robj *engine_name = c->argv[2]; + robj *function_name = c->argv[3]; + + int replace = 0; + int argc_pos = 4; + sds desc = NULL; + while (argc_pos < c->argc - 1) { + robj *next_arg = c->argv[argc_pos++]; + if (!strcasecmp(next_arg->ptr, "replace")) { + replace = 1; + continue; + } + if (!strcasecmp(next_arg->ptr, "description")) { + if (argc_pos >= c->argc) { + addReplyError(c, "Bad function description"); + return; + } + desc = c->argv[argc_pos++]->ptr; + continue; + } + } + + if (argc_pos >= c->argc) { + addReplyError(c, "Function code is missing"); + return; + } + + robj *code = c->argv[argc_pos]; + sds err = NULL; + if (functionsCreateWithFunctionCtx(function_name->ptr, engine_name->ptr, + desc, code->ptr, replace, &err, functions_ctx) != C_OK) + { + addReplyErrorSds(c, err); + return; + } + forceCommandPropagation(c, PROPAGATE_REPL | PROPAGATE_AOF); + addReply(c, shared.ok); +} + +/* Return memory usage of all the engines combine */ +unsigned long functionsMemory() { + dictIterator *iter = dictGetIterator(engines); + dictEntry *entry = NULL; + size_t engines_nemory = 0; + while ((entry = dictNext(iter))) { + engineInfo *ei = dictGetVal(entry); + engine *engine = ei->engine; + engines_nemory += engine->get_used_memory(engine->engine_ctx); + } + dictReleaseIterator(iter); + + return engines_nemory; +} + +/* Return memory overhead of all the engines combine */ +unsigned long functionsMemoryOverhead() { + size_t memory_overhead = dictSize(engines) * sizeof(dictEntry) + + dictSlots(engines) * sizeof(dictEntry*); + memory_overhead += dictSize(functions_ctx->functions) * sizeof(dictEntry) + + dictSlots(functions_ctx->functions) * sizeof(dictEntry*) + sizeof(functionsCtx); + memory_overhead += functions_ctx->cache_memory; + memory_overhead += engine_cache_memory; + + return memory_overhead; +} + +/* Returns the number of functions */ +unsigned long functionsNum() { + return dictSize(functions_ctx->functions); +} + +dict* functionsGet() { + return functions_ctx->functions; +} + +/* Initialize engine data structures. + * Should be called once on server initialization */ +int functionsInit() { + engines = dictCreate(&engineDictType); + functions_ctx = functionsCtxCreate(); + + if (luaEngineInitEngine() != C_OK) { + return C_ERR; + } + + return C_OK; +} diff --git a/src/functions.h b/src/functions.h new file mode 100644 index 000000000..8675883df --- /dev/null +++ b/src/functions.h @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2021, Redis Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FUNCTIONS_H_ +#define __FUNCTIONS_H_ + +/* + * functions.c unit provides the Redis Functions API: + * * FUNCTION CREATE + * * FUNCTION CALL + * * FUNCTION DELETE + * * FUNCTION KILL + * * FUNCTION INFO + * + * Also contains implementation for: + * * Save/Load function from rdb + * * Register engines + */ + +#include "server.h" +#include "script.h" +#include "redismodule.h" + +typedef struct engine { + /* engine specific context */ + void *engine_ctx; + + /* Create function callback, get the engine_ctx, and function code. + * returns NULL on error and set sds to be the error message */ + void* (*create)(void *engine_ctx, sds code, sds *err); + + /* Invoking a function, r_ctx is an opaque object (from engine POV). + * The r_ctx should be used by the engine to interaction with Redis, + * such interaction could be running commands, set resp, or set + * replication mode + */ + void (*call)(scriptRunCtx *r_ctx, void *engine_ctx, void *compiled_function, + robj **keys, size_t nkeys, robj **args, size_t nargs); + + /* get current used memory by the engine */ + size_t (*get_used_memory)(void *engine_ctx); + + /* Return memory overhead for a given function, + * such memory is not counted as engine memory but as general + * structs memory that hold different information */ + size_t (*get_function_memory_overhead)(void *compiled_function); + + /* Return memory overhead for engine (struct size holding the engine)*/ + size_t (*get_engine_memory_overhead)(void *engine_ctx); + + /* free the given function */ + void (*free_function)(void *engine_ctx, void *compiled_function); +} engine; + +/* Hold information about an engine. + * Used on rdb.c so it must be declared here. */ +typedef struct engineInfo { + sds name; /* Name of the engine */ + engine *engine; /* engine callbacks that allows to interact with the engine */ + client *c; /* Client that is used to run commands */ +} engineInfo; + +/* Hold information about the specific function. + * Used on rdb.c so it must be declared here. */ +typedef struct functionInfo { + sds name; /* Function name */ + void *function; /* Opaque object that set by the function's engine and allow it + to run the function, usually it's the function compiled code. */ + engineInfo *ei; /* Pointer to the function engine */ + sds code; /* Function code */ + sds desc; /* Function description */ +} functionInfo; + +int functionsRegisterEngine(const char *engine_name, engine *engine_ctx); +int functionsCreateWithFunctionCtx(sds function_name, sds engine_name, sds desc, sds code, + int replace, sds* err, functionsCtx *functions); +void functionsCreateCommand(client *c); +void fcallCommand(client *c); +void fcallCommandReadOnly(client *c); +void functionsDeleteCommand(client *c); +void functionsKillCommand(client *c); +void functionsStatsCommand(client *c); +void functionsInfoCommand(client *c); +void functionsListCommand(client *c); +void functionsHelpCommand(client *c); +unsigned long functionsMemory(); +unsigned long functionsMemoryOverhead(); +int functionsLoad(rio *rdb, int ver); +unsigned long functionsNum(); +dict* functionsGet(); +functionsCtx* functionsCtxGetCurrent(); +functionsCtx* functionsCtxCreate(); +void functionsCtxFree(functionsCtx *functions_ctx); +void functionsCtxClear(functionsCtx *functions_ctx); +void functionsCtxSwapWithCurrent(functionsCtx *functions_ctx); + +int luaEngineInitEngine(); +int functionsInit(); + +#endif /* __FUNCTIONS_H_ */ diff --git a/src/object.c b/src/object.c index dab0648a8..7a5563ccb 100644 --- a/src/object.c +++ b/src/object.c @@ -29,6 +29,7 @@ */ #include "server.h" +#include "functions.h" #include #include @@ -1212,6 +1213,8 @@ struct redisMemOverhead *getMemoryOverheadData(void) { } mh->lua_caches = mem; mem_total+=mem; + mh->functions_caches = functionsMemoryOverhead(); + mem_total+=mh->functions_caches; for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; @@ -1527,7 +1530,7 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) { struct redisMemOverhead *mh = getMemoryOverheadData(); - addReplyMapLen(c,25+mh->num_dbs); + addReplyMapLen(c,26+mh->num_dbs); addReplyBulkCString(c,"peak.allocated"); addReplyLongLong(c,mh->peak_allocated); @@ -1553,6 +1556,9 @@ NULL addReplyBulkCString(c,"lua.caches"); addReplyLongLong(c,mh->lua_caches); + addReplyBulkCString(c,"functions.caches"); + addReplyLongLong(c,mh->functions_caches); + for (size_t j = 0; j < mh->num_dbs; j++) { char dbname[32]; snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid); diff --git a/src/rdb.c b/src/rdb.c index dcbc83785..28b29d65d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -32,6 +32,7 @@ #include "zipmap.h" #include "endianconv.h" #include "stream.h" +#include "functions.h" #include #include @@ -1239,6 +1240,25 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { if (rdbSaveInfoAuxFields(rdb,rdbflags,rsi) == -1) goto werr; if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr; + /* save functions */ + dict *functions = functionsGet(); + dictIterator *iter = dictGetIterator(functions); + dictEntry *entry = NULL; + while ((entry = dictNext(iter))) { + rdbSaveType(rdb, RDB_OPCODE_FUNCTION); + functionInfo* fi = dictGetVal(entry); + if (rdbSaveRawString(rdb, (unsigned char *) fi->name, sdslen(fi->name)) == -1) goto werr; + if (rdbSaveRawString(rdb, (unsigned char *) fi->ei->name, sdslen(fi->ei->name)) == -1) goto werr; + if (fi->desc) { + if (rdbSaveLen(rdb, 1) == -1) goto werr; /* desc exists */ + if (rdbSaveRawString(rdb, (unsigned char *) fi->desc, sdslen(fi->desc)) == -1) goto werr; + } else { + if (rdbSaveLen(rdb, 0) == -1) goto werr; /* desc not exists */ + } + if (rdbSaveRawString(rdb, (unsigned char *) fi->code, sdslen(fi->code)) == -1) goto werr; + } + dictReleaseIterator(iter); + for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; dict *d = db->dict; @@ -2687,12 +2707,80 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { } } +static int rdbFunctionLoad(rio *rdb, int ver, functionsCtx* functions_ctx) { + UNUSED(ver); + sds name = NULL; + sds engine_name = NULL; + sds desc = NULL; + sds blob = NULL; + sds err = NULL; + uint64_t has_desc; + int res = C_ERR; + if (!(name = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) { + serverLog(LL_WARNING, "Failed loading function name"); + goto error; + } + + if (!(engine_name = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) { + serverLog(LL_WARNING, "Failed loading engine name"); + goto error; + } + + if ((has_desc = rdbLoadLen(rdb, NULL)) == RDB_LENERR) { + serverLog(LL_WARNING, "Failed loading function desc indicator"); + goto error; + } + + if (has_desc && !(desc = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) { + serverLog(LL_WARNING, "Failed loading function desc"); + goto error; + } + + if (!(blob = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) { + serverLog(LL_WARNING, "Failed loading function blob"); + goto error; + } + + if (functionsCreateWithFunctionCtx(name, engine_name, desc, blob, 0, &err, functions_ctx) != C_OK) { + serverLog(LL_WARNING, "Failed compiling and saving the function %s", err); + goto error; + } + + res = C_OK; + +error: + if (name) sdsfree(name); + if (engine_name) sdsfree(engine_name); + if (desc) sdsfree(desc); + if (blob) sdsfree(blob); + if (err) sdsfree(err); + return res; +} + /* Load an RDB file from the rio stream 'rdb'. On success C_OK is returned, * otherwise C_ERR is returned and 'errno' is set accordingly. */ -int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi, redisDb *dbarray) { +int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { + functionsCtx* functions_ctx = functionsCtxGetCurrent(); + functionsCtxClear(functions_ctx); + rdbLoadingCtx loading_ctx = { .dbarray = server.db, .functions_ctx = functions_ctx }; + int retval = rdbLoadRioWithLoadingCtx(rdb,rdbflags,rsi,&loading_ctx); + if (retval != C_OK) { + /* Loading failed, clear the function ctx */ + functionsCtxClear(functions_ctx); + } + return retval; +} + + +/* Load an RDB file from the rio stream 'rdb'. On success C_OK is returned, + * otherwise C_ERR is returned and 'errno' is set accordingly. + * The rdb_loading_ctx argument holds objects to which the rdb will be loaded to, + * currently it only allow to set db object and functionsCtx to which the data + * will be loaded (in the future it might contains more such objects). */ +int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx) { uint64_t dbid = 0; int type, rdbver; - redisDb *db = dbarray+0; + redisDb *db = rdb_loading_ctx->dbarray+0; char buf[1024]; int error; long long empty_keys_skipped = 0; @@ -2764,7 +2852,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi, redisDb *dbarray) { "databases. Exiting\n", server.dbnum); exit(1); } - db = dbarray+dbid; + db = rdb_loading_ctx->dbarray+dbid; continue; /* Read next opcode. */ } else if (type == RDB_OPCODE_RESIZEDB) { /* RESIZEDB: Hint about the size of the keys in the currently @@ -2895,6 +2983,12 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi, redisDb *dbarray) { decrRefCount(aux); continue; /* Read next opcode. */ } + } else if (type == RDB_OPCODE_FUNCTION) { + if (rdbFunctionLoad(rdb, rdbver, rdb_loading_ctx->functions_ctx) != C_OK) { + serverLog(LL_WARNING,"Failed loading function"); + goto eoferr; + } + continue; } /* Read key */ @@ -3044,7 +3138,9 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { if ((fp = fopen(filename,"r")) == NULL) return C_ERR; startLoadingFile(fp, filename,rdbflags); rioInitWithFile(&rdb,fp); - retval = rdbLoadRio(&rdb,rdbflags,rsi,server.db); + + retval = rdbLoadRio(&rdb,rdbflags,rsi); + fclose(fp); stopLoading(retval==C_OK); return retval; diff --git a/src/rdb.h b/src/rdb.h index f150bcb0d..66496bcec 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -100,6 +100,7 @@ #define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 18)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ +#define RDB_OPCODE_FUNCTION 246 /* engine data */ #define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */ #define RDB_OPCODE_IDLE 248 /* LRU idle time. */ #define RDB_OPCODE_FREQ 249 /* LFU frequency. */ @@ -166,7 +167,8 @@ int rdbSaveBinaryDoubleValue(rio *rdb, double val); int rdbLoadBinaryDoubleValue(rio *rdb, double *val); int rdbSaveBinaryFloatValue(rio *rdb, float val); int rdbLoadBinaryFloatValue(rio *rdb, float *val); -int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi, redisDb *db); +int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi); +int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx); int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi); rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi); diff --git a/src/replication.c b/src/replication.c index cfd7f1f37..66483d934 100644 --- a/src/replication.c +++ b/src/replication.c @@ -32,6 +32,7 @@ #include "server.h" #include "cluster.h" #include "bio.h" +#include "functions.h" #include #include @@ -1738,6 +1739,7 @@ void readSyncBulkPayload(connection *conn) { ssize_t nread, readlen, nwritten; int use_diskless_load = useDisklessLoad(); redisDb *diskless_load_tempDb = NULL; + functionsCtx* temp_functions_ctx = NULL; int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; off_t left; @@ -1913,6 +1915,7 @@ void readSyncBulkPayload(connection *conn) { if (use_diskless_load && server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) { /* Initialize empty tempDb dictionaries. */ diskless_load_tempDb = disklessLoadInitTempDb(); + temp_functions_ctx = functionsCtxCreate(); moduleFireServerEvent(REDISMODULE_EVENT_REPL_ASYNC_LOAD, REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_STARTED, @@ -1935,6 +1938,7 @@ void readSyncBulkPayload(connection *conn) { if (use_diskless_load) { rio rdb; redisDb *dbarray; + functionsCtx* functions_ctx; int asyncLoading = 0; if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) { @@ -1947,8 +1951,11 @@ void readSyncBulkPayload(connection *conn) { asyncLoading = 1; } dbarray = diskless_load_tempDb; + functions_ctx = temp_functions_ctx; } else { dbarray = server.db; + functions_ctx = functionsCtxGetCurrent(); + functionsCtxClear(functions_ctx); } rioInitWithConn(&rdb,conn,server.repl_transfer_size); @@ -1960,7 +1967,8 @@ void readSyncBulkPayload(connection *conn) { startLoading(server.repl_transfer_size, RDBFLAGS_REPLICATION, asyncLoading); int loadingFailed = 0; - if (rdbLoadRio(&rdb,RDBFLAGS_REPLICATION,&rsi,dbarray) != C_OK) { + rdbLoadingCtx loadingCtx = { .dbarray = dbarray, .functions_ctx = functions_ctx }; + if (rdbLoadRioWithLoadingCtx(&rdb,RDBFLAGS_REPLICATION,&rsi,&loadingCtx) != C_OK) { /* RDB loading failed. */ serverLog(LL_WARNING, "Failed trying to load the MASTER synchronization DB " @@ -1988,6 +1996,7 @@ void readSyncBulkPayload(connection *conn) { NULL); disklessLoadDiscardTempDb(diskless_load_tempDb); + functionsCtxFree(temp_functions_ctx); serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Discarding temporary DB in background"); } else { /* Remove the half-loaded data in case we started with an empty replica. */ @@ -2010,6 +2019,9 @@ void readSyncBulkPayload(connection *conn) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Swapping active DB with loaded DB"); swapMainDbWithTempDb(diskless_load_tempDb); + /* swap existing functions ctx with the temporary one */ + functionsCtxSwapWithCurrent(temp_functions_ctx); + moduleFireServerEvent(REDISMODULE_EVENT_REPL_ASYNC_LOAD, REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_COMPLETED, NULL); diff --git a/src/script.c b/src/script.c index 0d6014508..34e4953d3 100644 --- a/src/script.c +++ b/src/script.c @@ -60,6 +60,11 @@ client* scriptGetClient() { return curr_run_ctx->c; } +client* scriptGetCaller() { + serverAssert(scriptIsRunning()); + return curr_run_ctx->original_client; +} + /* interrupt function for scripts, should be call * from time to time to reply some special command (like ping) * and also check if the run should be terminated. */ @@ -78,8 +83,8 @@ int scriptInterrupt(scriptRunCtx *run_ctx) { serverLog(LL_WARNING, "Slow script detected: still in execution after %lld milliseconds. " - "You can try killing the script using the SCRIPT KILL command.", - elapsed); + "You can try killing the script using the %s command.", + elapsed, (run_ctx->flags & SCRIPT_EVAL_MODE) ? "SCRIPT KILL" : "FUNCTION KILL"); enterScriptTimedoutMode(run_ctx); /* Once the script timeouts we reenter the event loop to permit others @@ -159,8 +164,18 @@ int scriptIsRunning() { return curr_run_ctx != NULL; } +const char* scriptCurrFunction() { + serverAssert(scriptIsRunning()); + return curr_run_ctx->funcname; +} + +int scriptIsEval() { + serverAssert(scriptIsRunning()); + return curr_run_ctx->flags & SCRIPT_EVAL_MODE; +} + /* Kill the current running script */ -void scriptKill(client *c) { +void scriptKill(client *c, int is_eval) { if (!curr_run_ctx) { addReplyError(c, "-NOTBUSY No scripts in execution right now."); return; @@ -177,6 +192,16 @@ void scriptKill(client *c) { "using the SHUTDOWN NOSAVE command."); return; } + if (is_eval && !(curr_run_ctx->flags & SCRIPT_EVAL_MODE)) { + /* Kill a function with 'SCRIPT KILL' is not allow */ + addReplyErrorObject(c, shared.slowscripterr); + return; + } + if (!is_eval && (curr_run_ctx->flags & SCRIPT_EVAL_MODE)) { + /* Kill an eval with 'FUNCTION KILL' is not allow */ + addReplyErrorObject(c, shared.slowevalerr); + return; + } curr_run_ctx->flags |= SCRIPT_KILLED; addReply(c, shared.ok); } @@ -430,3 +455,10 @@ mstime_t scriptTimeSnapshot() { serverAssert(!curr_run_ctx); return curr_run_ctx->snapshot_time; } + +long long scriptRunDuration() { + serverAssert(scriptIsRunning()); + return elapsedMs(curr_run_ctx->start_time); +} + + diff --git a/src/script.h b/src/script.h index 4d5e92966..aeed72456 100644 --- a/src/script.h +++ b/src/script.h @@ -69,6 +69,7 @@ #define SCRIPT_READ_ONLY (1ULL<<5) /* indicate that the current script should only perform read commands */ #define SCRIPT_EVAL_REPLICATION (1ULL<<6) /* mode for eval, indicate that we replicate the script invocation and not the effects */ +#define SCRIPT_EVAL_MODE (1ULL<<7) /* Indicate that the current script called from legacy Lua */ typedef struct scriptRunCtx scriptRunCtx; struct scriptRunCtx { @@ -87,10 +88,14 @@ int scriptSetResp(scriptRunCtx *r_ctx, int resp); int scriptSetRepl(scriptRunCtx *r_ctx, int repl); void scriptCall(scriptRunCtx *r_ctx, robj **argv, int argc, sds *err); int scriptInterrupt(scriptRunCtx *r_ctx); -void scriptKill(client *c); +void scriptKill(client *c, int is_eval); int scriptIsRunning(); +const char* scriptCurrFunction(); +int scriptIsEval(); int scriptIsTimedout(); client* scriptGetClient(); +client* scriptGetCaller(); mstime_t scriptTimeSnapshot(); +long long scriptRunDuration(); #endif /* __SCRIPT_H_ */ diff --git a/src/script_lua.c b/src/script_lua.c index fb3fa397b..aa73c5fb3 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -867,6 +867,8 @@ cleanup: } c->user = NULL; + c->argv = NULL; + c->argc = 0; if (raise_error) { /* If we are here we should have an error in the stack, in the @@ -1067,8 +1069,12 @@ static void luaRemoveUnsupportedFunctions(lua_State *lua) { * the creation of globals accidentally. * * It should be the last to be called in the scripting engine initialization - * sequence, because it may interact with creation of globals. */ -void luaEnableGlobalsProtection(lua_State *lua) { + * sequence, because it may interact with creation of globals. + * + * On Legacy Lua (eval) we need to check 'w ~= \"main\"' otherwise we will not be able + * to create the global 'function ()' variable. On Lua engine we do not use this trick + * so its not needed. */ +void luaEnableGlobalsProtection(lua_State *lua, int is_eval) { char *s[32]; sds code = sdsempty(); int j = 0; @@ -1081,7 +1087,7 @@ void luaEnableGlobalsProtection(lua_State *lua) { s[j++]="mt.__newindex = function (t, n, v)\n"; s[j++]=" if dbg.getinfo(2) then\n"; s[j++]=" local w = dbg.getinfo(2, \"S\").what\n"; - s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; + s[j++]= is_eval ? " if w ~= \"main\" and w ~= \"C\" then\n" : " if w ~= \"C\" then\n"; s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"; s[j++]=" end\n"; s[j++]=" end\n"; @@ -1336,3 +1342,7 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t /* remove run_ctx from registry, its only applicable for the current script. */ luaSaveOnRegistry(lua, REGISTRY_RUN_CTX_NAME, NULL); } + +unsigned long luaMemory(lua_State *lua) { + return lua_gc(lua, LUA_GCCOUNT, 0) * 1024LL; +} diff --git a/src/script_lua.h b/src/script_lua.h index c18ad40bd..40cdd00b5 100644 --- a/src/script_lua.h +++ b/src/script_lua.h @@ -57,10 +57,11 @@ #define REGISTRY_RUN_CTX_NAME "__RUN_CTX__" void luaRegisterRedisAPI(lua_State* lua); -void luaEnableGlobalsProtection(lua_State *lua); +void luaEnableGlobalsProtection(lua_State *lua, int is_eval); void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr); void* luaGetFromRegistry(lua_State* lua, const char* name); void luaCallFunction(scriptRunCtx* r_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled); +unsigned long luaMemory(lua_State *lua); #endif /* __SCRIPT_LUA_H_ */ diff --git a/src/server.c b/src/server.c index 1fbf3fa3a..571eb1b06 100644 --- a/src/server.c +++ b/src/server.c @@ -35,7 +35,7 @@ #include "latency.h" #include "atomicvar.h" #include "mt19937-64.h" -#include "script.h" +#include "functions.h" #include #include @@ -472,6 +472,31 @@ struct redisCommand scriptSubcommands[] = { {NULL}, }; +struct redisCommand functionSubcommands[] = { + {"create",functionsCreateCommand,-5, + "may-replicate no-script @scripting"}, + + {"delete",functionsDeleteCommand,3, + "may-replicate no-script @scripting"}, + + {"kill",functionsKillCommand,2, + "no-script @scripting"}, + + {"info",functionsInfoCommand,-3, + "no-script @scripting"}, + + {"list",functionsListCommand,2, + "no-script @scripting"}, + + {"stats",functionsStatsCommand,2, + "no-script @scripting"}, + + {"help",functionsHelpCommand,2, + "ok-loading ok-stale @scripting"}, + + {NULL}, +}; + struct redisCommand clientSubcommands[] = { {"caching",clientCommand,3, "no-script ok-loading ok-stale @connection"}, @@ -2033,7 +2058,25 @@ struct redisCommand redisCommandTable[] = { "no-auth no-script ok-stale ok-loading fast @connection"}, {"failover",failoverCommand,-1, - "admin no-script ok-stale"} + "admin no-script ok-stale"}, + + {"function",NULL,-2, + "", + .subcommands=functionSubcommands}, + + {"fcall",fcallCommand,-3, + "no-script no-monitor may-replicate no-mandatory-keys @scripting", + {{"read write", /* We pass both read and write because these flag are worst-case-scenario */ + KSPEC_BS_INDEX,.bs.index={2}, + KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}}, + functionGetKeys}, + + {"fcall_ro",fcallCommandReadOnly,-3, + "no-script no-monitor no-mandatory-keys @scripting", + {{"read", + KSPEC_BS_INDEX,.bs.index={2}, + KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}}, + functionGetKeys}, }; /*============================ Utility functions ============================ */ @@ -2209,6 +2252,11 @@ void dictSdsDestructor(dict *d, void *val) sdsfree(val); } +void *dictSdsDup(dict *d, const void *key) { + UNUSED(d); + return sdsdup((const sds) key); +} + int dictObjKeyCompare(dict *d, const void *key1, const void *key2) { @@ -3517,8 +3565,10 @@ void createSharedObjects(void) { "-NOSCRIPT No matching script. Please use EVAL.\r\n")); shared.loadingerr = createObject(OBJ_STRING,sdsnew( "-LOADING Redis is loading the dataset in memory\r\n")); - shared.slowscripterr = createObject(OBJ_STRING,sdsnew( + shared.slowevalerr = createObject(OBJ_STRING,sdsnew( "-BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.\r\n")); + shared.slowscripterr = createObject(OBJ_STRING,sdsnew( + "-BUSY Redis is busy running a script. You can only call FUNCTION KILL or SHUTDOWN NOSAVE.\r\n")); shared.masterdownerr = createObject(OBJ_STRING,sdsnew( "-MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'.\r\n")); shared.bgsaveerr = createObject(OBJ_STRING,sdsnew( @@ -4345,6 +4395,7 @@ void initServer(void) { if (server.cluster_enabled) clusterInit(); replicationScriptCacheInit(); scriptingInit(1); + functionsInit(); slowlogInit(); latencyMonitorInit(); @@ -5448,9 +5499,15 @@ int processCommand(client *c) { tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && !(c->cmd->proc == scriptCommand && c->argc == 2 && - tolower(((char*)c->argv[1]->ptr)[0]) == 'k')) + tolower(((char*)c->argv[1]->ptr)[0]) == 'k') && + !(c->cmd->proc == functionsKillCommand) && + !(c->cmd->proc == functionsStatsCommand)) { - rejectCommand(c, shared.slowscripterr); + if (scriptIsEval()) { + rejectCommand(c, shared.slowevalerr); + } else { + rejectCommand(c, shared.slowscripterr); + } return C_OK; } @@ -6281,6 +6338,7 @@ sds genRedisInfoString(const char *section) { char peak_hmem[64]; char total_system_hmem[64]; char used_memory_lua_hmem[64]; + char used_memory_vm_total_hmem[64]; char used_memory_scripts_hmem[64]; char used_memory_rss_hmem[64]; char maxmemory_hmem[64]; @@ -6288,6 +6346,7 @@ sds genRedisInfoString(const char *section) { size_t total_system_mem = server.system_memory_size; const char *evict_policy = evictPolicyToString(); long long memory_lua = evalMemory(); + long long memory_functions = functionsMemory(); struct redisMemOverhead *mh = getMemoryOverheadData(); /* Peak memory is updated from time to time by serverCron() so it @@ -6301,7 +6360,8 @@ sds genRedisInfoString(const char *section) { bytesToHuman(peak_hmem,server.stat_peak_memory); bytesToHuman(total_system_hmem,total_system_mem); bytesToHuman(used_memory_lua_hmem,memory_lua); - bytesToHuman(used_memory_scripts_hmem,mh->lua_caches); + bytesToHuman(used_memory_vm_total_hmem,memory_functions + memory_lua); + bytesToHuman(used_memory_scripts_hmem,mh->lua_caches + mh->functions_caches); bytesToHuman(used_memory_rss_hmem,server.cron_malloc_stats.process_rss); bytesToHuman(maxmemory_hmem,server.maxmemory); @@ -6324,11 +6384,18 @@ sds genRedisInfoString(const char *section) { "allocator_resident:%zu\r\n" "total_system_memory:%lu\r\n" "total_system_memory_human:%s\r\n" - "used_memory_lua:%lld\r\n" - "used_memory_lua_human:%s\r\n" + "used_memory_lua:%lld\r\n" /* deprecated, renamed to used_memory_vm_eval */ + "used_memory_vm_eval:%lld\r\n" + "used_memory_lua_human:%s\r\n" /* deprecated */ + "used_memory_scripts_eval:%lld\r\n" + "number_of_cached_scripts:%lu\r\n" + "number_of_functions:%lu\r\n" + "used_memory_vm_functions:%lld\r\n" + "used_memory_vm_total:%lld\r\n" + "used_memory_vm_total_human:%s\r\n" + "used_memory_functions:%lld\r\n" "used_memory_scripts:%lld\r\n" "used_memory_scripts_human:%s\r\n" - "number_of_cached_scripts:%lu\r\n" "maxmemory:%lld\r\n" "maxmemory_human:%s\r\n" "maxmemory_policy:%s\r\n" @@ -6367,10 +6434,17 @@ sds genRedisInfoString(const char *section) { (unsigned long)total_system_mem, total_system_hmem, memory_lua, + memory_lua, used_memory_lua_hmem, (long long) mh->lua_caches, - used_memory_scripts_hmem, dictSize(evalScriptsDict()), + functionsNum(), + memory_functions, + memory_functions + memory_lua, + used_memory_vm_total_hmem, + (long long) mh->functions_caches, + (long long) mh->lua_caches + (long long) mh->functions_caches, + used_memory_scripts_hmem, server.maxmemory, maxmemory_hmem, evict_policy, diff --git a/src/server.h b/src/server.h index 3511ff04b..535d21dd9 100644 --- a/src/server.h +++ b/src/server.h @@ -822,6 +822,19 @@ typedef struct redisDb { clusterSlotToKeyMapping *slots_to_keys; /* Array of slots to keys. Only used in cluster mode (db 0). */ } redisDb; +/* forward declaration for functions ctx */ +typedef struct functionsCtx functionsCtx; + +/* Holding object that need to be populated during + * rdb loading. On loading end it is possible to decide + * whether not to set those objects on their rightful place. + * For example: dbarray need to be set as main database on + * successful loading and dropped on failure. */ +typedef struct rdbLoadingCtx { + redisDb* dbarray; + functionsCtx* functions_ctx; +}rdbLoadingCtx; + /* Client MULTI/EXEC state */ typedef struct multiCmd { robj **argv; @@ -1122,7 +1135,7 @@ struct sharedObjectsStruct { robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space, *queued, *null[4], *nullarray[4], *emptymap[4], *emptyset[4], *emptyarray, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, - *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, + *outofrangeerr, *noscripterr, *loadingerr, *slowevalerr, *slowscripterr, *bgsaveerr, *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, @@ -1203,6 +1216,7 @@ struct redisMemOverhead { size_t clients_normal; size_t aof_buffer; size_t lua_caches; + size_t functions_caches; size_t overhead_total; size_t dataset; size_t total_keys; @@ -2670,6 +2684,7 @@ int sintercardGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysRes int zunionInterDiffGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result); int zunionInterDiffStoreGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result); int evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); +int functionGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); int sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); int migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); int georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); @@ -2761,6 +2776,7 @@ uint64_t dictSdsCaseHash(const void *key); int dictSdsKeyCompare(dict *d, const void *key1, const void *key2); int dictSdsKeyCaseCompare(dict *d, const void *key1, const void *key2); void dictSdsDestructor(dict *d, void *val); +void *dictSdsDup(dict *d, const void *key); /* Git SHA1 */ char *redisGitSHA1(void); diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index fe4adbe93..1332c8380 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -521,6 +521,12 @@ foreach testType {Successful Aborted} { # Set a key value on replica to check status during loading, on failure and after swapping db $replica set mykey myvalue + # Set a function value on replica to check status during loading, on failure and after swapping db + $replica function create LUA test {return 'hello1'} + + # Set a function value on master to check it reaches the replica when replication ends + $master function create LUA test {return 'hello2'} + # Force the replica to try another full sync (this time it will have matching master replid) $master multi $master client kill type replica @@ -552,6 +558,9 @@ foreach testType {Successful Aborted} { # Ensure we still see old values while async_loading is in progress and also not LOADING status assert_equal [$replica get mykey] "myvalue" + # Ensure we still can call old function while async_loading is in progress + assert_equal [$replica fcall test 0] "hello1" + # Make sure we're still async_loading to validate previous assertion assert_equal [s -1 async_loading] 1 @@ -576,6 +585,9 @@ foreach testType {Successful Aborted} { # Ensure we see old values from replica assert_equal [$replica get mykey] "myvalue" + # Ensure we still can call old function + assert_equal [$replica fcall test 0] "hello1" + # Make sure amount of replica keys didn't change assert_equal [$replica dbsize] 2001 } @@ -595,6 +607,9 @@ foreach testType {Successful Aborted} { # Ensure we don't see anymore the key that was stored only to replica and also that we don't get LOADING status assert_equal [$replica GET mykey] "" + # Ensure we got the new function + assert_equal [$replica fcall test 0] "hello2" + # Make sure amount of keys matches master assert_equal [$replica dbsize] 1010 } @@ -624,6 +639,10 @@ test {diskless loading short read} { $replica config set dynamic-hz no # Try to fill the master with all types of data types / encodings set start [clock clicks -milliseconds] + + # Set a function value to check short read handling on functions + r function create LUA test {return 'hello1'} + for {set k 0} {$k < 3} {incr k} { for {set i 0} {$i < 10} {incr i} { r set "$k int_$i" [expr {int(rand()*10000)}] diff --git a/tests/unit/functions.tcl b/tests/unit/functions.tcl new file mode 100644 index 000000000..0736a44da --- /dev/null +++ b/tests/unit/functions.tcl @@ -0,0 +1,280 @@ +start_server {tags {"scripting"}} { + test {FUNCTION - Basic usage} { + r function create LUA test {return 'hello'} + r fcall test 0 + } {hello} + + test {FUNCTION - Create an already exiting function raise error} { + catch { + r function create LUA test {return 'hello1'} + } e + set _ $e + } {*Function already exists*} + + test {FUNCTION - Create function with unexisting engine} { + catch { + r function create bad_engine test {return 'hello1'} + } e + set _ $e + } {*Engine not found*} + + test {FUNCTION - Test uncompiled script} { + catch { + r function create LUA test1 {bad script} + } e + set _ $e + } {*Error compiling function*} + + test {FUNCTION - test replace argument} { + r function create LUA test REPLACE {return 'hello1'} + r fcall test 0 + } {hello1} + + test {FUNCTION - test replace argument with function creation failure keeps old function} { + catch {r function create LUA test REPLACE {error}} + r fcall test 0 + } {hello1} + + test {FUNCTION - test function delete} { + r function delete test + catch { + r fcall test 0 + } e + set _ $e + } {*Function not found*} + + test {FUNCTION - test description argument} { + r function create LUA test DESCRIPTION {some description} {return 'hello'} + r function list + } {{name test engine LUA description {some description}}} + + test {FUNCTION - test info specific function} { + r function info test WITHCODE + } {name test engine LUA description {some description} code {return 'hello'}} + + test {FUNCTION - test info without code} { + r function info test + } {name test engine LUA description {some description}} + + test {FUNCTION - test info on function that does not exists} { + catch { + r function info bad_function_name + } e + set _ $e + } {*Function does not exists*} + + test {FUNCTION - test info with bad number of arguments} { + catch { + r function info test WITHCODE bad_arg + } e + set _ $e + } {*wrong number of arguments*} + + test {FUNCTION - test fcall bad arguments} { + catch { + r fcall test bad_arg + } e + set _ $e + } {*Bad number of keys provided*} + + test {FUNCTION - test fcall bad number of keys arguments} { + catch { + r fcall test 10 key1 + } e + set _ $e + } {*Number of keys can't be greater than number of args*} + + test {FUNCTION - test fcall negative number of keys} { + catch { + r fcall test -1 key1 + } e + set _ $e + } {*Number of keys can't be negative*} + + test {FUNCTION - test function delete on not exiting function} { + catch { + r function delete test1 + } e + set _ $e + } {*Function not found*} + + test {FUNCTION - test function kill when function is not running} { + catch { + r function kill + } e + set _ $e + } {*No scripts in execution*} + + test {FUNCTION - test wrong subcommand} { + catch { + r function bad_subcommand + } e + set _ $e + } {*Unknown subcommand*} + + test {FUNCTION - test loading from rdb} { + r debug reload + r fcall test 0 + } {hello} + + test {FUNCTION - test fcall_ro with write command} { + r function create lua test REPLACE {return redis.call('set', 'x', '1')} + catch { r fcall_ro test 0 } e + set _ $e + } {*Write commands are not allowed from read-only scripts*} + + test {FUNCTION - test fcall_ro with read only commands} { + r function create lua test REPLACE {return redis.call('get', 'x')} + r set x 1 + r fcall_ro test 0 + } {1} + + test {FUNCTION - test keys and argv} { + r function create lua test REPLACE {return redis.call('set', KEYS[1], ARGV[1])} + r fcall test 1 x foo + r get x + } {foo} + + test {FUNCTION - test command get keys on fcall} { + r COMMAND GETKEYS fcall test 1 x foo + } {x} + + test {FUNCTION - test command get keys on fcall_ro} { + r COMMAND GETKEYS fcall_ro test 1 x foo + } {x} + + test {FUNCTION - test function kill} { + set rd [redis_deferring_client] + r config set script-time-limit 10 + r function create lua test REPLACE {local a = 1 while true do a = a + 1 end} + $rd fcall test 0 + after 200 + catch {r ping} e + assert_match {BUSY*} $e + assert_match {running_script {name test command {fcall test 0} duration_ms *} engines LUA} [r FUNCTION STATS] + r function kill + after 200 ; # Give some time to Lua to call the hook again... + assert_equal [r ping] "PONG" + } + + test {FUNCTION - test script kill not working on function} { + set rd [redis_deferring_client] + r config set script-time-limit 10 + r function create lua test REPLACE {local a = 1 while true do a = a + 1 end} + $rd fcall test 0 + after 200 + catch {r ping} e + assert_match {BUSY*} $e + catch {r script kill} e + assert_match {BUSY*} $e + r function kill + after 200 ; # Give some time to Lua to call the hook again... + assert_equal [r ping] "PONG" + } + + test {FUNCTION - test function kill not working on eval} { + set rd [redis_deferring_client] + r config set script-time-limit 10 + $rd eval {local a = 1 while true do a = a + 1 end} 0 + after 200 + catch {r ping} e + assert_match {BUSY*} $e + catch {r function kill} e + assert_match {BUSY*} $e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + assert_equal [r ping] "PONG" + } +} + +start_server {tags {"scripting repl"}} { + start_server {} { + test "Connect a replica to the master instance" { + r -1 slaveof [srv 0 host] [srv 0 port] + wait_for_condition 50 100 { + [s -1 role] eq {slave} && + [string match {*master_link_status:up*} [r -1 info replication]] + } else { + fail "Can't turn the instance into a replica" + } + } + + test {FUNCTION - creation is replicated to replica} { + r function create LUA test DESCRIPTION {some description} {return 'hello'} + wait_for_condition 50 100 { + [r -1 function list] eq {{name test engine LUA description {some description}}} + } else { + fail "Failed waiting for function to replicate to replica" + } + } + + test {FUNCTION - call on replica} { + r -1 fcall test 0 + } {hello} + + test {FUNCTION - delete is replicated to replica} { + r function delete test + wait_for_condition 50 100 { + [r -1 function list] eq {} + } else { + fail "Failed waiting for function to replicate to replica" + } + } + + test "Disconnecting the replica from master instance" { + r -1 slaveof no one + # creating a function after disconnect to make sure function + # is replicated on rdb phase + r function create LUA test DESCRIPTION {some description} {return 'hello'} + + # reconnect the replica + r -1 slaveof [srv 0 host] [srv 0 port] + wait_for_condition 50 100 { + [s -1 role] eq {slave} && + [string match {*master_link_status:up*} [r -1 info replication]] + } else { + fail "Can't turn the instance into a replica" + } + } + + test "FUNCTION - test replication to replica on rdb phase" { + r -1 fcall test 0 + } {hello} + + test "FUNCTION - test replication to replica on rdb phase info command" { + r -1 function info test WITHCODE + } {name test engine LUA description {some description} code {return 'hello'}} + + test "FUNCTION - create on read only replica" { + catch { + r -1 function create LUA test DESCRIPTION {some description} {return 'hello'} + } e + set _ $e + } {*Can not create a function on a read only replica*} + + test "FUNCTION - delete on read only replica" { + catch { + r -1 function delete test + } e + set _ $e + } {*Can not delete a function on a read only replica*} + + test "FUNCTION - function effect is replicated to replica" { + r function create LUA test REPLACE {return redis.call('set', 'x', '1')} + r fcall test 0 + assert {[r get x] eq {1}} + wait_for_condition 50 100 { + [r -1 get x] eq {1} + } else { + fail "Failed waiting function effect to be replicated to replica" + } + } + + test "FUNCTION - modify key space of read only replica" { + catch { + r -1 fcall test 0 + } e + set _ $e + } {*can't write against a read only replica*} + } +} \ No newline at end of file diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index f62f68970..09c021e56 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -1,48 +1,87 @@ +foreach is_eval {0 1} { + +if {$is_eval == 1} { + proc run_script {args} { + r eval {*}$args + } + proc run_script_ro {args} { + r eval_ro {*}$args + } + proc run_script_on_connection {args} { + [lindex $args 0] eval {*}[lrange $args 1 end] + } + proc kill_script {args} { + r script kill + } +} else { + proc run_script {args} { + r function create LUA test replace [lindex $args 0] + r fcall test {*}[lrange $args 1 end] + } + proc run_script_ro {args} { + r function create LUA test replace [lindex $args 0] + r fcall_ro test {*}[lrange $args 1 end] + } + proc run_script_on_connection {args} { + set rd [lindex $args 0] + $rd function create LUA test replace [lindex $args 1] + # read the ok reply of function create + $rd read + $rd fcall test {*}[lrange $args 2 end] + } + proc kill_script {args} { + r function kill + } +} + start_server {tags {"scripting"}} { + test {EVAL - Does Lua interpreter replies to our requests?} { - r eval {return 'hello'} 0 + run_script {return 'hello'} 0 } {hello} test {EVAL - Lua integer -> Redis protocol type conversion} { - r eval {return 100.5} 0 + run_script {return 100.5} 0 } {100} test {EVAL - Lua string -> Redis protocol type conversion} { - r eval {return 'hello world'} 0 + run_script {return 'hello world'} 0 } {hello world} test {EVAL - Lua true boolean -> Redis protocol type conversion} { - r eval {return true} 0 + run_script {return true} 0 } {1} test {EVAL - Lua false boolean -> Redis protocol type conversion} { - r eval {return false} 0 + run_script {return false} 0 } {} test {EVAL - Lua status code reply -> Redis protocol type conversion} { - r eval {return {ok='fine'}} 0 + run_script {return {ok='fine'}} 0 } {fine} test {EVAL - Lua error reply -> Redis protocol type conversion} { catch { - r eval {return {err='this is an error'}} 0 + run_script {return {err='this is an error'}} 0 } e set _ $e } {this is an error} test {EVAL - Lua table -> Redis protocol type conversion} { - r eval {return {1,2,3,'ciao',{1,2}}} 0 + run_script {return {1,2,3,'ciao',{1,2}}} 0 } {1 2 3 ciao {1 2}} test {EVAL - Are the KEYS and ARGV arrays populated correctly?} { - r eval {return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}} 2 a{t} b{t} c{t} d{t} + run_script {return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}} 2 a{t} b{t} c{t} d{t} } {a{t} b{t} c{t} d{t}} test {EVAL - is Lua able to call Redis API?} { r set mykey myval - r eval {return redis.call('get',KEYS[1])} 1 mykey + run_script {return redis.call('get',KEYS[1])} 1 mykey } {myval} + if {$is_eval eq 1} { + # eval sha is only relevant for is_eval Lua test {EVALSHA - Can we call a SHA1 if already defined?} { r evalsha fd758d1589d044dd850a6f05d52f2eefd27f033f 1 mykey } {myval} @@ -60,10 +99,11 @@ start_server {tags {"scripting"}} { catch {r evalsha ffd632c7d33e571e9f24556ebed26c3479a87130 0} e set _ $e } {NOSCRIPT*} + } ;# is_eval test {EVAL - Redis integer -> Lua type conversion} { r set x 0 - r eval { + run_script { local foo = redis.pcall('incr',KEYS[1]) return {type(foo),foo} } 1 x @@ -71,7 +111,7 @@ start_server {tags {"scripting"}} { test {EVAL - Redis bulk -> Lua type conversion} { r set mykey myval - r eval { + run_script { local foo = redis.pcall('get',KEYS[1]) return {type(foo),foo} } 1 mykey @@ -82,14 +122,14 @@ start_server {tags {"scripting"}} { r rpush mylist a r rpush mylist b r rpush mylist c - r eval { + run_script { local foo = redis.pcall('lrange',KEYS[1],0,-1) return {type(foo),foo[1],foo[2],foo[3],# foo} } 1 mylist } {table a b c 3} test {EVAL - Redis status reply -> Lua type conversion} { - r eval { + run_script { local foo = redis.pcall('set',KEYS[1],'myval') return {type(foo),foo['ok']} } 1 mykey @@ -97,7 +137,7 @@ start_server {tags {"scripting"}} { test {EVAL - Redis error reply -> Lua type conversion} { r set mykey myval - r eval { + run_script { local foo = redis.pcall('incr',KEYS[1]) return {type(foo),foo['err']} } 1 mykey @@ -105,7 +145,7 @@ start_server {tags {"scripting"}} { test {EVAL - Redis nil bulk reply -> Lua type conversion} { r del mykey - r eval { + run_script { local foo = redis.pcall('get',KEYS[1]) return {type(foo),foo == false} } 1 mykey @@ -115,13 +155,13 @@ start_server {tags {"scripting"}} { r set mykey "this is DB 9" r select 10 r set mykey "this is DB 10" - r eval {return redis.pcall('get',KEYS[1])} 1 mykey + run_script {return redis.pcall('get',KEYS[1])} 1 mykey } {this is DB 10} {singledb:skip} test {EVAL - SELECT inside Lua should not affect the caller} { # here we DB 10 is selected r set mykey "original value" - r eval {return redis.pcall('select','9')} 0 + run_script {return redis.pcall('select','9')} 0 set res [r get mykey] r select 9 set res @@ -131,7 +171,7 @@ start_server {tags {"scripting"}} { test {EVAL - Script can't run more than configured time limit} { r config set lua-time-limit 1 catch { - r eval { + run_script { local i = 0 while true do i=i+1 end } 0 @@ -142,71 +182,74 @@ start_server {tags {"scripting"}} { test {EVAL - Scripts can't run blpop command} { set e {} - catch {r eval {return redis.pcall('blpop','x',0)} 0} e + catch {run_script {return redis.pcall('blpop','x',0)} 0} e set e } {*not allowed*} test {EVAL - Scripts can't run brpop command} { set e {} - catch {r eval {return redis.pcall('brpop','empty_list',0)} 0} e + catch {run_script {return redis.pcall('brpop','empty_list',0)} 0} e set e } {*not allowed*} test {EVAL - Scripts can't run brpoplpush command} { set e {} - catch {r eval {return redis.pcall('brpoplpush','empty_list1', 'empty_list2',0)} 0} e + catch {run_script {return redis.pcall('brpoplpush','empty_list1', 'empty_list2',0)} 0} e set e } {*not allowed*} test {EVAL - Scripts can't run blmove command} { set e {} - catch {r eval {return redis.pcall('blmove','empty_list1', 'empty_list2', 'LEFT', 'LEFT', 0)} 0} e + catch {run_script {return redis.pcall('blmove','empty_list1', 'empty_list2', 'LEFT', 'LEFT', 0)} 0} e set e } {*not allowed*} test {EVAL - Scripts can't run bzpopmin command} { set e {} - catch {r eval {return redis.pcall('bzpopmin','empty_zset', 0)} 0} e + catch {run_script {return redis.pcall('bzpopmin','empty_zset', 0)} 0} e set e } {*not allowed*} test {EVAL - Scripts can't run bzpopmax command} { set e {} - catch {r eval {return redis.pcall('bzpopmax','empty_zset', 0)} 0} e + catch {run_script {return redis.pcall('bzpopmax','empty_zset', 0)} 0} e set e } {*not allowed*} test {EVAL - Scripts can't run XREAD and XREADGROUP with BLOCK option} { r del s r xgroup create s g $ MKSTREAM - set res [r eval {return redis.pcall('xread','STREAMS','s','$')} 1 s] + set res [run_script {return redis.pcall('xread','STREAMS','s','$')} 1 s] assert {$res eq {}} - assert_error "*xread command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xread','BLOCK',0,'STREAMS','s','$')} 1 s} - set res [r eval {return redis.pcall('xreadgroup','group','g','c','STREAMS','s','>')} 1 s] + assert_error "*xread command is not allowed with BLOCK option from scripts" {run_script {return redis.pcall('xread','BLOCK',0,'STREAMS','s','$')} 1 s} + set res [run_script {return redis.pcall('xreadgroup','group','g','c','STREAMS','s','>')} 1 s] assert {$res eq {}} - assert_error "*xreadgroup command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xreadgroup','group','g','c','BLOCK',0,'STREAMS','s','>')} 1 s} + assert_error "*xreadgroup command is not allowed with BLOCK option from scripts" {run_script {return redis.pcall('xreadgroup','group','g','c','BLOCK',0,'STREAMS','s','>')} 1 s} } + if {$is_eval eq 1} { + # only is_eval Lua can not execute randomkey test {EVAL - Scripts can't run certain commands} { set e {} r debug lua-always-replicate-commands 0 catch { - r eval "redis.pcall('randomkey'); return redis.pcall('set','x','ciao')" 0 + run_script "redis.pcall('randomkey'); return redis.pcall('set','x','ciao')" 0 } e r debug lua-always-replicate-commands 1 set e } {*not allowed after*} {needs:debug} + } ;# is_eval test {EVAL - No arguments to redis.call/pcall is considered an error} { set e {} - catch {r eval {return redis.call()} 0} e + catch {run_script {return redis.call()} 0} e set e } {*one argument*} test {EVAL - redis.call variant raises a Lua error on Redis cmd error (1)} { set e {} catch { - r eval "redis.call('nosuchcommand')" 0 + run_script "redis.call('nosuchcommand')" 0 } e set e } {*Unknown Redis*} @@ -214,7 +257,7 @@ start_server {tags {"scripting"}} { test {EVAL - redis.call variant raises a Lua error on Redis cmd error (1)} { set e {} catch { - r eval "redis.call('get','a','b','c')" 0 + run_script "redis.call('get','a','b','c')" 0 } e set e } {*number of args*} @@ -223,7 +266,7 @@ start_server {tags {"scripting"}} { set e {} r set foo bar catch { - r eval {redis.call('lpush',KEYS[1],'val')} 1 foo + run_script {redis.call('lpush',KEYS[1],'val')} 1 foo } e set e } {*against a key*} @@ -232,7 +275,7 @@ start_server {tags {"scripting"}} { # We must return the table as a string because otherwise # Redis converts floats to ints and we get 0 and 1023 instead # of 0.0003 and 1023.2 as the parsed output. - r eval {return + run_script {return table.concat( cjson.decode( "[0.0, -5e3, -1, 0.3e-3, 1023.2, 0e10]"), " ") @@ -240,13 +283,13 @@ start_server {tags {"scripting"}} { } {0 -5000 -1 0.0003 1023.2 0} test {EVAL - JSON string decoding} { - r eval {local decoded = cjson.decode('{"keya": "a", "keyb": "b"}') + run_script {local decoded = cjson.decode('{"keya": "a", "keyb": "b"}') return {decoded.keya, decoded.keyb} } 0 } {a b} test {EVAL - cmsgpack can pack double?} { - r eval {local encoded = cmsgpack.pack(0.1) + run_script {local encoded = cmsgpack.pack(0.1) local h = "" for i = 1, #encoded do h = h .. string.format("%02x",string.byte(encoded,i)) @@ -256,7 +299,7 @@ start_server {tags {"scripting"}} { } {cb3fb999999999999a} test {EVAL - cmsgpack can pack negative int64?} { - r eval {local encoded = cmsgpack.pack(-1099511627776) + run_script {local encoded = cmsgpack.pack(-1099511627776) local h = "" for i = 1, #encoded do h = h .. string.format("%02x",string.byte(encoded,i)) @@ -266,7 +309,7 @@ start_server {tags {"scripting"}} { } {d3ffffff0000000000} test {EVAL - cmsgpack can pack and unpack circular references?} { - r eval {local a = {x=nil,y=5} + run_script {local a = {x=nil,y=5} local b = {x=a} a['x'] = b local encoded = cmsgpack.pack(a) @@ -298,7 +341,7 @@ start_server {tags {"scripting"}} { } {82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0 1 1} test {EVAL - Numerical sanity check from bitop} { - r eval {assert(0x7fffffff == 2147483647, "broken hex literals"); + run_script {assert(0x7fffffff == 2147483647, "broken hex literals"); assert(0xffffffff == -1 or 0xffffffff == 2^32-1, "broken hex literals"); assert(tostring(-1) == "-1", "broken tostring()"); @@ -309,7 +352,7 @@ start_server {tags {"scripting"}} { } {} test {EVAL - Verify minimal bitop functionality} { - r eval {assert(bit.tobit(1) == 1); + run_script {assert(bit.tobit(1) == 1); assert(bit.band(1) == 1); assert(bit.bxor(1,2) == 3); assert(bit.bor(1,2,4,8,16,32,64,128) == 255) @@ -317,20 +360,22 @@ start_server {tags {"scripting"}} { } {} test {EVAL - Able to parse trailing comments} { - r eval {return 'hello' --trailing comment} 0 + run_script {return 'hello' --trailing comment} 0 } {hello} test {EVAL_RO - Successful case} { r set foo bar - assert_equal bar [r eval_ro {return redis.call('get', KEYS[1]);} 1 foo] + assert_equal bar [run_script_ro {return redis.call('get', KEYS[1]);} 1 foo] } test {EVAL_RO - Cannot run write commands} { r set foo bar - catch {r eval_ro {redis.call('del', KEYS[1]);} 1 foo} e + catch {run_script_ro {redis.call('del', KEYS[1]);} 1 foo} e set e } {*Write commands are not allowed from read-only scripts*} + if {$is_eval eq 1} { + # script command is only relevant for is_eval Lua test {SCRIPTING FLUSH - is able to clear the scripts cache?} { r set mykey myval set v [r evalsha fd758d1589d044dd850a6f05d52f2eefd27f033f 1 mykey] @@ -361,6 +406,7 @@ start_server {tags {"scripting"}} { [r evalsha b534286061d4b9e4026607613b95c06c06015ae8 0] } {b534286061d4b9e4026607613b95c06c06015ae8 loaded} + # reply oredering is only relevant for is_eval Lua test "In the context of Lua the output of random commands gets ordered" { r debug lua-always-replicate-commands 0 r del myset @@ -387,19 +433,20 @@ start_server {tags {"scripting"}} { r sadd myset a b c r eval {return redis.call('sort',KEYS[1],'by','_','get','#','get','_:*')} 1 myset } {a {} b {} c {}} {cluster:skip} + } ;# is_eval test "redis.sha1hex() implementation" { - list [r eval {return redis.sha1hex('')} 0] \ - [r eval {return redis.sha1hex('Pizza & Mandolino')} 0] + list [run_script {return redis.sha1hex('')} 0] \ + [run_script {return redis.sha1hex('Pizza & Mandolino')} 0] } {da39a3ee5e6b4b0d3255bfef95601890afd80709 74822d82031af7493c20eefa13bd07ec4fada82f} test {Globals protection reading an undeclared global variable} { - catch {r eval {return a} 0} e + catch {run_script {return a} 0} e set e } {*ERR*attempted to access * global*} test {Globals protection setting an undeclared global*} { - catch {r eval {a=10} 0} e + catch {run_script {a=10} 0} e set e } {*ERR*attempted to create global*} @@ -417,14 +464,16 @@ start_server {tags {"scripting"}} { } r set foo 5 set res {} - lappend res [r eval $decr_if_gt 1 foo 2] - lappend res [r eval $decr_if_gt 1 foo 2] - lappend res [r eval $decr_if_gt 1 foo 2] - lappend res [r eval $decr_if_gt 1 foo 2] - lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [run_script $decr_if_gt 1 foo 2] + lappend res [run_script $decr_if_gt 1 foo 2] + lappend res [run_script $decr_if_gt 1 foo 2] + lappend res [run_script $decr_if_gt 1 foo 2] + lappend res [run_script $decr_if_gt 1 foo 2] set res } {4 3 2 2 2} + if {$is_eval eq 1} { + # random handling is only relevant for is_eval Lua test {Scripting engine resets PRNG at every script execution} { set rand1 [r eval {return tostring(math.random())} 0] set rand2 [r eval {return tostring(math.random())} 0] @@ -444,13 +493,14 @@ start_server {tags {"scripting"}} { assert_equal $rand1 $rand2 assert {$rand2 ne $rand3} } + } ;# is_eval test {EVAL does not leak in the Lua stack} { r set x 0 # Use a non blocking client to speedup the loop. set rd [redis_deferring_client] for {set j 0} {$j < 10000} {incr j} { - $rd eval {return redis.call("incr",KEYS[1])} 1 x + run_script_on_connection $rd {return redis.call("incr",KEYS[1])} 1 x } for {set j 0} {$j < 10000} {incr j} { $rd read @@ -464,9 +514,9 @@ start_server {tags {"scripting"}} { r flushall r config set appendonly yes r config set aof-use-rdb-preamble no - r eval {redis.call("set",KEYS[1],"100")} 1 foo - r eval {redis.call("incr",KEYS[1])} 1 foo - r eval {redis.call("incr",KEYS[1])} 1 foo + run_script {redis.call("set",KEYS[1],"100")} 1 foo + run_script {redis.call("incr",KEYS[1])} 1 foo + run_script {redis.call("incr",KEYS[1])} 1 foo wait_for_condition 50 100 { [s aof_rewrite_in_progress] == 0 } else { @@ -481,6 +531,8 @@ start_server {tags {"scripting"}} { set res } {102} {external:skip} + if {$is_eval eq 1} { + # script propagation is irrelevant on functions test {EVAL timeout from AOF} { # generate a long running script that is propagated to the AOF as script # make sure that the script times out during loading @@ -528,9 +580,11 @@ start_server {tags {"scripting"}} { assert {[r mget a{t} b{t} c{t} d{t}] eq {1 2 3 4}} assert {[r spop myset] eq {}} } + } ;# is_eval + test {Call Redis command with many args from Lua (issue #1764)} { - r eval { + run_script { local i local x={} redis.call('del','mylist') @@ -543,7 +597,7 @@ start_server {tags {"scripting"}} { } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100} test {Number conversion precision test (issue #1118)} { - r eval { + run_script { local value = 9007199254740991 redis.call("set","foo",value) return redis.call("get","foo") @@ -551,19 +605,19 @@ start_server {tags {"scripting"}} { } {9007199254740991} test {String containing number precision test (regression of issue #1118)} { - r eval { + run_script { redis.call("set", "key", "12039611435714932082") return redis.call("get", "key") } 1 key } {12039611435714932082} test {Verify negative arg count is error instead of crash (issue #1842)} { - catch { r eval { return "hello" } -12 } e + catch { run_script { return "hello" } -12 } e set e } {ERR Number of keys can't be negative} test {Correct handling of reused argv (issue #1939)} { - r eval { + run_script { for i = 0, 10 do redis.call('SET', 'a{t}', '1') redis.call('MGET', 'a{t}', 'b{t}', 'c{t}') @@ -576,7 +630,7 @@ start_server {tags {"scripting"}} { test {Functions in the Redis namespace are able to report errors} { catch { - r eval { + run_script { redis.sha1hex() } 0 } e @@ -594,22 +648,22 @@ start_server {tags {"scripting"}} { assert_equal $res $expected_dict # Test RESP3 client with script in both RESP2 and RESP3 modes - set res [r eval {redis.setresp(3); return redis.call('hgetall', KEYS[1])} 1 hash] + set res [run_script {redis.setresp(3); return redis.call('hgetall', KEYS[1])} 1 hash] assert_equal $res $expected_dict - set res [r eval {redis.setresp(2); return redis.call('hgetall', KEYS[1])} 1 hash] + set res [run_script {redis.setresp(2); return redis.call('hgetall', KEYS[1])} 1 hash] assert_equal $res $expected_list # Test RESP2 client with script in both RESP2 and RESP3 modes r HELLO 2 - set res [r eval {redis.setresp(3); return redis.call('hgetall', KEYS[1])} 1 hash] + set res [run_script {redis.setresp(3); return redis.call('hgetall', KEYS[1])} 1 hash] assert_equal $res $expected_list - set res [r eval {redis.setresp(2); return redis.call('hgetall', KEYS[1])} 1 hash] + set res [run_script {redis.setresp(2); return redis.call('hgetall', KEYS[1])} 1 hash] assert_equal $res $expected_list } test {Script return recursive object} { r readraw 1 - set res [r eval {local a = {}; local b = {a}; a[1] = b; return a} 0] + set res [run_script {local a = {}; local b = {a}; a[1] = b; return a} 0] # drain the response while {true} { if {$res == "-ERR reached lua stack limit"} { @@ -640,11 +694,11 @@ start_server {tags {"scripting"}} { test {Timedout read-only scripts can be killed by SCRIPT KILL} { set rd [redis_deferring_client] r config set lua-time-limit 10 - $rd eval {while true do end} 0 + run_script_on_connection $rd {while true do end} 0 after 200 catch {r ping} e assert_match {BUSY*} $e - r script kill + kill_script after 200 ; # Give some time to Lua to call the hook again... assert_equal [r ping] "PONG" $rd close @@ -653,7 +707,7 @@ start_server {tags {"scripting"}} { test {Timedout read-only scripts can be killed by SCRIPT KILL even when use pcall} { set rd [redis_deferring_client] r config set lua-time-limit 10 - $rd eval {local f = function() while 1 do redis.call('ping') end end while 1 do pcall(f) end} 0 + run_script_on_connection $rd {local f = function() while 1 do redis.call('ping') end end while 1 do pcall(f) end} 0 wait_for_condition 50 100 { [catch {r ping} e] == 1 @@ -663,7 +717,7 @@ start_server {tags {"scripting"}} { catch {r ping} e assert_match {BUSY*} $e - r script kill + kill_script wait_for_condition 50 100 { [catch {r ping} e] == 0 @@ -685,8 +739,14 @@ start_server {tags {"scripting"}} { # senging (in a pipeline): # 1. eval "while 1 do redis.call('ping') end" 0 # 2. ping - set buf "*3\r\n\$4\r\neval\r\n\$33\r\nwhile 1 do redis.call('ping') end\r\n\$1\r\n0\r\n" - append buf "*1\r\n\$4\r\nping\r\n" + if {$is_eval == 1} { + set buf "*3\r\n\$4\r\neval\r\n\$33\r\nwhile 1 do redis.call('ping') end\r\n\$1\r\n0\r\n" + append buf "*1\r\n\$4\r\nping\r\n" + } else { + set buf "*6\r\n\$8\r\nfunction\r\n\$6\r\ncreate\r\n\$3\r\nlua\r\n\$4\r\ntest\r\n\$7\r\nreplace\r\n\$33\r\nwhile 1 do redis.call('ping') end\r\n" + append buf "*3\r\n\$5\r\nfcall\r\n\$4\r\ntest\r\n\$1\r\n0\r\n" + append buf "*1\r\n\$4\r\nping\r\n" + } $rd write $buf $rd flush @@ -698,7 +758,7 @@ start_server {tags {"scripting"}} { catch {r ping} e assert_match {BUSY*} $e - r script kill + kill_script wait_for_condition 50 100 { [catch {r ping} e] == 0 } else { @@ -706,6 +766,11 @@ start_server {tags {"scripting"}} { } assert_equal [r ping] "PONG" + if {$is_eval == 0} { + # read the ok reply of function create + assert_match {OK} [$rd read] + } + catch {$rd read} res assert_match {*killed by user*} $res @@ -717,18 +782,18 @@ start_server {tags {"scripting"}} { test {Timedout script link is still usable after Lua returns} { r config set lua-time-limit 10 - r eval {for i=1,100000 do redis.call('ping') end return 'ok'} 0 + run_script {for i=1,100000 do redis.call('ping') end return 'ok'} 0 r ping } {PONG} test {Timedout scripts that modified data can't be killed by SCRIPT KILL} { set rd [redis_deferring_client] r config set lua-time-limit 10 - $rd eval {redis.call('set',KEYS[1],'y'); while true do end} 1 x + run_script_on_connection $rd {redis.call('set',KEYS[1],'y'); while true do end} 1 x after 200 catch {r ping} e assert_match {BUSY*} $e - catch {r script kill} e + catch {kill_script} e assert_match {UNKILLABLE*} $e catch {r ping} e assert_match {BUSY*} $e @@ -761,11 +826,11 @@ foreach cmdrepl {0 1} { # One with an error, but still executing a command. # SHA is: 67164fc43fa971f76fd1aaeeaf60c1c178d25876 catch { - r eval {redis.call('incr',KEYS[1]); redis.call('nonexisting')} 1 x + run_script {redis.call('incr',KEYS[1]); redis.call('nonexisting')} 1 x } # One command is correct: # SHA is: 6f5ade10a69975e903c6d07b10ea44c6382381a5 - r eval {return redis.call('incr',KEYS[1])} 1 x + run_script {return redis.call('incr',KEYS[1])} 1 x } {2} test "Connect a replica to the master instance $rt" { @@ -778,6 +843,7 @@ foreach cmdrepl {0 1} { } } + if {$is_eval eq 1} { test "Now use EVALSHA against the master, with both SHAs $rt" { # The server should replicate successful and unsuccessful # commands as EVAL instead of EVALSHA. @@ -794,11 +860,12 @@ foreach cmdrepl {0 1} { fail "Expected 4 in x, but value is '[r -1 get x]'" } } + } ;# is_eval test "Replication of script multiple pushes to list with BLPOP $rt" { set rd [redis_deferring_client] $rd brpop a 0 - r eval { + run_script { redis.call("lpush",KEYS[1],"1"); redis.call("lpush",KEYS[1],"2"); } 1 a @@ -812,6 +879,7 @@ foreach cmdrepl {0 1} { set res } {a 1} + if {$is_eval eq 1} { test "EVALSHA replication when first call is readonly $rt" { r del x r eval {if tonumber(ARGV[1]) > 0 then redis.call('incr', KEYS[1]) end} 1 x 0 @@ -823,16 +891,17 @@ foreach cmdrepl {0 1} { fail "Expected 1 in x, but value is '[r -1 get x]'" } } + } ;# is_eval test "Lua scripts using SELECT are replicated correctly $rt" { - r eval { + run_script { redis.call("set","foo1","bar1") redis.call("select","10") redis.call("incr","x") redis.call("select","11") redis.call("incr","z") } 0 - r eval { + run_script { redis.call("set","foo1","bar1") redis.call("select","10") redis.call("incr","x") @@ -861,6 +930,8 @@ start_server {tags {"scripting repl external:skip"}} { } } + if {$is_eval eq 1} { + # replicate_commands is the default on Redis Function test "Redis.replicate_commands() must be issued before any write" { r eval { redis.call('set','foo','bar'); @@ -884,11 +955,11 @@ start_server {tags {"scripting repl external:skip"}} { r debug lua-always-replicate-commands 1 set e } {*only after turning on*} + } ;# is_eval test "Redis.set_repl() don't accept invalid values" { catch { - r eval { - redis.replicate_commands(); + run_script { redis.set_repl(12345); } 0 } e @@ -897,8 +968,7 @@ start_server {tags {"scripting repl external:skip"}} { test "Test selective replication of certain Redis commands from Lua" { r del a b c d - r eval { - redis.replicate_commands(); + run_script { redis.call('set','a','1'); redis.set_repl(redis.REPL_NONE); redis.call('set','b','2'); @@ -924,24 +994,37 @@ start_server {tags {"scripting repl external:skip"}} { } test "PRNG is seeded randomly for command replication" { - set a [ - r eval { - redis.replicate_commands(); - return math.random()*100000; - } 0 - ] - set b [ - r eval { - redis.replicate_commands(); - return math.random()*100000; - } 0 - ] + if {$is_eval eq 1} { + # on is_eval Lua we need to call redis.replicate_commands() to get real randomization + set a [ + run_script { + redis.replicate_commands() + return math.random()*100000; + } 0 + ] + set b [ + run_script { + redis.replicate_commands() + return math.random()*100000; + } 0 + ] + } else { + set a [ + run_script { + return math.random()*100000; + } 0 + ] + set b [ + run_script { + return math.random()*100000; + } 0 + ] + } assert {$a ne $b} } test "Using side effects is not a problem with command replication" { - r eval { - redis.replicate_commands(); + run_script { redis.call('set','time',redis.call('time')[1]) } 0 @@ -956,6 +1039,7 @@ start_server {tags {"scripting repl external:skip"}} { } } +if {$is_eval eq 1} { start_server {tags {"scripting external:skip"}} { r script debug sync r eval {return 'hello'} 0 @@ -984,12 +1068,13 @@ start_server {tags {"scripting needs:debug external:skip"}} { r write $cmd r flush set ret [r read] - assert_match {*Unknown Redis command called from*} $ret + assert_match {*Unknown Redis command called from script*} $ret # make sure the server is still ok reconnect assert_equal [r ping] {PONG} } } +} ;# is_eval start_server {tags {"scripting resp3 needs:debug"}} { r debug set-disable-deny-scripts 1 @@ -999,7 +1084,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { r readraw 1 test {test resp3 big number protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'bignum')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'bignum')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {$37} @@ -1021,7 +1106,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } test {test resp3 map protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'map')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'map')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {*6} @@ -1034,7 +1119,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } test {test resp3 set protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'set')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'set')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {*3} @@ -1047,7 +1132,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } test {test resp3 double protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'double')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'double')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {$5} @@ -1058,7 +1143,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } test {test resp3 null protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'null')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'null')" 0] if {$client_proto == 2} { # null is a special case in which a Lua client format does not effect the reply to the client assert_equal $ret {$-1} @@ -1068,7 +1153,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } {} test {test resp3 verbatim protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'verbatim')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'verbatim')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {$25} @@ -1082,7 +1167,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } test {test resp3 true protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'true')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'true')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {:1} @@ -1092,7 +1177,7 @@ start_server {tags {"scripting resp3 needs:debug"}} { } test {test resp3 false protocol parsing} { - set ret [r eval "redis.setresp($i);return redis.call('debug', 'protocol', 'false')" 0] + set ret [run_script "redis.setresp($i);return redis.call('debug', 'protocol', 'false')" 0] if {$client_proto == 2 || $i == 2} { # if either Lua or the clien is RESP2 the reply will be RESP2 assert_equal $ret {:0} @@ -1109,8 +1194,9 @@ start_server {tags {"scripting resp3 needs:debug"}} { test {test resp3 attribute protocol parsing} { # attributes are not (yet) expose to the script # So here we just check the parser handles them and they are ignored. - r eval "redis.setresp(3);return redis.call('debug', 'protocol', 'attrib')" 0 + run_script "redis.setresp(3);return redis.call('debug', 'protocol', 'attrib')" 0 } {Some real reply following the attribute} r debug set-disable-deny-scripts 0 } +} ;# foreach is_eval