caleb-brown/redict
1
0
Fork 0
mirror of https://codeberg.org/redict/redict.git synced 2025-01-22 08:08:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support function flags in script EVAL via shebang header (#10126)

Browse Source 
In #10025 we added a mechanism for flagging certain properties for Redis Functions.
This lead us to think we'd like to "port" this mechanism to Redis Scripts (`EVAL`) as well. 

One good reason for this, other than the added functionality is because it addresses the
poor behavior we currently have in `EVAL` in case the script performs a (non DENY_OOM) write operation
during OOM state. See #8478 (And a previous attempt to handle it via #10093) for details.
Note that in Redis Functions **all** write operations (including DEL) will return an error during OOM state
unless the function is flagged as `allow-oom` in which case no OOM checking is performed at all.

This PR:
- Enables setting `EVAL` (and `SCRIPT LOAD`) script flags as defined in #10025.
- Provides a syntactical framework via [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) for
  additional script annotations and even engine selection (instead of just lua) for scripts.
- Provides backwards compatibility so scripts without the new annotations will behave as they did before.
- Appropriate tests.
- Changes `EVAL[SHA]/_RO` to be flagged as `STALE` commands. This makes it possible to flag individual
  scripts as `allow-stale` or not flag them as such. In backwards compatibility mode these commands will
  return the `MASTERDOWN` error as before.
- Changes `SCRIPT LOAD` to be flagged as a `STALE` command. This is mainly to make it logically
  compatible with the change to `EVAL` in the previous point. It enables loading a script on a stale server
  which is technically okay it doesn't relate directly to the server's dataset. Running the script does, but that
  won't work unless the script is explicitly marked as `allow-stale`.

Note that even though the LUA syntax doesn't support hash tag comments `.lua` files do support a shebang
tag on the top so they can be executed on Unix systems like any shell script. LUA's `luaL_loadfile` handles
this as part of the LUA library. In the case of `luaL_loadbuffer`, which is what Redis uses, I needed to fix the
input script in case of a shebang manually. I did this the same way `luaL_loadfile` does, by replacing the
first line with a single line feed character.
This commit is contained in:
yoav-steinberg 2022-01-24 16:50:02 +02:00 committed by GitHub
parent 857dc5bacd
commit 7eadc5ee70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 370 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/commands.c
View File

@ -3579,7 +3579,7 @@ struct redisCommand SCRIPT_Subcommands[] = {
{"flush","Remove all the scripts from the script cache.","O(N) with N being the number of scripts in cache","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_FLUSH_History,SCRIPT_FLUSH_tips,scriptCommand,-2,CMD_NOSCRIPT,ACL_CATEGORY_SCRIPTING,.args=SCRIPT_FLUSH_Args},
{"help","Show helpful text about the different subcommands","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_HELP_History,SCRIPT_HELP_tips,scriptCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_SCRIPTING},
{"kill","Kill the script currently in execution.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_KILL_History,SCRIPT_KILL_tips,scriptCommand,2,CMD_NOSCRIPT|CMD_ALLOW_BUSY,ACL_CATEGORY_SCRIPTING},
{"load","Load the specified Lua script into the script cache.","O(N) with N being the length in bytes of the script body.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_LOAD_History,SCRIPT_LOAD_tips,scriptCommand,3,CMD_NOSCRIPT,ACL_CATEGORY_SCRIPTING,.args=SCRIPT_LOAD_Args},
{"load","Load the specified Lua script into the script cache.","O(N) with N being the length in bytes of the script body.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_LOAD_History,SCRIPT_LOAD_tips,scriptCommand,3,CMD_NOSCRIPT|CMD_STALE,ACL_CATEGORY_SCRIPTING,.args=SCRIPT_LOAD_Args},
{0}
};
@ -6923,10 +6923,10 @@ struct redisCommand redisCommandTable[] = {
{"sunsubscribe","Stop listening for messages posted to the given shard channels","O(N) where N is the number of clients already subscribed to a channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUNSUBSCRIBE_History,SUNSUBSCRIBE_tips,sunsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNSUBSCRIBE_Args},
{"unsubscribe","Stop listening for messages posted to the given channels","O(N) where N is the number of clients already subscribed to a channel.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,UNSUBSCRIBE_History,UNSUBSCRIBE_tips,unsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=UNSUBSCRIBE_Args},
/* scripting */
{"eval","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_History,EVAL_tips,evalCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_Args},
{"evalsha","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_History,EVALSHA_tips,evalShaCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_Args},
{"evalsha_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_RO_History,EVALSHA_RO_tips,evalShaRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_RO_Args},
{"eval_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_RO_History,EVAL_RO_tips,evalRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_RO_Args},
{"eval","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_History,EVAL_tips,evalCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_Args},
{"evalsha","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_History,EVALSHA_tips,evalShaCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_Args},
{"evalsha_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_RO_History,EVALSHA_RO_tips,evalShaRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_RO_Args},
{"eval_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_RO_History,EVAL_RO_tips,evalRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_RO_Args},
{"fcall","PATCH__TBD__38__","PATCH__TBD__37__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_tips,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args},
{"fcall_ro","PATCH__TBD__7__","PATCH__TBD__6__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_tips,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args},
{"function","A container for function commands","Depends on subcommand.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FUNCTION_History,FUNCTION_tips,NULL,-2,0,0,.subcommands=FUNCTION_Subcommands},

3
src/commands/eval.json
View File

@ -11,7 +11,8 @@
"NOSCRIPT",
"SKIP_MONITOR",
"MAY_REPLICATE",
"NO_MANDATORY_KEYS"
"NO_MANDATORY_KEYS",
"STALE"
],
"acl_categories": [
"SCRIPTING"

3
src/commands/eval_ro.json
View File

@ -10,7 +10,8 @@
"command_flags": [
"NOSCRIPT",
"SKIP_MONITOR",
"NO_MANDATORY_KEYS"
"NO_MANDATORY_KEYS",
"STALE"
],
"acl_categories": [
"SCRIPTING"

3
src/commands/evalsha.json
View File

@ -11,7 +11,8 @@
"NOSCRIPT",
"SKIP_MONITOR",
"MAY_REPLICATE",
"NO_MANDATORY_KEYS"
"NO_MANDATORY_KEYS",
"STALE"
],
"acl_categories": [
"SCRIPTING"

3
src/commands/evalsha_ro.json
View File

@ -10,7 +10,8 @@
"command_flags": [
"NOSCRIPT",
"SKIP_MONITOR",
"NO_MANDATORY_KEYS"
"NO_MANDATORY_KEYS",
"STALE"
],
"acl_categories": [
"SCRIPTING"

3
src/commands/script-load.json
View File

@ -8,7 +8,8 @@
"container": "SCRIPT",
"function": "scriptCommand",
"command_flags": [
"NOSCRIPT"
"NOSCRIPT",
"STALE"
],
"acl_categories": [
"SCRIPTING"

121
src/eval.c
View File

@ -47,11 +47,37 @@ void ldbEnable(client *c);
void evalGenericCommandWithDebugging(client *c, int evalsha);
sds ldbCatStackValue(sds s, lua_State *lua, int idx);
typedef struct luaScript {
uint64_t flags;
robj *body;
} luaScript;
static void dictLuaScriptDestructor(dict *d, void *val) {
UNUSED(d);
if (val == NULL) return; /* Lazy freeing will set value to NULL. */
decrRefCount(((luaScript*)val)->body);
zfree(val);
}
static uint64_t dictStrCaseHash(const void *key) {
return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key));
}
/* server.lua_scripts sha (as sds string) -> scripts (as robj) cache. */
dictType shaScriptObjectDictType = {
dictStrCaseHash, /* hash function */
NULL, /* key dup */
NULL, /* val dup */
dictSdsKeyCaseCompare, /* key compare */
dictSdsDestructor, /* key destructor */
dictLuaScriptDestructor, /* val destructor */
NULL /* allow to expand */
};
/* Lua context */
struct luaCtx {
lua_State *lua; /* The Lua interpreter. We use just one for all clients */
client *lua_client; /* The "fake client" to query Redis from Lua */
char *lua_cur_script; /* SHA1 of the script currently running, or NULL */
dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */
unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */
} lctx;
@ -165,7 +191,6 @@ void scriptingInit(int setup) {
if (setup) {
lctx.lua_client = NULL;
server.script_caller = NULL;
lctx.lua_cur_script = NULL;
server.script_disable_deny_script = 0;
ldbInit();
}
@ -291,22 +316,77 @@ void scriptingReset(int async) {
sds luaCreateFunction(client *c, robj *body) {
char funcname[43];
dictEntry *de;
uint64_t script_flags = SCRIPT_FLAG_EVAL_COMPAT_MODE;
funcname[0] = 'f';
funcname[1] = '_';
sha1hex(funcname+2,body->ptr,sdslen(body->ptr));
sds sha = sdsnewlen(funcname+2,40);
if ((de = dictFind(lctx.lua_scripts,sha)) != NULL) {
sdsfree(sha);
if ((de = dictFind(lctx.lua_scripts,funcname+2)) != NULL) {
return dictGetKey(de);
}
/* Handle shebang header in script code */
ssize_t shebang_len = 0;
if (!strncmp(body->ptr, "#!", 2)) {
int numparts,j;
char *shebang_end = strchr(body->ptr, '\n');
if (shebang_end == NULL) {
addReplyError(c,"Invalid script shebang");
return NULL;
}
shebang_len = shebang_end - (char*)body->ptr;
sds shebang = sdsnewlen(body->ptr, shebang_len);
sds *parts = sdssplitargs(shebang, &numparts);
sdsfree(shebang);
if (!parts || numparts == 0) {
addReplyError(c,"Invalid engine in script shebang");
sdsfreesplitres(parts, numparts);
return NULL;
}
/* Verify lua interpreter was specified */
if (strcmp(parts[0], "#!lua")) {
addReplyErrorFormat(c,"Unexpected engine in script shebang: %s", parts[0]);
sdsfreesplitres(parts, numparts);
return NULL;
}
script_flags &= ~SCRIPT_FLAG_EVAL_COMPAT_MODE;
for (j = 1; j < numparts; j++) {
if (!strncmp(parts[j], "flags=", 6)) {
sdsrange(parts[j], 6, -1);
int numflags, jj;
sds *flags = sdssplitlen(parts[j], sdslen(parts[j]), ",", 1, &numflags);
for (jj = 0; jj < numflags; jj++) {
scriptFlag *sf;
for (sf = scripts_flags_def; sf->flag; sf++) {
if (!strcmp(flags[jj], sf->str)) break;
}
if (!sf->flag) {
addReplyErrorFormat(c,"Unexpected flag in script shebang: %s", flags[jj]);
sdsfreesplitres(flags, numflags);
sdsfreesplitres(parts, numparts);
return NULL;
}
script_flags |= sf->flag;
}
sdsfreesplitres(flags, numflags);
} else {
/* We only support function flags options for lua scripts */
addReplyErrorFormat(c,"Unknown lua shebang option: %s", parts[j]);
sdsfreesplitres(parts, numparts);
return NULL;
}
}
sdsfreesplitres(parts, numparts);
}
/* Build the lua function to be loaded */
sds funcdef = sdsempty();
funcdef = sdscat(funcdef,"function ");
funcdef = sdscatlen(funcdef,funcname,42);
funcdef = sdscatlen(funcdef,"() ",3);
funcdef = sdscatlen(funcdef,body->ptr,sdslen(body->ptr));
/* Note that in case of a shebang line we skip it but keep the line feed to conserve the user's line numbers */
funcdef = sdscatlen(funcdef,(char*)body->ptr + shebang_len,sdslen(body->ptr) - shebang_len);
funcdef = sdscatlen(funcdef,"\nend",4);
if (luaL_loadbuffer(lctx.lua,funcdef,sdslen(funcdef),"@user_script")) {
@ -316,7 +396,6 @@ sds luaCreateFunction(client *c, robj *body) {
lua_tostring(lctx.lua,-1));
}
lua_pop(lctx.lua,1);
sdsfree(sha);
sdsfree(funcdef);
return NULL;
}
@ -328,14 +407,17 @@ sds luaCreateFunction(client *c, robj *body) {
lua_tostring(lctx.lua,-1));
}
lua_pop(lctx.lua,1);
sdsfree(sha);
return NULL;
}
/* We also save a SHA1 -> Original script map in a dictionary
* so that we can replicate / write in the AOF all the
* EVALSHA commands as EVAL using the original script. */
int retval = dictAdd(lctx.lua_scripts,sha,body);
luaScript *l = zcalloc(sizeof(luaScript));
l->body = body;
l->flags = script_flags;
sds sha = sdsnewlen(funcname+2,40);
int retval = dictAdd(lctx.lua_scripts,sha,l);
serverAssertWithInfo(c ? c : lctx.lua_client,NULL,retval == DICT_OK);
lctx.lua_scripts_mem += sdsZmallocSize(sha) + getStringObjectSdsUsedMemory(body);
incrRefCount(body);
@ -421,24 +503,21 @@ void evalGenericCommand(client *c, int evalsha) {
serverAssert(!lua_isnil(lua,-1));
}
lctx.lua_cur_script = funcname + 2;
char *lua_cur_script = funcname + 2;
dictEntry *de = dictFind(lctx.lua_scripts, lua_cur_script);
luaScript *l = dictGetVal(de);
int ro = c->cmd->proc == evalRoCommand || c->cmd->proc == evalShaRoCommand;
scriptRunCtx rctx;
scriptPrepareForRun(&rctx, lctx.lua_client, c, lctx.lua_cur_script);
rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as legacy so we
will get legacy error messages and logs */
/* This check is for EVAL_RO, EVALSHA_RO. We want to allow only read only commands */
if ((server.script_caller->cmd->proc == evalRoCommand ||
server.script_caller->cmd->proc == evalShaRoCommand)) {
rctx.flags |= SCRIPT_READ_ONLY;
if (scriptPrepareForRun(&rctx, lctx.lua_client, c, lua_cur_script, l->flags, ro) != C_OK) {
return;
}
rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as EVAL (as opposed to FCALL) so we'll
get appropriate error messages and logs */
luaCallFunction(&rctx, lua, c->argv+3, numkeys, c->argv+3+numkeys, c->argc-3-numkeys, ldb.active);
lua_pop(lua,1); /* Remove the error handler. */
scriptResetRun(&rctx);
lctx.lua_cur_script = NULL;
}
void evalCommand(client *c) {
@ -563,7 +642,7 @@ dict* evalScriptsDict() {
unsigned long evalScriptsMemory() {
return lctx.lua_scripts_mem +
dictSize(lctx.lua_scripts) * sizeof(dictEntry) +
dictSize(lctx.lua_scripts) * (sizeof(dictEntry) + sizeof(luaScript)) +
dictSlots(lctx.lua_scripts) * sizeof(dictEntry*);
}

64
src/functions.c
View File

@ -570,69 +570,11 @@ static void fcallCommandGeneric(client *c, int ro) {
return;
}
if ((fi->f_flags & SCRIPT_FLAG_NO_CLUSTER) && server.cluster_enabled) {
addReplyError(c, "Can not run function on cluster, 'no-cluster' flag is set.");
return;
}
if (!(fi->f_flags & SCRIPT_FLAG_ALLOW_OOM) && server.script_oom && server.maxmemory) {
addReplyError(c, "-OOM allow-oom flag is not set on the function, "
"can not run it when used memory > 'maxmemory'");
return;
}
if (server.masterhost && server.repl_state != REPL_STATE_CONNECTED &&
server.repl_serve_stale_data == 0 && !(fi->f_flags & SCRIPT_FLAG_ALLOW_STALE))
{
addReplyError(c, "-MASTERDOWN Link with MASTER is down, "
"replica-serve-stale-data is set to 'no' "
"and 'allow-stale' flag is not set on the function.");
return;
}
if (!(fi->f_flags & SCRIPT_FLAG_NO_WRITES)) {
/* Function may perform writes we need to verify:
* 1. we are not a readonly replica
* 2. no disk error detected
* 3. command is not 'fcall_ro' */
if (server.masterhost && server.repl_slave_ro && c->id != CLIENT_ID_AOF
&& !(c->flags & CLIENT_MASTER))
{
addReplyError(c, "Can not run a function with write flag on readonly replica");
return;
}
int deny_write_type = writeCommandsDeniedByDiskError();
if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL) {
if (deny_write_type == DISK_ERROR_TYPE_RDB)
addReplyError(c, "-MISCONF Redis is configured to save RDB snapshots, "
"but it is currently not able to persist on disk. "
"So its impossible to run functions that has 'write' flag on.");
else
addReplyErrorFormat(c, "-MISCONF Redis is configured to persist data to AOF, "
"but it is currently not able to persist on disk. "
"So its impossible to run functions that has 'write' flag on. "
"AOF error: %s", strerror(server.aof_last_write_errno));
return;
}
if (ro) {
addReplyError(c, "Can not execute a function with write flag using fcall_ro.");
return;
}
}
scriptRunCtx run_ctx;
scriptPrepareForRun(&run_ctx, fi->li->ei->c, c, fi->name);
if (ro || (fi->f_flags & SCRIPT_FLAG_NO_WRITES)) {
/* On fcall_ro or on functions that do not have the 'write'
* flag, we will not allow write commands. */
run_ctx.flags |= SCRIPT_READ_ONLY;
}
if (fi->f_flags & SCRIPT_FLAG_ALLOW_OOM) {
run_ctx.flags |= SCRIPT_ALLOW_OOM;
}
if (scriptPrepareForRun(&run_ctx, fi->li->ei->c, c, fi->name, fi->f_flags, ro) != C_OK)
return;
engine->call(&run_ctx, engine->engine_ctx, fi->function, c->argv + 3, numkeys,
c->argv + 3 + numkeys, c->argc - 3 - numkeys);
scriptResetRun(&run_ctx);

82
src/script.c
View File

@ -108,10 +108,70 @@ int scriptInterrupt(scriptRunCtx *run_ctx) {
}
/* Prepare the given run ctx for execution */
void scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *caller, const char *funcname) {
int scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *caller, const char *funcname, uint64_t script_flags, int ro) {
serverAssert(!curr_run_ctx);
/* set the curr_run_ctx so we can use it to kill the script if needed */
curr_run_ctx = run_ctx;
int running_stale = server.masterhost &&
server.repl_state != REPL_STATE_CONNECTED &&
server.repl_serve_stale_data == 0;
if (!(script_flags & SCRIPT_FLAG_EVAL_COMPAT_MODE)) {
if ((script_flags & SCRIPT_FLAG_NO_CLUSTER) && server.cluster_enabled) {
addReplyError(caller, "Can not run script on cluster, 'no-cluster' flag is set.");
return C_ERR;
}
if (!(script_flags & SCRIPT_FLAG_ALLOW_OOM) && server.script_oom && server.maxmemory) {
addReplyError(caller, "-OOM allow-oom flag is not set on the script, "
"can not run it when used memory > 'maxmemory'");
return C_ERR;
}
if (running_stale && !(script_flags & SCRIPT_FLAG_ALLOW_STALE)) {
addReplyError(caller, "-MASTERDOWN Link with MASTER is down, "
"replica-serve-stale-data is set to 'no' "
"and 'allow-stale' flag is not set on the script.");
return C_ERR;
}
if (!(script_flags & SCRIPT_FLAG_NO_WRITES)) {
/* Script may perform writes we need to verify:
* 1. we are not a readonly replica
* 2. no disk error detected
* 3. command is not `fcall_ro`/`eval[sha]_ro` */
if (server.masterhost && server.repl_slave_ro && caller->id != CLIENT_ID_AOF
&& !(caller->flags & CLIENT_MASTER))
{
addReplyError(caller, "Can not run script with write flag on readonly replica");
return C_ERR;
}
int deny_write_type = writeCommandsDeniedByDiskError();
if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL) {
if (deny_write_type == DISK_ERROR_TYPE_RDB)
addReplyError(caller, "-MISCONF Redis is configured to save RDB snapshots, "
"but it's currently unable to persist to disk. "
"Writable scripts are blocked. Use 'no-writes' flag for read only scripts.");
else
addReplyErrorFormat(caller, "-MISCONF Redis is configured to persist data to AOF, "
"but it's currently unable to persist to disk. "
"Writable scripts are blocked. Use 'no-writes' flag for read only scripts. "
"AOF error: %s", strerror(server.aof_last_write_errno));
return C_ERR;
}
if (ro) {
addReplyError(caller, "Can not execute a script with write flag using *_ro command.");
return C_ERR;
}
}
} else {
/* Special handling for backwards compatibility (no shebang eval[sha]) mode */
if (running_stale) {
addReplyErrorObject(caller, shared.masterdownerr);
return C_ERR;
}
}
run_ctx->c = engine_client;
run_ctx->original_client = caller;
@ -137,6 +197,20 @@ void scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *c
run_ctx->flags = 0;
run_ctx->repl_flags = PROPAGATE_AOF | PROPAGATE_REPL;
if (ro || (!(script_flags & SCRIPT_FLAG_EVAL_COMPAT_MODE) && (script_flags & SCRIPT_FLAG_NO_WRITES))) {
/* On fcall_ro or on functions that do not have the 'write'
* flag, we will not allow write commands. */
run_ctx->flags |= SCRIPT_READ_ONLY;
}
if (!(script_flags & SCRIPT_FLAG_EVAL_COMPAT_MODE) && (script_flags & SCRIPT_FLAG_ALLOW_OOM)) {
run_ctx->flags |= SCRIPT_ALLOW_OOM;
}
/* set the curr_run_ctx so we can use it to kill the script if needed */
curr_run_ctx = run_ctx;
return C_OK;
}
/* Reset the given run ctx after execution */
@ -279,7 +353,7 @@ static int scriptVerifyWriteCommandAllow(scriptRunCtx *run_ctx, char **err) {
} else {
*err = sdsempty();
*err = sdscatfmt(*err,
"MISCONF Errors writing to the AOF file: %s\r\n",
"-MISCONF Errors writing to the AOF file: %s\r\n",
strerror(server.aof_last_write_errno));
}
return C_ERR;

11
src/script.h
View File

@ -77,10 +77,11 @@ struct scriptRunCtx {
};
/* Scripts flags */
#define SCRIPT_FLAG_NO_WRITES (1ULL<<0)
#define SCRIPT_FLAG_ALLOW_OOM (1ULL<<1)
#define SCRIPT_FLAG_ALLOW_STALE (1ULL<<3)
#define SCRIPT_FLAG_NO_CLUSTER (1ULL<<4)
#define SCRIPT_FLAG_NO_WRITES (1ULL<<0)
#define SCRIPT_FLAG_ALLOW_OOM (1ULL<<1)
#define SCRIPT_FLAG_ALLOW_STALE (1ULL<<2)
#define SCRIPT_FLAG_NO_CLUSTER (1ULL<<3)
#define SCRIPT_FLAG_EVAL_COMPAT_MODE (1ULL<<4) /* EVAL Script backwards compatible behavior, no shebang provided */
/* Defines a script flags */
typedef struct scriptFlag {
@ -90,7 +91,7 @@ typedef struct scriptFlag {
extern scriptFlag scripts_flags_def[];
void scriptPrepareForRun(scriptRunCtx *r_ctx, client *engine_client, client *caller, const char *funcname);
int scriptPrepareForRun(scriptRunCtx *r_ctx, client *engine_client, client *caller, const char *funcname, uint64_t script_flags, int ro);
void scriptResetRun(scriptRunCtx *r_ctx);
int scriptSetResp(scriptRunCtx *r_ctx, int resp);
int scriptSetRepl(scriptRunCtx *r_ctx, int repl);

13
src/server.c
View File

@ -407,17 +407,6 @@ dictType dbDictType = {
dictEntryMetadataSize /* size of entry metadata in bytes */
};
/* server.lua_scripts sha (as sds string) -> scripts (as robj) cache. */
dictType shaScriptObjectDictType = {
dictSdsCaseHash, /* hash function */
NULL, /* key dup */
NULL, /* val dup */
dictSdsKeyCaseCompare, /* key compare */
dictSdsDestructor, /* key destructor */
dictObjectDestructor, /* val destructor */
NULL /* allow to expand */
};
/* Db->expires */
dictType dbExpiresDictType = {
dictSdsHash, /* hash function */
@ -1618,7 +1607,7 @@ void createSharedObjects(void) {
shared.masterdownerr = createObject(OBJ_STRING,sdsnew(
"-MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'.\r\n"));
shared.bgsaveerr = createObject(OBJ_STRING,sdsnew(
"-MISCONF Redis is configured to save RDB snapshots, but it is currently not able to persist on disk. Commands that may modify the data set are disabled, because this instance is configured to report errors during writes if RDB snapshotting fails (stop-writes-on-bgsave-error option). Please check the Redis logs for details about the RDB error.\r\n"));
"-MISCONF Redis is configured to save RDB snapshots, but it's currently unable to persist to disk. Commands that may modify the data set are disabled, because this instance is configured to report errors during writes if RDB snapshotting fails (stop-writes-on-bgsave-error option). Please check the Redis logs for details about the RDB error.\r\n"));
shared.roslaveerr = createObject(OBJ_STRING,sdsnew(
"-READONLY You can't write against a read only replica.\r\n"));
shared.noautherr = createObject(OBJ_STRING,sdsnew(

1
src/server.h
View File

@ -2293,7 +2293,6 @@ extern dictType objectKeyHeapPointerValueDictType;
extern dictType setDictType;
extern dictType zsetDictType;
extern dictType dbDictType;
extern dictType shaScriptObjectDictType;
extern double R_Zero, R_PosInf, R_NegInf, R_Nan;
extern dictType hashDictType;
extern dictType replScriptCacheDictType;

12
tests/cluster/tests/00-base.tcl
View File

@ -68,5 +68,15 @@ test "Function no-cluster flag" {
redis.register_function{function_name='f1', callback=function() return 'hello' end, flags={'no-cluster'}}
}
catch {R 1 fcall f1 0} e
assert_match {*Can not run function on cluster, 'no-cluster' flag is set*} $e
assert_match {*Can not run script on cluster, 'no-cluster' flag is set*} $e
}
test "Script no-cluster flag" {
catch {
R 1 eval {#!lua flags=no-cluster
return 1
} 0
} e
assert_match {*Can not run script on cluster, 'no-cluster' flag is set*} $e
}

6
tests/unit/functions.tcl
View File

@ -428,7 +428,7 @@ start_server {tags {"scripting repl external:skip"}} {
r -1 fcall test 0
} e
set _ $e
} {*Can not run a function with write flag on readonly replica*}
} {*Can not run script with write flag on readonly replica*}
}
}
@ -1023,7 +1023,7 @@ start_server {tags {"scripting"}} {
}}
catch {r fcall_ro f1 0} e
set _ $e
} {*Can not execute a function with write flag using fcall_ro*}
} {*Can not execute a script with write flag using \*_ro command*}
test {FUNCTION - write script with no-writes flag} {
r function load lua test replace {redis.register_function{
@ -1072,7 +1072,7 @@ start_server {tags {"scripting"}} {
r replicaof 127.0.0.1 1
catch {[r fcall f1 0]} e
assert_match {*'allow-stale' flag is not set on the function*} $e
assert_match {*'allow-stale' flag is not set on the script*} $e
assert_equal {hello} [r fcall f2 0]

153
tests/unit/scripting.tcl
View File

@ -1216,3 +1216,156 @@ start_server {tags {"scripting needs:debug"}} {
r debug set-disable-deny-scripts 0
}
} ;# foreach is_eval
# Scripting "shebang" notation tests
start_server {tags {"scripting"}} {
test "Shebang support for lua engine" {
catch {
r eval {#!not-lua
return 1
} 0
} e
assert_match {*Unexpected engine in script shebang*} $e
assert_equal [r eval {#!lua
return 1
} 0] 1
}
test "Unknown shebang option" {
catch {
r eval {#!lua badger=data
return 1
} 0
} e
assert_match {*Unknown lua shebang option*} $e
}
test "Unknown shebang flag" {
catch {
r eval {#!lua flags=allow-oom,what?
return 1
} 0
} e
assert_match {*Unexpected flag in script shebang*} $e
}
test "allow-oom shebang flag" {
r set x 123
r config set maxmemory 1
# Fail to execute deny-oom command in OOM condition (backwards compatibility mode without flags)
assert_error {ERR Error running script *OOM command not allowed when used memory > 'maxmemory'.} {
r eval {
redis.call('set','x',1)
return 1
} 1 x
}
# Can execute non deny-oom commands in OOM condition (backwards compatibility mode without flags)
assert_equal [
r eval {
return redis.call('get','x')
} 1 x
] {123}
# Fail to execute regardless of script content when we use default flags in OOM condition
assert_error {OOM allow-oom flag is not set on the script, can not run it when used memory > 'maxmemory'} {
r eval {#!lua flags=
return 1
} 0
}
assert_equal [
r eval {#!lua flags=allow-oom
redis.call('set','x',1)
return 1
} 0
] 1
r config set maxmemory 0
}
test "no-writes shebang flag" {
assert_error {ERR Error running script *Write commands are not allowed from read-only scripts.} {
r eval {#!lua flags=no-writes
redis.call('set','x',1)
return 1
} 1 x
}
}
start_server {tags {"external:skip"}} {
r -1 set x "some value"
test "no-writes shebang flag on replica" {
r replicaof [srv -1 host] [srv -1 port]
wait_for_condition 50 100 {
[s role] eq {slave} &&
[string match {*master_link_status:up*} [r info replication]]
} else {
fail "Can't turn the instance into a replica"
}
assert_equal [
r eval {#!lua flags=no-writes
return redis.call('get','x')
} 1 x
] "some value"
assert_error {ERR Can not run script with write flag on readonly replica} {
r eval {#!lua
return redis.call('get','x')
} 1 x
}
}
}
test "allow-stale shebang flag" {
r config set replica-serve-stale-data no
r replicaof 127.0.0.1 1
assert_error {MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'.} {
r eval {
return redis.call('get','x')
} 1 x
}
assert_error {*'allow-stale' flag is not set on the script*} {
r eval {#!lua flags=no-writes
return 1
} 0
}
assert_equal [
r eval {#!lua flags=allow-stale,no-writes
return 1
} 0
] 1
assert_error {*Can not execute the command on a stale replica*} {
r eval {#!lua flags=allow-stale,no-writes
return redis.call('get','x')
} 1 x
}
assert_match {*redis_version*} [
r eval {#!lua flags=allow-stale,no-writes
return redis.call('info','server')
} 0
]
# Test again with EVALSHA
set sha [
r script load {#!lua flags=allow-stale,no-writes
return redis.call('info','server')
}
]
assert_match {*redis_version*} [r evalsha $sha 0]
r replicaof no one
r config set replica-serve-stale-data yes
set _ {}
} {} {external:skip}
}