diff --git a/src/aof.c b/src/aof.c index 9e602ce01..9ad85c536 100644 --- a/src/aof.c +++ b/src/aof.c @@ -998,6 +998,7 @@ int rewriteAppendOnlyFileBackground(void) { * accumulated by the parent into server.aof_rewrite_buf will start * with a SELECT statement and it will be safe to merge. */ server.aof_selected_db = -1; + replicationScriptCacheFlush(); return REDIS_OK; } return REDIS_OK; /* unreached */ diff --git a/src/redis.c b/src/redis.c index 5928357ae..fdd30d148 100644 --- a/src/redis.c +++ b/src/redis.c @@ -591,10 +591,10 @@ dictType migrateCacheDictType = { * Keys are sds SHA1 strings, while values are not used at all in the current * implementation. */ dictType replScriptCacheDictType = { - dictSdsHash, /* hash function */ + dictSdsCaseHash, /* hash function */ NULL, /* key dup */ NULL, /* val dup */ - dictSdsKeyCompare, /* key compare */ + dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ NULL /* val destructor */ }; diff --git a/src/redis.h b/src/redis.h index e2cfbcccd..5d9dcd349 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1165,6 +1165,10 @@ void resizeReplicationBacklog(long long newsize); void replicationSetMaster(char *ip, int port); void replicationUnsetMaster(void); void refreshGoodSlavesCount(void); +void replicationScriptCacheInit(void); +void replicationScriptCacheFlush(void); +void replicationScriptCacheAdd(sds sha1); +int replicationScriptCacheExists(sds sha1); /* Generic persistence functions */ void startLoading(FILE *fp); diff --git a/src/replication.c b/src/replication.c index e2d5b3ec4..04a74dbdc 100644 --- a/src/replication.c +++ b/src/replication.c @@ -546,6 +546,8 @@ void syncCommand(redisClient *c) { return; } c->replstate = REDIS_REPL_WAIT_BGSAVE_END; + /* Flush the script cache for the new slave. */ + replicationScriptCacheFlush(); } if (server.repl_disable_tcp_nodelay) @@ -711,6 +713,11 @@ void updateSlavesWaitingBgsave(int bgsaveerr) { } } if (startbgsave) { + /* Since we are starting a new background save for one or more slaves, + * we flush the Replication Script Cache to use EVAL to propagate every + * new EVALSHA for the first time, since all the new slaves don't know + * about previous scripts. */ + replicationScriptCacheFlush(); if (rdbSaveBackground(server.rdb_filename) != REDIS_OK) { listIter li; @@ -1471,10 +1478,16 @@ void replicationScriptCacheInit(void) { } /* Empty the script cache. Should be called every time we are no longer sure - * that every slave knows about all the scripts in our set, for example - * every time a new slave connects to this master and performs a full - * resynchronization. There is no need to flush the cache when a partial - * resynchronization is performed. */ + * that every slave knows about all the scripts in our set, or when the + * current AOF "context" is no longer aware of the script. In general we + * should flush the cache: + * + * 1) Every time a new slave reconnects to this master and performs a + * full SYNC (PSYNC does not require flushing). + * 2) Every time an AOF rewrite is performed. + * 3) Every time we are left without slaves at all, and AOF is off, in order + * to reclaim otherwise unused memory. + */ void replicationScriptCacheFlush(void) { dictEmpty(server.repl_scriptcache_dict); listRelease(server.repl_scriptcache_fifo); @@ -1507,7 +1520,7 @@ void replicationScriptCacheAdd(sds sha1) { /* Returns non-zero if the specified entry exists inside the cache, that is, * if all the slaves are aware of this script SHA1. */ int replicationScriptCacheExists(sds sha1) { - return dictFetchValue(server.repl_scriptcache_dict,sha1) != NULL; + return dictFind(server.repl_scriptcache_dict,sha1) != NULL; } /* --------------------------- REPLICATION CRON ----------------------------- */ @@ -1624,6 +1637,16 @@ void replicationCron(void) { } } + /* If AOF is disabled and we no longer have attached slaves, we can + * free our Replication Script Cache as there is no need to propagate + * EVALSHA at all. */ + if (listLength(server.slaves) == 0 && + server.aof_state == REDIS_AOF_OFF && + listLength(server.repl_scriptcache_fifo) != 0) + { + replicationScriptCacheFlush(); + } + /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ refreshGoodSlavesCount(); } diff --git a/src/scripting.c b/src/scripting.c index 104bd3dde..f30956bd2 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -655,6 +655,10 @@ void scriptingInit(void) { * to global variables. */ scriptingEnableGlobalsProtection(lua); + /* Initialize the Replication Script Cache for EVALSHA propagation to + * slaves and AOF. */ + replicationScriptCacheInit(); + server.lua = lua; } @@ -931,23 +935,29 @@ void evalGenericCommand(redisClient *c, int evalsha) { luaReplyToRedisReply(c,lua); } - /* If we have slaves attached we want to replicate this command as - * EVAL instead of EVALSHA. We do this also in the AOF as currently there - * is no easy way to propagate a command in a different way in the AOF - * and in the replication link. + /* EVALSHA should be propagated to Slave and AOF file as full EVAL, unless + * we are sure that the script was already in the context of all the + * attached slaves *and* the current AOF file if enabled. * - * IMPROVEMENT POSSIBLE: - * 1) Replicate this command as EVALSHA in the AOF. - * 2) Remember what slave already received a given script, and replicate - * the EVALSHA against this slaves when possible. - */ + * To do so we use a cache of SHA1s of scripts that we already propagated + * as full EVAL, that's called the Replication Script Cache. + * + * For repliation, everytime a new slave attaches to the master, we need to + * flush our cache of scripts that can be replicated as EVALSHA, while + * for AOF we need to do so every time we rewrite the AOF file. */ if (evalsha) { - robj *script = dictFetchValue(server.lua_scripts,c->argv[1]->ptr); + if (!replicationScriptCacheExists(c->argv[1]->ptr)) { + /* This script is not in our script cache, replicate it as + * EVAL, then add it into the script cache, as from now on + * slaves and AOF know about it. */ + robj *script = dictFetchValue(server.lua_scripts,c->argv[1]->ptr); - redisAssertWithInfo(c,NULL,script != NULL); - rewriteClientCommandArgument(c,0, - resetRefCount(createStringObject("EVAL",4))); - rewriteClientCommandArgument(c,1,script); + replicationScriptCacheAdd(c->argv[1]->ptr); + redisAssertWithInfo(c,NULL,script != NULL); + rewriteClientCommandArgument(c,0, + resetRefCount(createStringObject("EVAL",4))); + rewriteClientCommandArgument(c,1,script); + } } }