redis-cli --replica reads dummy empty rdb instead of full snapshot (#10044)

This makes redis-cli --replica much faster and reduces COW/fork risks on server side.
This commit also improves the RDB filtering via REPLCONF rdb-filter-only to support no "include" specifiers at all.
This commit is contained in:
yoav-steinberg 2022-01-04 17:09:22 +02:00 committed by GitHub
parent d5a3b3f5ec
commit 65a7635793
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 23 additions and 15 deletions

View File

@ -1337,19 +1337,19 @@ int rdbSaveRio(int req, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION); snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION);
if (rdbWriteRaw(rdb,magic,9) == -1) goto werr; if (rdbWriteRaw(rdb,magic,9) == -1) goto werr;
if (rdbSaveInfoAuxFields(rdb,rdbflags,rsi) == -1) goto werr; if (rdbSaveInfoAuxFields(rdb,rdbflags,rsi) == -1) goto werr;
if (!(req & SLAVE_REQ_RDB_FUNCTIONS_ONLY) && rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr; if (!(req & SLAVE_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr;
/* save functions */ /* save functions */
if (rdbSaveFunctions(rdb) == -1) goto werr; if (!(req & SLAVE_REQ_RDB_EXCLUDE_FUNCTIONS) && rdbSaveFunctions(rdb) == -1) goto werr;
/* save all databases, skip this if we're in functions-only mode */ /* save all databases, skip this if we're in functions-only mode */
if (!(req & SLAVE_REQ_RDB_FUNCTIONS_ONLY)) { if (!(req & SLAVE_REQ_RDB_EXCLUDE_DATA)) {
for (j = 0; j < server.dbnum; j++) { for (j = 0; j < server.dbnum; j++) {
if (rdbSaveDb(rdb, j, rdbflags, &key_counter) == -1) goto werr; if (rdbSaveDb(rdb, j, rdbflags, &key_counter) == -1) goto werr;
} }
} }
if (!(req & SLAVE_REQ_RDB_FUNCTIONS_ONLY) && rdbSaveModulesAux(rdb, REDISMODULE_AUX_AFTER_RDB) == -1) goto werr; if (!(req & SLAVE_REQ_RDB_EXCLUDE_DATA) && rdbSaveModulesAux(rdb, REDISMODULE_AUX_AFTER_RDB) == -1) goto werr;
/* EOF opcode */ /* EOF opcode */
if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr; if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr;

View File

@ -8542,6 +8542,7 @@ int main(int argc, char **argv) {
if (config.slave_mode) { if (config.slave_mode) {
if (cliConnect(0) == REDIS_ERR) exit(1); if (cliConnect(0) == REDIS_ERR) exit(1);
sendCapa(); sendCapa();
sendReplconf("rdb-filter-only", "");
slaveMode(); slaveMode();
} }

View File

@ -833,11 +833,11 @@ int startBgsaveForReplication(int mincapa, int req) {
listNode *ln; listNode *ln;
/* We use a socket target if slave can handle the EOF marker and we're configured to do diskless syncs. /* We use a socket target if slave can handle the EOF marker and we're configured to do diskless syncs.
* Note that in case we're creating a "filtered" RDB (functions-only) we also force socket replication * Note that in case we're creating a "filtered" RDB (functions-only, for example) we also force socket replication
* to avoid overwriting the snapshot RDB file with filtered data. */ * to avoid overwriting the snapshot RDB file with filtered data. */
socket_target = (server.repl_diskless_sync || (req & SLAVE_REQ_RDB_FUNCTIONS_ONLY)) && (mincapa & SLAVE_CAPA_EOF); socket_target = (server.repl_diskless_sync || req & SLAVE_REQ_RDB_MASK) && (mincapa & SLAVE_CAPA_EOF);
/* `SYNC` should have failed with error if we don't support socket and require a filter, assert this here */ /* `SYNC` should have failed with error if we don't support socket and require a filter, assert this here */
serverAssert(socket_target || !(req & SLAVE_REQ_RDB_FUNCTIONS_ONLY)); serverAssert(socket_target || !(req & SLAVE_REQ_RDB_MASK));
serverLog(LL_NOTICE,"Starting BGSAVE for SYNC with target: %s", serverLog(LL_NOTICE,"Starting BGSAVE for SYNC with target: %s",
socket_target ? "replicas sockets" : "disk"); socket_target ? "replicas sockets" : "disk");
@ -958,7 +958,7 @@ void syncCommand(client *c) {
/* Fail sync if slave doesn't support EOF capability but wants a filtered RDB. This is because we force filtered /* Fail sync if slave doesn't support EOF capability but wants a filtered RDB. This is because we force filtered
* RDB's to be generated over a socket and not through a file to avoid conflicts with the snapshot files. Forcing * RDB's to be generated over a socket and not through a file to avoid conflicts with the snapshot files. Forcing
* use of a socket is handled, if needed, in `startBgsaveForReplication`. */ * use of a socket is handled, if needed, in `startBgsaveForReplication`. */
if ((c->slave_req & SLAVE_REQ_RDB_FUNCTIONS_ONLY) && !(c->slave_capa & SLAVE_CAPA_EOF)) { if (c->slave_req & SLAVE_REQ_RDB_MASK && !(c->slave_capa & SLAVE_CAPA_EOF)) {
addReplyError(c,"Filtered replica requires EOF capability"); addReplyError(c,"Filtered replica requires EOF capability");
return; return;
} }
@ -1124,7 +1124,8 @@ void syncCommand(client *c) {
* *
* - rdb-filter-only <include-filters> * - rdb-filter-only <include-filters>
* Define "include" filters for the RDB snapshot. Currently we only support * Define "include" filters for the RDB snapshot. Currently we only support
* a single include filter: "functions". */ * a single include filter: "functions". Passing an empty string "" will
* result in an empty RDB. */
void replconfCommand(client *c) { void replconfCommand(client *c) {
int j; int j;
@ -1214,9 +1215,12 @@ void replconfCommand(client *c) {
addReplyErrorFormat(c, "Missing rdb-filter-only values"); addReplyErrorFormat(c, "Missing rdb-filter-only values");
return; return;
} }
/* By default filter out all parts of the rdb */
c->slave_req |= SLAVE_REQ_RDB_EXCLUDE_DATA;
c->slave_req |= SLAVE_REQ_RDB_EXCLUDE_FUNCTIONS;
for (i = 0; i < filter_count; i++) { for (i = 0; i < filter_count; i++) {
if (!strcasecmp(filters[i], "functions")) if (!strcasecmp(filters[i], "functions"))
c->slave_req |= SLAVE_REQ_RDB_FUNCTIONS_ONLY; c->slave_req &= ~SLAVE_REQ_RDB_EXCLUDE_FUNCTIONS;
else { else {
addReplyErrorFormat(c, "Unsupported rdb-filter-only option: %s", (char*)filters[i]); addReplyErrorFormat(c, "Unsupported rdb-filter-only option: %s", (char*)filters[i]);
sdsfreesplitres(filters, filter_count); sdsfreesplitres(filters, filter_count);

View File

@ -387,7 +387,10 @@ typedef enum {
/* Slave requirements */ /* Slave requirements */
#define SLAVE_REQ_NONE 0 #define SLAVE_REQ_NONE 0
#define SLAVE_REQ_RDB_FUNCTIONS_ONLY (1 << 0) #define SLAVE_REQ_RDB_EXCLUDE_DATA (1 << 0) /* Exclude data from RDB */
#define SLAVE_REQ_RDB_EXCLUDE_FUNCTIONS (1 << 1) /* Exclude functions from RDB */
/* Mask of all bits in the slave requirements bitfield that represent non-standard (filtered) RDB requirements */
#define SLAVE_REQ_RDB_MASK (SLAVE_REQ_RDB_EXCLUDE_DATA | SLAVE_REQ_RDB_EXCLUDE_FUNCTIONS)
/* Synchronous read timeout - slave side */ /* Synchronous read timeout - slave side */
#define CONFIG_REPL_SYNCIO_TIMEOUT 5 #define CONFIG_REPL_SYNCIO_TIMEOUT 5