caleb-brown/redict
1
0
Fork 0
mirror of https://codeberg.org/redict/redict.git synced 2025-01-22 16:18:28 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Trim rdb loading code for pre-release formats (#11058)

Browse Source 
The initial module format introduced in 4.0  RC1 and was changed in RC2
The initial function format introduced in 7.0 RC1 and changed in RC3
This commit is contained in:
Oran Agra 2022-08-15 21:41:44 +03:00 committed by GitHub
parent 1189680edd
commit ac1cc5a6e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 52 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/functions.c
View File

@ -754,11 +754,15 @@ void functionRestoreCommand(client *c) {
err = sdsnew("can not read data type");
goto load_error;
}
if (type != RDB_OPCODE_FUNCTION && type != RDB_OPCODE_FUNCTION2) {
if (type == RDB_OPCODE_FUNCTION_PRE_GA) {
err = sdsnew("Pre-GA function format not supported");
goto load_error;
}
if (type != RDB_OPCODE_FUNCTION2) {
err = sdsnew("given type is not a function");
goto load_error;
}
if (rdbFunctionLoad(&payload, rdbver, functions_lib_ctx, type, RDBFLAGS_NONE, &err) != C_OK) {
if (rdbFunctionLoad(&payload, rdbver, functions_lib_ctx, RDBFLAGS_NONE, &err) != C_OK) {
if (!err) {
err = sdsnew("failed loading the given functions payload");
}

25
src/module.c
View File

@ -6543,10 +6543,8 @@ saveerr:
* new data types. */
uint64_t RM_LoadUnsigned(RedisModuleIO *io) {
if (io->error) return 0;
if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr;
}
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr;
uint64_t value;
int retval = rdbLoadLenByRef(io->rio, NULL, &value);
if (retval == -1) goto loaderr;
@ -6614,10 +6612,8 @@ saveerr:
/* Implements RM_LoadString() and RM_LoadStringBuffer() */
void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) {
if (io->error) return NULL;
if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr;
}
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr;
void *s = rdbGenericLoadStringObject(io->rio,
plain ? RDB_LOAD_PLAIN : RDB_LOAD_NONE, lenptr);
if (s == NULL) goto loaderr;
@ -6675,10 +6671,8 @@ saveerr:
* double value saved by RedisModule_SaveDouble(). */
double RM_LoadDouble(RedisModuleIO *io) {
if (io->error) return 0;
if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr;
}
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr;
double value;
int retval = rdbLoadBinaryDoubleValue(io->rio, &value);
if (retval == -1) goto loaderr;
@ -6712,10 +6706,8 @@ saveerr:
* float value saved by RedisModule_SaveFloat(). */
float RM_LoadFloat(RedisModuleIO *io) {
if (io->error) return 0;
if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr;
}
uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr;
float value;
int retval = rdbLoadBinaryFloatValue(io->rio, &value);
if (retval == -1) goto loaderr;
@ -6866,7 +6858,6 @@ void *RM_LoadDataTypeFromStringEncver(const RedisModuleString *str, const module
/* All RM_Save*() calls always write a version 2 compatible format, so we
* need to make sure we read the same.
*/
io.ver = 2;
ret = mt->rdb_load(&io,encver);
if (io.ctx) {
moduleFreeContext(io.ctx);

102
src/rdb.c
View File

@ -2605,7 +2605,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
raxStop(&ri_cg_pel);
}
}
} else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) {
} else if (rdbtype == RDB_TYPE_MODULE_PRE_GA) {
rdbReportCorruptRDB("Pre-release module format not supported");
return NULL;
} else if (rdbtype == RDB_TYPE_MODULE_2) {
uint64_t moduleid = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) {
rdbReportReadError("Short read module id");
@ -2613,7 +2616,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
}
moduleType *mt = moduleTypeLookupModuleByID(moduleid);
if (rdbCheckMode && rdbtype == RDB_TYPE_MODULE_2) {
if (rdbCheckMode) {
char name[10];
moduleTypeNameByID(name,moduleid);
return rdbLoadCheckModuleValue(rdb,name);
@ -2629,7 +2632,6 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
robj keyobj;
initStaticStringObject(keyobj,key);
moduleInitIOContext(io,mt,rdb,&keyobj,dbid);
io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2;
/* Call the rdb_load method of the module providing the 10 bit
* encoding version in the lower 10 bits of the module ID. */
void *ptr = mt->rdb_load(&io,moduleid&1023);
@ -2639,24 +2641,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
}
/* Module v2 serialization has an EOF mark at the end. */
if (io.ver == 2) {
uint64_t eof = rdbLoadLen(rdb,NULL);
if (eof == RDB_LENERR) {
if (ptr) {
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
decrRefCount(o);
}
return NULL;
uint64_t eof = rdbLoadLen(rdb,NULL);
if (eof == RDB_LENERR) {
if (ptr) {
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
decrRefCount(o);
}
if (eof != RDB_MODULE_OPCODE_EOF) {
rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by "
"the proper module value EOF marker", moduleTypeModuleName(mt));
if (ptr) {
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
decrRefCount(o);
}
return NULL;
return NULL;
}
if (eof != RDB_MODULE_OPCODE_EOF) {
rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by "
"the proper module value EOF marker", moduleTypeModuleName(mt));
if (ptr) {
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
decrRefCount(o);
}
return NULL;
}
if (ptr == NULL) {
@ -2794,62 +2794,14 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) {
*
* The lib_ctx argument is also optional. If NULL is given, only verify rdb
* structure with out performing the actual functions loading. */
int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int type, int rdbflags, sds *err) {
int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, sds *err) {
UNUSED(ver);
sds error = NULL;
sds final_payload = NULL;
int res = C_ERR;
if (type == RDB_OPCODE_FUNCTION) {
/* RDB that was generated on versions 7.0 rc1 and 7.0 rc2 has another
* an old format that contains the library name, engine and description.
* To support this format we must read those values. */
sds name = NULL;
sds engine_name = NULL;
sds desc = NULL;
sds blob = NULL;
uint64_t has_desc;
if (!(name = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library name");
goto cleanup;
}
if (!(engine_name = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading engine name");
goto cleanup;
}
if ((has_desc = rdbLoadLen(rdb, NULL)) == RDB_LENERR) {
error = sdsnew("Failed loading library description indicator");
goto cleanup;
}
if (has_desc && !(desc = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library description");
goto cleanup;
}
if (!(blob = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library blob");
goto cleanup;
}
/* Translate old format (versions 7.0 rc1 and 7.0 rc2) to new format.
* The new format has the library name and engine inside the script payload.
* Add those parameters to the original script payload (ignore the description if exists). */
final_payload = sdscatfmt(sdsempty(), "#!%s name=%s\n%s", engine_name, name, blob);
cleanup:
if (name) sdsfree(name);
if (engine_name) sdsfree(engine_name);
if (desc) sdsfree(desc);
if (blob) sdsfree(blob);
if (error) goto done;
} else if (type == RDB_OPCODE_FUNCTION2) {
if (!(final_payload = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library payload");
goto done;
}
} else {
serverPanic("Bad function type was given to rdbFunctionLoad");
if (!(final_payload = rdbGenericLoadStringObject(rdb, RDB_LOAD_SDS, NULL))) {
error = sdsnew("Failed loading library payload");
goto done;
}
if (lib_ctx) {
@ -3070,7 +3022,6 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
RedisModuleIO io;
moduleInitIOContext(io,mt,rdb,NULL,-1);
io.ver = 2;
/* Call the rdb_load method of the module providing the 10 bit
* encoding version in the lower 10 bits of the module ID. */
int rc = mt->aux_load(&io,moduleid&1023, when);
@ -3095,9 +3046,12 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
decrRefCount(aux);
continue; /* Read next opcode. */
}
} else if (type == RDB_OPCODE_FUNCTION || type == RDB_OPCODE_FUNCTION2) {
} else if (type == RDB_OPCODE_FUNCTION_PRE_GA) {
rdbReportCorruptRDB("Pre-release function format not supported.");
exit(1);
} else if (type == RDB_OPCODE_FUNCTION2) {
sds err = NULL;
if (rdbFunctionLoad(rdb, rdbver, rdb_loading_ctx->functions_lib_ctx, type, rdbflags, &err) != C_OK) {
if (rdbFunctionLoad(rdb, rdbver, rdb_loading_ctx->functions_lib_ctx, rdbflags, &err) != C_OK) {
serverLog(LL_WARNING,"Failed loading library, %s", err);
sdsfree(err);
goto eoferr;

6
src/rdb.h
View File

@ -78,7 +78,7 @@
#define RDB_TYPE_ZSET 3
#define RDB_TYPE_HASH 4
#define RDB_TYPE_ZSET_2 5 /* ZSET version 2 with doubles stored in binary. */
#define RDB_TYPE_MODULE 6
#define RDB_TYPE_MODULE_PRE_GA 6 /* Used in 4.0 release candidates */
#define RDB_TYPE_MODULE_2 7 /* Module value with annotations for parsing without
the generating module being loaded. */
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
@ -102,7 +102,7 @@
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_FUNCTION2 245 /* function library data */
#define RDB_OPCODE_FUNCTION 246 /* old function library data for 7.0 rc1 and rc2 */
#define RDB_OPCODE_FUNCTION_PRE_GA 246 /* old function library data for 7.0 rc1 and rc2 */
#define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */
#define RDB_OPCODE_IDLE 248 /* LRU idle time. */
#define RDB_OPCODE_FREQ 249 /* LFU frequency. */
@ -171,7 +171,7 @@ int rdbSaveBinaryFloatValue(rio *rdb, float val);
int rdbLoadBinaryFloatValue(rio *rdb, float *val);
int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi);
int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx);
int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int type, int rdbflags, sds *err);
int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, sds *err);
int rdbSaveRio(int req, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi);
ssize_t rdbSaveFunctions(rio *rdb);
rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi);

10
src/redis-check-rdb.c
View File

@ -85,8 +85,9 @@ char *rdb_type_string[] = {
"zset-v1",
"hash-hashtable",
"zset-v2",
"module-pre-release",
"module-value",
"","",
"",
"hash-zipmap",
"list-ziplist",
"set-intset",
@ -312,10 +313,13 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
robj *o = rdbLoadCheckModuleValue(&rdb,name);
decrRefCount(o);
continue; /* Read type again. */
} else if (type == RDB_OPCODE_FUNCTION || type == RDB_OPCODE_FUNCTION2) {
} else if (type == RDB_OPCODE_FUNCTION_PRE_GA) {
rdbCheckError("Pre-release function format not supported %d",rdbver);
goto err;
} else if (type == RDB_OPCODE_FUNCTION2) {
sds err = NULL;
rdbstate.doing = RDB_CHECK_DOING_READ_FUNCTIONS;
if (rdbFunctionLoad(&rdb, rdbver, NULL, type, 0, &err) != C_OK) {
if (rdbFunctionLoad(&rdb, rdbver, NULL, 0, &err) != C_OK) {
rdbCheckError("Failed loading library, %s", err);
sdsfree(err);
goto err;

3
src/server.h
View File

@ -791,8 +791,6 @@ typedef struct RedisModuleIO {
rio *rio; /* Rio stream. */
moduleType *type; /* Module type doing the operation. */
int error; /* True if error condition happened. */
int ver; /* Module serialization version: 1 (old),
* 2 (current version with opcodes annotation). */
struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/
struct redisObject *key; /* Optional name of key processed */
int dbid; /* The dbid of the key being processed, -1 when unknown. */
@ -805,7 +803,6 @@ typedef struct RedisModuleIO {
iovar.type = mtype; \
iovar.bytes = 0; \
iovar.error = 0; \
iovar.ver = 0; \
iovar.key = keyptr; \
iovar.dbid = db; \
iovar.ctx = NULL; \