From fc7ecd8d359160c5b74aa86c09eefdf4e9e2d653 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Jul 2017 13:38:23 +0200 Subject: [PATCH] AOF check utility: ability to check files with RDB preamble. --- src/Makefile | 11 +++++------ src/rdb.c | 2 +- src/redis-check-aof.c | 38 ++++++++++++++++++++++++++------------ src/redis-check-rdb.c | 32 +++++++++++++++++++++----------- src/server.c | 6 ++++-- src/server.h | 7 ++++--- 6 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/Makefile b/src/Makefile index 70574630a..86e0b3fe0 100644 --- a/src/Makefile +++ b/src/Makefile @@ -144,14 +144,13 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o REDIS_BENCHMARK_NAME=redis-benchmark REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o REDIS_CHECK_RDB_NAME=redis-check-rdb REDIS_CHECK_AOF_NAME=redis-check-aof -REDIS_CHECK_AOF_OBJ=redis-check-aof.o all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" @@ -207,6 +206,10 @@ $(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME) $(REDIS_CHECK_RDB_NAME): $(REDIS_SERVER_NAME) $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_CHECK_RDB_NAME) +# redis-check-aof +$(REDIS_CHECK_AOF_NAME): $(REDIS_SERVER_NAME) + $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) + # redis-cli $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS) @@ -215,10 +218,6 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) -# redis-check-aof -$(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ) - $(REDIS_LD) -o $@ $^ $(FINAL_LIBS) - dict-benchmark: dict.c zmalloc.c sds.c siphash.c $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) diff --git a/src/rdb.c b/src/rdb.c index 1341942c4..792c8ff94 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -61,7 +61,7 @@ void rdbCheckThenExit(int linenum, char *reason, ...) { if (!rdbCheckMode) { serverLog(LL_WARNING, "%s", msg); char *argv[2] = {"",server.rdb_filename}; - redis_check_rdb_main(2,argv); + redis_check_rdb_main(2,argv,NULL); } else { rdbCheckError("%s",msg); } diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index 6c8f55279..09a118eb7 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -28,13 +28,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#include "fmacros.h" -#include -#include -#include -#include +#include "server.h" #include -#include "config.h" #define ERROR(...) { \ char __buf[1024]; \ @@ -60,7 +55,7 @@ int readLong(FILE *fp, char prefix, long *target) { return 0; } if (buf[0] != prefix) { - ERROR("Expected prefix '%c', got: '%c'",buf[0],prefix); + ERROR("Expected prefix '%c', got: '%c'",prefix,buf[0]); return 0; } *target = strtol(buf+1,&eptr,10); @@ -87,7 +82,7 @@ int readString(FILE *fp, char** target) { /* Increase length to also consume \r\n */ len += 2; - *target = (char*)malloc(len); + *target = (char*)zmalloc(len); if (!readBytes(fp,*target,len)) { return 0; } @@ -127,12 +122,12 @@ off_t process(FILE *fp) { } } } - free(str); + zfree(str); } /* Stop if the loop did not finish */ if (i < argc) { - if (str) free(str); + if (str) zfree(str); break; } } @@ -146,7 +141,7 @@ off_t process(FILE *fp) { return pos; } -int main(int argc, char **argv) { +int redis_check_aof_main(int argc, char **argv) { char *filename; int fix = 0; @@ -185,6 +180,25 @@ int main(int argc, char **argv) { exit(1); } + /* This AOF file may have an RDB preamble. Check this to start, and if this + * is the case, start processing the RDB part. */ + if (size >= 8) { /* There must be at least room for the RDB header. */ + char sig[5]; + int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 && + memcmp(sig,"REDIS",sizeof(sig)) == 0; + rewind(fp); + if (has_preamble) { + printf("The AOF appears to start with an RDB preamble.\n" + "Checking the RDB preamble to start:\n"); + if (redis_check_rdb_main(argc,argv,fp) == C_ERR) { + printf("RDB preamble of AOF file is not sane, aborting.\n"); + exit(1); + } else { + printf("RDB preamble is OK, proceding with AOF tail...\n"); + } + } + } + off_t pos = process(fp); off_t diff = size-pos; printf("AOF analyzed: size=%lld, ok_up_to=%lld, diff=%lld\n", @@ -214,5 +228,5 @@ int main(int argc, char **argv) { } fclose(fp); - return 0; + exit(0); } diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 08be40f6a..4027536e5 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -173,16 +173,18 @@ void rdbCheckSetupSignals(void) { } /* Check the specified RDB file. Return 0 if the RDB looks sane, otherwise - * 1 is returned. */ -int redis_check_rdb(char *rdbfilename) { + * 1 is returned. + * The file is specified as a filename in 'rdbfilename' if 'fp' is not NULL, + * otherwise the already open file 'fp' is checked. */ +int redis_check_rdb(char *rdbfilename, FILE *fp) { uint64_t dbid; int type, rdbver; char buf[1024]; long long expiretime, now = mstime(); - FILE *fp; static rio rdb; /* Pointed by global struct riostate. */ - if ((fp = fopen(rdbfilename,"r")) == NULL) return 1; + int closefile = (fp == NULL); + if (fp == NULL && (fp = fopen(rdbfilename,"r")) == NULL) return 1; rioInitWithFile(&rdb,fp); rdbstate.rio = &rdb; @@ -310,7 +312,7 @@ int redis_check_rdb(char *rdbfilename) { } } - fclose(fp); + if (closefile) fclose(fp); return 0; eoferr: /* unexpected end of file is handled here with a fatal exit */ @@ -323,12 +325,19 @@ eoferr: /* unexpected end of file is handled here with a fatal exit */ } /* RDB check main: called form redis.c when Redis is executed with the - * redis-check-rdb alias. + * redis-check-rdb alias, on during RDB loading errors. * - * The function never returns, but exits with the status code according - * to success (RDB is sane) or error (RDB is corrupted). */ -int redis_check_rdb_main(int argc, char **argv) { - if (argc != 2) { + * The function works in two ways: can be called with argc/argv as a + * standalone executable, or called with a non NULL 'fp' argument if we + * already have an open file to check. This happens when the function + * is used to check an RDB preamble inside an AOF file. + * + * When called with fp = NULL, the function never returns, but exits with the + * status code according to success (RDB is sane) or error (RDB is corrupted). + * Otherwise if called with a non NULL fp, the function returns C_OK or + * C_ERR depending on the success or failure. */ +int redis_check_rdb_main(int argc, char **argv, FILE *fp) { + if (argc != 2 && fp == NULL) { fprintf(stderr, "Usage: %s \n", argv[0]); exit(1); } @@ -341,10 +350,11 @@ int redis_check_rdb_main(int argc, char **argv) { rdbCheckMode = 1; rdbCheckInfo("Checking RDB file %s", argv[1]); rdbCheckSetupSignals(); - int retval = redis_check_rdb(argv[1]); + int retval = redis_check_rdb(argv[1],fp); if (retval == 0) { rdbCheckInfo("\\o/ RDB looks OK! \\o/"); rdbShowGenericInfo(); } + if (fp) return (retval == 0) ? C_OK : C_ERR; exit(retval); } diff --git a/src/server.c b/src/server.c index 57ee43e38..2da6fb544 100644 --- a/src/server.c +++ b/src/server.c @@ -3711,11 +3711,13 @@ int main(int argc, char **argv) { initSentinel(); } - /* Check if we need to start in redis-check-rdb mode. We just execute + /* Check if we need to start in redis-check-rdb/aof mode. We just execute * the program main. However the program is part of the Redis executable * so that we can easily execute an RDB check on loading errors. */ if (strstr(argv[0],"redis-check-rdb") != NULL) - redis_check_rdb_main(argc,argv); + redis_check_rdb_main(argc,argv,NULL); + else if (strstr(argv[0],"redis-check-aof") != NULL) + redis_check_aof_main(argc,argv); if (argc >= 2) { j = 1; /* First option to parse in argv[] */ diff --git a/src/server.h b/src/server.h index 88690ace4..2be75f1a1 100644 --- a/src/server.h +++ b/src/server.h @@ -1770,9 +1770,10 @@ void sentinelTimer(void); char *sentinelHandleConfiguration(char **argv, int argc); void sentinelIsRunning(void); -/* redis-check-rdb */ -int redis_check_rdb(char *rdbfilename); -int redis_check_rdb_main(int argc, char **argv); +/* redis-check-rdb & aof */ +int redis_check_rdb(char *rdbfilename, FILE *fp); +int redis_check_rdb_main(int argc, char **argv, FILE *fp); +int redis_check_aof_main(int argc, char **argv); /* Scripting */ void scriptingInit(int setup);