redict/tests/unit/shutdown.tcl
Binbin 78f15b7ef1
Fix race in temp rdb delete shutdown test (#11840)
I saw this error once, in the FreeBSD Daily CI:
```
*** [err]: Temp rdb will be deleted if we use bg_unlink when shutdown in tests/unit/shutdown.tcl
Expected [file exists /xxx/temp-10336.rdb] (context: type eval line 15 cmd {assert {[file exists $temp_rdb]}} proc ::test)
```

The log shows that bgsave was executed, and it was successfully executed in the end:
```
Starting test Temp rdb will be deleted if we use bg_unlink when shutdown in tests/unit/shutdown.tcl
10251:M 22 Feb 2023 11:37:25.441 * Background saving started by pid 10336
10336:C 22 Feb 2023 11:37:27.949 * DB saved on disk
10336:C 22 Feb 2023 11:37:27.949 * Fork CoW for RDB: current 0 MB, peak 0 MB, average 0 MB
10251:M 22 Feb 2023 11:37:28.060 * Background saving terminated with success
```

There may be two reasons:
1. The child process has been created, but it has not created
   the temp rdb file yet, so [file exists $temp_rdb] check failed.
2. The child process bgsave has been executed successfully and the
   temp file has been deleted, so [file exists $temp_rdb] check failed.

From the logs pint, it should be the case 2, case 1 is too extreme,
set rdb-key-save-delay to a higher value to ensure bgsave does not
succeed early to avoid this case.
2023-03-21 17:51:47 +02:00

110 lines
3.6 KiB
Tcl

start_server {tags {"shutdown external:skip"}} {
test {Temp rdb will be deleted if we use bg_unlink when shutdown} {
for {set i 0} {$i < 20} {incr i} {
r set $i $i
}
r config set rdb-key-save-delay 10000000
# Child is dumping rdb
r bgsave
wait_for_condition 1000 10 {
[s rdb_bgsave_in_progress] eq 1
} else {
fail "bgsave did not start in time"
}
after 100 ;# give the child a bit of time for the file to be created
set dir [lindex [r config get dir] 1]
set child_pid [get_child_pid 0]
set temp_rdb [file join [lindex [r config get dir] 1] temp-${child_pid}.rdb]
# Temp rdb must be existed
assert {[file exists $temp_rdb]}
catch {r shutdown nosave}
# Make sure the server was killed
catch {set rd [redis_deferring_client]} e
assert_match {*connection refused*} $e
# Temp rdb file must be deleted
assert {![file exists $temp_rdb]}
}
}
start_server {tags {"shutdown external:skip"}} {
test {SHUTDOWN ABORT can cancel SIGTERM} {
r debug pause-cron 1
set pid [s process_id]
exec kill -SIGTERM $pid
after 10; # Give signal handler some time to run
r shutdown abort
verify_log_message 0 "*Shutdown manually aborted*" 0
r debug pause-cron 0
r ping
} {PONG}
test {Temp rdb will be deleted in signal handle} {
for {set i 0} {$i < 20} {incr i} {
r set $i $i
}
# It will cost 2s (20 * 100ms) to dump rdb
r config set rdb-key-save-delay 100000
set pid [s process_id]
set temp_rdb [file join [lindex [r config get dir] 1] temp-${pid}.rdb]
# trigger a shutdown which will save an rdb
exec kill -SIGINT $pid
# Wait for creation of temp rdb
wait_for_condition 50 10 {
[file exists $temp_rdb]
} else {
fail "Can't trigger rdb save on shutdown"
}
# Insist on immediate shutdown, temp rdb file must be deleted
exec kill -SIGINT $pid
# wait for the rdb file to be deleted
wait_for_condition 50 10 {
![file exists $temp_rdb]
} else {
fail "Can't trigger rdb save on shutdown"
}
}
}
start_server {tags {"shutdown external:skip"}} {
set pid [s process_id]
set dump_rdb [file join [lindex [r config get dir] 1] dump.rdb]
test {RDB save will be failed in shutdown} {
for {set i 0} {$i < 20} {incr i} {
r set $i $i
}
# create a folder called 'dump.rdb' to trigger temp-rdb rename failure
# and it will cause rdb save to fail eventually.
if {[file exists $dump_rdb]} {
exec rm -f $dump_rdb
}
exec mkdir -p $dump_rdb
}
test {SHUTDOWN will abort if rdb save failed on signal} {
# trigger a shutdown which will save an rdb
exec kill -SIGINT $pid
wait_for_log_messages 0 {"*Error trying to save the DB, can't exit*"} 0 100 10
}
test {SHUTDOWN will abort if rdb save failed on shutdown command} {
catch {[r shutdown]} err
assert_match {*Errors trying to SHUTDOWN*} $err
# make sure the server is still alive
assert_equal [r ping] {PONG}
}
test {SHUTDOWN can proceed if shutdown command was with nosave} {
catch {[r shutdown nosave]}
wait_for_log_messages 0 {"*ready to exit, bye bye*"} 0 100 10
}
test {Clean up rdb same named folder} {
exec rm -r $dump_rdb
}
}