mirror of
https://codeberg.org/redict/redict.git
synced 2025-01-23 00:28:26 -05:00
8a86bca5ed
This commit revives the improves the ability to run the test suite against external servers, instead of launching and managing `redis-server` processes as part of the test fixture. This capability existed in the past, using the `--host` and `--port` options. However, it was quite limited and mostly useful when running a specific tests. Attempting to run larger chunks of the test suite experienced many issues: * Many tests depend on being able to start and control `redis-server` themselves, and there's no clear distinction between external server compatible and other tests. * Cluster mode is not supported (resulting with `CROSSSLOT` errors). This PR cleans up many things and makes it possible to run the entire test suite against an external server. It also provides more fine grained controls to handle cases where the external server supports a subset of the Redis commands, limited number of databases, cluster mode, etc. The tests directory now contains a `README.md` file that describes how this works. This commit also includes additional cleanups and fixes: * Tests can now be tagged. * Tag-based selection is now unified across `start_server`, `tags` and `test`. * More information is provided about skipped or ignored tests. * Repeated patterns in tests have been extracted to common procedures, both at a global level and on a per-test file basis. * Cleaned up some cases where test setup was based on a previous test executing (a major anti-pattern that repeats itself in many places). * Cleaned up some cases where test teardown was not part of a test (in the future we should have dedicated teardown code that executes even when tests fail). * Fixed some tests that were flaky running on external servers.
295 lines
9.8 KiB
Tcl
295 lines
9.8 KiB
Tcl
start_server {tags {"failover external:skip"}} {
|
|
start_server {} {
|
|
start_server {} {
|
|
set node_0 [srv 0 client]
|
|
set node_0_host [srv 0 host]
|
|
set node_0_port [srv 0 port]
|
|
set node_0_pid [srv 0 pid]
|
|
|
|
set node_1 [srv -1 client]
|
|
set node_1_host [srv -1 host]
|
|
set node_1_port [srv -1 port]
|
|
set node_1_pid [srv -1 pid]
|
|
|
|
set node_2 [srv -2 client]
|
|
set node_2_host [srv -2 host]
|
|
set node_2_port [srv -2 port]
|
|
set node_2_pid [srv -2 pid]
|
|
|
|
proc assert_digests_match {n1 n2 n3} {
|
|
assert_equal [$n1 debug digest] [$n2 debug digest]
|
|
assert_equal [$n2 debug digest] [$n3 debug digest]
|
|
}
|
|
|
|
test {failover command fails without connected replica} {
|
|
catch { $node_0 failover to $node_1_host $node_1_port } err
|
|
if {! [string match "ERR*" $err]} {
|
|
fail "failover command succeeded when replica not connected"
|
|
}
|
|
}
|
|
|
|
test {setup replication for following tests} {
|
|
$node_1 replicaof $node_0_host $node_0_port
|
|
$node_2 replicaof $node_0_host $node_0_port
|
|
wait_for_sync $node_1
|
|
wait_for_sync $node_2
|
|
}
|
|
|
|
test {failover command fails with invalid host} {
|
|
catch { $node_0 failover to invalidhost $node_1_port } err
|
|
assert_match "ERR*" $err
|
|
}
|
|
|
|
test {failover command fails with invalid port} {
|
|
catch { $node_0 failover to $node_1_host invalidport } err
|
|
assert_match "ERR*" $err
|
|
}
|
|
|
|
test {failover command fails with just force and timeout} {
|
|
catch { $node_0 FAILOVER FORCE TIMEOUT 100} err
|
|
assert_match "ERR*" $err
|
|
}
|
|
|
|
test {failover command fails when sent to a replica} {
|
|
catch { $node_1 failover to $node_1_host $node_1_port } err
|
|
assert_match "ERR*" $err
|
|
}
|
|
|
|
test {failover command fails with force without timeout} {
|
|
catch { $node_0 failover to $node_1_host $node_1_port FORCE } err
|
|
assert_match "ERR*" $err
|
|
}
|
|
|
|
test {failover command to specific replica works} {
|
|
set initial_psyncs [s -1 sync_partial_ok]
|
|
set initial_syncs [s -1 sync_full]
|
|
|
|
# Generate a delta between primary and replica
|
|
set load_handler [start_write_load $node_0_host $node_0_port 5]
|
|
exec kill -SIGSTOP [srv -1 pid]
|
|
wait_for_condition 50 100 {
|
|
[s 0 total_commands_processed] > 100
|
|
} else {
|
|
fail "Node 0 did not accept writes"
|
|
}
|
|
exec kill -SIGCONT [srv -1 pid]
|
|
|
|
# Execute the failover
|
|
$node_0 failover to $node_1_host $node_1_port
|
|
|
|
# Wait for failover to end
|
|
wait_for_condition 50 100 {
|
|
[s 0 master_failover_state] == "no-failover"
|
|
} else {
|
|
fail "Failover from node 0 to node 1 did not finish"
|
|
}
|
|
|
|
# stop the write load and make sure no more commands processed
|
|
stop_write_load $load_handler
|
|
wait_load_handlers_disconnected
|
|
|
|
$node_2 replicaof $node_1_host $node_1_port
|
|
wait_for_sync $node_0
|
|
wait_for_sync $node_2
|
|
|
|
assert_match *slave* [$node_0 role]
|
|
assert_match *master* [$node_1 role]
|
|
assert_match *slave* [$node_2 role]
|
|
|
|
# We should accept psyncs from both nodes
|
|
assert_equal [expr [s -1 sync_partial_ok] - $initial_psyncs] 2
|
|
assert_equal [expr [s -1 sync_full] - $initial_psyncs] 0
|
|
assert_digests_match $node_0 $node_1 $node_2
|
|
}
|
|
|
|
test {failover command to any replica works} {
|
|
set initial_psyncs [s -2 sync_partial_ok]
|
|
set initial_syncs [s -2 sync_full]
|
|
|
|
wait_for_ofs_sync $node_1 $node_2
|
|
# We stop node 0 to and make sure node 2 is selected
|
|
exec kill -SIGSTOP $node_0_pid
|
|
$node_1 set CASE 1
|
|
$node_1 FAILOVER
|
|
|
|
# Wait for failover to end
|
|
wait_for_condition 50 100 {
|
|
[s -1 master_failover_state] == "no-failover"
|
|
} else {
|
|
fail "Failover from node 1 to node 2 did not finish"
|
|
}
|
|
exec kill -SIGCONT $node_0_pid
|
|
$node_0 replicaof $node_2_host $node_2_port
|
|
|
|
wait_for_sync $node_0
|
|
wait_for_sync $node_1
|
|
|
|
assert_match *slave* [$node_0 role]
|
|
assert_match *slave* [$node_1 role]
|
|
assert_match *master* [$node_2 role]
|
|
|
|
# We should accept Psyncs from both nodes
|
|
assert_equal [expr [s -2 sync_partial_ok] - $initial_psyncs] 2
|
|
assert_equal [expr [s -1 sync_full] - $initial_psyncs] 0
|
|
assert_digests_match $node_0 $node_1 $node_2
|
|
}
|
|
|
|
test {failover to a replica with force works} {
|
|
set initial_psyncs [s 0 sync_partial_ok]
|
|
set initial_syncs [s 0 sync_full]
|
|
|
|
exec kill -SIGSTOP $node_0_pid
|
|
# node 0 will never acknowledge this write
|
|
$node_2 set case 2
|
|
$node_2 failover to $node_0_host $node_0_port TIMEOUT 100 FORCE
|
|
|
|
# Wait for node 0 to give up on sync attempt and start failover
|
|
wait_for_condition 50 100 {
|
|
[s -2 master_failover_state] == "failover-in-progress"
|
|
} else {
|
|
fail "Failover from node 2 to node 0 did not timeout"
|
|
}
|
|
|
|
# Quick check that everyone is a replica, we never want a
|
|
# state where there are two masters.
|
|
assert_match *slave* [$node_1 role]
|
|
assert_match *slave* [$node_2 role]
|
|
|
|
exec kill -SIGCONT $node_0_pid
|
|
|
|
# Wait for failover to end
|
|
wait_for_condition 50 100 {
|
|
[s -2 master_failover_state] == "no-failover"
|
|
} else {
|
|
fail "Failover from node 2 to node 0 did not finish"
|
|
}
|
|
$node_1 replicaof $node_0_host $node_0_port
|
|
|
|
wait_for_sync $node_1
|
|
wait_for_sync $node_2
|
|
|
|
assert_match *master* [$node_0 role]
|
|
assert_match *slave* [$node_1 role]
|
|
assert_match *slave* [$node_2 role]
|
|
|
|
assert_equal [count_log_message -2 "time out exceeded, failing over."] 1
|
|
|
|
# We should accept both psyncs, although this is the condition we might not
|
|
# since we didn't catch up.
|
|
assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 2
|
|
assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
|
|
assert_digests_match $node_0 $node_1 $node_2
|
|
}
|
|
|
|
test {failover with timeout aborts if replica never catches up} {
|
|
set initial_psyncs [s 0 sync_partial_ok]
|
|
set initial_syncs [s 0 sync_full]
|
|
|
|
# Stop replica so it never catches up
|
|
exec kill -SIGSTOP [srv -1 pid]
|
|
$node_0 SET CASE 1
|
|
|
|
$node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 500
|
|
# Wait for failover to end
|
|
wait_for_condition 50 20 {
|
|
[s 0 master_failover_state] == "no-failover"
|
|
} else {
|
|
fail "Failover from node_0 to replica did not finish"
|
|
}
|
|
|
|
exec kill -SIGCONT [srv -1 pid]
|
|
|
|
# We need to make sure the nodes actually sync back up
|
|
wait_for_ofs_sync $node_0 $node_1
|
|
wait_for_ofs_sync $node_0 $node_2
|
|
|
|
assert_match *master* [$node_0 role]
|
|
assert_match *slave* [$node_1 role]
|
|
assert_match *slave* [$node_2 role]
|
|
|
|
# Since we never caught up, there should be no syncs
|
|
assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 0
|
|
assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
|
|
assert_digests_match $node_0 $node_1 $node_2
|
|
}
|
|
|
|
test {failovers can be aborted} {
|
|
set initial_psyncs [s 0 sync_partial_ok]
|
|
set initial_syncs [s 0 sync_full]
|
|
|
|
# Stop replica so it never catches up
|
|
exec kill -SIGSTOP [srv -1 pid]
|
|
$node_0 SET CASE 2
|
|
|
|
$node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 60000
|
|
assert_match [s 0 master_failover_state] "waiting-for-sync"
|
|
|
|
# Sanity check that read commands are still accepted
|
|
$node_0 GET CASE
|
|
|
|
$node_0 failover abort
|
|
assert_match [s 0 master_failover_state] "no-failover"
|
|
|
|
exec kill -SIGCONT [srv -1 pid]
|
|
|
|
# Just make sure everything is still synced
|
|
wait_for_ofs_sync $node_0 $node_1
|
|
wait_for_ofs_sync $node_0 $node_2
|
|
|
|
assert_match *master* [$node_0 role]
|
|
assert_match *slave* [$node_1 role]
|
|
assert_match *slave* [$node_2 role]
|
|
|
|
# Since we never caught up, there should be no syncs
|
|
assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 0
|
|
assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
|
|
assert_digests_match $node_0 $node_1 $node_2
|
|
}
|
|
|
|
test {failover aborts if target rejects sync request} {
|
|
set initial_psyncs [s 0 sync_partial_ok]
|
|
set initial_syncs [s 0 sync_full]
|
|
|
|
# We block psync, so the failover will fail
|
|
$node_1 acl setuser default -psync
|
|
|
|
# We pause the target long enough to send a write command
|
|
# during the pause. This write will not be interrupted.
|
|
exec kill -SIGSTOP [srv -1 pid]
|
|
set rd [redis_deferring_client]
|
|
$rd SET FOO BAR
|
|
$node_0 failover to $node_1_host $node_1_port
|
|
exec kill -SIGCONT [srv -1 pid]
|
|
|
|
# Wait for failover to end
|
|
wait_for_condition 50 100 {
|
|
[s 0 master_failover_state] == "no-failover"
|
|
} else {
|
|
fail "Failover from node_0 to replica did not finish"
|
|
}
|
|
|
|
assert_equal [$rd read] "OK"
|
|
$rd close
|
|
|
|
# restore access to psync
|
|
$node_1 acl setuser default +psync
|
|
|
|
# We need to make sure the nodes actually sync back up
|
|
wait_for_sync $node_1
|
|
wait_for_sync $node_2
|
|
|
|
assert_match *master* [$node_0 role]
|
|
assert_match *slave* [$node_1 role]
|
|
assert_match *slave* [$node_2 role]
|
|
|
|
# We will cycle all of our replicas here and force a psync.
|
|
assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 2
|
|
assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
|
|
|
|
assert_equal [count_log_message 0 "Failover target rejected psync request"] 1
|
|
assert_digests_match $node_0 $node_1 $node_2
|
|
}
|
|
}
|
|
}
|
|
}
|