mirror of
https://codeberg.org/redict/redict.git
synced 2025-01-23 00:28:26 -05:00
8a86bca5ed
This commit revives the improves the ability to run the test suite against external servers, instead of launching and managing `redis-server` processes as part of the test fixture. This capability existed in the past, using the `--host` and `--port` options. However, it was quite limited and mostly useful when running a specific tests. Attempting to run larger chunks of the test suite experienced many issues: * Many tests depend on being able to start and control `redis-server` themselves, and there's no clear distinction between external server compatible and other tests. * Cluster mode is not supported (resulting with `CROSSSLOT` errors). This PR cleans up many things and makes it possible to run the entire test suite against an external server. It also provides more fine grained controls to handle cases where the external server supports a subset of the Redis commands, limited number of databases, cluster mode, etc. The tests directory now contains a `README.md` file that describes how this works. This commit also includes additional cleanups and fixes: * Tests can now be tagged. * Tag-based selection is now unified across `start_server`, `tags` and `test`. * More information is provided about skipped or ignored tests. * Repeated patterns in tests have been extracted to common procedures, both at a global level and on a per-test file basis. * Cleaned up some cases where test setup was based on a previous test executing (a major anti-pattern that repeats itself in many places). * Cleaned up some cases where test teardown was not part of a test (in the future we should have dedicated teardown code that executes even when tests fail). * Fixed some tests that were flaky running on external servers.
169 lines
7.9 KiB
Tcl
169 lines
7.9 KiB
Tcl
source tests/support/benchmark.tcl
|
|
|
|
|
|
proc cmdstat {cmd} {
|
|
return [cmdrstat $cmd r]
|
|
}
|
|
|
|
start_server {tags {"benchmark network external:skip"}} {
|
|
start_server {} {
|
|
set master_host [srv 0 host]
|
|
set master_port [srv 0 port]
|
|
|
|
test {benchmark: set,get} {
|
|
r config resetstat
|
|
r flushall
|
|
set cmd [redisbenchmark $master_host $master_port "-c 5 -n 10 -e -t set,get"]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
assert_match {*calls=10,*} [cmdstat set]
|
|
assert_match {*calls=10,*} [cmdstat get]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat lrange]
|
|
}
|
|
|
|
test {benchmark: full test suite} {
|
|
r config resetstat
|
|
set cmd [redisbenchmark $master_host $master_port "-c 10 -n 100 -e"]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
# ping total calls are 2*issued commands per test due to PING_INLINE and PING_MBULK
|
|
assert_match {*calls=200,*} [cmdstat ping]
|
|
assert_match {*calls=100,*} [cmdstat set]
|
|
assert_match {*calls=100,*} [cmdstat get]
|
|
assert_match {*calls=100,*} [cmdstat incr]
|
|
# lpush total calls are 2*issued commands per test due to the lrange tests
|
|
assert_match {*calls=200,*} [cmdstat lpush]
|
|
assert_match {*calls=100,*} [cmdstat rpush]
|
|
assert_match {*calls=100,*} [cmdstat lpop]
|
|
assert_match {*calls=100,*} [cmdstat rpop]
|
|
assert_match {*calls=100,*} [cmdstat sadd]
|
|
assert_match {*calls=100,*} [cmdstat hset]
|
|
assert_match {*calls=100,*} [cmdstat spop]
|
|
assert_match {*calls=100,*} [cmdstat zadd]
|
|
assert_match {*calls=100,*} [cmdstat zpopmin]
|
|
assert_match {*calls=400,*} [cmdstat lrange]
|
|
assert_match {*calls=100,*} [cmdstat mset]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat rpoplpush]
|
|
}
|
|
|
|
test {benchmark: multi-thread set,get} {
|
|
r config resetstat
|
|
r flushall
|
|
set cmd [redisbenchmark $master_host $master_port "--threads 10 -c 5 -n 10 -e -t set,get"]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
assert_match {*calls=10,*} [cmdstat set]
|
|
assert_match {*calls=10,*} [cmdstat get]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat lrange]
|
|
|
|
# ensure only one key was populated
|
|
assert_match {1} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
|
|
}
|
|
|
|
test {benchmark: pipelined full set,get} {
|
|
r config resetstat
|
|
r flushall
|
|
set cmd [redisbenchmark $master_host $master_port "-P 5 -c 10 -n 10010 -e -t set,get"]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
assert_match {*calls=10010,*} [cmdstat set]
|
|
assert_match {*calls=10010,*} [cmdstat get]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat lrange]
|
|
|
|
# ensure only one key was populated
|
|
assert_match {1} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
|
|
}
|
|
|
|
test {benchmark: arbitrary command} {
|
|
r config resetstat
|
|
r flushall
|
|
set cmd [redisbenchmark $master_host $master_port "-c 5 -n 150 -e INCRBYFLOAT mykey 10.0"]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
assert_match {*calls=150,*} [cmdstat incrbyfloat]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat get]
|
|
|
|
# ensure only one key was populated
|
|
assert_match {1} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
|
|
}
|
|
|
|
test {benchmark: keyspace length} {
|
|
r flushall
|
|
r config resetstat
|
|
set cmd [redisbenchmark $master_host $master_port "-r 50 -t set -n 1000"]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
assert_match {*calls=1000,*} [cmdstat set]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat get]
|
|
|
|
# ensure the keyspace has the desired size
|
|
assert_match {50} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
|
|
}
|
|
|
|
# tls specific tests
|
|
if {$::tls} {
|
|
test {benchmark: specific tls-ciphers} {
|
|
r flushall
|
|
r config resetstat
|
|
set cmd [redisbenchmark $master_host $master_port "-r 50 -t set -n 1000 --tls-ciphers \"DEFAULT:-AES128-SHA256\""]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
assert_match {*calls=1000,*} [cmdstat set]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat get]
|
|
}
|
|
|
|
test {benchmark: specific tls-ciphersuites} {
|
|
r flushall
|
|
r config resetstat
|
|
set ciphersuites_supported 1
|
|
set cmd [redisbenchmark $master_host $master_port "-r 50 -t set -n 1000 --tls-ciphersuites \"TLS_AES_128_GCM_SHA256\""]
|
|
if {[catch { exec {*}$cmd } error]} {
|
|
set first_line [lindex [split $error "\n"] 0]
|
|
if {[string match "*Invalid option*" $first_line]} {
|
|
set ciphersuites_supported 0
|
|
if {$::verbose} {
|
|
puts "Skipping test, TLSv1.3 not supported."
|
|
}
|
|
} else {
|
|
puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
|
|
fail "redis-benchmark non zero code. first line: $first_line"
|
|
}
|
|
}
|
|
if {$ciphersuites_supported} {
|
|
assert_match {*calls=1000,*} [cmdstat set]
|
|
# assert one of the non benchmarked commands is not present
|
|
assert_match {} [cmdstat get]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|