mirror of
https://codeberg.org/redict/redict.git
synced 2025-01-22 16:18:28 -05:00
c31055db61
The test creates keys with various encodings, DUMP them, corrupt the payload and RESTORES it. It utilizes the recently added use-exit-on-panic config to distinguish between asserts and segfaults. If the restore succeeds, it runs random commands on the key to attempt to trigger a crash. It runs in two modes, one with deep sanitation enabled and one without. In the first one we don't expect any assertions or segfaults, in the second one we expect assertions, but no segfaults. We also check for leaks and invalid reads using valgrind, and if we find them we print the commands that lead to that issue. Changes in the code (other than the test): - Replace a few NPD (null pointer deference) flows and division by zero with an assertion, so that it doesn't fail the test. (since we set the server to use `exit` rather than `abort` on assertion). - Fix quite a lot of flows in rdb.c that could have lead to memory leaks in RESTORE command (since it now responds with an error rather than panic) - Add a DEBUG flag for SET-SKIP-CHECKSUM-VALIDATION so that the test don't need to bother with faking a valid checksum - Remove a pile of code in serverLogObjectDebugInfo which is actually unsafe to run in the crash report (see comments in the code) - fix a missing boundary check in lzf_decompress test suite infra improvements: - be able to run valgrind checks before the process terminates - rotate log files when restarting servers
194 lines
7.9 KiB
Tcl
194 lines
7.9 KiB
Tcl
# tests of corrupt ziplist payload with valid CRC
|
|
|
|
tags {"dump" "corruption"} {
|
|
|
|
proc generate_collections {suffix elements} {
|
|
set rd [redis_deferring_client]
|
|
for {set j 0} {$j < $elements} {incr j} {
|
|
# add both string values and integers
|
|
if {$j % 2 == 0} {set val $j} else {set val "_$j"}
|
|
$rd hset hash$suffix $j $val
|
|
$rd lpush list$suffix $val
|
|
$rd zadd zset$suffix $j $val
|
|
$rd sadd set$suffix $val
|
|
$rd xadd stream$suffix * item 1 value $val
|
|
}
|
|
for {set j 0} {$j < $elements * 5} {incr j} {
|
|
$rd read ; # Discard replies
|
|
}
|
|
$rd close
|
|
}
|
|
|
|
# generate keys with various types and encodings
|
|
proc generate_types {} {
|
|
r config set list-max-ziplist-size 5
|
|
r config set hash-max-ziplist-entries 5
|
|
r config set zset-max-ziplist-entries 5
|
|
r config set stream-node-max-entries 5
|
|
|
|
# create small (ziplist / listpack encoded) objects with 3 items
|
|
generate_collections "" 3
|
|
|
|
# add some metadata to the stream
|
|
r xgroup create stream mygroup 0
|
|
set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >]
|
|
r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0]
|
|
|
|
# create other non-collection types
|
|
r incr int
|
|
r set string str
|
|
|
|
# create bigger objects with 10 items (more than a single ziplist / listpack)
|
|
generate_collections big 10
|
|
|
|
# make sure our big stream also has a listpack record that has different
|
|
# field names than the master recored
|
|
r xadd streambig * item 1 value 1
|
|
r xadd streambig * item 1 unique value
|
|
}
|
|
|
|
proc corrupt_payload {payload} {
|
|
set len [string length $payload]
|
|
set count 1 ;# usually corrupt only one byte
|
|
if {rand() > 0.9} { set count 2 }
|
|
while { $count > 0 } {
|
|
set idx [expr {int(rand() * $len)}]
|
|
set ch [binary format c [expr {int(rand()*255)}]]
|
|
set payload [string replace $payload $idx $idx $ch]
|
|
incr count -1
|
|
}
|
|
return $payload
|
|
}
|
|
|
|
# fuzzy tester for corrupt RESTORE payloads
|
|
# valgrind will make sure there were no leaks in the rdb loader error handling code
|
|
foreach sanitize_dump {no yes} {
|
|
if {$::accurate} {
|
|
set min_duration [expr {60 * 10}] ;# run at least 10 minutes
|
|
set min_cycles 1000 ;# run at least 1k cycles (max 16 minutes)
|
|
} else {
|
|
set min_duration 10 ; # run at least 10 seconds
|
|
set min_cycles 10 ; # run at least 10 cycles
|
|
}
|
|
|
|
test "Fuzzer corrupt restore payloads - sanitize_dump: $sanitize_dump" {
|
|
if {$min_duration * 2 > $::timeout} {
|
|
fail "insufficient timeout"
|
|
}
|
|
# start a server, fill with data and save an RDB file once (avoid re-save)
|
|
start_server [list overrides [list "save" "" use-exit-on-panic yes crash-memcheck-enabled no loglevel verbose] ] {
|
|
set stdout [srv 0 stdout]
|
|
r config set sanitize-dump-payload $sanitize_dump
|
|
r debug set-skip-checksum-validation 1
|
|
set start_time [clock seconds]
|
|
generate_types
|
|
r save
|
|
set cycle 0
|
|
set stat_terminated_in_restore 0
|
|
set stat_terminated_in_traffic 0
|
|
set stat_terminated_by_signal 0
|
|
set stat_successful_restore 0
|
|
set stat_rejected_restore 0
|
|
set stat_traffic_commands_sent 0
|
|
# repeatedly DUMP a random key, corrupt it and try RESTORE into a new key
|
|
while true {
|
|
set k [r randomkey]
|
|
set dump [r dump $k]
|
|
set dump [corrupt_payload $dump]
|
|
set printable_dump [string2printable $dump]
|
|
set restore_failed false
|
|
set report_and_restart false
|
|
set sent {}
|
|
# RESTORE can fail, but hopefully not terminate
|
|
if { [catch { r restore "_$k" 0 $dump REPLACE } err] } {
|
|
set restore_failed true
|
|
# skip if return failed with an error response.
|
|
if {[string match "ERR*" $err]} {
|
|
incr stat_rejected_restore
|
|
} else {
|
|
set report_and_restart true
|
|
incr stat_terminated_in_restore
|
|
write_log_line 0 "corrupt payload: $printable_dump"
|
|
if {$sanitize_dump == 1} {
|
|
puts "Server crashed in RESTORE with payload: $printable_dump"
|
|
}
|
|
}
|
|
} else {
|
|
r ping ;# an attempt to check if the server didn't terminate (this will throw an error that will terminate the tests)
|
|
}
|
|
|
|
set print_commands false
|
|
if {!$restore_failed} {
|
|
# if RESTORE didn't fail or terminate, run some random traffic on the new key
|
|
incr stat_successful_restore
|
|
if { [ catch {
|
|
set sent [generate_fuzzy_traffic_on_key "_$k" 1] ;# traffic for 1 second
|
|
incr stat_traffic_commands_sent [llength $sent]
|
|
r del "_$k" ;# in case the server terminated, here's where we'll detect it.
|
|
} err ] } {
|
|
# if the server terminated update stats and restart it
|
|
set report_and_restart true
|
|
incr stat_terminated_in_traffic
|
|
set by_signal [count_log_message 0 "crashed by signal"]
|
|
incr stat_terminated_by_signal $by_signal
|
|
|
|
if {$by_signal != 0 || $sanitize_dump == 1 } {
|
|
puts "Server crashed (by signal: $by_signal), with payload: $printable_dump"
|
|
set print_commands true
|
|
}
|
|
}
|
|
}
|
|
|
|
# check valgrind report for invalid reads after each RESTORE
|
|
# payload so that we have a report that is easier to reproduce
|
|
set valgrind_errors [find_valgrind_errors [srv 0 stderr] false]
|
|
if {$valgrind_errors != ""} {
|
|
puts "valgrind found an issue for payload: $printable_dump"
|
|
set report_and_restart true
|
|
set print_commands true
|
|
}
|
|
|
|
if {$report_and_restart} {
|
|
if {$print_commands} {
|
|
puts "violating commands:"
|
|
foreach cmd $sent {
|
|
foreach arg $cmd {
|
|
puts -nonewline "[string2printable $arg] "
|
|
}
|
|
puts ""
|
|
}
|
|
}
|
|
|
|
# restart the server and re-apply debug configuration
|
|
write_log_line 0 "corrupt payload: $printable_dump"
|
|
restart_server 0 true true
|
|
r config set sanitize-dump-payload $sanitize_dump
|
|
r debug set-skip-checksum-validation 1
|
|
}
|
|
|
|
incr cycle
|
|
if { ([clock seconds]-$start_time) >= $min_duration && $cycle >= $min_cycles} {
|
|
break
|
|
}
|
|
}
|
|
if {$::verbose} {
|
|
puts "Done $cycle cycles in [expr {[clock seconds]-$start_time}] seconds."
|
|
puts "RESTORE: successful: $stat_successful_restore, rejected: $stat_rejected_restore"
|
|
puts "Total commands sent in traffic: $stat_traffic_commands_sent, crashes during traffic: $stat_terminated_in_traffic ($stat_terminated_by_signal by signal)."
|
|
}
|
|
}
|
|
# if we run sanitization we never expect the server to crash at runtime
|
|
if { $sanitize_dump == 1} {
|
|
assert_equal $stat_terminated_in_restore 0
|
|
assert_equal $stat_terminated_in_traffic 0
|
|
}
|
|
# make sure all terminations where due to assertion and not a SIGSEGV
|
|
assert_equal $stat_terminated_by_signal 0
|
|
}
|
|
}
|
|
|
|
|
|
|
|
} ;# tags
|
|
|