mirror of
https://codeberg.org/redict/redict.git
synced 2025-01-23 08:38:27 -05:00
349ef3f6a0
When sanitizing the stream listpack, we need to count the deleted records too. otherwise the last line that checks the next pointer fails. Add test to cover that state in the stream tests.
202 lines
8.2 KiB
Tcl
202 lines
8.2 KiB
Tcl
# tests of corrupt ziplist payload with valid CRC
|
|
|
|
tags {"dump" "corruption"} {
|
|
|
|
proc generate_collections {suffix elements} {
|
|
set rd [redis_deferring_client]
|
|
for {set j 0} {$j < $elements} {incr j} {
|
|
# add both string values and integers
|
|
if {$j % 2 == 0} {set val $j} else {set val "_$j"}
|
|
$rd hset hash$suffix $j $val
|
|
$rd lpush list$suffix $val
|
|
$rd zadd zset$suffix $j $val
|
|
$rd sadd set$suffix $val
|
|
$rd xadd stream$suffix * item 1 value $val
|
|
}
|
|
for {set j 0} {$j < $elements * 5} {incr j} {
|
|
$rd read ; # Discard replies
|
|
}
|
|
$rd close
|
|
}
|
|
|
|
# generate keys with various types and encodings
|
|
proc generate_types {} {
|
|
r config set list-max-ziplist-size 5
|
|
r config set hash-max-ziplist-entries 5
|
|
r config set zset-max-ziplist-entries 5
|
|
r config set stream-node-max-entries 5
|
|
|
|
# create small (ziplist / listpack encoded) objects with 3 items
|
|
generate_collections "" 3
|
|
|
|
# add some metadata to the stream
|
|
r xgroup create stream mygroup 0
|
|
set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >]
|
|
r xdel stream [lindex [lindex [lindex [lindex $records 0] 1] 1] 0]
|
|
r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0]
|
|
|
|
# create other non-collection types
|
|
r incr int
|
|
r set string str
|
|
|
|
# create bigger objects with 10 items (more than a single ziplist / listpack)
|
|
generate_collections big 10
|
|
|
|
# make sure our big stream also has a listpack record that has different
|
|
# field names than the master recored
|
|
r xadd streambig * item 1 value 1
|
|
r xadd streambig * item 1 unique value
|
|
}
|
|
|
|
proc corrupt_payload {payload} {
|
|
set len [string length $payload]
|
|
set count 1 ;# usually corrupt only one byte
|
|
if {rand() > 0.9} { set count 2 }
|
|
while { $count > 0 } {
|
|
set idx [expr {int(rand() * $len)}]
|
|
set ch [binary format c [expr {int(rand()*255)}]]
|
|
set payload [string replace $payload $idx $idx $ch]
|
|
incr count -1
|
|
}
|
|
return $payload
|
|
}
|
|
|
|
# fuzzy tester for corrupt RESTORE payloads
|
|
# valgrind will make sure there were no leaks in the rdb loader error handling code
|
|
foreach sanitize_dump {no yes} {
|
|
if {$::accurate} {
|
|
set min_duration [expr {60 * 10}] ;# run at least 10 minutes
|
|
set min_cycles 1000 ;# run at least 1k cycles (max 16 minutes)
|
|
} else {
|
|
set min_duration 10 ; # run at least 10 seconds
|
|
set min_cycles 10 ; # run at least 10 cycles
|
|
}
|
|
|
|
# Don't execute this on FreeBSD due to a yet-undiscovered memory issue
|
|
# which causes tclsh to bloat.
|
|
if {[exec uname] == "FreeBSD"} {
|
|
set min_cycles 1
|
|
set min_duration 1
|
|
}
|
|
|
|
test "Fuzzer corrupt restore payloads - sanitize_dump: $sanitize_dump" {
|
|
if {$min_duration * 2 > $::timeout} {
|
|
fail "insufficient timeout"
|
|
}
|
|
# start a server, fill with data and save an RDB file once (avoid re-save)
|
|
start_server [list overrides [list "save" "" use-exit-on-panic yes crash-memcheck-enabled no loglevel verbose] ] {
|
|
set stdout [srv 0 stdout]
|
|
r config set sanitize-dump-payload $sanitize_dump
|
|
r debug set-skip-checksum-validation 1
|
|
set start_time [clock seconds]
|
|
generate_types
|
|
r save
|
|
set cycle 0
|
|
set stat_terminated_in_restore 0
|
|
set stat_terminated_in_traffic 0
|
|
set stat_terminated_by_signal 0
|
|
set stat_successful_restore 0
|
|
set stat_rejected_restore 0
|
|
set stat_traffic_commands_sent 0
|
|
# repeatedly DUMP a random key, corrupt it and try RESTORE into a new key
|
|
while true {
|
|
set k [r randomkey]
|
|
set dump [r dump $k]
|
|
set dump [corrupt_payload $dump]
|
|
set printable_dump [string2printable $dump]
|
|
set restore_failed false
|
|
set report_and_restart false
|
|
set sent {}
|
|
# RESTORE can fail, but hopefully not terminate
|
|
if { [catch { r restore "_$k" 0 $dump REPLACE } err] } {
|
|
set restore_failed true
|
|
# skip if return failed with an error response.
|
|
if {[string match "ERR*" $err]} {
|
|
incr stat_rejected_restore
|
|
} else {
|
|
set report_and_restart true
|
|
incr stat_terminated_in_restore
|
|
write_log_line 0 "corrupt payload: $printable_dump"
|
|
if {$sanitize_dump == 1} {
|
|
puts "Server crashed in RESTORE with payload: $printable_dump"
|
|
}
|
|
}
|
|
} else {
|
|
r ping ;# an attempt to check if the server didn't terminate (this will throw an error that will terminate the tests)
|
|
}
|
|
|
|
set print_commands false
|
|
if {!$restore_failed} {
|
|
# if RESTORE didn't fail or terminate, run some random traffic on the new key
|
|
incr stat_successful_restore
|
|
if { [ catch {
|
|
set sent [generate_fuzzy_traffic_on_key "_$k" 1] ;# traffic for 1 second
|
|
incr stat_traffic_commands_sent [llength $sent]
|
|
r del "_$k" ;# in case the server terminated, here's where we'll detect it.
|
|
} err ] } {
|
|
# if the server terminated update stats and restart it
|
|
set report_and_restart true
|
|
incr stat_terminated_in_traffic
|
|
set by_signal [count_log_message 0 "crashed by signal"]
|
|
incr stat_terminated_by_signal $by_signal
|
|
|
|
if {$by_signal != 0 || $sanitize_dump == 1 } {
|
|
puts "Server crashed (by signal: $by_signal), with payload: $printable_dump"
|
|
set print_commands true
|
|
}
|
|
}
|
|
}
|
|
|
|
# check valgrind report for invalid reads after each RESTORE
|
|
# payload so that we have a report that is easier to reproduce
|
|
set valgrind_errors [find_valgrind_errors [srv 0 stderr] false]
|
|
if {$valgrind_errors != ""} {
|
|
puts "valgrind found an issue for payload: $printable_dump"
|
|
set report_and_restart true
|
|
set print_commands true
|
|
}
|
|
|
|
if {$report_and_restart} {
|
|
if {$print_commands} {
|
|
puts "violating commands:"
|
|
foreach cmd $sent {
|
|
foreach arg $cmd {
|
|
puts -nonewline "[string2printable $arg] "
|
|
}
|
|
puts ""
|
|
}
|
|
}
|
|
|
|
# restart the server and re-apply debug configuration
|
|
write_log_line 0 "corrupt payload: $printable_dump"
|
|
restart_server 0 true true
|
|
r config set sanitize-dump-payload $sanitize_dump
|
|
r debug set-skip-checksum-validation 1
|
|
}
|
|
|
|
incr cycle
|
|
if { ([clock seconds]-$start_time) >= $min_duration && $cycle >= $min_cycles} {
|
|
break
|
|
}
|
|
}
|
|
if {$::verbose} {
|
|
puts "Done $cycle cycles in [expr {[clock seconds]-$start_time}] seconds."
|
|
puts "RESTORE: successful: $stat_successful_restore, rejected: $stat_rejected_restore"
|
|
puts "Total commands sent in traffic: $stat_traffic_commands_sent, crashes during traffic: $stat_terminated_in_traffic ($stat_terminated_by_signal by signal)."
|
|
}
|
|
}
|
|
# if we run sanitization we never expect the server to crash at runtime
|
|
if { $sanitize_dump == 1} {
|
|
assert_equal $stat_terminated_in_restore 0
|
|
assert_equal $stat_terminated_in_traffic 0
|
|
}
|
|
# make sure all terminations where due to assertion and not a SIGSEGV
|
|
assert_equal $stat_terminated_by_signal 0
|
|
}
|
|
}
|
|
|
|
|
|
|
|
} ;# tags
|
|
|