tests/valgrind: don't use debug restart (#7404)

* tests/valgrind: don't use debug restart

DEBUG REATART causes two issues:
1. it uses execve which replaces the original process and valgrind doesn't
   have a chance to check for errors, so leaks go unreported.
2. valgrind report invalid calls to close() which we're unable to resolve.

So now the tests use restart_server mechanism in the tests, that terminates
the old server and starts a new one, new PID, but same stdout, stderr.

since the stderr can contain two or more valgrind report, it is not enough
to just check for the absence of leaks, we also need to check for some known
errors, we do both, and fail if we either find an error, or can't find a
report saying there are no leaks.

other changes:
- when killing a server that was already terminated we check for leaks too.
- adding DEBUG LEAK which was used to test it.
- adding --trace-children to valgrind, although no longer needed.
- since the stdout contains two or more runs, we need slightly different way
  of checking if the new process is up (explicitly looking for the new PID)
- move the code that handles --wait-server to happen earlier (before
  watching the startup message in the log), and serve the restarted server too.

* squashme - CR fixes

src/debug.c

@@ -378,6 +378,7 @@ void debugCommand(client *c) {
 "DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]",
 "ERROR <string> -- Return a Redis protocol error with <string> as message. Useful for clients unit tests to simulate Redis errors.",
 "LOG <message> -- write message to the server log.",
+"LEAK <string> -- Create a memory leak of the input string.",
 "HTSTATS <dbid> -- Return hash table statistics of the specified Redis database.",
 "HTSTATS-KEY <key> -- Like htstats but for the hash table stored as key's value.",
 "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.",
@@ -430,6 +431,9 @@ NULL
     } else if (!strcasecmp(c->argv[1]->ptr,"log") && c->argc == 3) {
         serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr);
         addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"leak") && c->argc == 3) {
+        sdsdup(c->argv[2]->ptr);
+        addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
         int flush = 1, save = 1;
         int flags = RDBFLAGS_NONE;

tests/integration/psync2.tcl

@@ -280,7 +280,8 @@ start_server {} {
         set sync_partial_err [status $R($master_id) sync_partial_err]
         catch {
             $R($slave_id) config rewrite
-            $R($slave_id) debug restart
+            restart_server [expr {0-$slave_id}] true
+            set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
         # note: just waiting for connected_slaves==4 has a race condition since
         # we might do the check before the master realized that the slave disconnected
@@ -328,7 +329,8 @@ start_server {} {
 
         catch {
             $R($slave_id) config rewrite
-            $R($slave_id) debug restart
+            restart_server [expr {0-$slave_id}] true
+            set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
 
         # Reconfigure the slave correctly again, when it's back online.

tests/integration/rdb.tcl

@@ -137,18 +137,8 @@ test {client freed during loading} {
         # 100mb of rdb, 100k keys will load in more than 1 second
         r debug populate 100000 key 1000
 
-        catch {
+        restart_server 0 false
-            r debug restart
-        }
 
-        set stdout [srv 0 stdout]
-        while 1 {
-            # check that the new server actually started and is ready for connections
-            if {[exec grep -i "Server initialized" | wc -l < $stdout] > 1} {
-                break
-            }
-            after 10
-        }
         # make sure it's still loading
         assert_equal [s loading] 1
 
@@ -180,4 +170,4 @@ test {client freed during loading} {
         # no need to keep waiting for loading to complete
         exec kill [srv 0 pid]
     }
-}
+}

tests/support/server.tcl

@@ -17,7 +17,14 @@ proc check_valgrind_errors stderr {
     set buf [read $fd]
     close $fd
 
+    # look for stack trace and other errors, or the absense of a leak free summary
     if {[regexp -- { at 0x} $buf] ||
+        [regexp -- {Warning} $buf] ||
+        [regexp -- {Invalid} $buf] ||
+        [regexp -- {Mismatched} $buf] ||
+        [regexp -- {uninitialized} $buf] ||
+        [regexp -- {has a fishy} $buf] ||
+        [regexp -- {overlap} $buf] ||
         (![regexp -- {definitely lost: 0 bytes} $buf] &&
          ![regexp -- {no leaks are possible} $buf])} {
         send_data_packet $::test_server_fd err "Valgrind error: $buf\n"
@@ -29,7 +36,13 @@ proc kill_server config {
     if {$::external} return
 
     # nevermind if its already dead
-    if {![is_alive $config]} { return }
+    if {![is_alive $config]} {
+        # Check valgrind errors if needed
+        if {$::valgrind} {
+            check_valgrind_errors [dict get $config stderr]
+        }
+        return
+    }
     set pid [dict get $config pid]
 
     # check for leaks
@@ -153,6 +166,55 @@ proc create_server_config_file {filename config} {
     close $fp
 }
 
+proc spawn_server {config_file stdout stderr} {
+    if {$::valgrind} {
+        set pid [exec valgrind --track-origins=yes --trace-children=yes --suppressions=[pwd]/src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file >> $stdout 2>> $stderr &]
+    } elseif ($::stack_logging) {
+        set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file >> $stdout 2>> $stderr &]
+    } else {
+        set pid [exec src/redis-server $config_file >> $stdout 2>> $stderr &]
+    }
+
+    if {$::wait_server} {
+        set msg "server started PID: $pid. press any key to continue..."
+        puts $msg
+        read stdin 1
+    }
+
+    # Tell the test server about this new instance.
+    send_data_packet $::test_server_fd server-spawned $pid
+    return $pid
+}
+
+# Wait for actual startup, return 1 if port is busy, 0 otherwise
+proc wait_server_started {config_file stdout pid} {
+    set checkperiod 100; # Milliseconds
+    set maxiter [expr {120*1000/$checkperiod}] ; # Wait up to 2 minutes.
+    set port_busy 0
+    while 1 {
+        if {[regexp -- " PID: $pid" [exec cat $stdout]]} {
+            break
+        }
+        after $checkperiod
+        incr maxiter -1
+        if {$maxiter == 0} {
+            start_server_error $config_file "No PID detected in log $stdout"
+            puts "--- LOG CONTENT ---"
+            puts [exec cat $stdout]
+            puts "-------------------"
+            break
+        }
+
+        # Check if the port is actually busy and the server failed
+        # for this reason.
+        if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
+            set port_busy 1
+            break
+        }
+    }
+    return $port_busy
+}
+
 proc start_server {options {code undefined}} {
     # If we are running against an external server, we just push the
     # host/port pair in the stack the first time
@@ -248,44 +310,10 @@ proc start_server {options {code undefined}} {
 
         send_data_packet $::test_server_fd "server-spawning" "port $port"
 
-        if {$::valgrind} {
+        set pid [spawn_server $config_file $stdout $stderr]
-            set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
-        } elseif ($::stack_logging) {
-            set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &]
-        } else {
-            set pid [exec src/redis-server $config_file > $stdout 2> $stderr &]
-        }
-
-        # Tell the test server about this new instance.
-        send_data_packet $::test_server_fd server-spawned $pid
 
         # check that the server actually started
-        # ugly but tries to be as fast as possible...
+        set port_busy [wait_server_started $config_file $stdout $pid]
-        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
-
-        # Wait for actual startup
-        set checkperiod 100; # Milliseconds
-        set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes.
-        set port_busy 0
-        while {![info exists _pid]} {
-            regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid
-            after $checkperiod
-            incr maxiter -1
-            if {$maxiter == 0} {
-                start_server_error $config_file "No PID detected in log $stdout"
-                puts "--- LOG CONTENT ---"
-                puts [exec cat $stdout]
-                puts "-------------------"
-                break
-            }
-
-            # Check if the port is actually busy and the server failed
-            # for this reason.
-            if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
-                set port_busy 1
-                break
-            }
-        }
 
         # Sometimes we have to try a different port, even if we checked
         # for availability. Other test clients may grab the port before we
@@ -302,6 +330,7 @@ proc start_server {options {code undefined}} {
             continue; # Try again
         }
 
+        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
         if {$code ne "undefined"} {
             set serverisup [server_is_up $::host $port $retrynum]
         } else {
@@ -345,12 +374,6 @@ proc start_server {options {code undefined}} {
             error_and_quit $config_file $line
         }
 
-        if {$::wait_server} {
-            set msg "server started PID: [dict get $srv "pid"]. press any key to continue..."
-            puts $msg
-            read stdin 1
-        }
-
         while 1 {
             # check that the server actually started and is ready for connections
             if {[exec grep -i "Ready to accept" | wc -l < $stdout] > 0} {
@@ -370,6 +393,9 @@ proc start_server {options {code undefined}} {
         if {[catch { uplevel 1 $code } error]} {
             set backtrace $::errorInfo
 
+            # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
+            set srv [lindex $::servers end]
+
             # Kill the server without checking for leaks
             dict set srv "skipleaks" 1
             kill_server $srv
@@ -387,6 +413,9 @@ proc start_server {options {code undefined}} {
             error $error $backtrace
         }
 
+        # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
+        set srv [lindex $::servers end]
+
         # Don't do the leak check when no tests were run
         if {$num_tests == $::num_tests} {
             dict set srv "skipleaks" 1
@@ -402,3 +431,35 @@ proc start_server {options {code undefined}} {
         set _ $srv
     }
 }
+
+proc restart_server {level wait_ready} {
+    set srv [lindex $::servers end+$level]
+    kill_server $srv
+
+    set stdout [dict get $srv "stdout"]
+    set stderr [dict get $srv "stderr"]
+    set config_file [dict get $srv "config_file"]
+
+    set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout]
+
+    set pid [spawn_server $config_file $stdout $stderr]
+
+    # check that the server actually started
+    wait_server_started $config_file $stdout $pid
+
+    # update the pid in the servers list
+    dict set srv "pid" $pid
+    # re-set $srv in the servers list
+    lset ::servers end+$level $srv
+
+    if {$wait_ready} {
+        while 1 {
+            # check that the server actually started and is ready for connections
+            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > $prev_ready_count + 1} {
+                break
+            }
+            after 10
+        }
+    }
+    reconnect $level
+}