redict/tests/support/aofmanifest.tcl

174 lines
4.3 KiB
Tcl
Raw Normal View History

Implement Multi Part AOF mechanism to avoid AOFRW overheads. (#9788) Implement Multi-Part AOF mechanism to avoid overheads during AOFRW. Introducing a folder with multiple AOF files tracked by a manifest file. The main issues with the the original AOFRW mechanism are: * buffering of commands that are processed during rewrite (consuming a lot of RAM) * freezes of the main process when the AOFRW completes to drain the remaining part of the buffer and fsync it. * double disk IO for the data that arrives during AOFRW (had to be written to both the old and new AOF files) The main modifications of this PR: 1. Remove the AOF rewrite buffer and related code. 2. Divide the AOF into multiple files, they are classified as two types, one is the the `BASE` type, it represents the full amount of data (Maybe AOF or RDB format) after each AOFRW, there is only one `BASE` file at most. The second is `INCR` type, may have more than one. They represent the incremental commands since the last AOFRW. 3. Use a AOF manifest file to record and manage these AOF files mentioned above. 4. The original configuration of `appendfilename` will be the base part of the new file name, for example: `appendonly.aof.1.base.rdb` and `appendonly.aof.2.incr.aof` 5. Add manifest-related TCL tests, and modified some existing tests that depend on the `appendfilename` 6. Remove the `aof_rewrite_buffer_length` field in info. 7. Add `aof-disable-auto-gc` configuration. By default we're automatically deleting HISTORY type AOFs. It also gives users the opportunity to preserve the history AOFs. just for testing use now. 8. Add AOFRW limiting measure. When the AOFRW failures reaches the threshold (3 times now), we will delay the execution of the next AOFRW by 1 minute. If the next AOFRW also fails, it will be delayed by 2 minutes. The next is 4, 8, 16, the maximum delay is 60 minutes (1 hour). During the limit period, we can still use the 'bgrewriteaof' command to execute AOFRW immediately. 9. Support upgrade (load) data from old version redis. 10. Add `appenddirname` configuration, as the directory name of the append only files. All AOF files and manifest file will be placed in this directory. 11. Only the last AOF file (BASE or INCR) can be truncated. Otherwise redis will exit even if `aof-load-truncated` is enabled. Co-authored-by: Oran Agra <oran@redislabs.com>
2022-01-03 12:14:13 -05:00
set ::base_aof_sufix ".base"
set ::incr_aof_sufix ".incr"
set ::manifest_suffix ".manifest"
set ::aof_format_suffix ".aof"
set ::rdb_format_suffix ".rdb"
proc get_full_path {dir filename} {
set _ [format "%s/%s" $dir $filename]
}
proc join_path {dir1 dir2} {
return [format "%s/%s" $dir1 $dir2]
}
proc get_redis_dir {} {
set config [srv config]
set _ [dict get $config "dir"]
}
proc check_file_exist {dir filename} {
set file_path [get_full_path $dir $filename]
return [file exists $file_path]
}
proc del_file {dir filename} {
set file_path [get_full_path $dir $filename]
catch {exec rm -rf $file_path}
}
proc get_cur_base_aof_name {manifest_filepath} {
set fp [open $manifest_filepath r+]
set lines {}
while {1} {
set line [gets $fp]
if {[eof $fp]} {
close $fp
break;
}
lappend lines $line
}
if {[llength $lines] == 0} {
return ""
}
set first_line [lindex $lines 0]
set aofname [lindex [split $first_line " "] 1]
set aoftype [lindex [split $first_line " "] 5]
if { $aoftype eq "b" } {
return $aofname
}
return ""
}
proc get_last_incr_aof_name {manifest_filepath} {
set fp [open $manifest_filepath r+]
set lines {}
while {1} {
set line [gets $fp]
if {[eof $fp]} {
close $fp
break;
}
lappend lines $line
}
if {[llength $lines] == 0} {
return ""
}
set len [llength $lines]
set last_line [lindex $lines [expr $len - 1]]
set aofname [lindex [split $last_line " "] 1]
set aoftype [lindex [split $last_line " "] 5]
if { $aoftype eq "i" } {
return $aofname
}
return ""
}
proc get_last_incr_aof_path {r} {
set dir [lindex [$r config get dir] 1]
set appenddirname [lindex [$r config get appenddirname] 1]
set appendfilename [lindex [$r config get appendfilename] 1]
set manifest_filepath [file join $dir $appenddirname $appendfilename$::manifest_suffix]
set last_incr_aof_name [get_last_incr_aof_name $manifest_filepath]
if {$last_incr_aof_name == ""} {
return ""
}
return [file join $dir $appenddirname $last_incr_aof_name]
}
proc get_base_aof_path {r} {
set dir [lindex [$r config get dir] 1]
set appenddirname [lindex [$r config get appenddirname] 1]
set appendfilename [lindex [$r config get appendfilename] 1]
set manifest_filepath [file join $dir $appenddirname $appendfilename$::manifest_suffix]
set cur_base_aof_name [get_cur_base_aof_name $manifest_filepath]
if {$cur_base_aof_name == ""} {
return ""
}
return [file join $dir $appenddirname $cur_base_aof_name]
}
proc assert_aof_manifest_content {manifest_path content} {
set fp [open $manifest_path r+]
set lines {}
while {1} {
set line [gets $fp]
if {[eof $fp]} {
close $fp
break;
}
lappend lines $line
}
assert_equal [llength $lines] [llength $content]
for { set i 0 } { $i < [llength $lines] } {incr i} {
assert_equal [lindex $lines $i] [lindex $content $i]
}
}
proc clean_aof_persistence {aof_dirpath} {
catch {eval exec rm -rf [glob $aof_dirpath]}
}
proc append_to_manifest {str} {
upvar fp fp
puts -nonewline $fp $str
}
proc create_aof_manifest {dir aof_manifest_file code} {
create_aof_dir $dir
upvar fp fp
set fp [open $aof_manifest_file w+]
uplevel 1 $code
close $fp
}
proc append_to_aof {str} {
upvar fp fp
puts -nonewline $fp $str
}
proc create_aof {dir aof_file code} {
create_aof_dir $dir
upvar fp fp
set fp [open $aof_file w+]
uplevel 1 $code
close $fp
}
proc create_aof_dir {dir_path} {
file mkdir $dir_path
}
proc start_server_aof {overrides code} {
upvar defaults defaults srv srv server_path server_path aof_basename aof_basename aof_dirpath aof_dirpath aof_manifest_file aof_manifest_file aof_manifest_file2 aof_manifest_file2
set config [concat $defaults $overrides]
start_server [list overrides $config keep_persistence true] $code
}
proc start_server_aof_ex {overrides options code} {
Implement Multi Part AOF mechanism to avoid AOFRW overheads. (#9788) Implement Multi-Part AOF mechanism to avoid overheads during AOFRW. Introducing a folder with multiple AOF files tracked by a manifest file. The main issues with the the original AOFRW mechanism are: * buffering of commands that are processed during rewrite (consuming a lot of RAM) * freezes of the main process when the AOFRW completes to drain the remaining part of the buffer and fsync it. * double disk IO for the data that arrives during AOFRW (had to be written to both the old and new AOF files) The main modifications of this PR: 1. Remove the AOF rewrite buffer and related code. 2. Divide the AOF into multiple files, they are classified as two types, one is the the `BASE` type, it represents the full amount of data (Maybe AOF or RDB format) after each AOFRW, there is only one `BASE` file at most. The second is `INCR` type, may have more than one. They represent the incremental commands since the last AOFRW. 3. Use a AOF manifest file to record and manage these AOF files mentioned above. 4. The original configuration of `appendfilename` will be the base part of the new file name, for example: `appendonly.aof.1.base.rdb` and `appendonly.aof.2.incr.aof` 5. Add manifest-related TCL tests, and modified some existing tests that depend on the `appendfilename` 6. Remove the `aof_rewrite_buffer_length` field in info. 7. Add `aof-disable-auto-gc` configuration. By default we're automatically deleting HISTORY type AOFs. It also gives users the opportunity to preserve the history AOFs. just for testing use now. 8. Add AOFRW limiting measure. When the AOFRW failures reaches the threshold (3 times now), we will delay the execution of the next AOFRW by 1 minute. If the next AOFRW also fails, it will be delayed by 2 minutes. The next is 4, 8, 16, the maximum delay is 60 minutes (1 hour). During the limit period, we can still use the 'bgrewriteaof' command to execute AOFRW immediately. 9. Support upgrade (load) data from old version redis. 10. Add `appenddirname` configuration, as the directory name of the append only files. All AOF files and manifest file will be placed in this directory. 11. Only the last AOF file (BASE or INCR) can be truncated. Otherwise redis will exit even if `aof-load-truncated` is enabled. Co-authored-by: Oran Agra <oran@redislabs.com>
2022-01-03 12:14:13 -05:00
upvar defaults defaults srv srv server_path server_path
set config [concat $defaults $overrides]
start_server [concat [list overrides $config keep_persistence true] $options] $code
Implement Multi Part AOF mechanism to avoid AOFRW overheads. (#9788) Implement Multi-Part AOF mechanism to avoid overheads during AOFRW. Introducing a folder with multiple AOF files tracked by a manifest file. The main issues with the the original AOFRW mechanism are: * buffering of commands that are processed during rewrite (consuming a lot of RAM) * freezes of the main process when the AOFRW completes to drain the remaining part of the buffer and fsync it. * double disk IO for the data that arrives during AOFRW (had to be written to both the old and new AOF files) The main modifications of this PR: 1. Remove the AOF rewrite buffer and related code. 2. Divide the AOF into multiple files, they are classified as two types, one is the the `BASE` type, it represents the full amount of data (Maybe AOF or RDB format) after each AOFRW, there is only one `BASE` file at most. The second is `INCR` type, may have more than one. They represent the incremental commands since the last AOFRW. 3. Use a AOF manifest file to record and manage these AOF files mentioned above. 4. The original configuration of `appendfilename` will be the base part of the new file name, for example: `appendonly.aof.1.base.rdb` and `appendonly.aof.2.incr.aof` 5. Add manifest-related TCL tests, and modified some existing tests that depend on the `appendfilename` 6. Remove the `aof_rewrite_buffer_length` field in info. 7. Add `aof-disable-auto-gc` configuration. By default we're automatically deleting HISTORY type AOFs. It also gives users the opportunity to preserve the history AOFs. just for testing use now. 8. Add AOFRW limiting measure. When the AOFRW failures reaches the threshold (3 times now), we will delay the execution of the next AOFRW by 1 minute. If the next AOFRW also fails, it will be delayed by 2 minutes. The next is 4, 8, 16, the maximum delay is 60 minutes (1 hour). During the limit period, we can still use the 'bgrewriteaof' command to execute AOFRW immediately. 9. Support upgrade (load) data from old version redis. 10. Add `appenddirname` configuration, as the directory name of the append only files. All AOF files and manifest file will be placed in this directory. 11. Only the last AOF file (BASE or INCR) can be truncated. Otherwise redis will exit even if `aof-load-truncated` is enabled. Co-authored-by: Oran Agra <oran@redislabs.com>
2022-01-03 12:14:13 -05:00
}