redict/tests/unit/moduleapi/list.tcl

167 lines
5.2 KiB
Tcl
Raw Normal View History

2024-03-21 09:30:47 -04:00
# SPDX-FileCopyrightText: 2024 Redict Contributors
# SPDX-FileCopyrightText: 2024 Salvatore Sanfilippo <antirez at gmail dot com>
#
# SPDX-License-Identifier: BSD-3-Clause
# SPDX-License-Identifier: GPL-3.0-only
set testmodule [file normalize tests/modules/list.so]
# The following arguments can be passed to args:
# i -- the number of inserts
# d -- the number of deletes
# r -- the number of replaces
# index -- the last index
# entry -- The entry pointed to by index
proc verify_list_edit_reply {reply argv} {
foreach {k v} $argv {
assert_equal [dict get $reply $k] $v
}
}
start_server {tags {"modules"}} {
r module load $testmodule
test {Module list set, get, insert, delete} {
r del k
Add listpack encoding for list (#11303) Improve memory efficiency of list keys ## Description of the feature The new listpack encoding uses the old `list-max-listpack-size` config to perform the conversion, which we can think it of as a node inside a quicklist, but without 80 bytes overhead (internal fragmentation included) of quicklist and quicklistNode structs. For example, a list key with 5 items of 10 chars each, now takes 128 bytes instead of 208 it used to take. ## Conversion rules * Convert listpack to quicklist When the listpack length or size reaches the `list-max-listpack-size` limit, it will be converted to a quicklist. * Convert quicklist to listpack When a quicklist has only one node, and its length or size is reduced to half of the `list-max-listpack-size` limit, it will be converted to a listpack. This is done to avoid frequent conversions when we add or remove at the bounding size or length. ## Interface changes 1. add list entry param to listTypeSetIteratorDirection When list encoding is listpack, `listTypeIterator->lpi` points to the next entry of current entry, so when changing the direction, we need to use the current node (listTypeEntry->p) to update `listTypeIterator->lpi` to the next node in the reverse direction. ## Benchmark ### Listpack VS Quicklist with one node * LPUSH - roughly 0.3% improvement * LRANGE - roughly 13% improvement ### Both are quicklist * LRANGE - roughly 3% improvement * LRANGE without pipeline - roughly 3% improvement From the benchmark, as we can see from the results 1. When list is quicklist encoding, LRANGE improves performance by <5%. 2. When list is listpack encoding, LRANGE improves performance by ~13%, the main enhancement is brought by `addListListpackRangeReply()`. ## Memory usage 1M lists(key:0~key:1000000) with 5 items of 10 chars ("hellohello") each. shows memory usage down by 35.49%, from 214MB to 138MB. ## Note 1. Add conversion callback to support doing some work before conversion Since the quicklist iterator decompresses the current node when it is released, we can no longer decompress the quicklist after we convert the list.
2022-11-16 13:29:46 -05:00
assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} {r list.set k 1 xyz}
r rpush k x
# insert, set, get
r list.insert k 0 foo
r list.insert k -1 bar
r list.set k 1 xyz
assert_equal {foo xyz bar} [r list.getall k]
assert_equal {foo} [r list.get k 0]
assert_equal {xyz} [r list.get k 1]
assert_equal {bar} [r list.get k 2]
assert_equal {bar} [r list.get k -1]
assert_equal {foo} [r list.get k -3]
assert_error {ERR index out*} {r list.get k -4}
assert_error {ERR index out*} {r list.get k 3}
# remove
assert_error {ERR index out*} {r list.delete k -4}
assert_error {ERR index out*} {r list.delete k 3}
r list.delete k 0
r list.delete k -1
assert_equal {xyz} [r list.getall k]
# removing the last element deletes the list
r list.delete k 0
assert_equal 0 [r exists k]
}
test {Module list iteration} {
r del k
r rpush k x y z
assert_equal {x y z} [r list.getall k]
assert_equal {z y x} [r list.getall k REVERSE]
}
test {Module list insert & delete} {
r del k
r rpush k x y z
verify_list_edit_reply [r list.edit k ikikdi foo bar baz] {i 3 index 5}
r list.getall k
} {foo x bar y baz}
test {Module list insert & delete, neg index} {
r del k
r rpush k x y z
verify_list_edit_reply [r list.edit k REVERSE ikikdi foo bar baz] {i 3 index -6}
r list.getall k
} {baz y bar z foo}
test {Module list set while iterating} {
r del k
r rpush k x y z
verify_list_edit_reply [r list.edit k rkr foo bar] {r 2 index 3}
r list.getall k
} {foo y bar}
test {Module list set while iterating, neg index} {
r del k
r rpush k x y z
verify_list_edit_reply [r list.edit k reverse rkr foo bar] {r 2 index -4}
r list.getall k
} {bar y foo}
Add listpack encoding for list (#11303) Improve memory efficiency of list keys ## Description of the feature The new listpack encoding uses the old `list-max-listpack-size` config to perform the conversion, which we can think it of as a node inside a quicklist, but without 80 bytes overhead (internal fragmentation included) of quicklist and quicklistNode structs. For example, a list key with 5 items of 10 chars each, now takes 128 bytes instead of 208 it used to take. ## Conversion rules * Convert listpack to quicklist When the listpack length or size reaches the `list-max-listpack-size` limit, it will be converted to a quicklist. * Convert quicklist to listpack When a quicklist has only one node, and its length or size is reduced to half of the `list-max-listpack-size` limit, it will be converted to a listpack. This is done to avoid frequent conversions when we add or remove at the bounding size or length. ## Interface changes 1. add list entry param to listTypeSetIteratorDirection When list encoding is listpack, `listTypeIterator->lpi` points to the next entry of current entry, so when changing the direction, we need to use the current node (listTypeEntry->p) to update `listTypeIterator->lpi` to the next node in the reverse direction. ## Benchmark ### Listpack VS Quicklist with one node * LPUSH - roughly 0.3% improvement * LRANGE - roughly 13% improvement ### Both are quicklist * LRANGE - roughly 3% improvement * LRANGE without pipeline - roughly 3% improvement From the benchmark, as we can see from the results 1. When list is quicklist encoding, LRANGE improves performance by <5%. 2. When list is listpack encoding, LRANGE improves performance by ~13%, the main enhancement is brought by `addListListpackRangeReply()`. ## Memory usage 1M lists(key:0~key:1000000) with 5 items of 10 chars ("hellohello") each. shows memory usage down by 35.49%, from 214MB to 138MB. ## Note 1. Add conversion callback to support doing some work before conversion Since the quicklist iterator decompresses the current node when it is released, we can no longer decompress the quicklist after we convert the list.
2022-11-16 13:29:46 -05:00
test {Module list - encoding conversion while inserting} {
r config set list-max-listpack-size 4
r del k
r rpush k a b c d
assert_encoding listpack k
# Converts to quicklist after inserting.
r list.edit k dii foo bar
assert_encoding quicklist k
assert_equal [r list.getall k] {foo bar b c d}
# Converts to listpack after deleting three entries.
r list.edit k ddd e
assert_encoding listpack k
assert_equal [r list.getall k] {c d}
}
test {Module list - encoding conversion while replacing} {
r config set list-max-listpack-size -1
r del k
r rpush k x y z
assert_encoding listpack k
# Converts to quicklist after replacing.
set big [string repeat "x" 4096]
r list.edit k r $big
assert_encoding quicklist k
assert_equal [r list.getall k] "$big y z"
# Converts to listpack after deleting the big entry.
r list.edit k d
assert_encoding listpack k
assert_equal [r list.getall k] {y z}
}
test {Module list - list entry and index should be updated when deletion} {
set original_config [config_get_set list-max-listpack-size 1]
# delete from start (index 0)
r del l
r rpush l x y z
verify_list_edit_reply [r list.edit l dd] {d 2 index 0 entry z}
assert_equal [r list.getall l] {z}
# delete from start (index -3)
r del l
r rpush l x y z
verify_list_edit_reply [r list.edit l reverse kkd] {d 1 index -3}
assert_equal [r list.getall l] {y z}
# # delete from tail (index 2)
r del l
r rpush l x y z
verify_list_edit_reply [r list.edit l kkd] {d 1 index 2}
assert_equal [r list.getall l] {x y}
# # delete from tail (index -1)
r del l
r rpush l x y z
verify_list_edit_reply [r list.edit l reverse dd] {d 2 index -1 entry x}
assert_equal [r list.getall l] {x}
# # delete from middle (index 1)
r del l
r rpush l x y z
verify_list_edit_reply [r list.edit l kdd] {d 2 index 1}
assert_equal [r list.getall l] {x}
# # delete from middle (index -2)
r del l
r rpush l x y z
verify_list_edit_reply [r list.edit l reverse kdd] {d 2 index -2}
assert_equal [r list.getall l] {z}
config_set list-max-listpack-size $original_config
}
test "Unload the module - list" {
assert_equal {OK} [r module unload list]
}
}