redict/tests/unit/cluster/cluster-response-tls.tcl
Chen Tianjie 22a29935ff
Support TLS service when "tls-cluster" is not enabled and persist both plain and TLS port in nodes.conf (#12233)
Originally, when "tls-cluster" is enabled, `port` is set to TLS port. In order to support non-TLS clients, `pport` is used to propagate TCP port across cluster nodes. However when "tls-cluster" is disabled, `port` is set to TCP port, and `pport` is not used, which means the cluster cannot provide TLS service unless "tls-cluster" is on.
```
typedef struct {
    // ...
    uint16_t port;  /* Latest known clients port (TLS or plain). */
    uint16_t pport; /* Latest known clients plaintext port. Only used if the main clients port is for TLS. */
    // ...
} clusterNode;
```
```
typedef struct {
    // ...
    uint16_t port;   /* TCP base port number. */
    uint16_t pport;  /* Sender TCP plaintext port, if base port is TLS */
    // ...
} clusterMsg;
```
This PR renames `port` and `pport` in `clusterNode` to `tcp_port` and `tls_port`, to record both ports no matter "tls-cluster" is enabled or disabled.

This allows to provide TLS service to clients when "tls-cluster" is disabled: when displaying cluster topology, or giving `MOVED` error, server can provide TLS or TCP port according to client's connection type, no matter what type of connection cluster bus is using.

For backwards compatibility, `port` and `pport` in `clusterMsg` are preserved, when "tls-cluster" is enabled, `port` is set to TLS port and `pport` is set to TCP port, when "tls-cluster" is disabled, `port` is set to TCP port and `pport` is set to TLS port (instead of 0).

Also, in the nodes.conf file, a new aux field displaying an extra port is added to complete the persisted info. We may have `tls_port=xxxxx` or `tcp_port=xxxxx` in the aux field, to complete the cluster topology, while the other port is stored in the normal `<ip>:<port>` field. The format is shown below.
```
<node-id> <ip>:<tcp_port>@<cport>,<hostname>,shard-id=...,tls-port=6379 myself,master - 0 0 0 connected 0-1000
```
Or we can switch the position of two ports, both can be correctly resolved.
```
<node-id> <ip>:<tls_port>@<cport>,<hostname>,shard-id=...,tcp-port=6379 myself,master - 0 0 0 connected 0-1000
```
2023-06-26 07:43:38 -07:00

111 lines
4.0 KiB
Tcl

source tests/support/cluster.tcl
proc get_port_from_moved_error {e} {
set ip_port [lindex [split $e " "] 2]
return [lindex [split $ip_port ":"] 1]
}
proc get_pport_by_port {port} {
foreach srv $::servers {
set srv_port [dict get $srv port]
if {$port == $srv_port} {
return [dict get $srv pport]
}
}
return 0
}
proc get_port_from_node_info {line} {
set fields [split $line " "]
set addr [lindex $fields 1]
set ip_port [lindex [split $addr "@"] 0]
return [lindex [split $ip_port ":"] 1]
}
proc cluster_response_tls {tls_cluster} {
test "CLUSTER SLOTS with different connection type -- tls-cluster $tls_cluster" {
set slots1 [R 0 cluster slots]
set pport [srv 0 pport]
set cluster_client [redis_cluster 127.0.0.1:$pport 0]
set slots2 [$cluster_client cluster slots]
$cluster_client close
# Compare the ports in the first row
assert_no_match [lindex $slots1 0 2 1] [lindex $slots2 0 2 1]
}
test "CLUSTER NODES return port according to connection type -- tls-cluster $tls_cluster" {
set nodes [R 0 cluster nodes]
set port1 [get_port_from_node_info [lindex [split $nodes "\r\n"] 0]]
set pport [srv 0 pport]
set cluster_client [redis_cluster 127.0.0.1:$pport 0]
set nodes [$cluster_client cluster nodes]
set port2 [get_port_from_node_info [lindex [split $nodes "\r\n"] 0]]
$cluster_client close
assert_not_equal $port1 $port2
}
set cluster [redis_cluster 127.0.0.1:[srv 0 port]]
set cluster_pport [redis_cluster 127.0.0.1:[srv 0 pport] 0]
$cluster refresh_nodes_map
test "Set many keys in the cluster -- tls-cluster $tls_cluster" {
for {set i 0} {$i < 5000} {incr i} {
$cluster set $i $i
assert { [$cluster get $i] eq $i }
}
}
test "Test cluster responses during migration of slot x -- tls-cluster $tls_cluster" {
set slot 10
array set nodefrom [$cluster masternode_for_slot $slot]
array set nodeto [$cluster masternode_notfor_slot $slot]
$nodeto(link) cluster setslot $slot importing $nodefrom(id)
$nodefrom(link) cluster setslot $slot migrating $nodeto(id)
# Get a key from that slot
set key [$nodefrom(link) cluster GETKEYSINSLOT $slot "1"]
# MOVED REPLY
catch {$nodeto(link) set $key "newVal"} e_moved1
assert_match "*MOVED*" $e_moved1
# ASK REPLY
catch {$nodefrom(link) set "abc{$key}" "newVal"} e_ask1
assert_match "*ASK*" $e_ask1
# UNSTABLE REPLY
assert_error "*TRYAGAIN*" {$nodefrom(link) mset "a{$key}" "newVal" $key "newVal2"}
# Connecting using another protocol
array set nodefrom_pport [$cluster_pport masternode_for_slot $slot]
array set nodeto_pport [$cluster_pport masternode_notfor_slot $slot]
# MOVED REPLY
catch {$nodeto_pport(link) set $key "newVal"} e_moved2
assert_match "*MOVED*" $e_moved2
# ASK REPLY
catch {$nodefrom_pport(link) set "abc{$key}" "newVal"} e_ask2
assert_match "*ASK*" $e_ask2
# Compare MOVED error's port
set port1 [get_port_from_moved_error $e_moved1]
set port2 [get_port_from_moved_error $e_moved2]
assert_not_equal $port1 $port2
assert_equal $port1 $nodefrom(port)
assert_equal $port2 [get_pport_by_port $nodefrom(port)]
# Compare ASK error's port
set port1 [get_port_from_moved_error $e_ask1]
set port2 [get_port_from_moved_error $e_ask2]
assert_not_equal $port1 $port2
assert_equal $port1 $nodeto(port)
assert_equal $port2 [get_pport_by_port $nodeto(port)]
}
}
if {$::tls} {
start_cluster 3 3 {tags {external:skip cluster tls} overrides {tls-cluster yes tls-replication yes}} {
cluster_response_tls yes
}
start_cluster 3 3 {tags {external:skip cluster tls} overrides {tls-cluster no tls-replication no}} {
cluster_response_tls no
}
}