redict/utils/generate-command-code.py
Jason Elbaum 1f76bb17dd
Reimplement cli hints based on command arg docs (#10515)
Now that the command argument specs are available at runtime (#9656), this PR addresses
#8084 by implementing a complete solution for command-line hinting in `redis-cli`.

It correctly handles nearly every case in Redis's complex command argument definitions, including
`BLOCK` and `ONEOF` arguments, reordering of optional arguments, and repeated arguments
(even when followed by mandatory arguments). It also validates numerically-typed arguments.
It may not correctly handle all possible combinations of those, but overall it is quite robust.

Arguments are only matched after the space bar is typed, so partial word matching is not
supported - that proved to be more confusing than helpful. When the user's current input
cannot be matched against the argument specs, hinting is disabled.

Partial support has been implemented for legacy (pre-7.0) servers that do not support
`COMMAND DOCS`, by falling back to a statically-compiled command argument table.
On startup, if the server does not support `COMMAND DOCS`, `redis-cli` will now issue
an `INFO SERVER` command to retrieve the server version (unless `HELLO` has already
been sent, in which case the server version will be extracted from the reply to `HELLO`).
The server version will be used to filter the commands and arguments in the command table,
removing those not supported by that version of the server. However, the static table only
includes core Redis commands, so with a legacy server hinting will not be supported for
module commands. The auto generated help.h and the scripts that generates it are gone.

Command and argument tables for the server and CLI use different structs, due primarily
to the need to support different runtime data. In order to generate code for both, macros
have been added to `commands.def` (previously `commands.c`) to make it possible to
configure the code generation differently for different use cases (one linked with redis-server,
and one with redis-cli).

Also adding a basic testing framework for the command hints based on new (undocumented)
command line options to `redis-cli`: `--test_hint 'INPUT'` prints out the command-line hint for
a given input string, and `--test_hint_file <filename>` runs a suite of test cases for the hinting
mechanism. The test suite is in `tests/assets/test_cli_hint_suite.txt`, and it is run from
`tests/integration/redis-cli.tcl`.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2023-03-30 19:03:56 +03:00

624 lines
22 KiB
Python
Executable File

#!/usr/bin/env python3
import glob
import json
import os
import argparse
ARG_TYPES = {
"string": "ARG_TYPE_STRING",
"integer": "ARG_TYPE_INTEGER",
"double": "ARG_TYPE_DOUBLE",
"key": "ARG_TYPE_KEY",
"pattern": "ARG_TYPE_PATTERN",
"unix-time": "ARG_TYPE_UNIX_TIME",
"pure-token": "ARG_TYPE_PURE_TOKEN",
"oneof": "ARG_TYPE_ONEOF",
"block": "ARG_TYPE_BLOCK",
}
GROUPS = {
"generic": "COMMAND_GROUP_GENERIC",
"string": "COMMAND_GROUP_STRING",
"list": "COMMAND_GROUP_LIST",
"set": "COMMAND_GROUP_SET",
"sorted_set": "COMMAND_GROUP_SORTED_SET",
"hash": "COMMAND_GROUP_HASH",
"pubsub": "COMMAND_GROUP_PUBSUB",
"transactions": "COMMAND_GROUP_TRANSACTIONS",
"connection": "COMMAND_GROUP_CONNECTION",
"server": "COMMAND_GROUP_SERVER",
"scripting": "COMMAND_GROUP_SCRIPTING",
"hyperloglog": "COMMAND_GROUP_HYPERLOGLOG",
"cluster": "COMMAND_GROUP_CLUSTER",
"sentinel": "COMMAND_GROUP_SENTINEL",
"geo": "COMMAND_GROUP_GEO",
"stream": "COMMAND_GROUP_STREAM",
"bitmap": "COMMAND_GROUP_BITMAP",
}
def get_optional_desc_string(desc, field, force_uppercase=False):
v = desc.get(field, None)
if v and force_uppercase:
v = v.upper()
ret = "\"%s\"" % v if v else "NULL"
return ret.replace("\n", "\\n")
def check_command_args_key_specs(args, command_key_specs_index_set, command_arg_key_specs_index_set):
if not args:
return True
for arg in args:
if arg.key_spec_index is not None:
assert isinstance(arg.key_spec_index, int)
if arg.key_spec_index not in command_key_specs_index_set:
print("command: %s arg: %s key_spec_index error" % (command.fullname(), arg.name))
return False
command_arg_key_specs_index_set.add(arg.key_spec_index)
if not check_command_args_key_specs(arg.subargs, command_key_specs_index_set, command_arg_key_specs_index_set):
return False
return True
def check_command_key_specs(command):
if not command.key_specs:
return True
assert isinstance(command.key_specs, list)
for cmd_key_spec in command.key_specs:
if "flags" not in cmd_key_spec:
print("command: %s key_specs missing flags" % command.fullname())
return False
if "NOT_KEY" in cmd_key_spec["flags"]:
# Like SUNSUBSCRIBE / SPUBLISH / SSUBSCRIBE
return True
command_key_specs_index_set = set(range(len(command.key_specs)))
command_arg_key_specs_index_set = set()
# Collect key_spec used for each arg, including arg.subarg
if not check_command_args_key_specs(command.args, command_key_specs_index_set, command_arg_key_specs_index_set):
return False
# Check if we have key_specs not used
if command_key_specs_index_set != command_arg_key_specs_index_set:
print("command: %s may have unused key_spec" % command.fullname())
return False
return True
# Globals
subcommands = {} # container_name -> dict(subcommand_name -> Subcommand) - Only subcommands
commands = {} # command_name -> Command - Only commands
class KeySpec(object):
def __init__(self, spec):
self.spec = spec
def struct_code(self):
def _flags_code():
s = ""
for flag in self.spec.get("flags", []):
s += "CMD_KEY_%s|" % flag
return s[:-1] if s else 0
def _begin_search_code():
if self.spec["begin_search"].get("index"):
return "KSPEC_BS_INDEX,.bs.index={%d}" % (
self.spec["begin_search"]["index"]["pos"]
)
elif self.spec["begin_search"].get("keyword"):
return "KSPEC_BS_KEYWORD,.bs.keyword={\"%s\",%d}" % (
self.spec["begin_search"]["keyword"]["keyword"],
self.spec["begin_search"]["keyword"]["startfrom"],
)
elif "unknown" in self.spec["begin_search"]:
return "KSPEC_BS_UNKNOWN,{{0}}"
else:
print("Invalid begin_search! value=%s" % self.spec["begin_search"])
exit(1)
def _find_keys_code():
if self.spec["find_keys"].get("range"):
return "KSPEC_FK_RANGE,.fk.range={%d,%d,%d}" % (
self.spec["find_keys"]["range"]["lastkey"],
self.spec["find_keys"]["range"]["step"],
self.spec["find_keys"]["range"]["limit"]
)
elif self.spec["find_keys"].get("keynum"):
return "KSPEC_FK_KEYNUM,.fk.keynum={%d,%d,%d}" % (
self.spec["find_keys"]["keynum"]["keynumidx"],
self.spec["find_keys"]["keynum"]["firstkey"],
self.spec["find_keys"]["keynum"]["step"]
)
elif "unknown" in self.spec["find_keys"]:
return "KSPEC_FK_UNKNOWN,{{0}}"
else:
print("Invalid find_keys! value=%s" % self.spec["find_keys"])
exit(1)
return "%s,%s,%s,%s" % (
get_optional_desc_string(self.spec, "notes"),
_flags_code(),
_begin_search_code(),
_find_keys_code()
)
def verify_no_dup_names(container_fullname, args):
name_list = [arg.name for arg in args]
name_set = set(name_list)
if len(name_list) != len(name_set):
print("{}: Dup argument names: {}".format(container_fullname, name_list))
exit(1)
class Argument(object):
def __init__(self, parent_name, desc):
self.parent_name = parent_name
self.desc = desc
self.name = self.desc["name"].lower()
if "_" in self.name:
print("{}: name ({}) should not contain underscores".format(self.fullname(), self.name))
exit(1)
self.type = self.desc["type"]
self.key_spec_index = self.desc.get("key_spec_index", None)
self.subargs = []
if self.type in ["oneof", "block"]:
self.display = None
for subdesc in self.desc["arguments"]:
self.subargs.append(Argument(self.fullname(), subdesc))
if len(self.subargs) < 2:
print("{}: oneof or block arg contains less than two subargs".format(self.fullname()))
exit(1)
verify_no_dup_names(self.fullname(), self.subargs)
else:
self.display = self.desc.get("display")
def fullname(self):
return ("%s %s" % (self.parent_name, self.name)).replace("-", "_")
def struct_name(self):
return "%s_Arg" % (self.fullname().replace(" ", "_"))
def subarg_table_name(self):
assert self.subargs
return "%s_Subargs" % (self.fullname().replace(" ", "_"))
def struct_code(self):
"""
Output example:
MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=GETEX_expiration_Subargs
"""
def _flags_code():
s = ""
if self.desc.get("optional", False):
s += "CMD_ARG_OPTIONAL|"
if self.desc.get("multiple", False):
s += "CMD_ARG_MULTIPLE|"
if self.desc.get("multiple_token", False):
assert self.desc.get("multiple", False) # Sanity
s += "CMD_ARG_MULTIPLE_TOKEN|"
return s[:-1] if s else "CMD_ARG_NONE"
s = "MAKE_ARG(\"%s\",%s,%d,%s,%s,%s,%s,%d,%s)" % (
self.name,
ARG_TYPES[self.type],
self.desc.get("key_spec_index", -1),
get_optional_desc_string(self.desc, "token", force_uppercase=True),
get_optional_desc_string(self.desc, "summary"),
get_optional_desc_string(self.desc, "since"),
_flags_code(),
len(self.subargs),
get_optional_desc_string(self.desc, "deprecated_since"),
)
if "display" in self.desc:
s += ",.display_text=\"%s\"" % self.desc["display"].lower()
if self.subargs:
s += ",.subargs=%s" % self.subarg_table_name()
return s
def write_internal_structs(self, f):
if self.subargs:
for subarg in self.subargs:
subarg.write_internal_structs(f)
f.write("/* %s argument table */\n" % self.fullname())
f.write("struct COMMAND_ARG %s[] = {\n" % self.subarg_table_name())
for subarg in self.subargs:
f.write("{%s},\n" % subarg.struct_code())
f.write("};\n\n")
def to_c_name(str):
return str.replace(":", "").replace(".", "_").replace("$", "_")\
.replace("^", "_").replace("*", "_").replace("-", "_")
class ReplySchema(object):
def __init__(self, name, desc):
self.name = to_c_name(name)
self.schema = {}
if desc.get("type") == "object":
if desc.get("properties") and desc.get("additionalProperties") is None:
print("%s: Any object that has properties should have the additionalProperties field" % self.name)
exit(1)
elif desc.get("type") == "array":
if desc.get("items") and isinstance(desc["items"], list) and any([desc.get(k) is None for k in ["minItems", "maxItems"]]):
print("%s: Any array that has items should have the minItems and maxItems fields" % self.name)
exit(1)
for k, v in desc.items():
if isinstance(v, dict):
self.schema[k] = ReplySchema("%s_%s" % (self.name, k), v)
elif isinstance(v, list):
self.schema[k] = []
for i, subdesc in enumerate(v):
self.schema[k].append(ReplySchema("%s_%s_%i" % (self.name, k,i), subdesc))
else:
self.schema[k] = v
def write(self, f):
def struct_code(name, k, v):
if isinstance(v, ReplySchema):
t = "JSON_TYPE_OBJECT"
vstr = ".value.object=&%s" % name
elif isinstance(v, list):
t = "JSON_TYPE_ARRAY"
vstr = ".value.array={.objects=%s,.length=%d}" % (name, len(v))
elif isinstance(v, bool):
t = "JSON_TYPE_BOOLEAN"
vstr = ".value.boolean=%d" % int(v)
elif isinstance(v, str):
t = "JSON_TYPE_STRING"
vstr = ".value.string=\"%s\"" % v
elif isinstance(v, int):
t = "JSON_TYPE_INTEGER"
vstr = ".value.integer=%d" % v
return "%s,\"%s\",%s" % (t, k, vstr)
for k, v in self.schema.items():
if isinstance(v, ReplySchema):
v.write(f)
elif isinstance(v, list):
for i, schema in enumerate(v):
schema.write(f)
name = to_c_name("%s_%s" % (self.name, k))
f.write("/* %s array reply schema */\n" % name)
f.write("struct jsonObject *%s[] = {\n" % name)
for i, schema in enumerate(v):
f.write("&%s,\n" % schema.name)
f.write("};\n\n")
f.write("/* %s reply schema */\n" % self.name)
f.write("struct jsonObjectElement %s_elements[] = {\n" % self.name)
for k, v in self.schema.items():
name = to_c_name("%s_%s" % (self.name, k))
f.write("{%s},\n" % struct_code(name, k, v))
f.write("};\n\n")
f.write("struct jsonObject %s = {%s_elements,.length=%d};\n\n" % (self.name, self.name, len(self.schema)))
class Command(object):
def __init__(self, name, desc):
self.name = name.upper()
self.desc = desc
self.group = self.desc["group"]
self.key_specs = self.desc.get("key_specs", [])
self.subcommands = []
self.args = []
for arg_desc in self.desc.get("arguments", []):
self.args.append(Argument(self.fullname(), arg_desc))
verify_no_dup_names(self.fullname(), self.args)
self.reply_schema = None
if "reply_schema" in self.desc:
self.reply_schema = ReplySchema(self.reply_schema_name(), self.desc["reply_schema"])
def fullname(self):
return self.name.replace("-", "_").replace(":", "")
def return_types_table_name(self):
return "%s_ReturnInfo" % self.fullname().replace(" ", "_")
def subcommand_table_name(self):
assert self.subcommands
return "%s_Subcommands" % self.name
def history_table_name(self):
return "%s_History" % (self.fullname().replace(" ", "_"))
def tips_table_name(self):
return "%s_Tips" % (self.fullname().replace(" ", "_"))
def arg_table_name(self):
return "%s_Args" % (self.fullname().replace(" ", "_"))
def key_specs_table_name(self):
return "%s_Keyspecs" % (self.fullname().replace(" ", "_"))
def reply_schema_name(self):
return "%s_ReplySchema" % (self.fullname().replace(" ", "_"))
def struct_name(self):
return "%s_Command" % (self.fullname().replace(" ", "_"))
def history_code(self):
if not self.desc.get("history"):
return ""
s = ""
for tupl in self.desc["history"]:
s += "{\"%s\",\"%s\"},\n" % (tupl[0], tupl[1])
return s
def num_history(self):
if not self.desc.get("history"):
return 0
return len(self.desc["history"])
def tips_code(self):
if not self.desc.get("command_tips"):
return ""
s = ""
for hint in self.desc["command_tips"]:
s += "\"%s\",\n" % hint.lower()
return s
def num_tips(self):
if not self.desc.get("command_tips"):
return 0
return len(self.desc["command_tips"])
def key_specs_code(self):
s = ""
for spec in self.key_specs:
s += "{%s}," % KeySpec(spec).struct_code()
return s[:-1]
def struct_code(self):
"""
Output example:
MAKE_CMD("set","Set the string value of a key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,4,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_STRING,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args
"""
def _flags_code():
s = ""
for flag in self.desc.get("command_flags", []):
s += "CMD_%s|" % flag
return s[:-1] if s else 0
def _acl_categories_code():
s = ""
for cat in self.desc.get("acl_categories", []):
s += "ACL_CATEGORY_%s|" % cat
return s[:-1] if s else 0
def _doc_flags_code():
s = ""
for flag in self.desc.get("doc_flags", []):
s += "CMD_DOC_%s|" % flag
return s[:-1] if s else "CMD_DOC_NONE"
s = "MAKE_CMD(\"%s\",%s,%s,%s,%s,%s,%s,%s,%s,%s,%d,%s,%d,%s,%d,%s,%s,%s,%d,%s,%d)," % (
self.name.lower(),
get_optional_desc_string(self.desc, "summary"),
get_optional_desc_string(self.desc, "complexity"),
get_optional_desc_string(self.desc, "since"),
_doc_flags_code(),
get_optional_desc_string(self.desc, "replaced_by"),
get_optional_desc_string(self.desc, "deprecated_since"),
"\"%s\"" % self.group,
GROUPS[self.group],
self.history_table_name(),
self.num_history(),
self.tips_table_name(),
self.num_tips(),
self.desc.get("function", "NULL"),
self.desc["arity"],
_flags_code(),
_acl_categories_code(),
self.key_specs_table_name(),
len(self.key_specs),
self.desc.get("get_keys_function", "NULL"),
len(self.args),
)
if self.subcommands:
s += ".subcommands=%s," % self.subcommand_table_name()
if self.args:
s += ".args=%s," % self.arg_table_name()
if self.reply_schema and args.with_reply_schema:
s += ".reply_schema=&%s," % self.reply_schema_name()
return s[:-1]
def write_internal_structs(self, f):
if self.subcommands:
subcommand_list = sorted(self.subcommands, key=lambda cmd: cmd.name)
for subcommand in subcommand_list:
subcommand.write_internal_structs(f)
f.write("/* %s command table */\n" % self.fullname())
f.write("struct COMMAND_STRUCT %s[] = {\n" % self.subcommand_table_name())
for subcommand in subcommand_list:
f.write("{%s},\n" % subcommand.struct_code())
f.write("{0}\n")
f.write("};\n\n")
f.write("/********** %s ********************/\n\n" % self.fullname())
f.write("#ifndef SKIP_CMD_HISTORY_TABLE\n")
f.write("/* %s history */\n" % self.fullname())
code = self.history_code()
if code:
f.write("commandHistory %s[] = {\n" % self.history_table_name())
f.write("%s" % code)
f.write("};\n")
else:
f.write("#define %s NULL\n" % self.history_table_name())
f.write("#endif\n\n")
f.write("#ifndef SKIP_CMD_TIPS_TABLE\n")
f.write("/* %s tips */\n" % self.fullname())
code = self.tips_code()
if code:
f.write("const char *%s[] = {\n" % self.tips_table_name())
f.write("%s" % code)
f.write("};\n")
else:
f.write("#define %s NULL\n" % self.tips_table_name())
f.write("#endif\n\n")
f.write("#ifndef SKIP_CMD_KEY_SPECS_TABLE\n")
f.write("/* %s key specs */\n" % self.fullname())
code = self.key_specs_code()
if code:
f.write("keySpec %s[%d] = {\n" % (self.key_specs_table_name(), len(self.key_specs)))
f.write("%s\n" % code)
f.write("};\n")
else:
f.write("#define %s NULL\n" % self.key_specs_table_name())
f.write("#endif\n\n")
if self.args:
for arg in self.args:
arg.write_internal_structs(f)
f.write("/* %s argument table */\n" % self.fullname())
f.write("struct COMMAND_ARG %s[] = {\n" % self.arg_table_name())
for arg in self.args:
f.write("{%s},\n" % arg.struct_code())
f.write("};\n\n")
if self.reply_schema and args.with_reply_schema:
self.reply_schema.write(f)
class Subcommand(Command):
def __init__(self, name, desc):
self.container_name = desc["container"].upper()
super(Subcommand, self).__init__(name, desc)
def fullname(self):
return "%s %s" % (self.container_name, self.name.replace("-", "_").replace(":", ""))
def create_command(name, desc):
if desc.get("container"):
cmd = Subcommand(name.upper(), desc)
subcommands.setdefault(desc["container"].upper(), {})[name] = cmd
else:
cmd = Command(name.upper(), desc)
commands[name.upper()] = cmd
# MAIN
# Figure out where the sources are
srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")
parser = argparse.ArgumentParser()
parser.add_argument('--with-reply-schema', action='store_true')
args = parser.parse_args()
# Create all command objects
print("Processing json files...")
for filename in glob.glob('%s/commands/*.json' % srcdir):
with open(filename, "r") as f:
try:
d = json.load(f)
for name, desc in d.items():
create_command(name, desc)
except json.decoder.JSONDecodeError as err:
print("Error processing %s: %s" % (filename, err))
exit(1)
# Link subcommands to containers
print("Linking container command to subcommands...")
for command in commands.values():
assert command.group
if command.name not in subcommands:
continue
for subcommand in subcommands[command.name].values():
assert not subcommand.group or subcommand.group == command.group
subcommand.group = command.group
command.subcommands.append(subcommand)
check_command_error_counter = 0 # An error counter is used to count errors in command checking.
print("Checking all commands...")
for command in commands.values():
if not check_command_key_specs(command):
check_command_error_counter += 1
if check_command_error_counter != 0:
print("Error: There are errors in the commands check, please check the above logs.")
exit(1)
commands_filename = "commands_with_reply_schema" if args.with_reply_schema else "commands"
print("Generating %s.def..." % commands_filename)
with open("%s/%s.def" % (srcdir, commands_filename), "w") as f:
f.write("/* Automatically generated by %s, do not edit. */\n\n" % os.path.basename(__file__))
f.write(
"""
/* We have fabulous commands from
* the fantastic
* Redis Command Table! */
/* Must match redisCommandGroup */
const char *COMMAND_GROUP_STR[] = {
"generic",
"string",
"list",
"set",
"sorted-set",
"hash",
"pubsub",
"transactions",
"connection",
"server",
"scripting",
"hyperloglog",
"cluster",
"sentinel",
"geo",
"stream",
"bitmap",
"module"
};
const char *commandGroupStr(int index) {
return COMMAND_GROUP_STR[index];
}
"""
)
command_list = sorted(commands.values(), key=lambda cmd: (cmd.group, cmd.name))
for command in command_list:
command.write_internal_structs(f)
f.write("/* Main command table */\n")
f.write("struct COMMAND_STRUCT redisCommandTable[] = {\n")
curr_group = None
for command in command_list:
if curr_group != command.group:
curr_group = command.group
f.write("/* %s */\n" % curr_group)
f.write("{%s},\n" % command.struct_code())
f.write("{0}\n")
f.write("};\n")
print("All done, exiting.")