| Message ID | 20250110104946.74960-2-victortoso@redhat.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | qapi-go: add generator for Golang interfaces | expand |
Victor Toso <victortoso@redhat.com> writes: > This patch handles QAPI enum types and generates its equivalent in Go. > We sort the output based on enum's type name. Any particular reason for sorting? The existing backends generate output it source order, on the (bold?) assumption that developers care to pick an order that makes sense. > Enums are being handled as strings in Golang. > > 1. For each QAPI enum, we will define a string type in Go to be the > assigned type of this specific enum. > > 2. Naming: CamelCase will be used in any identifier that we want to > export, which is everything. > > Example: > > qapi: > | ## > | # @DisplayProtocol: > | # > | # Display protocols which support changing password options. > | # > | # Since: 7.0 > | ## > | { 'enum': 'DisplayProtocol', > | 'data': [ 'vnc', 'spice' ] } > > go: > | // Display protocols which support changing password options. > | // > | // Since: 7.0 > | type DisplayProtocol string > | > | const ( > | DisplayProtocolVnc DisplayProtocol = "vnc" > | DisplayProtocolSpice DisplayProtocol = "spice" > | ) > > Signed-off-by: Victor Toso <victortoso@redhat.com> > --- > scripts/qapi/golang.py | 266 +++++++++++++++++++++++++++++++++++++++++ > scripts/qapi/main.py | 3 + > 2 files changed, 269 insertions(+) > create mode 100644 scripts/qapi/golang.py > > diff --git a/scripts/qapi/golang.py b/scripts/qapi/golang.py > new file mode 100644 > index 0000000000..1e04c99f1c > --- /dev/null > +++ b/scripts/qapi/golang.py [...] > +class QAPISchemaGenGolangVisitor(QAPISchemaVisitor): [...] > + def write(self, output_dir: str) -> None: > + for module_name, content in self.target.items(): > + go_module = module_name + "s.go" > + go_dir = "go" > + pathname = os.path.join(output_dir, go_dir, go_module) > + odir = os.path.dirname(pathname) > + os.makedirs(odir, exist_ok=True) > + > + with open(pathname, "w", encoding="utf8") as outfile: > + outfile.write(content) Your write() serves the same purpose as QAPIGen.write(). The latter touches output files only when their contents actually changes. Have you considered use of QAPIGen? The backends generating C use QAPISchemaMonolithicCVisitor or QAPISchemaModularCVisitor, which use QAPIGenC, QAPIGenH and QAPIGenTrace, all specializations of QAPIGen. > diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py > index 316736b6a2..f1f813b466 100644 > --- a/scripts/qapi/main.py > +++ b/scripts/qapi/main.py > @@ -15,6 +15,7 @@ > from .common import must_match > from .error import QAPIError > from .events import gen_events > +from .golang import gen_golang > from .introspect import gen_introspect > from .schema import QAPISchema > from .types import gen_types > @@ -54,6 +55,8 @@ def generate(schema_file: str, > gen_events(schema, output_dir, prefix) > gen_introspect(schema, output_dir, prefix, unmask) > > + gen_golang(schema, output_dir, prefix) > + > > def main() -> int: > """
Hi, On Tue, Jan 14, 2025 at 09:52:23AM +0100, Markus Armbruster wrote: > Victor Toso <victortoso@redhat.com> writes: > > > This patch handles QAPI enum types and generates its equivalent in Go. > > We sort the output based on enum's type name. > > Any particular reason for sorting? It was a request from Daniel that I've accepted. https://lists.gnu.org/archive/html/qemu-devel/2023-09/msg07042.html We did the same thing in the code generator in libvirt-go-module and the result helps with navigating the code and also with the diff itself when we regenerate. > The existing backends generate output it source order, on the (bold?) > assumption that developers care to pick an order that makes sense. > > > Enums are being handled as strings in Golang. > > > > 1. For each QAPI enum, we will define a string type in Go to be the > > assigned type of this specific enum. > > > > 2. Naming: CamelCase will be used in any identifier that we want to > > export, which is everything. > > > > Example: > > > > qapi: > > | ## > > | # @DisplayProtocol: > > | # > > | # Display protocols which support changing password options. > > | # > > | # Since: 7.0 > > | ## > > | { 'enum': 'DisplayProtocol', > > | 'data': [ 'vnc', 'spice' ] } > > > > go: > > | // Display protocols which support changing password options. > > | // > > | // Since: 7.0 > > | type DisplayProtocol string > > | > > | const ( > > | DisplayProtocolVnc DisplayProtocol = "vnc" > > | DisplayProtocolSpice DisplayProtocol = "spice" > > | ) > > > > Signed-off-by: Victor Toso <victortoso@redhat.com> > > --- > > scripts/qapi/golang.py | 266 +++++++++++++++++++++++++++++++++++++++++ > > scripts/qapi/main.py | 3 + > > 2 files changed, 269 insertions(+) > > create mode 100644 scripts/qapi/golang.py > > > > diff --git a/scripts/qapi/golang.py b/scripts/qapi/golang.py > > new file mode 100644 > > index 0000000000..1e04c99f1c > > --- /dev/null > > +++ b/scripts/qapi/golang.py > > [...] > > > +class QAPISchemaGenGolangVisitor(QAPISchemaVisitor): > > [...] > > > + def write(self, output_dir: str) -> None: > > + for module_name, content in self.target.items(): > > + go_module = module_name + "s.go" > > + go_dir = "go" > > + pathname = os.path.join(output_dir, go_dir, go_module) > > + odir = os.path.dirname(pathname) > > + os.makedirs(odir, exist_ok=True) > > + > > + with open(pathname, "w", encoding="utf8") as outfile: > > + outfile.write(content) > > Your write() serves the same purpose as QAPIGen.write(). The latter > touches output files only when their contents actually changes. > > Have you considered use of QAPIGen? I have not, didn't know that actually. I have to investigate if it'd be beneficial. Considering this is once per release execution, shouldn't be a big problem either way. > The backends generating C use QAPISchemaMonolithicCVisitor or > QAPISchemaModularCVisitor, which use QAPIGenC, QAPIGenH and > QAPIGenTrace, all specializations of QAPIGen. > > > diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py > > index 316736b6a2..f1f813b466 100644 > > --- a/scripts/qapi/main.py > > +++ b/scripts/qapi/main.py > > @@ -15,6 +15,7 @@ > > from .common import must_match > > from .error import QAPIError > > from .events import gen_events > > +from .golang import gen_golang > > from .introspect import gen_introspect > > from .schema import QAPISchema > > from .types import gen_types > > @@ -54,6 +55,8 @@ def generate(schema_file: str, > > gen_events(schema, output_dir, prefix) > > gen_introspect(schema, output_dir, prefix, unmask) > > > > + gen_golang(schema, output_dir, prefix) > > + > > > > def main() -> int: > > """ Thanks for the review, Victor
On Tue, Jan 14, 2025 at 09:52:23AM +0100, Markus Armbruster wrote: > Victor Toso <victortoso@redhat.com> writes: > > > This patch handles QAPI enum types and generates its equivalent in Go. > > We sort the output based on enum's type name. > > Any particular reason for sorting? > > The existing backends generate output it source order, on the (bold?) > assumption that developers care to pick an order that makes sense. Even if that assumption is valid (and I question whether it really is), what makes sense to a QEMU maintainer is not the same as what makes sense to a consumer of QEMU. Our approach to file splitting is fairly arbitrary First, we've got a split across sub-systems, except where we don't. Sometimes one subsystem splits into many files. eg block.json vs block-core.json vs block-export.json Sometimes we just pull something into a standalone file even when it is not really a subsystem just common code. eg sockets.json Sometimes we have a split because its convenient for QEMU linux-user vs system compilation & ELF linker needs. eg misc.json vs misc-target.json At best we can say that the split of files is intentionally done for the convenience of the QEMU maintainers. This is irrevelant (and harmful) for anyone who isn't a QEMU maintainer, which covers any consumer of the Go code. IMHO it is also a bug that this file split leaks out into our existing QAPI docs. Then when adding to files, sometimes people append to the end of a file, sometimes it is injected in the middle. I can't say that is an intentional/concious ordering decision. The overall result is that wearing my hat of a consumer of QEMU, when looking at the QAPI schema there is no discernable ordering. Given the source ordering is not useful, there are two choices * Sort to have declarations before use as primary key, and then alphabetical as a secondary sort key * Sort alphabetically as the primary key Fortunately the Go compiler has no requirement for forward declarations. Sorting declarations before use also doesn't really have much effect on most declarations, so the first option ends up 90% alphabetical sorted anyway. Might as well just go the whole way and do pure alphabetical sorting. Comparing the enums code in "source order": https://gitlab.com/victortoso/qapi-go/-/blob/qapi-golang-v1-by-tags/pkg/qapi/enums.go vs alphabetical order: https://gitlab.com/victortoso/qapi-go/-/blob/qapi-golang-v3-by-tags/pkg/qapi/enums.go the improvement is massive IMHO. With regards, Daniel
diff --git a/scripts/qapi/golang.py b/scripts/qapi/golang.py new file mode 100644 index 0000000000..1e04c99f1c --- /dev/null +++ b/scripts/qapi/golang.py @@ -0,0 +1,266 @@ +""" +Golang QAPI generator +""" + +# Copyright (c) 2025 Red Hat Inc. +# +# Authors: +# Victor Toso <victortoso@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +# Just for type hint on self +from __future__ import annotations + +import os, textwrap +from typing import List, Optional + +from .schema import ( + QAPISchema, + QAPISchemaBranches, + QAPISchemaEnumMember, + QAPISchemaFeature, + QAPISchemaIfCond, + QAPISchemaObjectType, + QAPISchemaObjectTypeMember, + QAPISchemaType, + QAPISchemaVariants, + QAPISchemaVisitor, +) +from .source import QAPISourceInfo + + +TEMPLATE_ENUM = """ +{maindoc} +type {name} string + +const ( +{fields} +) +""" + + +# Takes the documentation object of a specific type and returns +# that type's documentation and its member's docs. +def qapi_to_golang_struct_docs(doc: QAPIDoc) -> (str, Dict[str, str]): + if doc is None: + return "", {} + + cmt = "// " + fmt = textwrap.TextWrapper( + width=70, initial_indent=cmt, subsequent_indent=cmt + ) + main = fmt.fill(doc.body.text) + + for section in doc.sections: + # TODO is not a relevant section to Go applications + if section.tag in ["TODO"]: + continue + + if main != "": + # Give empty line as space for the tag. + main += "\n//\n" + + tag = "" if section.tag is None else f"{section.tag}: " + text = section.text.replace(" ", " ") + main += fmt.fill(f"{tag}{text}") + + fields = {} + for key, value in doc.args.items(): + if len(value.text) > 0: + fields[key] = " ".join(value.text.replace("\n", " ").split()) + + return main, fields + + +def gen_golang(schema: QAPISchema, output_dir: str, prefix: str) -> None: + vis = QAPISchemaGenGolangVisitor(prefix) + schema.visit(vis) + vis.write(output_dir) + + +def qapi_to_field_name_enum(name: str) -> str: + return name.title().replace("-", "") + + +def fetch_indent_blocks_over_enum_with_docs( + name: str, members: List[QAPISchemaEnumMember], docfields: Dict[str, str] +) -> Tuple[int]: + maxname = 0 + blocks: List[int] = [0] + for member in members: + # For simplicity, every time we have doc, we add a new indent block + hasdoc = member.name is not None and member.name in docfields + + enum_name = f"{name}{qapi_to_field_name_enum(member.name)}" + maxname = ( + max(maxname, len(enum_name)) if not hasdoc else len(enum_name) + ) + + if hasdoc: + blocks.append(maxname) + else: + blocks[-1] = maxname + + return blocks + + +def generate_content_from_dict(data: dict[str, str]) -> str: + content = "" + + for name in sorted(data): + content += data[name] + + return content.replace("\n\n\n", "\n\n") + + +class QAPISchemaGenGolangVisitor(QAPISchemaVisitor): + # pylint: disable=too-many-arguments + def __init__(self, _: str): + super().__init__() + types = ("enum",) + self.target = dict.fromkeys(types, "") + self.schema: QAPISchema + self.golang_package_name = "qapi" + self.enums: dict[str, str] = {} + self.docmap = {} + + def visit_begin(self, schema: QAPISchema) -> None: + self.schema = schema + + # iterate once in schema.docs to map doc objects to its name + for doc in schema.docs: + if doc.symbol is None: + continue + self.docmap[doc.symbol] = doc + + # Every Go file needs to reference its package name + for target in self.target: + self.target[target] = f"package {self.golang_package_name}" + + def visit_end(self) -> None: + del self.schema + self.target["enum"] += generate_content_from_dict(self.enums) + + def visit_object_type( + self, + name: str, + info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + base: Optional[QAPISchemaObjectType], + members: List[QAPISchemaObjectTypeMember], + branches: Optional[QAPISchemaBranches], + ) -> None: + pass + + def visit_alternate_type( + self, + name: str, + info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + variants: QAPISchemaVariants, + ) -> None: + pass + + def visit_enum_type( + self, + name: str, + info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + members: List[QAPISchemaEnumMember], + prefix: Optional[str], + ) -> None: + assert name not in self.enums + doc = self.docmap.get(name, None) + maindoc, docfields = qapi_to_golang_struct_docs(doc) + + # The logic below is to generate QAPI enums as blocks of Go consts + # each with its own type for type safety inside Go applications. + # + # Block of const() blocks are vertically indented so we have to + # first iterate over all names to calculate space between + # $var_name and $var_type. This is achieved by helper function + # @fetch_indent_blocks_over_enum_with_docs() + # + # A new indentation block is defined by empty line or a comment. + + indent_block = iter( + fetch_indent_blocks_over_enum_with_docs(name, members, docfields) + ) + maxname = next(indent_block) + fields = "" + for index, member in enumerate(members): + # For simplicity, every time we have doc, we go to next indent block + hasdoc = member.name is not None and member.name in docfields + + if hasdoc: + maxname = next(indent_block) + + enum_name = f"{name}{qapi_to_field_name_enum(member.name)}" + name2type = " " * (maxname - len(enum_name) + 1) + + if hasdoc: + docstr = ( + textwrap.TextWrapper(width=80) + .fill(docfields[member.name]) + .replace("\n", "\n\t// ") + ) + fields += f"""\t// {docstr}\n""" + + fields += f"""\t{enum_name}{name2type}{name} = "{member.name}"\n""" + + self.enums[name] = TEMPLATE_ENUM.format( + maindoc=maindoc, name=name, fields=fields[:-1] + ) + + def visit_array_type( + self, + name: str, + info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + element_type: QAPISchemaType, + ) -> None: + pass + + def visit_command( + self, + name: str, + info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + arg_type: Optional[QAPISchemaObjectType], + ret_type: Optional[QAPISchemaType], + gen: bool, + success_response: bool, + boxed: bool, + allow_oob: bool, + allow_preconfig: bool, + coroutine: bool, + ) -> None: + pass + + def visit_event( + self, + name: str, + info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + arg_type: Optional[QAPISchemaObjectType], + boxed: bool, + ) -> None: + pass + + def write(self, output_dir: str) -> None: + for module_name, content in self.target.items(): + go_module = module_name + "s.go" + go_dir = "go" + pathname = os.path.join(output_dir, go_dir, go_module) + odir = os.path.dirname(pathname) + os.makedirs(odir, exist_ok=True) + + with open(pathname, "w", encoding="utf8") as outfile: + outfile.write(content) diff --git a/scripts/qapi/main.py b/scripts/qapi/main.py index 316736b6a2..f1f813b466 100644 --- a/scripts/qapi/main.py +++ b/scripts/qapi/main.py @@ -15,6 +15,7 @@ from .common import must_match from .error import QAPIError from .events import gen_events +from .golang import gen_golang from .introspect import gen_introspect from .schema import QAPISchema from .types import gen_types @@ -54,6 +55,8 @@ def generate(schema_file: str, gen_events(schema, output_dir, prefix) gen_introspect(schema, output_dir, prefix, unmask) + gen_golang(schema, output_dir, prefix) + def main() -> int: """
This patch handles QAPI enum types and generates its equivalent in Go. We sort the output based on enum's type name. Enums are being handled as strings in Golang. 1. For each QAPI enum, we will define a string type in Go to be the assigned type of this specific enum. 2. Naming: CamelCase will be used in any identifier that we want to export, which is everything. Example: qapi: | ## | # @DisplayProtocol: | # | # Display protocols which support changing password options. | # | # Since: 7.0 | ## | { 'enum': 'DisplayProtocol', | 'data': [ 'vnc', 'spice' ] } go: | // Display protocols which support changing password options. | // | // Since: 7.0 | type DisplayProtocol string | | const ( | DisplayProtocolVnc DisplayProtocol = "vnc" | DisplayProtocolSpice DisplayProtocol = "spice" | ) Signed-off-by: Victor Toso <victortoso@redhat.com> --- scripts/qapi/golang.py | 266 +++++++++++++++++++++++++++++++++++++++++ scripts/qapi/main.py | 3 + 2 files changed, 269 insertions(+) create mode 100644 scripts/qapi/golang.py