~bigbes/tarantool

tarantool-protobuf

b3d342d67c5e2fcce101fdafe3aae3c5120f1e73 — Eugene Blikh 3 months ago 78d234c
codegen: protoc-gen-tarantool-doc — Markdown reference plugin

Sibling Go plugin under cmd/protoc-gen-tarantool-doc that emits one
Markdown file per input .proto. Sections (omitted when empty):

  - Header (path, package, imports)
  - Messages (per-message description + field table:
    # | Field | Type | Label | Description)
  - Enums (value table)
  - Services (method table with unary/client/server/bidi label)

Field type cells render scalar names, full type names for message/enum
references, and `map<K, V>` for maps. Synthetic map-entry messages are
skipped. Leading comments preserved via SourceCodeInfo (squashed to a
single line inside table cells).

Built via `make build-doc`; sample output committed at
examples/docs/hello.md via `make gen-docs`. Smoke tests in
test/doc_test.lua build the plugin if needed and assert the expected
sections and labels appear. 403/403 luatest green.
5 files changed, 563 insertions(+), 2 deletions(-)

M .gitignore
M Makefile
A cmd/protoc-gen-tarantool-doc/main.go
A examples/docs/hello.md
A test/doc_test.lua
M .gitignore => .gitignore +1 -0
@@ 7,3 7,4 @@
.vscode/
*.swp
*.swo
/protoc-gen-tarantool-doc

M Makefile => Makefile +17 -2
@@ 1,5 1,7 @@
PLUGIN := protoc-gen-tarantool
DOC_PLUGIN := protoc-gen-tarantool-doc
GEN_DIR := examples/expected
DOCS_DIR := examples/docs
PROTO_DIR := examples/proto
CONFORMANCE_PROTO_DIR := test/conformance/proto



@@ 18,16 20,29 @@ empty :=
space := $(empty) $(empty)
LUA_PATH_JOINED := $(subst $(space),;,$(strip $(LUA_PATH_PARTS)));;

.PHONY: all build gen gen-full gen-runtime goldens test test-suite \
        bench bench-baseline bench-compare jit-trace clean
.PHONY: all build build-doc gen gen-full gen-runtime gen-docs goldens \
        test test-suite bench bench-baseline bench-compare jit-trace clean

all: build gen test

build:
	go build -o $(PLUGIN) ./cmd/protoc-gen-tarantool

build-doc:
	go build -o $(DOC_PLUGIN) ./cmd/protoc-gen-tarantool-doc

gen: gen-full gen-runtime gen-conformance

# Render Markdown reference docs for example protos. Output is committed
# so the doc plugin's behavior is visible in PR diffs.
gen-docs: build-doc
	mkdir -p $(DOCS_DIR)
	protoc \
		--plugin=./$(DOC_PLUGIN) \
		--tarantool-doc_out=$(DOCS_DIR) \
		-I $(PROTO_DIR) -I options \
		$(PROTO_DIR)/*.proto

gen-full: build
	mkdir -p $(GEN_DIR)
	protoc \

A cmd/protoc-gen-tarantool-doc/main.go => cmd/protoc-gen-tarantool-doc/main.go +308 -0
@@ 0,0 1,308 @@
// protoc-gen-tarantool-doc generates Markdown reference documentation from
// proto3 .proto files. One Markdown file per input proto, named after the
// proto file (foo.proto -> foo.md). Output mirrors protoc's path layout
// relative to the -I directory.
//
// Usage:
//   protoc --tarantool-doc_out=./docs \
//          --plugin=./protoc-gen-tarantool-doc foo.proto
//
// Output sections (in order, omitted when empty):
//   - Header with package + imports
//   - Messages (per-message: leading comment, fields table, oneofs, nested)
//   - Enums (per-enum: leading comment, values table)
//   - Services (per-service: leading comment, methods table)
//
// Leading comments on messages/enums/fields/services/methods are preserved
// from SourceCodeInfo via protogen.
package main

import (
	"fmt"
	"io"
	"os"
	"path"
	"sort"
	"strings"

	"google.golang.org/protobuf/compiler/protogen"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/reflect/protoreflect"
	"google.golang.org/protobuf/types/descriptorpb"
	"google.golang.org/protobuf/types/pluginpb"
)

func main() {
	in, err := io.ReadAll(os.Stdin)
	if err != nil {
		fail("read stdin: %v", err)
	}
	req := &pluginpb.CodeGeneratorRequest{}
	if err := proto.Unmarshal(in, req); err != nil {
		fail("parse CodeGeneratorRequest: %v", err)
	}

	// Inject a synthetic go_package so protogen accepts the input even when
	// none of the files declare one (same trick as the codegen plugin).
	for _, f := range req.ProtoFile {
		if f.Options == nil {
			f.Options = &descriptorpb.FileOptions{}
		}
		if f.Options.GoPackage == nil {
			stub := "tarantooldoc_synthetic/" + strings.TrimSuffix(f.GetName(), ".proto")
			f.Options.GoPackage = proto.String(stub)
		}
	}

	plugin, err := protogen.Options{}.New(req)
	if err != nil {
		fail("init protogen: %v", err)
	}
	plugin.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL)

	for _, file := range plugin.Files {
		if !file.Generate {
			continue
		}
		writeFile(plugin, file)
	}

	out, err := proto.Marshal(plugin.Response())
	if err != nil {
		fail("marshal CodeGeneratorResponse: %v", err)
	}
	if _, err := os.Stdout.Write(out); err != nil {
		fail("write stdout: %v", err)
	}
}

func writeFile(plugin *protogen.Plugin, file *protogen.File) {
	base := strings.TrimSuffix(path.Base(file.Desc.Path()), ".proto") + ".md"
	relDir := path.Dir(file.Desc.Path())
	outPath := base
	if relDir != "" && relDir != "." {
		outPath = path.Join(relDir, base)
	}
	g := plugin.NewGeneratedFile(outPath, "")

	var b strings.Builder
	fmt.Fprintf(&b, "# %s\n\n", file.Desc.Path())
	if pkg := file.Desc.Package(); pkg != "" {
		fmt.Fprintf(&b, "**Package:** `%s`\n\n", pkg)
	}

	if imports := file.Desc.Imports(); imports.Len() > 0 {
		b.WriteString("**Imports:**\n\n")
		for i := 0; i < imports.Len(); i++ {
			fmt.Fprintf(&b, "- `%s`\n", imports.Get(i).Path())
		}
		b.WriteString("\n")
	}

	if len(file.Messages) > 0 {
		b.WriteString("## Messages\n\n")
		for _, m := range file.Messages {
			renderMessage(&b, m, 3)
		}
	}

	if len(file.Enums) > 0 {
		b.WriteString("## Enums\n\n")
		for _, e := range file.Enums {
			renderEnum(&b, e, 3)
		}
	}

	if len(file.Services) > 0 {
		b.WriteString("## Services\n\n")
		for _, s := range file.Services {
			renderService(&b, s)
		}
	}

	g.P(b.String())
}

// ----- rendering helpers ----------------------------------------------------

func heading(level int) string { return strings.Repeat("#", level) + " " }

func cleanComment(c protogen.Comments) string {
	// protogen.Comments leaves each line with a leading space and trailing
	// newline. Strip these and join with single newlines.
	raw := strings.TrimRight(string(c), "\n")
	if raw == "" {
		return ""
	}
	lines := strings.Split(raw, "\n")
	for i, l := range lines {
		lines[i] = strings.TrimSpace(strings.TrimPrefix(l, " "))
	}
	return strings.Join(lines, " ")
}

// renderMessage emits a section for one message and recurses into nested
// types. `level` controls Markdown heading depth.
func renderMessage(b *strings.Builder, m *protogen.Message, level int) {
	// Skip synthetic map-entry messages.
	if m.Desc.IsMapEntry() {
		return
	}

	full := string(m.Desc.FullName())
	fmt.Fprintf(b, "%s`%s`\n\n", heading(level), full)
	if c := cleanComment(m.Comments.Leading); c != "" {
		fmt.Fprintf(b, "%s\n\n", c)
	}

	// Collect oneof groupings to annotate fields.
	oneofByField := map[string]string{}
	for _, oo := range m.Oneofs {
		if oo.Desc.IsSynthetic() {
			continue
		}
		for _, f := range oo.Fields {
			oneofByField[string(f.Desc.Name())] = string(oo.Desc.Name())
		}
	}

	if len(m.Fields) > 0 {
		b.WriteString("| # | Field | Type | Label | Description |\n")
		b.WriteString("|---|-------|------|-------|-------------|\n")
		// Display in declaration order.
		fields := append([]*protogen.Field(nil), m.Fields...)
		sort.SliceStable(fields, func(i, j int) bool {
			return fields[i].Desc.Number() < fields[j].Desc.Number()
		})
		for _, f := range fields {
			fmt.Fprintf(b, "| %d | `%s` | %s | %s | %s |\n",
				f.Desc.Number(),
				f.Desc.Name(),
				fieldTypeText(f),
				labelText(f, oneofByField[string(f.Desc.Name())]),
				inlineComment(f.Comments.Leading))
		}
		b.WriteString("\n")
	}

	// Recurse into nested messages and enums under deeper headings.
	for _, nm := range m.Messages {
		renderMessage(b, nm, level+1)
	}
	for _, ne := range m.Enums {
		renderEnum(b, ne, level+1)
	}
}

func renderEnum(b *strings.Builder, e *protogen.Enum, level int) {
	fmt.Fprintf(b, "%s`%s`\n\n", heading(level), e.Desc.FullName())
	if c := cleanComment(e.Comments.Leading); c != "" {
		fmt.Fprintf(b, "%s\n\n", c)
	}
	b.WriteString("| Value | Name | Description |\n")
	b.WriteString("|-------|------|-------------|\n")
	for _, v := range e.Values {
		fmt.Fprintf(b, "| %d | `%s` | %s |\n",
			v.Desc.Number(),
			v.Desc.Name(),
			inlineComment(v.Comments.Leading))
	}
	b.WriteString("\n")
}

func renderService(b *strings.Builder, s *protogen.Service) {
	fmt.Fprintf(b, "### `%s`\n\n", s.Desc.FullName())
	if c := cleanComment(s.Comments.Leading); c != "" {
		fmt.Fprintf(b, "%s\n\n", c)
	}
	if len(s.Methods) == 0 {
		return
	}
	b.WriteString("| Method | Request | Response | Streaming | Description |\n")
	b.WriteString("|--------|---------|----------|-----------|-------------|\n")
	for _, m := range s.Methods {
		streaming := streamingText(m)
		fmt.Fprintf(b, "| `%s` | `%s` | `%s` | %s | %s |\n",
			m.Desc.Name(),
			m.Desc.Input().FullName(),
			m.Desc.Output().FullName(),
			streaming,
			inlineComment(m.Comments.Leading))
	}
	b.WriteString("\n")
}

// fieldTypeText renders a field's type as a Markdown fragment. Scalars stay
// lowercase; messages/enums become full-name code spans. Map fields render
// as `map<K, V>`.
func fieldTypeText(f *protogen.Field) string {
	d := f.Desc
	if d.IsMap() {
		return fmt.Sprintf("`map<%s, %s>`",
			scalarOrName(d.MapKey()),
			scalarOrName(d.MapValue()))
	}
	return "`" + scalarOrName(d) + "`"
}

// scalarOrName resolves a FieldDescriptor's type as either the lowercase
// proto3 scalar name or the full name of the referenced message/enum.
func scalarOrName(d protoreflect.FieldDescriptor) string {
	switch d.Kind() {
	case protoreflect.MessageKind, protoreflect.GroupKind:
		return string(d.Message().FullName())
	case protoreflect.EnumKind:
		return string(d.Enum().FullName())
	}
	return d.Kind().String()
}

func labelText(f *protogen.Field, oneof string) string {
	parts := []string{}
	d := f.Desc
	if d.IsList() && !d.IsMap() {
		parts = append(parts, "repeated")
	}
	if d.HasOptionalKeyword() {
		parts = append(parts, "optional")
	}
	if oneof != "" {
		parts = append(parts, fmt.Sprintf("oneof `%s`", oneof))
	}
	if len(parts) == 0 {
		return "—"
	}
	return strings.Join(parts, ", ")
}

func streamingText(m *protogen.Method) string {
	cs := m.Desc.IsStreamingClient()
	ss := m.Desc.IsStreamingServer()
	switch {
	case cs && ss:
		return "bidi"
	case cs:
		return "client"
	case ss:
		return "server"
	}
	return "unary"
}

// inlineComment squashes a leading comment to a single line for use inside a
// Markdown table cell (newlines and pipes break table rendering). Empty
// comments render as an em-dash to keep the column non-empty.
func inlineComment(c protogen.Comments) string {
	s := cleanComment(c)
	if s == "" {
		return "—"
	}
	s = strings.ReplaceAll(s, "|", "\\|")
	s = strings.ReplaceAll(s, "\n", " ")
	return s
}

func fail(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "protoc-gen-tarantool-doc: "+format+"\n", args...)
	os.Exit(1)
}

A examples/docs/hello.md => examples/docs/hello.md +111 -0
@@ 0,0 1,111 @@
# hello.proto

**Package:** `hello`

**Imports:**

- `google/protobuf/timestamp.proto`
- `google/protobuf/duration.proto`
- `google/protobuf/empty.proto`
- `google/protobuf/wrappers.proto`
- `google/protobuf/struct.proto`
- `google/protobuf/any.proto`
- `google/protobuf/field_mask.proto`

## Messages

### `hello.Result`

Demo message for oneof handling.

| # | Field | Type | Label | Description |
|---|-------|------|-------|-------------|
| 1 | `id` | `int32` | — | — |
| 2 | `text` | `string` | oneof `outcome` | — |
| 3 | `code` | `int32` | oneof `outcome` | — |
| 4 | `details` | `hello.Address` | oneof `outcome` | — |

### `hello.HelloRequest`

gRPC service demo. Covers unary + all three streaming flavors so the loopback transport exercises every codegen branch.

| # | Field | Type | Label | Description |
|---|-------|------|-------|-------------|
| 1 | `name` | `string` | — | — |

### `hello.HelloReply`

| # | Field | Type | Label | Description |
|---|-------|------|-------|-------------|
| 1 | `greeting` | `string` | — | — |

### `hello.Event`

Demo message exercising well-known types (M3).

| # | Field | Type | Label | Description |
|---|-------|------|-------|-------------|
| 1 | `title` | `string` | — | — |
| 2 | `created_at` | `google.protobuf.Timestamp` | — | — |
| 3 | `duration` | `google.protobuf.Duration` | — | — |
| 4 | `ack` | `google.protobuf.Empty` | — | — |
| 5 | `retry_count` | `google.protobuf.Int32Value` | — | — |
| 6 | `note` | `google.protobuf.StringValue` | — | — |
| 7 | `is_admin` | `google.protobuf.BoolValue` | — | — |
| 8 | `payload` | `google.protobuf.Struct` | — | — |
| 9 | `attribute` | `google.protobuf.Value` | — | — |
| 10 | `tags` | `google.protobuf.ListValue` | — | — |
| 11 | `extension` | `google.protobuf.Any` | — | — |
| 12 | `update_mask` | `google.protobuf.FieldMask` | — | — |

### `hello.Address`

| # | Field | Type | Label | Description |
|---|-------|------|-------|-------------|
| 1 | `street` | `string` | — | — |
| 2 | `city` | `string` | — | — |
| 3 | `zip` | `int32` | — | — |
| 4 | `apartment` | `string` | optional | Explicit-optional: presence is meaningful (distinct from default). |

### `hello.Person`

| # | Field | Type | Label | Description |
|---|-------|------|-------|-------------|
| 1 | `name` | `string` | — | — |
| 2 | `age` | `int32` | — | — |
| 3 | `emails` | `string` | repeated | — |
| 4 | `status` | `hello.Status` | — | — |
| 5 | `address` | `hello.Address` | — | — |
| 6 | `friends` | `hello.Person` | repeated | — |
| 7 | `lucky_numbers` | `int32` | repeated | — |
| 8 | `avatar` | `bytes` | — | — |
| 9 | `user_id` | `fixed64` | — | — |
| 10 | `balance` | `sint32` | — | — |
| 11 | `weight_kg` | `double` | — | — |
| 13 | `ages_by_nickname` | `map<string, int32>` | — | Map fields (M2) |
| 14 | `nickname_by_age` | `map<int32, string>` | — | — |
| 15 | `addresses_by_label` | `map<string, hello.Address>` | — | — |

## Enums

### `hello.Status`

| Value | Name | Description |
|-------|------|-------------|
| 0 | `UNKNOWN` | — |
| 1 | `OK` | — |
| 2 | `ERROR` | — |

## Services

### `hello.Greeter`

| Method | Request | Response | Streaming | Description |
|--------|---------|----------|-----------|-------------|
| `SayHello` | `hello.HelloRequest` | `hello.HelloReply` | unary | — |
| `Echo` | `hello.HelloRequest` | `hello.HelloRequest` | unary | — |
| `StreamHellos` | `hello.HelloRequest` | `hello.HelloReply` | server | Server-streaming: one request, server pushes N replies. |
| `CollectHellos` | `hello.HelloRequest` | `hello.HelloReply` | client | Client-streaming: client pushes N requests, server returns one reply. |
| `Chat` | `hello.HelloRequest` | `hello.HelloReply` | bidi | Bidirectional: both sides push and pull independently. |



A test/doc_test.lua => test/doc_test.lua +126 -0
@@ 0,0 1,126 @@
-- Smoke test for protoc-gen-tarantool-doc.
--
-- Builds the doc plugin (if not already built), runs it against
-- examples/proto/hello.proto, and asserts the expected Markdown sections
-- and entries appear in the output.
local t = require('luatest')
local fio = require('fio')

local REPO_ROOT = fio.abspath(fio.pathjoin(
    fio.dirname(debug.getinfo(1, 'S').source:sub(2)), '..'))
local PROTO_DIR = fio.pathjoin(REPO_ROOT, 'examples', 'proto')
local OPTIONS_DIR = fio.pathjoin(REPO_ROOT, 'options')
local PLUGIN = fio.pathjoin(REPO_ROOT, 'protoc-gen-tarantool-doc')

local function slurp(path)
    local f = assert(io.open(path, 'rb'))
    local s = f:read('*a')
    f:close()
    return s
end

local function ensure_plugin()
    if fio.path.exists(PLUGIN) then return end
    local cmd = string.format('cd %q && go build -o %s ./cmd/protoc-gen-tarantool-doc',
                              REPO_ROOT, fio.basename(PLUGIN))
    assert(os.execute(cmd) == 0 or os.execute(cmd) == true,
           'failed to build doc plugin: ' .. cmd)
end

local OUT_DIR = fio.tempdir()
local OUT_FILE = fio.pathjoin(OUT_DIR, 'hello.md')

do
    ensure_plugin()
    local cmd = string.format(
        'protoc --plugin=%q --tarantool-doc_out=%q -I %q -I %q %q',
        PLUGIN, OUT_DIR, PROTO_DIR, OPTIONS_DIR,
        fio.pathjoin(PROTO_DIR, 'hello.proto'))
    local ok = os.execute(cmd)
    assert(ok == 0 or ok == true, 'doc plugin failed: ' .. cmd)
end

local g = t.group('doc')

g.test_header = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '# hello.proto')
    t.assert_str_contains(md, '**Package:** `hello`')
end

g.test_imports_listed = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '**Imports:**')
    t.assert_str_contains(md, '`google/protobuf/timestamp.proto`')
end

g.test_message_section = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '## Messages')
    t.assert_str_contains(md, '### `hello.Person`')
    t.assert_str_contains(md, '### `hello.Address`')
end

g.test_field_table = function()
    local md = slurp(OUT_FILE)
    -- Header row.
    t.assert_str_contains(md, '| # | Field | Type | Label | Description |')
    -- A scalar field.
    t.assert_str_contains(md, '| 1 | `name` | `string` |')
    -- An enum reference.
    t.assert_str_contains(md, '`hello.Status`')
    -- A WKT reference.
    t.assert_str_contains(md, '`google.protobuf.Timestamp`')
end

g.test_optional_label = function()
    local md = slurp(OUT_FILE)
    -- apartment is explicit-optional.
    t.assert_str_contains(md, '| 4 | `apartment` | `string` | optional |')
end

g.test_repeated_label = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '`lucky_numbers` | `int32` | repeated |')
end

g.test_oneof_label = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, 'oneof `outcome`')
end

g.test_map_field_type = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '`map<string, int32>`')
    t.assert_str_contains(md, '`map<string, hello.Address>`')
end

g.test_no_map_entry_message = function()
    -- Synthetic map entry messages must not appear as their own section.
    local md = slurp(OUT_FILE)
    t.assert_equals(md:find('AgesByNicknameEntry'), nil)
end

g.test_enum_table = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '## Enums')
    t.assert_str_contains(md, '### `hello.Status`')
    t.assert_str_contains(md, '| 0 | `UNKNOWN`')
    t.assert_str_contains(md, '| 2 | `ERROR`')
end

g.test_service_table = function()
    local md = slurp(OUT_FILE)
    t.assert_str_contains(md, '## Services')
    t.assert_str_contains(md, '### `hello.Greeter`')
    t.assert_str_contains(md, '| `SayHello` | `hello.HelloRequest` | `hello.HelloReply` | unary |')
    t.assert_str_contains(md, '| `Chat` | `hello.HelloRequest` | `hello.HelloReply` | bidi |')
    t.assert_str_contains(md, '| `StreamHellos` | `hello.HelloRequest` | `hello.HelloReply` | server |')
    t.assert_str_contains(md, '| `CollectHellos` | `hello.HelloRequest` | `hello.HelloReply` | client |')
end

g.test_leading_comment_preserved = function()
    local md = slurp(OUT_FILE)
    -- The leading comment on `apartment` should appear in the table row.
    t.assert_str_contains(md, 'Explicit-optional: presence is meaningful')
end