~bigbes/confluence-md-utilities

b3d8390484fc02ba8fff65877f5e24bd87cbbc9b — Eugene Blikh 2 months ago 798bc95
feat: add pull --with-comments sidecar

`mdcx pull --with-comments` fetches inline + page comments via
`/rest/api/content/{id}/child/comment` (paginated, two-level reply
expansion) and writes them to `<output>.comments.json` alongside the
Markdown file. Inline comments correlate to the markdown's
`data-inline-comment="UUID"` spans through their `marker_ref` field.

Comment bodies are stored as raw Confluence storage XML; pushing
comments back is not implemented. Requires `--output`.
5 files changed, 604 insertions(+), 2 deletions(-)

M CLAUDE.md
M README.md
A api/comments.go
A api/comments_test.go
M cmd/mdcx/pull.go
M CLAUDE.md => CLAUDE.md +2 -0
@@ 44,6 44,8 @@ confluence/

**Template markers** — `<!-- MD_CONTENT_START -->` / `<!-- MD_CONTENT_END -->` delimit the editable region in Confluence pages. The `push --template` flow: fetch page → replace between markers → update with version increment.

**Comment sidecar** — Confluence comment threads live outside the page body. `pull --with-comments` calls `/rest/api/content/{id}/child/comment` (paginated, with `children.comment` expanded two levels deep) and writes them to `<output>.comments.json`. Inline comments link back to body markers via the `marker_ref` UUID, which equals the `data-inline-comment="..."` attribute the renderer emits. Comment round-trip (push) is not implemented.

### Semantic mapping (non-obvious)

- Markdown blockquotes → Confluence info panel macro (not `<blockquote>`)

M README.md => README.md +5 -0
@@ 61,8 61,13 @@ Fetch a page from Confluence and convert to Markdown.
mdcx pull "https://confluence.example.com/pages/viewpage.action?pageId=12345" -o page.md
mdcx pull "https://confluence.example.com/display/TEAM/Page+Title" -o page.md
mdcx pull "https://confluence.example.com/display/TEAM/Page+Title" --raw -o page.xml

# Also save inline + page comments to page.md.comments.json
mdcx pull "https://confluence.example.com/display/TEAM/Page+Title" -o page.md --with-comments
```

`--with-comments` writes a sidecar JSON file at `<output>.comments.json` with two arrays — `inline_comments` and `page_comments` — each containing nested `replies`. Inline comments correlate to the markdown's `data-inline-comment="UUID"` spans through the `marker_ref` field. Comment bodies stay as raw Confluence storage XML; pushing comments back is not supported. Requires `--output` (does not work with stdout).

### push

Convert local Markdown and update a Confluence page.

A api/comments.go => api/comments.go +207 -0
@@ 0,0 1,207 @@
package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// Comment is the public representation of a Confluence comment thread node.
//
// BodyStorage is the raw Confluence storage XML for the comment body; it is not
// converted to Markdown. Replies are nested up to two levels deep (the depth
// supported by the children.comment expand chain); deeper threads are not
// captured.
type Comment struct {
	ID          string     `json:"id"`
	Title       string     `json:"title,omitempty"`
	Status      string     `json:"status,omitempty"`
	Location    string     `json:"location"`
	MarkerRef   string     `json:"marker_ref,omitempty"`
	Resolved    bool       `json:"resolved,omitempty"`
	Resolution  string     `json:"resolution,omitempty"`
	Author      User       `json:"author"`
	Created     time.Time  `json:"created"`
	Updated     time.Time  `json:"updated,omitempty"`
	BodyStorage string     `json:"body_storage"`
	Replies     []*Comment `json:"replies,omitempty"`
}

// User identifies a Confluence user attached to a comment.
type User struct {
	Username    string `json:"username,omitempty"`
	DisplayName string `json:"display_name,omitempty"`
	UserKey     string `json:"user_key,omitempty"`
	AccountID   string `json:"account_id,omitempty"`
}

const (
	commentLimit = 200

	commentExpand = "body.storage,version,history,history.createdBy," +
		"extensions.location,extensions.inlineProperties,extensions.resolution," +
		"children.comment.body.storage,children.comment.version,children.comment.history,children.comment.history.createdBy," +
		"children.comment.extensions.location,children.comment.extensions.inlineProperties,children.comment.extensions.resolution," +
		"children.comment.children.comment.body.storage,children.comment.children.comment.version," +
		"children.comment.children.comment.history,children.comment.children.comment.history.createdBy," +
		"children.comment.children.comment.extensions.location," +
		"children.comment.children.comment.extensions.inlineProperties," +
		"children.comment.children.comment.extensions.resolution"
)

type apiCommentList struct {
	Results []apiComment `json:"results"`
	Size    int          `json:"size"`
	Limit   int          `json:"limit"`
	Start   int          `json:"start"`
	Links   apiLinks     `json:"_links"`
}

type apiLinks struct {
	Next    string `json:"next"`
	Base    string `json:"base"`
	Context string `json:"context"`
}

type apiComment struct {
	ID         string         `json:"id"`
	Type       string         `json:"type"`
	Status     string         `json:"status"`
	Title      string         `json:"title"`
	Body       Body           `json:"body"`
	Version    apiVersionMeta `json:"version"`
	History    apiHistory     `json:"history"`
	Extensions apiExtensions  `json:"extensions"`
	Children   apiChildren    `json:"children"`
}

type apiVersionMeta struct {
	Number int       `json:"number"`
	When   time.Time `json:"when"`
	By     apiUser   `json:"by"`
}

type apiHistory struct {
	CreatedDate time.Time `json:"createdDate"`
	CreatedBy   apiUser   `json:"createdBy"`
}

type apiUser struct {
	Username    string `json:"username"`
	DisplayName string `json:"displayName"`
	UserKey     string `json:"userKey"`
	AccountID   string `json:"accountId"`
}

type apiExtensions struct {
	Location         string              `json:"location"`
	InlineProperties apiInlineProperties `json:"inlineProperties"`
	Resolution       apiResolution       `json:"resolution"`
}

type apiInlineProperties struct {
	MarkerRef         string `json:"markerRef"`
	OriginalSelection string `json:"originalSelection,omitempty"`
}

type apiResolution struct {
	Status       string    `json:"status"`
	LastModifier apiUser   `json:"lastModifier"`
	LastModified time.Time `json:"lastModified"`
}

type apiChildren struct {
	Comment apiCommentList `json:"comment"`
}

// GetComments fetches all comments (inline + footer) for the given page,
// following pagination and converting nested reply trees.
func (c *Client) GetComments(pageID string) ([]*Comment, error) {
	params := url.Values{
		"depth":  {"all"},
		"limit":  {fmt.Sprintf("%d", commentLimit)},
		"expand": {commentExpand},
	}
	path := fmt.Sprintf("/rest/api/content/%s/child/comment?%s", pageID, params.Encode())

	var out []*Comment
	for path != "" {
		page, err := c.fetchCommentPage(path, pageID)
		if err != nil {
			return nil, err
		}
		for i := range page.Results {
			out = append(out, convertComment(&page.Results[i]))
		}
		path = c.nextLinkPath(page.Links.Next)
	}
	return out, nil
}

func (c *Client) fetchCommentPage(path, pageID string) (*apiCommentList, error) {
	req, err := c.newRequest(http.MethodGet, path)
	if err != nil {
		return nil, err
	}

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("fetching comments for page %s: %w", pageID, err)
	}
	defer resp.Body.Close()

	if err := checkResponse(resp); err != nil {
		return nil, err
	}

	var list apiCommentList
	if err := json.NewDecoder(resp.Body).Decode(&list); err != nil {
		return nil, fmt.Errorf("decoding comments response: %w", err)
	}
	return &list, nil
}

// nextLinkPath normalizes the _links.next value into a path the client can
// pass to newRequest. Confluence DC sometimes returns it as a relative path
// ("/rest/api/...") and sometimes prefixed with the base URL.
func (c *Client) nextLinkPath(next string) string {
	if next == "" {
		return ""
	}
	if strings.HasPrefix(next, c.BaseURL) {
		return strings.TrimPrefix(next, c.BaseURL)
	}
	return next
}

func convertComment(in *apiComment) *Comment {
	out := &Comment{
		ID:          in.ID,
		Title:       in.Title,
		Status:      in.Status,
		Location:    in.Extensions.Location,
		MarkerRef:   in.Extensions.InlineProperties.MarkerRef,
		Resolution:  in.Extensions.Resolution.Status,
		Resolved:    in.Extensions.Resolution.Status == "resolved",
		Author:      convertUser(in.History.CreatedBy),
		Created:     in.History.CreatedDate,
		Updated:     in.Version.When,
		BodyStorage: in.Body.Storage.Value,
	}
	for i := range in.Children.Comment.Results {
		out.Replies = append(out.Replies, convertComment(&in.Children.Comment.Results[i]))
	}
	return out
}

func convertUser(in apiUser) User {
	return User{
		Username:    in.Username,
		DisplayName: in.DisplayName,
		UserKey:     in.UserKey,
		AccountID:   in.AccountID,
	}
}

A api/comments_test.go => api/comments_test.go +324 -0
@@ 0,0 1,324 @@
package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync/atomic"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// commentJSON builds a single comment node as a generic map. Helpers below
// compose these into a paginated list response.
type commentJSON map[string]any

func footerComment(id, body, displayName string, replies ...commentJSON) commentJSON {
	return commentJSON{
		"id":     id,
		"type":   "comment",
		"status": "current",
		"body": map[string]any{
			"storage": map[string]any{
				"value":          body,
				"representation": "storage",
			},
		},
		"version": map[string]any{
			"number": 1,
			"when":   "2025-12-02T09:00:00.000Z",
		},
		"history": map[string]any{
			"createdDate": "2025-12-02T09:00:00.000Z",
			"createdBy": map[string]any{
				"username":    "u-" + id,
				"displayName": displayName,
				"userKey":     "key-" + id,
			},
		},
		"extensions": map[string]any{
			"location": "footer",
		},
		"children": childrenWrap(replies),
	}
}

func inlineComment(id, body, markerRef, resolution, displayName string, replies ...commentJSON) commentJSON {
	ext := map[string]any{
		"location": "inline",
		"inlineProperties": map[string]any{
			"markerRef": markerRef,
		},
	}
	if resolution != "" {
		ext["resolution"] = map[string]any{"status": resolution}
	}
	return commentJSON{
		"id":     id,
		"type":   "comment",
		"status": "current",
		"body": map[string]any{
			"storage": map[string]any{
				"value":          body,
				"representation": "storage",
			},
		},
		"version": map[string]any{
			"number": 1,
			"when":   "2025-12-01T11:00:00.000Z",
		},
		"history": map[string]any{
			"createdDate": "2025-12-01T10:15:00.000Z",
			"createdBy": map[string]any{
				"username":    "u-" + id,
				"displayName": displayName,
			},
		},
		"extensions": ext,
		"children":   childrenWrap(replies),
	}
}

func childrenWrap(replies []commentJSON) map[string]any {
	results := make([]any, 0, len(replies))
	for _, r := range replies {
		results = append(results, r)
	}
	return map[string]any{
		"comment": map[string]any{
			"results": results,
			"size":    len(results),
		},
	}
}

func listResponse(results []commentJSON, next string) map[string]any {
	out := make([]any, 0, len(results))
	for _, r := range results {
		out = append(out, r)
	}
	resp := map[string]any{
		"results": out,
		"size":    len(out),
		"limit":   200,
		"start":   0,
		"_links":  map[string]any{},
	}
	if next != "" {
		resp["_links"] = map[string]any{"next": next}
	}
	return resp
}

func writeJSON(t *testing.T, w http.ResponseWriter, payload any) {
	t.Helper()
	w.Header().Set("Content-Type", "application/json")
	require.NoError(t, json.NewEncoder(w).Encode(payload))
}

func TestGetComments_Empty(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, "/rest/api/content/12345/child/comment", r.URL.Path)
		writeJSON(t, w, listResponse(nil, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	assert.Empty(t, result)
}

func TestGetComments_FooterOnly(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeJSON(t, w, listResponse([]commentJSON{
			footerComment("100", "<p>LGTM</p>", "Bob"),
			footerComment("101", "<p>nice</p>", "Eve"),
		}, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 2)
	assert.Equal(t, "footer", result[0].Location)
	assert.Empty(t, result[0].MarkerRef)
	assert.Equal(t, "<p>LGTM</p>", result[0].BodyStorage)
	assert.Equal(t, "Bob", result[0].Author.DisplayName)
}

func TestGetComments_InlineWithMarker(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeJSON(t, w, listResponse([]commentJSON{
			inlineComment("200", "<p>clarify?</p>", "uuid-A", "open", "Jane"),
		}, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 1)
	assert.Equal(t, "inline", result[0].Location)
	assert.Equal(t, "uuid-A", result[0].MarkerRef)
	assert.False(t, result[0].Resolved)
	assert.Equal(t, "open", result[0].Resolution)
	assert.Equal(t, "Jane", result[0].Author.DisplayName)
}

func TestGetComments_Resolved(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeJSON(t, w, listResponse([]commentJSON{
			inlineComment("300", "<p>fixed</p>", "uuid-B", "resolved", "Jane"),
		}, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 1)
	assert.True(t, result[0].Resolved)
	assert.Equal(t, "resolved", result[0].Resolution)
}

func TestGetComments_NestedReplies(t *testing.T) {
	grandchild := inlineComment("403", "<p>thanks</p>", "uuid-C", "open", "Jane")
	child1 := inlineComment("401", "<p>agreed</p>", "uuid-C", "open", "Alice", grandchild)
	child2 := inlineComment("402", "<p>also</p>", "uuid-C", "open", "Bob")
	parent := inlineComment("400", "<p>q?</p>", "uuid-C", "open", "Jane", child1, child2)

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeJSON(t, w, listResponse([]commentJSON{parent}, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 1)
	assert.Equal(t, "400", result[0].ID)
	require.Len(t, result[0].Replies, 2)
	assert.Equal(t, "401", result[0].Replies[0].ID)
	require.Len(t, result[0].Replies[0].Replies, 1)
	assert.Equal(t, "403", result[0].Replies[0].Replies[0].ID)
	assert.Equal(t, "402", result[0].Replies[1].ID)
}

func TestGetComments_PaginationRelative(t *testing.T) {
	var hits int32
	var server *httptest.Server
	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		n := atomic.AddInt32(&hits, 1)
		switch n {
		case 1:
			writeJSON(t, w, listResponse([]commentJSON{
				footerComment("1", "<p>a</p>", "A"),
			}, "/rest/api/content/12345/child/comment?start=1&limit=200"))
		case 2:
			assert.Equal(t, "1", r.URL.Query().Get("start"))
			writeJSON(t, w, listResponse([]commentJSON{
				footerComment("2", "<p>b</p>", "B"),
			}, ""))
		default:
			t.Fatalf("unexpected request to %s", r.URL.String())
		}
		_ = server
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 2)
	assert.Equal(t, int32(2), atomic.LoadInt32(&hits))
}

func TestGetComments_PaginationAbsolute(t *testing.T) {
	var hits int32
	var serverURL string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		n := atomic.AddInt32(&hits, 1)
		switch n {
		case 1:
			next := serverURL + "/rest/api/content/12345/child/comment?start=1"
			writeJSON(t, w, listResponse([]commentJSON{
				footerComment("1", "<p>a</p>", "A"),
			}, next))
		case 2:
			assert.Equal(t, "1", r.URL.Query().Get("start"))
			writeJSON(t, w, listResponse([]commentJSON{
				footerComment("2", "<p>b</p>", "B"),
			}, ""))
		}
	}))
	defer server.Close()
	serverURL = server.URL

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 2)
	assert.Equal(t, int32(2), atomic.LoadInt32(&hits))
}

func TestGetComments_ExpandParam(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, "all", r.URL.Query().Get("depth"))
		assert.Equal(t, fmt.Sprintf("%d", commentLimit), r.URL.Query().Get("limit"))
		expand := r.URL.Query().Get("expand")
		assert.Equal(t, commentExpand, expand)
		assert.True(t, strings.Contains(expand, "extensions.inlineProperties"))
		assert.True(t, strings.Contains(expand, "children.comment.children.comment"))
		writeJSON(t, w, listResponse(nil, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	_, err := client.GetComments("12345")
	require.NoError(t, err)
}

func TestGetComments_NotFound(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	_, err := client.GetComments("12345")
	require.Error(t, err)
	assert.Contains(t, err.Error(), "not found")
}

func TestGetComments_MissingOptionalFields(t *testing.T) {
	// Footer comment without resolution/inlineProperties; minimum viable shape.
	c := commentJSON{
		"id":     "999",
		"type":   "comment",
		"status": "current",
		"body": map[string]any{
			"storage": map[string]any{"value": "<p>hi</p>", "representation": "storage"},
		},
		"extensions": map[string]any{"location": "footer"},
	}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeJSON(t, w, listResponse([]commentJSON{c}, ""))
	}))
	defer server.Close()

	client := NewClient(server.URL, "test-token")
	result, err := client.GetComments("12345")
	require.NoError(t, err)
	require.Len(t, result, 1)
	assert.Equal(t, "footer", result[0].Location)
	assert.Empty(t, result[0].MarkerRef)
	assert.False(t, result[0].Resolved)
	assert.Empty(t, result[0].Resolution)
	assert.Empty(t, result[0].Author.DisplayName)
}

M cmd/mdcx/pull.go => cmd/mdcx/pull.go +66 -2
@@ 1,6 1,7 @@
package main

import (
	"encoding/json"
	"fmt"
	"os"



@@ 11,8 12,9 @@ import (
)

var (
	pullOutput string
	pullRaw    bool
	pullOutput       string
	pullRaw          bool
	pullWithComments bool
)

var pullCmd = &cobra.Command{


@@ 35,6 37,10 @@ Authentication via --token flag or CONFLUENCE_TOKEN environment variable.`,
			return fmt.Errorf("Confluence token required: use --token flag or set CONFLUENCE_TOKEN env var")
		}

		if pullWithComments && pullOutput == "" {
			return fmt.Errorf("--with-comments requires --output")
		}

		ref, err := api.ParsePageURL(args[0])
		if err != nil {
			return err


@@ 64,6 70,12 @@ Authentication via --token flag or CONFLUENCE_TOKEN environment variable.`,
				return err
			}
			fmt.Fprintf(os.Stderr, "Written to %s\n", pullOutput)

			if pullWithComments {
				if err := pullSaveComments(client, page, pullOutput); err != nil {
					return fmt.Errorf("fetching comments: %w", err)
				}
			}
			return nil
		}
		fmt.Print(result)


@@ 71,8 83,60 @@ Authentication via --token flag or CONFLUENCE_TOKEN environment variable.`,
	},
}

type commentSidecar struct {
	Page           sidecarPage    `json:"page"`
	InlineComments []*api.Comment `json:"inline_comments"`
	PageComments   []*api.Comment `json:"page_comments"`
}

type sidecarPage struct {
	ID      string `json:"id"`
	Title   string `json:"title"`
	Version int    `json:"version"`
}

func pullSaveComments(client *api.Client, page *api.ContentResponse, mdPath string) error {
	comments, err := client.GetComments(page.ID)
	if err != nil {
		return err
	}
	inline, footer := splitComments(comments)
	sidecar := commentSidecar{
		Page: sidecarPage{
			ID:      page.ID,
			Title:   page.Title,
			Version: page.Version.Number,
		},
		InlineComments: inline,
		PageComments:   footer,
	}
	data, err := json.MarshalIndent(sidecar, "", "  ")
	if err != nil {
		return err
	}
	sidecarPath := mdPath + ".comments.json"
	if err := os.WriteFile(sidecarPath, data, 0644); err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "Saved %d inline + %d page comments to %s\n",
		len(inline), len(footer), sidecarPath)
	return nil
}

func splitComments(all []*api.Comment) (inline, footer []*api.Comment) {
	for _, c := range all {
		if c.Location == "inline" {
			inline = append(inline, c)
		} else {
			footer = append(footer, c)
		}
	}
	return
}

func init() {
	pullCmd.Flags().StringVarP(&pullOutput, "output", "o", "", "Output file (default: stdout)")
	pullCmd.Flags().BoolVar(&pullRaw, "raw", false, "Output raw Confluence storage XML instead of Markdown")
	pullCmd.Flags().BoolVar(&pullWithComments, "with-comments", false, "Also save inline + page comments to <output>.comments.json (requires --output)")
	rootCmd.AddCommand(pullCmd)
}