package webhooks import ( "bytes" "context" "database/sql" "fmt" "io" "log" "net/http" "slices" "strings" "time" sq "github.com/Masterminds/squirrel" "github.com/google/uuid" "github.com/vaughan0/go-ini" "git.sr.ht/~sircmpwn/core-go/config" "git.sr.ht/~sircmpwn/core-go/crypto" "git.sr.ht/~sircmpwn/core-go/database" work "git.sr.ht/~sircmpwn/dowork" ) type LegacyQueue struct { Queue *work.Queue } type LegacySubscription struct { ID int Created time.Time URL string Events []string } // Creates a new worker for delivering legacy webhooks. The caller must start // the worker themselves. func NewLegacyQueue(conf ini.File) *LegacyQueue { queueSize := config.GetInt(conf, "webhooks", "queue-size", config.DefaultQueueSize) return &LegacyQueue{ work.NewQueue("webhooks_legacy", queueSize), } } // Schedules delivery of a legacy webhook to a set of subscribers. // // The select builder should not return any columns, i.e. the caller should use // squirrel.Select() with no parameters. The caller should prepare FROM and any // WHERE clauses which are necessary to refine the subscriber list (e.g. by // affected resource ID). The caller must alias the webhook table to "sub", e.g. // sq.Select().From("my_webhook_subscription sub"). // // Name shall be the prefix of the webhook tables, e.g. "user" for // "user_webhook_{delivery,subscription}". func (lq *LegacyQueue) Schedule(ctx context.Context, q sq.SelectBuilder, name, event string, payload []byte) { // The following tasks are done during this process: // // 1. Fetch subscription details from the database // 2. Prepare deliveries and create delivery records // 3. Deliver the webhooks // // The first two steps are done in this task, then N tasks are created for // step 3 where N = number of subscriptions. subs, err := fetchSubscriptions(ctx, q, event) if err != nil { panic(err) } if len(subs) == 0 { return } task := work.NewTask(func(ctx context.Context) error { tasks := make([]*work.Task, len(subs)) if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { var err error for i, sub := range subs { tasks[i], err = lq.queueStage2(ctx, tx, name, event, sub, payload) if err != nil { return err } } return nil }); err != nil { log.Printf("Failed to enqueue webhooks: %v", err) return err } for _, task := range tasks { lq.Queue.Enqueue(task) } log.Printf("Enqueued %s %s webhook delivery for %d subscriptions", name, event, len(subs)) return nil }) lq.Queue.Enqueue(task) } func fetchSubscriptions(ctx context.Context, q sq.SelectBuilder, event string) ([]*LegacySubscription, error) { var subs []*LegacySubscription if err := database.WithTx(ctx, &sql.TxOptions{ Isolation: 0, ReadOnly: true, }, func(tx *sql.Tx) error { var ( err error rows *sql.Rows ) if rows, err = q. Columns("sub.id", "sub.created", "sub.url", "sub.events"). Where(sq.Like{"sub.events": "%" + event + "%"}). PlaceholderFormat(sq.Dollar). RunWith(tx). QueryContext(ctx); err != nil { return err } defer rows.Close() var events string for rows.Next() { var sub LegacySubscription if err := rows.Scan(&sub.ID, &sub.Created, &sub.URL, &events); err != nil { panic(err) } // The LIKE clause gets us an approximate list of implicated // subscriptions, so we quickly decode the event list and // double check here to get the final list. sub.Events = strings.Split(events, ",") valid := slices.Contains(sub.Events, event) if valid { subs = append(subs, &sub) } } return nil }); err != nil { return nil, err } return subs, nil } // Inserts the delivery record and schedules the actual delivery task func (lq *LegacyQueue) queueStage2(ctx context.Context, tx *sql.Tx, name, event string, sub *LegacySubscription, payload []byte) (*work.Task, error) { deliveryUUID := uuid.New().String() headers := make(http.Header) headers.Set("Content-Type", "application/json") headers.Set("User-Agent", "SourceHut core-go (https://git.sr.ht/~sircmpwn/core-go) webhook queue") headers.Set("X-Webhook-Event", event) headers.Set("X-Webhook-Delivery", deliveryUUID) var sb strings.Builder headers.Write(&sb) var deliveryID int err := sq. Insert(name+"_webhook_delivery"). Columns("uuid", "created", "event", "url", "payload", "payload_headers", "response_status", "subscription_id"). Values(deliveryUUID, sq.Expr("NOW() at time zone 'utc'"), event, sub.URL, string(payload), sb.String(), -2, sub.ID). Suffix(`RETURNING (id)`). PlaceholderFormat(sq.Dollar). RunWith(tx). ScanContext(ctx, &deliveryID) if err != nil { return nil, err } return work.NewTask(func(ctx context.Context) error { return deliverPayload(ctx, name, sub.URL, headers, payload, deliveryID) }).Retries(5).After(func(ctx context.Context, task *work.Task) { if task.Result() == nil { log.Printf("%s: webhook delivery complete after %d attempts", deliveryUUID, task.Attempts()) } else { log.Printf("%s: webhook delivery failed after %d attempts: %v", deliveryUUID, task.Attempts(), task.Result()) } }), nil } // Performs a webhook delivery and updates the delivery record in the database func deliverPayload(ctx context.Context, name, url string, headers http.Header, payload []byte, deliveryID int) error { client := &http.Client{ Timeout: 30 * time.Second, } rctx, cancel := context.WithDeadline(ctx, time.Now().Add(30*time.Second)) req, err := http.NewRequestWithContext(rctx, http.MethodPost, url, bytes.NewReader(payload)) defer cancel() if err != nil { return fmt.Errorf("http.NewRequestWithContext: %v: %w", err, work.ErrDoNotReattempt) } req.Header = make(http.Header) for key, values := range headers { for _, value := range values { req.Header.Add(key, value) } } nonce, sig := crypto.SignWebhook(payload) req.Header.Add("X-Payload-Nonce", nonce) req.Header.Add("X-Payload-Signature", sig) var ours strings.Builder req.Header.Write(&ours) resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 65536) // No more than 64 KiB body, err := io.ReadAll(reader) if err != nil { return fmt.Errorf("Error reading response body: %v: %w", err, work.ErrDoNotReattempt) } if err = database.WithTx(ctx, nil, func(tx *sql.Tx) error { var theirs strings.Builder resp.Header.Write(&theirs) _, err := sq. Update(name+"_webhook_delivery"). Set("response", string(body)). Set("response_status", resp.StatusCode). Set("response_headers", theirs.String()). Set("payload_headers", ours.String()). Where("id = ?", deliveryID). PlaceholderFormat(sq.Dollar). RunWith(tx). ExecContext(ctx) return err }); err != nil { log.Printf("Warning: webhook delivered, but updating delivery record failed: %v", err) return nil } if resp.StatusCode == http.StatusBadGateway || resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusGatewayTimeout { // Retry return fmt.Errorf("Server returned status %d: %s", resp.StatusCode, resp.Status) } return nil }