package webhooks import ( "bytes" "context" "database/sql" "fmt" "io" "io/ioutil" "log" "net/http" "strings" "time" "git.sr.ht/~sircmpwn/dowork" sq "github.com/Masterminds/squirrel" "github.com/google/uuid" "git.sr.ht/~sircmpwn/core-go/crypto" "git.sr.ht/~sircmpwn/core-go/database" ) type LegacyQueue struct { Queue *work.Queue } type LegacySubscription struct { ID int Created time.Time URL string Events []string } // Creates a new worker for delivering legacy webhooks. The caller must start // the worker themselves. func NewLegacyQueue() *LegacyQueue { return &LegacyQueue{ work.NewQueue("webhooks_legacy"), } } // Schedules delivery of a legacy webhook to a set of subscribers. // // The select builder should not return any columns, i.e. the caller should use // squirrel.Select() with no parameters. The caller should prepare FROM and any // WHERE clauses which are necessary to refine the subscriber list (e.g. by // affected resource ID). // // Name shall be the prefix of the webhook tables, e.g. "user" for // "user_webhook_{delivery,subscription}". func (lq *LegacyQueue) Schedule(q sq.SelectBuilder, name, event string, payload []byte) { // The following tasks are done during this process: // // 1. Fetch subscription details from the database // 2. Prepare deliveries and create delivery records // 3. Deliver the webhooks // // The first two steps are done in this task, then N tasks are created for // step 3 where N = number of subscriptions. task := work.NewTask(func(ctx context.Context) error { subs, err := fetchSubscriptions(ctx, q, event) if err != nil { return err } if len(subs) == 0 { return nil } tasks := make([]*work.Task, len(subs)) if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { var err error for i, sub := range subs { tasks[i], err = lq.queueStage2(ctx, tx, name, event, sub, payload) if err != nil { return err } } return nil }); err != nil { return err } for _, task := range tasks { lq.Queue.Enqueue(task) } log.Printf("Enqueued %s %s webhook delivery for %d subscriptions", name, event, len(subs)) return nil }) lq.Queue.Enqueue(task) } func fetchSubscriptions(ctx context.Context, q sq.SelectBuilder, event string) ([]*LegacySubscription, error) { var subs []*LegacySubscription if err := database.WithTx(ctx, &sql.TxOptions{ Isolation: 0, ReadOnly: true, }, func(tx *sql.Tx) error { var ( err error rows *sql.Rows ) if rows, err = q. Columns("id", "created", "url", "events"). Where(sq.Like{"events": "%" + event + "%"}). RunWith(tx). QueryContext(ctx); err != nil { panic(err) } defer rows.Close() var events string for rows.Next() { var sub LegacySubscription if err := rows.Scan(&sub.ID, &sub.Created, &sub.URL, &events); err != nil { panic(err) } // The LIKE clause gets us an approximate list of implicated // subscriptions, so we quickly decode the event list and // double check here to get the final list. sub.Events = strings.Split(events, ",") var valid bool for _, e := range sub.Events { if e == event { valid = true break } } if valid { subs = append(subs, &sub) } } return nil }); err != nil { return nil, err } return subs, nil } // Inserts the delivery record and schedules the actual delivery task func (lq *LegacyQueue) queueStage2(ctx context.Context, tx *sql.Tx, name, event string, sub *LegacySubscription, payload []byte) (*work.Task, error) { deliveryUUID := uuid.New().String() headers := make(http.Header) headers.Set("Content-Type", "application/json") headers.Set("X-Webhook-Event", event) headers.Set("X-Webhook-Delivery", deliveryUUID) var sb strings.Builder headers.Write(&sb) var deliveryID int sq.Insert(name+"_webhook_delivery"). Columns("uuid", "created", "event", "url", "payload", "payload_headers", "response_status", "subscription_id"). Values(deliveryUUID, "NOW() at time zone 'utc'", event, sub.URL, string(payload), sb.String(), -2, sub.ID). Suffix(`RETURNING (id)`). RunWith(tx). ScanContext(ctx, &deliveryID) return work.NewTask(func(ctx context.Context) error { return deliverPayload(ctx, name, sub.URL, headers, payload, deliveryID) }).Retries(5).After(func(ctx context.Context, task *work.Task) { if task.Result() == nil { log.Printf("LEGACY WEBHOOK: %s: delivery complete after %d attempts", deliveryUUID, task.Attempts()) } else { log.Printf("LEGACY WEBHOOK: %s: delivery failed after %d attempts: %v", deliveryUUID, task.Attempts(), task.Result()) } }), nil } // Performs a webhook delivery and updates the delivery record in the database func deliverPayload(ctx context.Context, name, url string, headers http.Header, payload []byte, deliveryID int) error { client := &http.Client{ Timeout: 30 * time.Second, } rctx, cancel := context.WithDeadline(ctx, time.Now().Add(30*time.Second)) req, err := http.NewRequestWithContext(rctx, http.MethodPost, url, bytes.NewReader(payload)) defer cancel() if err != nil { return fmt.Errorf("http.NewRequestWithContext: %v: %e", err, work.ErrDoNotReattempt) } req.Header = headers for key, values := range headers { for _, value := range values { req.Header.Add(key, value) } } nonce, sig := crypto.SignWebhook(payload) req.Header.Add("X-Payload-Nonce", nonce) req.Header.Add("X-Payload-Signature", sig) resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() reader := io.LimitReader(resp.Body, 65536) // No more than 64 KiB body, err := ioutil.ReadAll(reader) if err != nil { return fmt.Errorf("Error reading response body: %v: %e", err, work.ErrDoNotReattempt) } if err = database.WithTx(ctx, nil, func(tx *sql.Tx) error { var sb strings.Builder resp.Header.Write(&sb) _, err := sq.Update(name+"_webhook_delivery"). Set("response", string(body)). Set("response_status", resp.StatusCode). Set("response_headers", sb.String()). RunWith(tx). ExecContext(ctx) return err }); err != nil { log.Printf("Warning: webhook delivered, but updating delivery record failed: %v", err) return nil } if resp.StatusCode == http.StatusBadGateway || resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusGatewayTimeout { // Retry return fmt.Errorf("Server returned status %d: %s", resp.StatusCode, resp.Status) } return nil }