From 2b781a27e6fd7eaa68ee4e692fe0da7348569daf Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sat, 18 Jul 2026 21:11:08 +0300 Subject: [PATCH] remoteapi: authz interceptors, remotesrv assembly, credentials service --- authn/backend.go | 13 + remoteapi/credsvc.go | 141 +++++++++ remoteapi/integration_test.go | 503 +++++++++++++++++++++++++++++++++ remoteapi/interceptors.go | 330 +++++++++++++++++++++ remoteapi/interceptors_test.go | 310 ++++++++++++++++++++ remoteapi/keystore.go | 60 ++++ remoteapi/server.go | 165 +++++++++++ 7 files changed, 1522 insertions(+) create mode 100644 remoteapi/credsvc.go create mode 100644 remoteapi/integration_test.go create mode 100644 remoteapi/interceptors.go create mode 100644 remoteapi/interceptors_test.go create mode 100644 remoteapi/keystore.go create mode 100644 remoteapi/server.go diff --git a/authn/backend.go b/authn/backend.go index 656a81a75cf0c942dd4b0db2374616bfc4fab135..e071c1590c96ae9ca00cefb23002624d0b7da62a 100644 --- a/authn/backend.go +++ b/authn/backend.go @@ -52,6 +52,19 @@ func (coreMetaBackend) IsRevoked(ctx context.Context, username string, hash [64] // meta.sr.ht implementation; tests reassign it (white-box) and restore it. var meta MetaBackend = coreMetaBackend{} +// SetMetaBackend swaps the package-level meta backend used by the resolvers and +// returns a function that restores the previous one. It is a wiring/test seam: +// integration tests living in OTHER packages (e.g. remoteapi) need to inject an +// in-memory MetaBackend so they can exercise the full auth stack without a live +// meta.sr.ht or the internal-network trust it requires. Production code never +// calls it, and it is not safe for concurrent use — a test installs a backend, +// runs, and restores it via the returned func (typically with t.Cleanup). +func SetMetaBackend(b MetaBackend) (restore func()) { + prev := meta + meta = b + return func() { meta = prev } +} + // equalUsername reports whether two SourceHut usernames refer to the same user, // ignoring a leading "~" (the canonical-name sigil) and ASCII case. func equalUsername(a, b string) bool { diff --git a/remoteapi/credsvc.go b/remoteapi/credsvc.go new file mode 100644 index 0000000000000000000000000000000000000000..d244ef526773747fbe97657f69eadbc1a825d6b3 --- /dev/null +++ b/remoteapi/credsvc.go @@ -0,0 +1,141 @@ +package remoteapi + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "strings" + + "git.sr.ht/~sircmpwn/core-go/config" + "git.sr.ht/~sircmpwn/core-go/database" + remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/sirupsen/logrus" + "github.com/vaughan0/go-ini" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "go.bigb.es/sourcehut-dolt/authn" +) + +// credService implements remotesapi.CredentialsServiceServer.WhoAmI. `dolt +// login` polls WhoAmI with the user's keypair Bearer JWT until the key is +// associated with a SourceHut account through the web UI; once association +// happens, the JWT verifies and WhoAmI returns the account identity, which the +// CLI prints as confirmation. +type credService struct { + remotesapi.UnimplementedCredentialsServiceServer + + conf ini.File + service string + expectedAud string + pool *sql.DB + keys authn.KeyStore + logger *logrus.Entry +} + +// WhoAmI verifies the request's Bearer keypair JWT (exactly as the remotesapi +// interceptors do) and returns the owning SourceHut user's identity. It is +// keypair-only: a missing or non-Bearer authorization header, or an invalid +// token, is Unauthenticated — the state the CLI polls through until the web UI +// associates the key. A transient backend failure is Unavailable. +func (c *credService) WhoAmI(ctx context.Context, _ *remotesapi.WhoAmIRequest) (*remotesapi.WhoAmIResponse, error) { + ctx = database.Context(config.Context(ctx, c.conf, c.service), c.pool) + + token, err := bearerToken(ctx) + if err != nil { + return nil, err + } + + ac, err := authn.ResolveDoltJWT(ctx, token, c.expectedAud, c.keys) + if err != nil { + if errors.Is(err, authn.ErrInvalidToken) { + return nil, status.Error(codes.Unauthenticated, "invalid or expired credentials") + } + c.logger.Errorf("credentials WhoAmI backend error: %v", err) + return nil, status.Error(codes.Unavailable, "authentication temporarily unavailable") + } + + // The meta mirror carries no separate display name, so we surface the + // username for DisplayName; EmailAddress comes from the mirrored profile. + return &remotesapi.WhoAmIResponse{ + Username: ac.Username, + DisplayName: ac.Username, + EmailAddress: ac.Email, + }, nil +} + +// bearerToken extracts the raw JWT from an incoming "authorization: Bearer " +// metadata header. A missing header or non-Bearer scheme is Unauthenticated. +func bearerToken(ctx context.Context) (string, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "", status.Error(codes.Unauthenticated, "missing credentials") + } + vals := md.Get("authorization") + if len(vals) == 0 || vals[0] == "" { + return "", status.Error(codes.Unauthenticated, "missing credentials") + } + scheme, value, found := strings.Cut(vals[0], " ") + if !found || !strings.EqualFold(scheme, "bearer") || value == "" { + return "", status.Error(codes.Unauthenticated, "expected a Bearer keypair token") + } + return value, nil +} + +// CredServer is the small standalone gRPC server hosting CredentialsService on +// its own port (nginx path-routes it separately from the chunk-store server). +type CredServer struct { + grpc *grpc.Server + addr string + logger *logrus.Entry +} + +// NewCredServer assembles the CredentialsService server. It shares the keystore +// derivation and audience normalization with the chunk-store server so both +// verify keypair JWTs identically. +func NewCredServer(cfg Config) (*CredServer, error) { + if cfg.DB == nil { + return nil, fmt.Errorf("remoteapi: NewCredServer requires a non-nil DB") + } + if cfg.CredsListenAddr == "" { + return nil, fmt.Errorf("remoteapi: NewCredServer requires a CredsListenAddr") + } + logger := cfg.Logger + if logger == nil { + logger = logrus.NewEntry(logrus.StandardLogger()) + } + + svc := &credService{ + conf: cfg.Conf, + service: serviceName, + expectedAud: normalizeAud(cfg.HttpHost), + pool: cfg.DB, + keys: newKeyStore(cfg.DB), + logger: logger, + } + + gsrv := grpc.NewServer() + remotesapi.RegisterCredentialsServiceServer(gsrv, svc) + + return &CredServer{grpc: gsrv, addr: cfg.CredsListenAddr, logger: logger}, nil +} + +// Serve binds the listener and serves until GracefulStop. It blocks. It returns +// an error if binding fails or the gRPC server exits with one. +func (s *CredServer) Serve() error { + lis, err := net.Listen("tcp", s.addr) + if err != nil { + return fmt.Errorf("remoteapi: bind credentials %q: %w", s.addr, err) + } + if err := s.grpc.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + return fmt.Errorf("remoteapi: credentials serve: %w", err) + } + return nil +} + +// GracefulStop stops the credentials server. +func (s *CredServer) GracefulStop() { s.grpc.GracefulStop() } diff --git a/remoteapi/integration_test.go b/remoteapi/integration_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f7504f078fbf3ba809b85a9a58ce1f2f2c28652e --- /dev/null +++ b/remoteapi/integration_test.go @@ -0,0 +1,503 @@ +//go:build integration + +// Integration test for the full remotesapi stack: real Postgres (Docker), the +// real dolt CLI (v2.1.10) driving clone/push over our auth interceptors, and the +// CredentialsService.WhoAmI RPC. It exercises every auth path end to end: +// +// (1) anonymous clone of a PUBLIC db succeeds; +// (2) anonymous push is denied; +// (3) Basic auth (--user + DOLT_REMOTE_PASSWORD, forged PAT) push to own repo; +// (4) PRIVATE anonymous clone fails "not found"; +// (5) keypair (Bearer EdDSA JWT) clone of a PRIVATE own db succeeds after the +// key is registered (dolt creds flow, no --user); +// (6) WhoAmI answers with the account identity for a keypair JWT; +// (7) an ACL RO grantee can clone but not push (private repo stays "found"). +// +// Run with: +// +// CGO_CPPFLAGS=-I/opt/homebrew/opt/icu4c@78/include \ +// CGO_LDFLAGS=-L/opt/homebrew/opt/icu4c@78/lib \ +// go test -tags integration -mod=readonly ./remoteapi/ -run TestRemoteAPIIntegration -v +// +// It skips cleanly when the dolt CLI or Docker (and DOLTSRHT_TEST_PG) are absent. +package remoteapi + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + "encoding/hex" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "git.sr.ht/~sircmpwn/core-go/auth" + "git.sr.ht/~sircmpwn/core-go/crypto" + remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/dolthub/dolt/go/libraries/doltcore/creds" + "github.com/fernet/fernet-go" + _ "github.com/lib/pq" + "github.com/sirupsen/logrus" + "github.com/vaughan0/go-ini" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "go.bigb.es/sourcehut-dolt/authn" + "go.bigb.es/sourcehut-dolt/core" + "go.bigb.es/sourcehut-dolt/db" + "go.bigb.es/sourcehut-dolt/storage" +) + +const doltBin = "/opt/homebrew/bin/dolt" + +// stubMeta is an in-memory MetaBackend: it mirrors a fixed user table and never +// reports a token revoked. It lets the full auth stack run without a live +// meta.sr.ht or the internal-network trust it needs. +type stubMeta struct { + users map[string]auth.AuthContext +} + +func (s *stubMeta) LookupUser(_ context.Context, username string, out *auth.AuthContext) error { + u, ok := s.users[strings.ToLower(strings.TrimPrefix(username, "~"))] + if !ok { + return fmt.Errorf("stubMeta: unknown user %q", username) + } + *out = u + return nil +} + +func (s *stubMeta) IsRevoked(_ context.Context, _ string, _ [64]byte, _ string) (bool, error) { + return false, nil +} + +func TestRemoteAPIIntegration(t *testing.T) { + if _, err := os.Stat(doltBin); err != nil { + t.Skipf("dolt CLI not found at %s (%v); skipping integration test", doltBin, err) + } + + dsn, stopPG := startPostgres(t) + defer stopPG() + + pool, err := sql.Open("postgres", dsn) + if err != nil { + t.Fatalf("open pool: %v", err) + } + defer pool.Close() + waitPG(t, pool) + applySchema(t, pool) + + // Shared crypto keyset (fernet network key + ed25519 webhooks seed) so forged + // PATs validate; the same ini is threaded into the server as its config. + conf := synthConf(t) + crypto.InitCrypto(conf) + + // Install the in-memory meta backend for the duration of the test. + restore := authn.SetMetaBackend(&stubMeta{users: map[string]auth.AuthContext{ + "alice": {UserID: 1, Username: "alice", Email: "alice@example.test", UserType: auth.USER_TYPE_USER}, + "bob": {UserID: 2, Username: "bob", Email: "bob@example.test", UserType: auth.USER_TYPE_USER}, + }}) + defer restore() + + ctx := context.Background() + store := db.NewStore(pool) + + insertUser(t, pool, 1, "alice", core.UserTypeUser) + insertUser(t, pool, 2, "bob", core.UserTypeUser) + + reposRoot := filepath.Join(t.TempDir(), "repos") + if err := os.MkdirAll(reposRoot, 0o755); err != nil { + t.Fatal(err) + } + + pubRepo := mkRepo(t, ctx, store, reposRoot, 1, "alice", "pubdb", core.VisibilityPublic) + privRepo := mkRepo(t, ctx, store, reposRoot, 1, "alice", "privdb", core.VisibilityPrivate) + // A separate private repo that bob will hold an RO ACL on (scenario 7). + roRepo := mkRepo(t, ctx, store, reposRoot, 1, "alice", "rodb", core.VisibilityPrivate) + if err := store.UpsertACL(ctx, roRepo.ID, 2, core.AccessRO); err != nil { + t.Fatalf("grant bob RO on rodb: %v", err) + } + _ = pubRepo + _ = privRepo + + // Start both servers on ephemeral ports. HttpHost is the remotesapi listen + // addr so chunk URLs carry the right host:port AND the derived JWT audience + // (the bare host) matches what the CLI sends. + addr := freeAddr(t) + credAddr := freeAddr(t) + logger := logrus.NewEntry(logrus.New()) + logger.Logger.SetLevel(logrus.ErrorLevel) + + cfg := Config{ + Conf: conf, + DB: pool, + ReposRoot: reposRoot, + ListenAddr: addr, + CredsListenAddr: credAddr, + HttpHost: addr, + Logger: logger, + } + srv, err := New(cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + go func() { + if err := srv.Serve(); err != nil { + logger.Errorf("remotesapi serve: %v", err) + } + }() + defer srv.GracefulStop() + + csrv, err := NewCredServer(cfg) + if err != nil { + t.Fatalf("NewCredServer: %v", err) + } + go func() { + if err := csrv.Serve(); err != nil { + logger.Errorf("credentials serve: %v", err) + } + }() + defer csrv.GracefulStop() + + waitTCP(t, addr) + waitTCP(t, credAddr) + + // Scratch HOME so the developer's real dolt config is untouched. + home := filepath.Join(t.TempDir(), "home") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatal(err) + } + baseEnv := append(os.Environ(), "HOME="+home) + runDolt(t, baseEnv, home, "config", "--global", "--add", "user.email", "alice@example.test") + runDolt(t, baseEnv, home, "config", "--global", "--add", "user.name", "alice") + + pubURL := fmt.Sprintf("http://%s/~alice/pubdb", addr) + privURL := fmt.Sprintf("http://%s/~alice/privdb", addr) + roURL := fmt.Sprintf("http://%s/~alice/rodb", addr) + + pat := forgePAT("alice", "dolt.sr.ht/repos:RW", time.Now().Add(time.Hour)) + + // (1) Anonymous clone of PUBLIC succeeds. + t.Run("anonymous_public_clone", func(t *testing.T) { + dir := t.TempDir() + runDolt(t, baseEnv, dir, "clone", pubURL) + if _, err := os.Stat(filepath.Join(dir, "pubdb", ".dolt")); err != nil { + t.Fatalf("clone did not produce a working dir: %v", err) + } + }) + + // (2) Anonymous push is denied. + t.Run("anonymous_push_denied", func(t *testing.T) { + dir := t.TempDir() + runDolt(t, baseEnv, dir, "clone", pubURL) + work := filepath.Join(dir, "pubdb") + runDolt(t, baseEnv, work, "sql", "-q", "create table t(i int primary key); insert into t values (1);") + runDolt(t, baseEnv, work, "commit", "-Am", "anon change") + out, err := tryDolt(baseEnv, work, "push", "origin", "main") + if err == nil { + t.Fatalf("anonymous push unexpectedly succeeded:\n%s", out) + } + t.Logf("anonymous push correctly denied:\n%s", out) + }) + + // (3) Basic auth (forged PAT) push to own repo succeeds. + t.Run("basic_auth_push", func(t *testing.T) { + dir := t.TempDir() + env := append(append([]string{}, baseEnv...), "DOLT_REMOTE_PASSWORD="+pat) + runDolt(t, env, dir, "clone", "--user", "alice", pubURL) + work := filepath.Join(dir, "pubdb") + runDolt(t, env, work, "sql", "-q", "create table basic(i int primary key); insert into basic values (7);") + runDolt(t, env, work, "commit", "-Am", "basic push") + runDolt(t, env, work, "push", "--user", "alice", "origin", "main") + + // Re-clone anonymously and verify the pushed row landed. + dir2 := t.TempDir() + runDolt(t, baseEnv, dir2, "clone", pubURL) + out := runDolt(t, baseEnv, filepath.Join(dir2, "pubdb"), "sql", "-q", "select i from basic", "-r", "csv") + if !strings.Contains(out, "7") { + t.Fatalf("pushed row not present after re-clone; got:\n%s", out) + } + }) + + // (4) PRIVATE anonymous clone fails "not found". + t.Run("private_anonymous_notfound", func(t *testing.T) { + dir := t.TempDir() + out, err := tryDolt(baseEnv, dir, "clone", privURL) + if err == nil { + t.Fatalf("anonymous clone of PRIVATE unexpectedly succeeded:\n%s", out) + } + if !strings.Contains(strings.ToLower(out), "not found") { + t.Fatalf("expected a not-found style failure, got:\n%s", out) + } + t.Logf("private anonymous clone correctly not found:\n%s", out) + }) + + // (5) Keypair (Bearer JWT) clone of PRIVATE own db succeeds. Mint a keypair, + // register it for alice, then clone with no --user so the CLI uses the JWK. + t.Run("keypair_private_clone", func(t *testing.T) { + dc, err := creds.GenerateCredentials() + if err != nil { + t.Fatalf("generate creds: %v", err) + } + kid := dc.KeyIDBase32Str() + if _, err := store.InsertKey(ctx, 1, kid, dc.PubKey, "it-key"); err != nil { + t.Fatalf("register dolt key: %v", err) + } + writeJWK(t, home, dc) + runDolt(t, baseEnv, home, "config", "--global", "--add", "user.creds", kid) + + dir := t.TempDir() + runDolt(t, baseEnv, dir, "clone", privURL) + if _, err := os.Stat(filepath.Join(dir, "privdb", ".dolt")); err != nil { + t.Fatalf("keypair clone of PRIVATE did not produce a working dir: %v", err) + } + t.Log("keypair clone of PRIVATE own db succeeded") + + // Unregister user.creds so later CLI runs are anonymous again. + runDolt(t, baseEnv, home, "config", "--global", "--unset", "user.creds") + }) + + // (6) WhoAmI answers with alice's identity for a keypair JWT. + t.Run("whoami", func(t *testing.T) { + dc, err := creds.GenerateCredentials() + if err != nil { + t.Fatalf("generate creds: %v", err) + } + kid := dc.KeyIDBase32Str() + if _, err := store.InsertKey(ctx, 1, kid, dc.PubKey, "whoami-key"); err != nil { + t.Fatalf("register dolt key: %v", err) + } + + conn, err := grpc.NewClient(credAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("dial credentials: %v", err) + } + defer conn.Close() + client := remotesapi.NewCredentialsServiceClient(conn) + + // aud = the bare host the CLI derives from the endpoint (host:port). + host, _, _ := net.SplitHostPort(credAddr) + rpc := dc.RPCCreds(host) + reqMD, err := rpc.GetRequestMetadata(ctx) + if err != nil { + t.Fatalf("mint JWT: %v", err) + } + outCtx := metadata.AppendToOutgoingContext(ctx, "authorization", reqMD["authorization"]) + resp, err := client.WhoAmI(outCtx, &remotesapi.WhoAmIRequest{}) + if err != nil { + t.Fatalf("WhoAmI: %v", err) + } + if resp.Username != "alice" || resp.EmailAddress != "alice@example.test" { + t.Fatalf("WhoAmI returned %+v, want alice/alice@example.test", resp) + } + + // A request with no token is Unauthenticated. + if _, err := client.WhoAmI(ctx, &remotesapi.WhoAmIRequest{}); status.Code(err) != codes.Unauthenticated { + t.Fatalf("unauthenticated WhoAmI: want Unauthenticated, got %v", err) + } + }) + + // (7) ACL RO grantee (bob) can clone the private rodb but cannot push. + t.Run("acl_ro_clone_not_push", func(t *testing.T) { + bobPAT := forgePAT("bob", "dolt.sr.ht/repos:RW", time.Now().Add(time.Hour)) + env := append(append([]string{}, baseEnv...), "DOLT_REMOTE_PASSWORD="+bobPAT) + + dir := t.TempDir() + runDolt(t, env, dir, "clone", "--user", "bob", roURL) + work := filepath.Join(dir, "rodb") + runDolt(t, env, work, "sql", "-q", "create table t(i int primary key); insert into t values (1);") + runDolt(t, env, work, "commit", "-Am", "bob change") + out, err := tryDolt(env, work, "push", "--user", "bob", "origin", "main") + if err == nil { + t.Fatalf("RO grantee push unexpectedly succeeded:\n%s", out) + } + t.Logf("RO grantee push correctly denied:\n%s", out) + }) +} + +// --- helpers --- + +func synthConf(t *testing.T) ini.File { + t.Helper() + var fk fernet.Key + if err := fk.Generate(); err != nil { + t.Fatal(err) + } + seed := make([]byte, 32) + if _, err := rand.Read(seed); err != nil { + t.Fatal(err) + } + return ini.File{ + "sr.ht": ini.Section{"network-key": fk.Encode()}, + "webhooks": ini.Section{"private-key": base64.StdEncoding.EncodeToString(seed)}, + } +} + +func forgePAT(username, grants string, expires time.Time) string { + bt := auth.BearerToken{ + Version: auth.TokenVersion, + Expires: auth.ToTimestamp(expires), + Grants: grants, + Username: username, + } + return bt.Encode() +} + +func writeJWK(t *testing.T, home string, dc creds.DoltCreds) { + t.Helper() + dir := filepath.Join(home, ".dolt", "creds") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + data, err := creds.JWKCredSerialize(dc) + if err != nil { + t.Fatalf("serialize JWK: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, dc.KeyIDBase32Str()+".jwk"), data, 0o600); err != nil { + t.Fatalf("write JWK: %v", err) + } +} + +func mkRepo(t *testing.T, ctx context.Context, store *db.Store, reposRoot string, ownerID int, owner, name string, vis core.Visibility) *core.Repo { + t.Helper() + abs := storage.RepoDiskPath(reposRoot, owner, name) + if err := storage.InitStore(ctx, abs, owner, owner+"@example.test"); err != nil { + t.Fatalf("init store %s: %v", name, err) + } + repo, err := store.CreateRepo(ctx, &core.Repo{ + Name: name, OwnerID: ownerID, OwnerName: owner, Path: abs, Visibility: vis, + }) + if err != nil { + t.Fatalf("create repo %s: %v", name, err) + } + return repo +} + +func insertUser(t *testing.T, pool *sql.DB, id int, username string, ut core.UserType) { + t.Helper() + now := time.Now().UTC() + _, err := pool.Exec(` +INSERT INTO "user" (id, username, created, updated, email, user_type) +VALUES ($1, $2, $3, $3, $4, $5)`, id, username, now, username+"@example.test", string(ut)) + if err != nil { + t.Fatalf("insert user %s: %v", username, err) + } +} + +// startPostgres returns a DSN for a Postgres to test against. It prefers an +// existing DOLTSRHT_TEST_PG DSN; otherwise it spins postgres:16-alpine via +// Docker. The test skips cleanly if neither is available. +func startPostgres(t *testing.T) (dsn string, cleanup func()) { + t.Helper() + if env := os.Getenv("DOLTSRHT_TEST_PG"); env != "" { + return env, func() {} + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("neither DOLTSRHT_TEST_PG nor docker available; skipping integration test") + } + port := freePort(t) + name := "doltsrht-it-" + randToken() + out, err := exec.Command("docker", "run", "-d", "--name", name, + "-e", "POSTGRES_PASSWORD=postgres", "-e", "POSTGRES_DB=doltsrht", + "-p", fmt.Sprintf("127.0.0.1:%d:5432", port), "postgres:16-alpine").CombinedOutput() + if err != nil { + t.Skipf("docker run postgres failed (%v):\n%s", err, out) + } + cleanup = func() { + if o, err := exec.Command("docker", "rm", "-f", name).CombinedOutput(); err != nil { + t.Logf("docker rm -f %s: %v\n%s", name, err, o) + } + } + return fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/doltsrht?sslmode=disable", port), cleanup +} + +func waitPG(t *testing.T, pool *sql.DB) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + if err := pool.Ping(); err == nil { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatal("postgres never became ready") +} + +func applySchema(t *testing.T, pool *sql.DB) { + t.Helper() + ddl, err := os.ReadFile("../schema.sql") + if err != nil { + t.Fatalf("read schema.sql: %v", err) + } + if _, err := pool.Exec(string(ddl)); err != nil { + t.Fatalf("apply schema.sql: %v", err) + } +} + +func freePort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("probe listen: %v", err) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +func freeAddr(t *testing.T) string { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("probe listen: %v", err) + } + addr := l.Addr().String() + l.Close() + return addr +} + +func waitTCP(t *testing.T, addr string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) + if err == nil { + conn.Close() + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("server at %s never became reachable", addr) +} + +func runDolt(t *testing.T, env []string, dir string, args ...string) string { + t.Helper() + out, err := tryDolt(env, dir, args...) + if err != nil { + t.Fatalf("dolt %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return out +} + +func tryDolt(env []string, dir string, args ...string) (string, error) { + cmd := exec.Command(doltBin, args...) + cmd.Dir = dir + cmd.Env = env + out, err := cmd.CombinedOutput() + return string(out), err +} + +func randToken() string { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return hex.EncodeToString(b) +} diff --git a/remoteapi/interceptors.go b/remoteapi/interceptors.go new file mode 100644 index 0000000000000000000000000000000000000000..3e255d2fa64868e73ef65e33d24f365c56920449 --- /dev/null +++ b/remoteapi/interceptors.go @@ -0,0 +1,330 @@ +package remoteapi + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "git.sr.ht/~sircmpwn/core-go/auth" + "git.sr.ht/~sircmpwn/core-go/config" + "git.sr.ht/~sircmpwn/core-go/database" + remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/sirupsen/logrus" + "github.com/vaughan0/go-ini" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "go.bigb.es/sourcehut-dolt/authn" + "go.bigb.es/sourcehut-dolt/core" + "go.bigb.es/sourcehut-dolt/db" +) + +// Method-classification sets, copied verbatim from upstream remotesrv's +// interceptors.go (SUPER_USER_RPC_METHODS / CLONE_ADMIN_RPC_METHODS). Upstream +// drops the authenticated context and never sees the repo path, so it can only +// make a binary superuser/clone-admin decision; we keep its exact method lists +// but layer our own per-repo ACL check on top (see authorize). +// +// - writeMethods are pushes: they require OpPush / AccessRW. +// - readMethods are clone/pull/fetch reads: OpCloneRead / AccessRO. +// - anything else is an unknown method and is denied (PermissionDenied), +// matching upstream's "unknown rpc method" hard failure. +var ( + writeMethods = map[string]bool{ + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/AddTableFiles": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/Commit": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/GetUploadLocations": true, + } + readMethods = map[string]bool{ + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/GetDownloadLocations": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/GetRepoMetadata": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/HasChunks": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/ListTableFiles": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/RefreshTableFileUrl": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/Root": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/StreamDownloadLocations": true, + "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/StreamChunkLocations": true, + } + + // rootMethod is the only read RPC the dolt client may send with no repo path + // (a bare "what is the current root" ping used during dial/handshake). It is + // treated as an unauthenticated-OK ping: it still authenticates the caller + // (so a bad token is rejected) but skips the per-repo ACL check when no path + // is present. Every real read carries a repo path and is checked normally. + rootMethod = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/Root" +) + +// repoRequest is the subset of every ChunkStoreService request message that +// carries the target repository, mirrored from upstream remotesrv's private +// repoRequest interface. All request types implement it. +type repoRequest interface { + GetRepoId() *remotesapi.RepoId + GetRepoPath() string +} + +// repoStore is the narrow slice of db.Store the interceptor needs. Declaring it +// locally (rather than depending on *db.Store directly) keeps the authorization +// logic unit-testable with an in-memory stub — no Postgres required. +type repoStore interface { + GetRepoByOwnerAndName(ctx context.Context, ownerUsername, name string) (*core.Repo, error) + EffectiveAccess(ctx context.Context, userID, repoID int) (*core.AccessMode, error) +} + +// interceptor holds the collaborators the per-RPC auth/authz decision needs. It +// is installed on the remotesrv gRPC server via Options(). +type interceptor struct { + conf ini.File + service string + expectedAud string + keys authn.KeyStore + logger *logrus.Entry + + // pool is the shared database pool, threaded into the request context via + // database.Context so db.FromContext-style lookups and the token resolvers + // can reach it. + pool *sql.DB + + // stores returns a repoStore for a request. Production binds a fresh + // db.Store to the shared pool; tests inject a stub. + stores func() repoStore +} + +// newInterceptor builds the interceptor over the shared pool. It binds a fresh +// db.Store per request (the pool owns connection lifetime, ctx bounds each +// query). pool and keys must be non-nil. +func newInterceptor(conf ini.File, service, expectedAud string, pool *sql.DB, keys authn.KeyStore, logger *logrus.Entry) *interceptor { + if pool == nil { + panic("remoteapi: newInterceptor requires a non-nil *sql.DB") + } + if logger == nil { + logger = logrus.NewEntry(logrus.StandardLogger()) + } + i := &interceptor{ + conf: conf, + service: service, + expectedAud: expectedAud, + keys: keys, + logger: logger, + pool: pool, + } + i.stores = func() repoStore { return db.NewStore(pool) } + return i +} + +// dbHandle returns the shared pool threaded into request contexts. +func (i *interceptor) dbHandle() *sql.DB { return i.pool } + +// Options returns the gRPC server options that install both interceptors, +// matching upstream ServerInterceptor.Options. +func (i *interceptor) Options() []grpc.ServerOption { + return []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(i.unary()), + grpc.ChainStreamInterceptor(i.stream()), + } +} + +// classify maps a full gRPC method to its access op/mode. ok is false for an +// unknown method, which the caller denies. +func classify(fullMethod string) (op core.Op, mode core.AccessMode, ok bool) { + if writeMethods[fullMethod] { + return core.OpPush, core.AccessRW, true + } + if readMethods[fullMethod] { + return core.OpCloneRead, core.AccessRO, true + } + return 0, "", false +} + +// withServiceCtx augments the live per-RPC context (which carries the gRPC +// deadline, cancellation and incoming metadata) with the config and database +// values the authn resolvers and db.Store read from context. We augment the +// handler's context rather than starting from a stored base context so request +// deadlines and the incoming "authorization" metadata are preserved. +func (i *interceptor) withServiceCtx(ctx context.Context) context.Context { + ctx = config.Context(ctx, i.conf, i.service) + ctx = database.Context(ctx, i.dbHandle()) + return ctx +} + +// authenticate resolves the caller from the incoming "authorization" metadata. +// It returns the caller (nil for an anonymous request) or a gRPC status error: +// Unauthenticated for a bad/forged/revoked credential, Unavailable for a +// transient backend failure (meta.sr.ht or the database unreachable). +func (i *interceptor) authenticate(ctx context.Context) (*auth.AuthContext, error) { + header := "" + if md, ok := metadata.FromIncomingContext(ctx); ok { + if vals := md.Get("authorization"); len(vals) > 0 { + header = vals[0] + } + } + ac, err := authn.ResolveGRPCAuth(ctx, header, i.expectedAud, i.keys) + if err != nil { + if errors.Is(err, authn.ErrInvalidToken) { + i.logger.Warnf("remotesapi authentication rejected: %v", err) + return nil, status.Error(codes.Unauthenticated, "invalid or expired credentials") + } + // Transient: meta.sr.ht or the database is unreachable. Never surface as a + // hard credential rejection — the client should retry. + i.logger.Errorf("remotesapi authentication backend error: %v", err) + return nil, status.Error(codes.Unavailable, "authentication temporarily unavailable") + } + return ac, nil +} + +// authorize applies the grant gate and the per-repo ACL to an authenticated +// caller and returns a context carrying the resolved caller for the handler, or +// a gRPC status error. It is called once per unary request and once per stream +// message. The order mirrors the plan: +// +// 1. classify the method (unknown ⇒ PermissionDenied); +// 2. OAuth grant gate (TokenGrantsAllow) — PAT callers must carry +// dolt.sr.ht/repos:RO for reads / :RW for pushes; anonymous, cookie and +// dolt-key callers pass trivially; +// 3. extract the repo path (Root with no path ⇒ unauthenticated-OK ping); +// 4. load the repository row and the caller's effective ACL; +// 5. core.Allowed — on denial, NotFound for a PRIVATE repo the caller cannot +// even see (no existence leak), PermissionDenied for a visible repo. +func (i *interceptor) authorize(ctx context.Context, ac *auth.AuthContext, fullMethod string, req any) (context.Context, error) { + op, mode, ok := classify(fullMethod) + if !ok { + return nil, status.Errorf(codes.PermissionDenied, "unknown rpc method: %s", fullMethod) + } + + // Grant gate is a global property of the token (independent of any repo), so + // running it first leaks nothing about repository existence. + if !authn.TokenGrantsAllow(ac, mode) { + return nil, status.Errorf(codes.PermissionDenied, + "token grants do not permit %s on %s repositories", mode, i.service) + } + + rr, isRepoReq := req.(repoRequest) + repoPath := "" + if isRepoReq { + repoPath = repoPathOf(rr) + } + if repoPath == "" { + if fullMethod == rootMethod { + // Handshake ping with no repo: authenticated but not repo-scoped. + return authn.WithCaller(ctx, ac), nil + } + return nil, status.Error(codes.InvalidArgument, "request is missing a repository path") + } + + owner, name, err := core.ParseRepoPath(repoPath) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid repository path %q: %v", repoPath, err) + } + + store := i.stores() + repo, err := store.GetRepoByOwnerAndName(ctx, owner, name) + if err != nil { + if errors.Is(err, db.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "repository %s/%s not found", owner, name) + } + i.logger.Errorf("remotesapi repo lookup %s/%s: %v", owner, name, err) + return nil, status.Error(codes.Unavailable, "repository lookup temporarily unavailable") + } + + caller := authn.AsCoreCaller(ac) + var aclMode *core.AccessMode + if caller != nil { + aclMode, err = store.EffectiveAccess(ctx, caller.UserID, repo.ID) + if err != nil { + i.logger.Errorf("remotesapi effective-access user=%d repo=%d: %v", caller.UserID, repo.ID, err) + return nil, status.Error(codes.Unavailable, "authorization temporarily unavailable") + } + } + + if !core.Allowed(caller, repo, aclMode, op) { + if core.NotFoundForPrivate(caller, repo, aclMode) { + return nil, status.Errorf(codes.NotFound, "repository %s/%s not found", owner, name) + } + return nil, status.Errorf(codes.PermissionDenied, "%s denied on %s/%s", op, owner, name) + } + + return authn.WithCaller(ctx, ac), nil +} + +// repoPathOf extracts the target repo path from a request without panicking on +// an absent path (upstream's getRepoPath panics on empty path + nil repo id). +// It returns "" when neither is present so the caller can treat Root specially. +func repoPathOf(req repoRequest) string { + if p := req.GetRepoPath(); p != "" { + return p + } + if id := req.GetRepoId(); id != nil { + return fmt.Sprintf("%s/%s", id.Org, id.RepoName) + } + return "" +} + +// unary is the unary server interceptor: authenticate once, authorize the +// request, then invoke the handler with the caller-carrying context. +func (i *interceptor) unary() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + ctx = i.withServiceCtx(ctx) + ac, err := i.authenticate(ctx) + if err != nil { + return nil, err + } + authedCtx, err := i.authorize(ctx, ac, info.FullMethod, req) + if err != nil { + return nil, err + } + return handler(authedCtx, req) + } +} + +// stream is the stream server interceptor. Authentication happens once at +// stream start; the repo-path authorization is re-checked for every message the +// client sends (the streaming reads carry a repo path per message), matching the +// plan's "override Context() and check each RecvMsg" design. +func (i *interceptor) stream() grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + ctx := i.withServiceCtx(ss.Context()) + ac, err := i.authenticate(ctx) + if err != nil { + return err + } + // A stream to an unknown method is denied before the handler runs. + if _, _, ok := classify(info.FullMethod); !ok { + return status.Errorf(codes.PermissionDenied, "unknown rpc method: %s", info.FullMethod) + } + wrapped := &authStream{ + ServerStream: ss, + ctx: authn.WithCaller(ctx, ac), + i: i, + ac: ac, + fullMethod: info.FullMethod, + } + return handler(srv, wrapped) + } +} + +// authStream wraps a grpc.ServerStream so the handler sees the caller-carrying +// context and every received message is re-authorized against its repo path. +type authStream struct { + grpc.ServerStream + ctx context.Context + i *interceptor + ac *auth.AuthContext + fullMethod string +} + +// Context returns the authenticated, caller-carrying context. +func (s *authStream) Context() context.Context { return s.ctx } + +// RecvMsg receives the next message and re-authorizes it against its repo path +// before handing it to the handler. An authorization failure aborts the stream. +func (s *authStream) RecvMsg(m any) error { + if err := s.ServerStream.RecvMsg(m); err != nil { + return err + } + if _, err := s.i.authorize(s.ctx, s.ac, s.fullMethod, m); err != nil { + return err + } + return nil +} diff --git a/remoteapi/interceptors_test.go b/remoteapi/interceptors_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5e662cec51ec02de1b2e9c345920c7f18aba3ec5 --- /dev/null +++ b/remoteapi/interceptors_test.go @@ -0,0 +1,310 @@ +package remoteapi + +import ( + "context" + "testing" + + "git.sr.ht/~sircmpwn/core-go/auth" + "git.sr.ht/~sircmpwn/core-go/config" + remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/sirupsen/logrus" + "github.com/vaughan0/go-ini" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.bigb.es/sourcehut-dolt/authn" + "go.bigb.es/sourcehut-dolt/core" + "go.bigb.es/sourcehut-dolt/db" +) + +const ( + mRoot = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/Root" + mGetMeta = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/GetRepoMetadata" + mCommit = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/Commit" + mUpload = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/GetUploadLocations" + mDownload = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/GetDownloadLocations" + mUnknown = "/dolt.services.remotesapi.v1alpha1.ChunkStoreService/Frobnicate" +) + +// fakeReq implements repoRequest with a fixed path/id so tests need not depend +// on which concrete proto message carries the repo for a given method. +type fakeReq struct { + path string + id *remotesapi.RepoId +} + +func (f fakeReq) GetRepoPath() string { return f.path } +func (f fakeReq) GetRepoId() *remotesapi.RepoId { return f.id } + +// stubStore is an in-memory repoStore: no Postgres. +type stubStore struct { + repo *core.Repo + repoErr error + acl *core.AccessMode + aclErr error +} + +func (s stubStore) GetRepoByOwnerAndName(_ context.Context, _, _ string) (*core.Repo, error) { + if s.repoErr != nil { + return nil, s.repoErr + } + return s.repo, nil +} + +func (s stubStore) EffectiveAccess(_ context.Context, _, _ int) (*core.AccessMode, error) { + return s.acl, s.aclErr +} + +func testInterceptor(store repoStore) *interceptor { + return &interceptor{ + conf: ini.File{}, + service: "dolt.sr.ht", + expectedAud: "dolt.srht.bigb.es", + logger: logrus.NewEntry(logrus.New()), + stores: func() repoStore { return store }, + } +} + +func repo(id, ownerID int, vis core.Visibility) *core.Repo { + return &core.Repo{ID: id, Name: "db", OwnerID: ownerID, OwnerName: "alice", Visibility: vis} +} + +func acl(m core.AccessMode) *core.AccessMode { return &m } + +// patCaller builds an authenticated PAT caller with the given grant string +// (empty ⇒ universal token). config context is needed by DecodeGrants. +func patCaller(t *testing.T, userID int, grants string) *auth.AuthContext { + t.Helper() + ctx := config.Context(context.Background(), ini.File{}, "dolt.sr.ht") + g, err := auth.DecodeGrants(ctx, grants) + if err != nil { + t.Fatalf("DecodeGrants(%q): %v", grants, err) + } + return &auth.AuthContext{ + UserID: userID, + Username: "alice", + UserType: auth.USER_TYPE_USER, + AuthMethod: auth.AUTH_OAUTH2, + BearerToken: &auth.BearerToken{}, + Grants: g, + } +} + +// cookieCaller builds an authenticated non-token caller (cookie/keypair): no +// BearerToken, so the grant gate passes trivially. +func cookieCaller(userID int) *auth.AuthContext { + return &auth.AuthContext{ + UserID: userID, + Username: "alice", + UserType: auth.USER_TYPE_USER, + AuthMethod: auth.AUTH_COOKIE, + } +} + +func wantCode(t *testing.T, err error, code codes.Code) { + t.Helper() + if status.Code(err) != code { + t.Fatalf("want gRPC code %s, got %s (err=%v)", code, status.Code(err), err) + } +} + +func TestClassify(t *testing.T) { + cases := []struct { + method string + op core.Op + mode core.AccessMode + ok bool + }{ + {mCommit, core.OpPush, core.AccessRW, true}, + {mUpload, core.OpPush, core.AccessRW, true}, + {mRoot, core.OpCloneRead, core.AccessRO, true}, + {mGetMeta, core.OpCloneRead, core.AccessRO, true}, + {mDownload, core.OpCloneRead, core.AccessRO, true}, + {mUnknown, 0, "", false}, + } + for _, c := range cases { + op, mode, ok := classify(c.method) + if ok != c.ok || (ok && (op != c.op || mode != c.mode)) { + t.Errorf("classify(%s) = (%v,%v,%v), want (%v,%v,%v)", + c.method, op, mode, ok, c.op, c.mode, c.ok) + } + } +} + +func TestAuthorize(t *testing.T) { + ctx := context.Background() + req := fakeReq{path: "~alice/db"} + + t.Run("anonymous public read allowed", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 100, core.VisibilityPublic)}) + if _, err := i.authorize(ctx, nil, mGetMeta, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("anonymous public push denied (visible)", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 100, core.VisibilityPublic)}) + _, err := i.authorize(ctx, nil, mCommit, req) + wantCode(t, err, codes.PermissionDenied) + }) + + t.Run("anonymous private read is not found", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 100, core.VisibilityPrivate)}) + _, err := i.authorize(ctx, nil, mGetMeta, req) + wantCode(t, err, codes.NotFound) + }) + + t.Run("owner push allowed", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPrivate)}) + if _, err := i.authorize(ctx, cookieCaller(42), mCommit, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("non-owner private read no acl is not found", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPrivate)}) + _, err := i.authorize(ctx, cookieCaller(7), mGetMeta, req) + wantCode(t, err, codes.NotFound) + }) + + t.Run("acl RO reads private", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPrivate), acl: acl(core.AccessRO)}) + if _, err := i.authorize(ctx, cookieCaller(7), mGetMeta, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("acl RO cannot push private (visible ⇒ PermissionDenied)", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPrivate), acl: acl(core.AccessRO)}) + _, err := i.authorize(ctx, cookieCaller(7), mCommit, req) + wantCode(t, err, codes.PermissionDenied) + }) + + t.Run("acl RW pushes private", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPrivate), acl: acl(core.AccessRW)}) + if _, err := i.authorize(ctx, cookieCaller(7), mCommit, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("unknown method denied", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 100, core.VisibilityPublic)}) + _, err := i.authorize(ctx, nil, mUnknown, req) + wantCode(t, err, codes.PermissionDenied) + }) + + t.Run("malformed path is invalid argument", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 100, core.VisibilityPublic)}) + _, err := i.authorize(ctx, nil, mGetMeta, fakeReq{path: "no-slash"}) + wantCode(t, err, codes.InvalidArgument) + }) + + t.Run("missing repo row is not found", func(t *testing.T) { + i := testInterceptor(stubStore{repoErr: db.ErrNotFound}) + _, err := i.authorize(ctx, nil, mGetMeta, req) + wantCode(t, err, codes.NotFound) + }) + + t.Run("root ping with no path is allowed", func(t *testing.T) { + i := testInterceptor(stubStore{}) + got, err := i.authorize(ctx, nil, mRoot, fakeReq{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got == nil { + t.Fatal("expected a caller context") + } + }) + + t.Run("non-root with no path is invalid argument", func(t *testing.T) { + i := testInterceptor(stubStore{}) + _, err := i.authorize(ctx, nil, mGetMeta, fakeReq{}) + wantCode(t, err, codes.InvalidArgument) + }) + + t.Run("repo id shape resolves path", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 100, core.VisibilityPublic)}) + r := fakeReq{id: &remotesapi.RepoId{Org: "alice", RepoName: "db"}} + if _, err := i.authorize(ctx, nil, mGetMeta, r); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +func TestAuthorizeGrantGate(t *testing.T) { + ctx := context.Background() + req := fakeReq{path: "~alice/db"} + + t.Run("PAT without RO grant denied on read", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPublic)}) + _, err := i.authorize(ctx, patCaller(t, 42, "meta.sr.ht/profile:RO"), mGetMeta, req) + wantCode(t, err, codes.PermissionDenied) + }) + + t.Run("PAT with RO grant allowed on read of own repo", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPublic)}) + if _, err := i.authorize(ctx, patCaller(t, 42, "dolt.sr.ht/repos:RO"), mGetMeta, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("PAT with RO grant cannot push", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPublic)}) + _, err := i.authorize(ctx, patCaller(t, 42, "dolt.sr.ht/repos:RO"), mCommit, req) + wantCode(t, err, codes.PermissionDenied) + }) + + t.Run("PAT with RW grant pushes own repo", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPublic)}) + if _, err := i.authorize(ctx, patCaller(t, 42, "dolt.sr.ht/repos:RW"), mCommit, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("universal PAT (empty grants) passes gate", func(t *testing.T) { + i := testInterceptor(stubStore{repo: repo(1, 42, core.VisibilityPublic)}) + if _, err := i.authorize(ctx, patCaller(t, 42, ""), mCommit, req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +// TestUnaryAnonymous exercises the full unary path (authenticate + authorize) +// for an anonymous request: no metadata ⇒ anonymous caller, no crypto/PG. +func TestUnaryAnonymous(t *testing.T) { + i := &interceptor{ + conf: ini.File{}, + service: "dolt.sr.ht", + logger: logrus.NewEntry(logrus.New()), + stores: func() repoStore { return stubStore{repo: repo(1, 100, core.VisibilityPublic)} }, + } + var gotCaller bool + handler := func(ctx context.Context, _ any) (any, error) { + // The handler must see a caller-carrying context (anonymous ⇒ nil ac). + gotCaller = authn.CallerFromContext(ctx) == nil + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: mGetMeta} + resp, err := i.unary()(context.Background(), fakeReq{path: "~alice/db"}, info, handler) + if err != nil { + t.Fatalf("unary: %v", err) + } + if resp != "ok" || !gotCaller { + t.Fatalf("handler not invoked with anonymous caller context (resp=%v)", resp) + } +} + +func TestNormalizeAud(t *testing.T) { + cases := map[string]string{ + "dolt.srht.bigb.es": "dolt.srht.bigb.es", + "dolt.srht.bigb.es:443": "dolt.srht.bigb.es", + "127.0.0.1:5306": "127.0.0.1", + "": "", + } + for in, want := range cases { + if got := normalizeAud(in); got != want { + t.Errorf("normalizeAud(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/remoteapi/keystore.go b/remoteapi/keystore.go new file mode 100644 index 0000000000000000000000000000000000000000..cbd8c83c52a81b71bcd18aa01b760cca5a44e31b --- /dev/null +++ b/remoteapi/keystore.go @@ -0,0 +1,60 @@ +// Package remoteapi assembles dolt.sr.ht's remotesapi subsystem: the gRPC +// authentication/authorization interceptors, the importable remotesrv server +// that serves the bare NBS chunk stores, and the small separate +// CredentialsService (WhoAmI) server that backs `dolt login`. +// +// The two gRPC surfaces run on two ports (nginx path-routes between them): +// +// - the remotesapi ChunkStoreService (clone/push data plane) on ListenAddr, +// wrapped by our per-repo authz interceptors (see interceptors.go); +// - the CredentialsService.WhoAmI RPC on CredsListenAddr (see credsvc.go), +// which lets the dolt CLI discover which SourceHut user a keypair maps to +// while it polls for the key to be associated through the web UI. +package remoteapi + +import ( + "context" + "database/sql" + + "go.bigb.es/sourcehut-dolt/authn" + "go.bigb.es/sourcehut-dolt/db" +) + +// keyStore adapts the SQL persistence layer (db.Store over the shared *sql.DB +// pool) to the authn.KeyStore interface the Bearer-JWT resolver consumes. It +// wraps the pool rather than a context-bound Store so it can be constructed once +// at startup and shared by both the interceptor path and the credentials +// service; each call binds a fresh db.Store to the pool (which manages +// connection lifetime) and threads ctx for cancellation. +type keyStore struct { + db *sql.DB +} + +var _ authn.KeyStore = (*keyStore)(nil) + +// newKeyStore builds a keyStore over the shared database pool. pool must be +// non-nil. +func newKeyStore(pool *sql.DB) *keyStore { + if pool == nil { + panic("remoteapi: newKeyStore requires a non-nil *sql.DB") + } + return &keyStore{db: pool} +} + +// ByKID resolves a dolt key id to the owner's raw Ed25519 public key and +// username, mapping the db layer's KeyAuth onto authn.KeyStore's contract. A +// missing key surfaces as db.ErrNotFound (returned unchanged), which the JWT +// resolver wraps into an invalid-token rejection. +func (k *keyStore) ByKID(ctx context.Context, kid string) (pubkey []byte, username string, err error) { + ka, err := db.NewStore(k.db).KeyByKID(ctx, kid) + if err != nil { + return nil, "", err + } + return ka.PubKey, ka.Username, nil +} + +// TouchLastUsed stamps the key's last_used column after a successful keypair +// authentication. +func (k *keyStore) TouchLastUsed(ctx context.Context, kid string) error { + return db.NewStore(k.db).TouchKeyLastUsed(ctx, kid) +} diff --git a/remoteapi/server.go b/remoteapi/server.go new file mode 100644 index 0000000000000000000000000000000000000000..5f16303d6798cce02e82c6542b016f366f1ee65c --- /dev/null +++ b/remoteapi/server.go @@ -0,0 +1,165 @@ +package remoteapi + +import ( + "context" + "database/sql" + "fmt" + "net" + + remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/dolthub/dolt/go/libraries/doltcore/remotesrv" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/sirupsen/logrus" + "github.com/vaughan0/go-ini" + + "go.bigb.es/sourcehut-dolt/db" + "go.bigb.es/sourcehut-dolt/storage" +) + +// serviceName is the SourceHut service identifier for dolt.sr.ht. It selects +// the config section and OAuth grant namespace used across the auth stack. +const serviceName = "dolt.sr.ht" + +// Config configures the remotesapi server assembly. It is the single struct the +// main wiring populates for both New (the chunk-store server) and NewCredServer +// (the credentials server); each uses the subset it needs. +type Config struct { + // Conf is the loaded instance config (the shared config.ini as ini.File), + // threaded into request contexts so the auth resolvers can read + // [webhooks]private-key, network-key, meta origin, etc. + Conf ini.File + // DB is the shared Postgres pool. + DB *sql.DB + // ReposRoot is the absolute directory under which bare NBS stores live + // ("/~/"). The remotesrv filesys is rooted here. + ReposRoot string + // ListenAddr is the host:port for the remotesapi (gRPC + HTTP chunk data + // plane multiplexed on one h2c port). + ListenAddr string + // CredsListenAddr is the host:port for the separate CredentialsService + // (WhoAmI) gRPC server. + CredsListenAddr string + // HttpHost is the authority the server stamps into sealed chunk-download + // URLs (e.g. "dolt.srht.bigb.es"; may carry a port for local testing, e.g. + // "127.0.0.1:5306"). It also seeds the expected JWT audience: dolt's client + // derives the audience with net.SplitHostPort(endpoint), i.e. the bare host + // with any port stripped, so the audience the server must expect is + // normalizeAud(HttpHost). Leave empty to echo the request :authority into + // chunk URLs (auth then cannot be host-checked; used only by tests without a + // stable host). + HttpHost string + // Logger is the base logger; may be nil (a default is used). + Logger *logrus.Entry +} + +// Server is the assembled remotesapi server: the remotesrv chunk-store server +// plus the storage cache it serves from. The cache is exposed so the web delete +// flow can evict a store when its repository is removed. +type Server struct { + srv *remotesrv.Server + cache *storage.Cache + logger *logrus.Entry + addr string +} + +// New assembles the remotesapi chunk-store server: a db-backed repo lookup, the +// storage cache, our auth/authz interceptors, and the importable remotesrv +// server bound to a single h2c port. It does not start listening; call Serve. +func New(cfg Config) (*Server, error) { + if cfg.DB == nil { + return nil, fmt.Errorf("remoteapi: New requires a non-nil DB") + } + if cfg.ReposRoot == "" { + return nil, fmt.Errorf("remoteapi: New requires a ReposRoot") + } + logger := cfg.Logger + if logger == nil { + logger = logrus.NewEntry(logrus.StandardLogger()) + } + + // Repo lookup: resolve owner/name to the absolute on-disk store dir via the + // repository row. v1 has no push-to-create, so a missing row is an error the + // cache propagates (mapped to gRPC NotFound at the interceptor layer, which + // runs first anyway). + lookup := func(ctx context.Context, owner, name string) (string, error) { + repo, err := db.NewStore(cfg.DB).GetRepoByOwnerAndName(ctx, owner, name) + if err != nil { + return "", err + } + return repo.Path, nil + } + cache := storage.NewCache(lookup) + + keys := newKeyStore(cfg.DB) + icept := newInterceptor(cfg.Conf, serviceName, normalizeAud(cfg.HttpHost), cfg.DB, keys, logger) + + // Load-bearing (see storage/init.go): the FS MUST be rooted at ReposRoot via + // LocalFilesysWithWorkingDir so sealed chunk-download URLs carry clean + // relative prefixes; a bare LocalFS breaks every clone/push at chunk + // transfer. + fs, err := filesys.LocalFilesysWithWorkingDir(cfg.ReposRoot) + if err != nil { + return nil, fmt.Errorf("remoteapi: root filesys at %q: %w", cfg.ReposRoot, err) + } + + srv, err := remotesrv.NewServer(remotesrv.ServerArgs{ + Logger: logger, + HttpHost: cfg.HttpHost, + HttpListenAddr: cfg.ListenAddr, + GrpcListenAddr: cfg.ListenAddr, // == HttpListenAddr ⇒ single h2c port + FS: fs, + DBCache: cache, + ReadOnly: false, + Options: icept.Options(), + ConcurrencyControl: remotesapi.PushConcurrencyControl_PUSH_CONCURRENCY_CONTROL_IGNORE_WORKING_SET, + }) + if err != nil { + if cerr := cache.Close(); cerr != nil { + logger.Warnf("remoteapi: closing cache after NewServer failure: %v", cerr) + } + return nil, fmt.Errorf("remoteapi: remotesrv.NewServer: %w", err) + } + + return &Server{srv: srv, cache: cache, logger: logger, addr: cfg.ListenAddr}, nil +} + +// Cache returns the storage cache backing this server so the web delete flow +// can evict a store on repository removal. +func (s *Server) Cache() *storage.Cache { return s.cache } + +// Serve binds the listeners and serves until GracefulStop. It blocks. It +// returns an error only if binding the listeners fails; the underlying +// remotesrv.Serve blocks until shutdown and does not return an error. +func (s *Server) Serve() error { + listeners, err := s.srv.Listeners() + if err != nil { + return fmt.Errorf("remoteapi: bind %q: %w", s.addr, err) + } + s.srv.Serve(listeners) + return nil +} + +// GracefulStop stops the server and closes every memoized chunk store. +func (s *Server) GracefulStop() { + s.srv.GracefulStop() + if err := s.cache.Close(); err != nil { + s.logger.Warnf("remoteapi: closing cache on shutdown: %v", err) + } +} + +// normalizeAud reduces a configured HttpHost to the bare host the dolt client +// puts in a JWT audience. Verified against dolt's grpc_dial_provider: +// getHostFromEndpoint(endpoint) calls net.SplitHostPort and returns the host +// with any port stripped, so the audience the server receives is always the +// bare host. Normalizing here lets operators write either "dolt.srht.bigb.es" +// or "dolt.srht.bigb.es:443" (or a "127.0.0.1:PORT" test host) and get the same +// expected audience. An empty host yields an empty audience (no keypair auth). +func normalizeAud(host string) string { + if host == "" { + return "" + } + if h, _, err := net.SplitHostPort(host); err == nil { + return h + } + return host +}