Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,12 @@ jobs:

- name: Install golang-migrate
run: |
go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1
# -tags 'pgx5', matching the Dockerfile and the library import in
# internal/database/postgres/migrations: the postgres tag links
# lib/pq, which carries unfixable advisories (issue #1849). No
# @version suffix: installed as a package of this module, so the
# version and dependency set come from go.mod.
go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate
- name: Run database migrations
env:
Expand All @@ -248,7 +253,7 @@ jobs:
run: |
if [ -d "internal/database/postgres/migrations" ]; then
migrate -path internal/database/postgres/migrations \
-database "postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" \
-database "pgx5://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" \
up
fi
Expand Down
37 changes: 30 additions & 7 deletions .github/workflows/database-migration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,12 @@ jobs:
go-version-file: go.mod

- name: Install golang-migrate
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1
# No @version suffix: installed as a package of this module, so the
# migrate version and its dependency set come from go.mod, the same way
# the Dockerfile and `make install-tools` build it. -tags 'pgx5' keeps
# lib/pq out of the binary that talks to the production database.
# See issue #1849.
run: go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1
Expand Down Expand Up @@ -304,7 +309,9 @@ jobs:
STEPS: ${{ inputs.steps }}
run: |
set -uo pipefail
DB_URL="postgresql://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"
# pgx5://, not postgresql://: migrate is installed with -tags 'pgx5'
# above and golang-migrate dispatches on the URL scheme (issue #1849).
DB_URL="pgx5://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"

# `validate` already rejects a malformed `steps`/`direction` before this
# job is ever reached, but this job sits behind an environment gate and
Expand Down Expand Up @@ -339,7 +346,9 @@ jobs:
DB_ENDPOINT: ${{ steps.get-endpoint.outputs.endpoint }}
run: |
set -uo pipefail
DB_URL="postgresql://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"
# pgx5://, not postgresql://: migrate is installed with -tags 'pgx5'
# above and golang-migrate dispatches on the URL scheme (issue #1849).
DB_URL="pgx5://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"
VERSION=$(migrate -path "$MIGRATIONS_PATH" -database "$DB_URL" version 2>&1 || echo "unknown")
echo "Current migration version: $VERSION"
echo "MIGRATION_VERSION=$VERSION" >> "$GITHUB_ENV"
Expand Down Expand Up @@ -378,7 +387,12 @@ jobs:
go-version-file: go.mod

- name: Install golang-migrate
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1
# No @version suffix: installed as a package of this module, so the
# migrate version and its dependency set come from go.mod, the same way
# the Dockerfile and `make install-tools` build it. -tags 'pgx5' keeps
# lib/pq out of the binary that talks to the production database.
# See issue #1849.
run: go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate

- name: Authenticate to Google Cloud
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
Expand Down Expand Up @@ -409,7 +423,9 @@ jobs:
STEPS: ${{ inputs.steps }}
run: |
set -uo pipefail
DB_URL="postgresql://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"
# pgx5://, not postgresql://: migrate is installed with -tags 'pgx5'
# above and golang-migrate dispatches on the URL scheme (issue #1849).
DB_URL="pgx5://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"

STEPS="${STEPS:-0}"

Expand Down Expand Up @@ -465,7 +481,12 @@ jobs:
go-version-file: go.mod

- name: Install golang-migrate
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1
# No @version suffix: installed as a package of this module, so the
# migrate version and its dependency set come from go.mod, the same way
# the Dockerfile and `make install-tools` build it. -tags 'pgx5' keeps
# lib/pq out of the binary that talks to the production database.
# See issue #1849.
run: go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate

- name: Azure Login
uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0
Expand Down Expand Up @@ -494,7 +515,9 @@ jobs:
STEPS: ${{ inputs.steps }}
run: |
set -uo pipefail
DB_URL="postgresql://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"
# pgx5://, not postgresql://: migrate is installed with -tags 'pgx5'
# above and golang-migrate dispatches on the URL scheme (issue #1849).
DB_URL="pgx5://cudly:${DB_PASSWORD}@${DB_ENDPOINT}:5432/cudly?sslmode=require"

STEPS="${STEPS:-0}"

Expand Down
49 changes: 32 additions & 17 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -38,23 +38,6 @@ RUN apk add --no-cache \
# Set shell with pipefail for safer pipe operations
SHELL ["/bin/ash", "-eo", "pipefail", "-c"]

# Build golang-migrate from source on this stage's pinned Go toolchain, the same
# way `make install-tools` does. Upstream's prebuilt release tarballs carry
# whatever toolchain upstream built them with (v4.19.1 ships go1.25.4), which is
# how issue #1833's stdlib CVEs reached the runtime image, where entrypoint.sh
# runs `migrate up` against the database on every container start.
# `go install` refuses GOBIN when cross-compiling and writes to
# bin/${GOOS}_${GOARCH}/ instead, so resolve both layouts; the final `mv` fails
# the build if neither produced a binary.
# Keep this version in step with MIGRATE_VERSION in the Makefile.
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go install -tags=postgres -ldflags="-s -w -X main.Version=v4.19.1" \
github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1 && \
GOPATH_BIN="$(go env GOPATH)/bin" && \
MIGRATE_BIN="${GOPATH_BIN}/${TARGETOS}_${TARGETARCH}/migrate" && \
{ [ -x "${MIGRATE_BIN}" ] || MIGRATE_BIN="${GOPATH_BIN}/migrate"; } && \
mv "${MIGRATE_BIN}" /usr/local/bin/migrate

WORKDIR /app

# Copy go module files
Expand All @@ -69,6 +52,38 @@ COPY providers/gcp/go.mod providers/gcp/go.sum providers/gcp/
# Download dependencies
RUN go mod download

# Build golang-migrate from source on this stage's pinned Go toolchain, the same
# way `make install-tools` does. Upstream's prebuilt release tarballs carry
# whatever toolchain upstream built them with (v4.19.1 ships go1.25.4), which is
# how issue #1833's stdlib CVEs reached the runtime image, where entrypoint.sh
# runs `migrate up` against the database on every container start.
#
# Built as a package of THIS module (no @version suffix) rather than
# `go install ...@v4.19.1`, and therefore placed after `go mod download`.
# The @version form resolves dependencies from golang-migrate's own go.mod,
# which pins jackc/pgx v5.5.4, x/crypto v0.45.0 and x/text v0.31.0 - all
# superseded, and together 17 advisories with published fixes that
# scripts/scan-shipped-image.sh correctly fails on. Building from the main
# module instead resolves them at this repo's own versions, which are already
# above every one of those fixes. The migrate version therefore comes from
# go.mod, and is read back out of the build list rather than restated.
#
# -tags=pgx5, not postgres: the postgres tag links the lib/pq driver, which
# carries advisories with no fixed version in any release (GO-2026-6166, 6168,
# 6170, 6171, 6172) reached through Driver.Open and conn.Exec on every
# container start. The pgx5 tag builds the same driver on jackc/pgx v5, which
# the application already uses. It registers the "pgx5" URL scheme only, so
# scripts/entrypoint.sh and .github/workflows/database-migration.yml must build
# pgx5:// URLs - a postgres:// URL against this binary fails at runtime with
# "unknown driver". See issue #1849.
RUN MIGRATE_VERSION="$(go list -m -f '{{.Version}}' github.com/golang-migrate/migrate/v4)" && \
echo "Building golang-migrate ${MIGRATE_VERSION} for ${TARGETOS}/${TARGETARCH}" && \
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build \
-tags=pgx5 \
-ldflags="-s -w -X main.Version=${MIGRATE_VERSION}" \
-o /usr/local/bin/migrate \
github.com/golang-migrate/migrate/v4/cmd/migrate

# Copy source code
COPY . .

Expand Down
8 changes: 5 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ GIT_SHA?=$(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
GOLANGCI_LINT_VERSION?=v2.10.1
GOSEC_VERSION?=v2.28.0
GOCYCLO_VERSION?=v0.6.0
MIGRATE_VERSION?=v4.19.1
# golang-migrate deliberately has no version variable: it is installed as a
# package of this module (see install-tools), so its version and its whole
# dependency set come from go.mod. See issue #1849.
# staticcheck has no CI pin; it is used by scripts/security-scan.sh
STATICCHECK_VERSION?=v0.7.0
LDFLAGS=-ldflags "-s -w -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitSHA=$(GIT_SHA)"
Expand Down Expand Up @@ -246,8 +248,8 @@ install-dev-tools:
@go install honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VERSION)
@echo "Installing gocyclo $(GOCYCLO_VERSION)..."
@go install github.com/fzipp/gocyclo/cmd/gocyclo@$(GOCYCLO_VERSION)
@echo "Installing golang-migrate $(MIGRATE_VERSION)..."
@go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@$(MIGRATE_VERSION)
@echo "Installing golang-migrate $$(go list -m -f '{{.Version}}' github.com/golang-migrate/migrate/v4)..."
@go install -tags 'pgx5' github.com/golang-migrate/migrate/v4/cmd/migrate
@echo "✓ Development tools installed"
@echo ""
@echo "Additional tools to install manually:"
Expand Down
9 changes: 7 additions & 2 deletions docs/DEPLOYMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -637,12 +637,17 @@ Multi-account support requires an AES-256-GCM encryption key for stored cloud ac

If `DB_AUTO_MIGRATE=true` (the default), migration 000011 runs automatically on Lambda cold start. To run manually:

> The URL scheme is `pgx5://`, not `postgres://`. golang-migrate selects its
> driver by scheme, and `migrate` is built here with `-tags pgx5` so it does not
> link `lib/pq` (issue #1849). A `postgres://` URL fails with
> `unknown driver postgres`.

```bash
DB_PASSWORD=$(aws secretsmanager get-secret-value --secret-id cudly-dev-db-password-* --query SecretString --output text | jq -r .password)
RDS_ENDPOINT=$(cd terraform/environments/aws && terraform output -raw database_proxy_endpoint)

migrate -path internal/database/postgres/migrations \
-database "postgresql://cudly:${DB_PASSWORD}@${RDS_ENDPOINT}:5432/cudly?sslmode=require" up
-database "pgx5://cudly:${DB_PASSWORD}@${RDS_ENDPOINT}:5432/cudly?sslmode=require" up
```

---
Expand All @@ -656,7 +661,7 @@ DB_PASSWORD=$(aws secretsmanager get-secret-value --secret-id cudly-dev-db-passw
RDS_ENDPOINT=$(cd terraform/environments/aws && terraform output -raw database_proxy_endpoint)

migrate -path internal/database/postgres/migrations \
-database "postgresql://cudly:${DB_PASSWORD}@${RDS_ENDPOINT}:5432/cudly?sslmode=require" up
-database "pgx5://cudly:${DB_PASSWORD}@${RDS_ENDPOINT}:5432/cudly?sslmode=require" up
```

### Terraform State Lock
Expand Down
16 changes: 11 additions & 5 deletions docs/DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,23 @@ docker-compose down

### 3. Database Access

> Migration URLs use the `pgx5://` scheme, not `postgres://`. golang-migrate
> picks its database driver by URL scheme, and this repo builds `migrate` with
> `-tags pgx5` so the binary does not link `lib/pq`, which carries advisories
> with no published fix (issue #1849). A `postgres://` URL fails against it with
> `unknown driver postgres`.

```bash
# Connect to PostgreSQL using psql
docker-compose exec postgres psql -U cudly -d cudly

# Run migrations manually
docker-compose exec app migrate -path /app/internal/database/postgres/migrations \
-database "postgresql://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" up
-database "pgx5://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" up

# Check migration status
docker-compose exec app migrate -path /app/internal/database/postgres/migrations \
-database "postgresql://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" version
-database "pgx5://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" version
```

## Development Workflow
Expand All @@ -68,11 +74,11 @@ migrate create -ext sql -dir internal/database/postgres/migrations -seq add_new_

# Run migrations
docker-compose exec app migrate -path /app/internal/database/postgres/migrations \
-database "postgresql://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" up
-database "pgx5://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" up

# Rollback last migration
docker-compose exec app migrate -path /app/internal/database/postgres/migrations \
-database "postgresql://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" down 1
-database "pgx5://cudly:cudly_local_dev@postgres:5432/cudly?sslmode=disable" down 1
```

## Environment Variables
Expand Down Expand Up @@ -374,7 +380,7 @@ docker-compose exec postgres psql -U cudly -d cudly -c "SELECT * FROM schema_mig

# Force migration version (use with caution)
migrate -path internal/database/postgres/migrations \
-database "postgresql://cudly:cudly_local_dev@localhost:5432/cudly?sslmode=disable" \
-database "pgx5://cudly:cudly_local_dev@localhost:5432/cudly?sslmode=disable" \
force <version>
```

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,11 @@ require (
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/microsoft/kiota-abstractions-go v1.9.4 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,8 @@ github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl
github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
Expand Down
17 changes: 13 additions & 4 deletions internal/database/postgres/migrations/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (
"strconv"

"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/postgres" // postgres driver
_ "github.com/golang-migrate/migrate/v4/source/file" // file source
_ "github.com/golang-migrate/migrate/v4/database/pgx/v5" // postgres driver (pgx/v5)
_ "github.com/golang-migrate/migrate/v4/source/file" // file source
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/crypto/bcrypt"
)
Expand Down Expand Up @@ -540,6 +540,15 @@ func GetMigrationVersion(ctx context.Context, pool *pgxpool.Pool, migrationsPath
return version, dirty, nil
}

// migrateURLScheme selects which golang-migrate database driver handles the
// DSN. golang-migrate dispatches purely on the URL scheme, and the pgx/v5
// driver registers itself as "pgx5" only - "postgres"/"postgresql" belong to
// the lib/pq-backed driver this package deliberately does not import (issue
// #1849). A mismatch between this constant and the imported driver fails at
// migration time, not at compile time, which is why
// TestBuildMigrateDSN_SchemeMatchesRegisteredDriver pins the two together.
const migrateURLScheme = "pgx5"

// buildMigrateDSN builds a connection string for golang-migrate from pgx config.
// The SSL mode is recovered from the parsed TLSConfig so strict modes
// (verify-ca / verify-full) are preserved rather than silently downgraded to
Expand All @@ -558,10 +567,10 @@ func buildMigrateDSN(config *pgxpool.Config) string {

sslMode := sslModeFromTLSConfig(config.ConnConfig.TLSConfig)

// Build DSN (golang-migrate uses postgres:// format)
// Don't add connection options - RDS Proxy doesn't support them
return fmt.Sprintf(
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
"%s://%s:%s@%s:%d/%s?sslmode=%s",
migrateURLScheme,
encodedUser,
encodedPassword,
host,
Expand Down
Loading
Loading