Skip to content
11 changes: 10 additions & 1 deletion .github/actions/build-ami/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ inputs:
description: 'Prefix for the AMI name'
required: false
default: 'supabase-postgres'
packer_template:
description: 'Packer template for stage 1 (e.g., amazon-arm64-nix.pkr.hcl)'
required: false
default: 'amazon-arm64-nix.pkr.hcl'
instance_type:
description: 'EC2 instance type for the build'
required: false
default: 'c6g.4xlarge'

outputs:
stage2_ami_id:
Expand Down Expand Up @@ -65,7 +73,7 @@ runs:
-var "packer-execution-id=${{ env.EXECUTION_ID }}" \
-var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \
-var 'ami_regions=${{ inputs.ami_regions }}' \
amazon-arm64-nix.pkr.hcl
${{ inputs.packer_template }}

- name: Build AMI stage 2
id: build-stage2
Expand All @@ -84,4 +92,5 @@ runs:
-var "postgres_major_version=${{ inputs.postgres_version }}" \
-var "ami_name=${{ inputs.ami_name_prefix }}" \
-var "git_sha=${{ inputs.git_sha }}" \
-var "instance_type=${{ inputs.instance_type }}" \
stage2-nix-psql.pkr.hcl
55 changes: 44 additions & 11 deletions .github/workflows/ami-release-nix-single.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,22 @@ on:
required: true
type: string
default: 'main'
arch:
description: 'Architecture to build'
required: true
type: choice
options:
- arm64
- amd64
default: arm64

permissions:
contents: write
id-token: write

jobs:
build:
runs-on: large-linux-arm
runs-on: ${{ github.event.inputs.arch == 'amd64' && 'blacksmith-2vcpu-ubuntu-2404' || 'large-linux-arm' }}
timeout-minutes: 150

steps:
Expand All @@ -41,6 +49,26 @@ jobs:
run: |
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

- name: Set arch-specific variables
id: arch_vars
run: |
ARCH="${{ github.event.inputs.arch }}"
if [ "$ARCH" = "amd64" ]; then
{
echo "packer_template=amazon-amd64-nix.pkr.hcl"
echo "instance_type=c6i.4xlarge"
echo "ami_name_prefix=supabase-postgres-x86"
echo "version_suffix=-x86"
} >> "$GITHUB_OUTPUT"
else
{
echo "packer_template=amazon-arm64-nix.pkr.hcl"
echo "instance_type=c6g.4xlarge"
echo "ami_name_prefix=supabase-postgres"
echo "version_suffix="
} >> "$GITHUB_OUTPUT"
fi

- name: Install nix
uses: ./.github/actions/nix-install-ephemeral
with:
Expand All @@ -57,6 +85,9 @@ jobs:
region: us-east-1
ami_regions: '["us-east-1"]'
git_sha: ${{ steps.get_sha.outputs.sha }}
packer_template: ${{ steps.arch_vars.outputs.packer_template }}
instance_type: ${{ steps.arch_vars.outputs.instance_type }}
ami_name_prefix: ${{ steps.arch_vars.outputs.ami_name_prefix }}

- name: Grab release version
id: process_release_version
Expand All @@ -83,16 +114,17 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}" \
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
-e "arch=${{ github.event.inputs.arch }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 staging
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/upgrade_bundle.tar.gz

- name: configure aws credentials - prod
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
Expand All @@ -104,22 +136,23 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}" \
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
-e "arch=${{ github.event.inputs.arch }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 prod
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}/upgrade_bundle.tar.gz

- name: Create release
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0
with:
name: ${{ steps.process_release_version.outputs.version }}
tag_name: ${{ steps.process_release_version.outputs.version }}
name: ${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}
tag_name: ${{ steps.process_release_version.outputs.version }}${{ steps.arch_vars.outputs.version_suffix }}
target_commitish: ${{ steps.get_sha.outputs.sha }}

- name: Slack Notification on Failure
Expand Down
78 changes: 56 additions & 22 deletions .github/workflows/ami-release-nix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,24 @@ jobs:
strategy:
matrix:
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
include:
- runner: blacksmith-2vcpu-ubuntu-2404-arm
runs-on: ${{ matrix.runner }}
arch:
- name: arm64
runner: blacksmith-2vcpu-ubuntu-2404-arm
packer_template: amazon-arm64-nix.pkr.hcl
vars_file: development-arm.vars.pkr.hcl
instance_type: c6g.4xlarge
nix_system: aarch64-linux
ami_name: supabase-postgres
ami_arch_filter: arm64
- name: amd64
runner: blacksmith-2vcpu-ubuntu-2404
packer_template: amazon-amd64-nix.pkr.hcl
vars_file: development-x86.vars.pkr.hcl
instance_type: c6i.4xlarge
nix_system: x86_64-linux
ami_name: supabase-postgres-x86
ami_arch_filter: x86_64
runs-on: ${{ matrix.arch.runner }}
timeout-minutes: 150

steps:
Expand All @@ -67,7 +82,7 @@ jobs:
- name: Set PostgreSQL version environment variable
run: |
echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV"
echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> "$GITHUB_ENV"
echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}-${{ matrix.arch.name }}" >> "$GITHUB_ENV"

- name: Generate common-nix.vars.pkr.hcl
run: |
Expand All @@ -79,9 +94,8 @@ jobs:
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
run: |
GIT_SHA=${{github.sha}}
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
# why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
nix run github:supabase/postgres/${GIT_SHA}#packer -- init ${{ matrix.arch.packer_template }}
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "ami_name=${{ matrix.arch.ami_name }}" ${{ matrix.arch.packer_template }}

- name: Find stage 1 AMI
run: |
Expand All @@ -98,6 +112,7 @@ jobs:
"Name=tag:postgresVersion,Values=${PG_VERSION}-stage1" \
"Name=tag:sourceSha,Values=${GIT_SHA}" \
"Name=state,Values=available" \
"Name=architecture,Values=${{ matrix.arch.ami_arch_filter }}" \
--query 'Images[0].ImageId' \
--output text)

Expand All @@ -115,8 +130,7 @@ jobs:
run: |
GIT_SHA=${{github.sha}}
nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" stage2-nix-psql.pkr.hcl
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var "instance_type=${{ matrix.arch.instance_type }}" -var "ami_name=${{ matrix.arch.ami_name }}" stage2-nix-psql.pkr.hcl

- name: Grab release version
id: process_release_version
Expand All @@ -125,6 +139,22 @@ jobs:
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "::notice title=AMI Published::Postgres AMI version: $VERSION"

- name: Set arch-qualified version
id: arch_version
run: |
VERSION="${{ steps.process_release_version.outputs.version }}"
if [ "${{ matrix.arch.name }}" = "amd64" ]; then
{
echo "s3_version=${VERSION}-x86"
echo "release_tag=${VERSION}-x86"
} >> "$GITHUB_OUTPUT"
else
{
echo "s3_version=${VERSION}"
echo "release_tag=${VERSION}"
} >> "$GITHUB_OUTPUT"
fi

- name: Create nix flake revision tarball
run: |
GIT_SHA=${{github.sha}}
Expand All @@ -144,16 +174,17 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.arch_version.outputs.s3_version }}" \
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ matrix.postgres_version }}" \
-e "arch=${{ matrix.arch.name }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 staging
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/upgrade_bundle.tar.gz

- name: configure aws credentials - prod
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
Expand All @@ -165,16 +196,17 @@ jobs:
run: |
cd ansible
ansible-playbook -i localhost \
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
-e "ami_release_version=${{ steps.arch_version.outputs.s3_version }}" \
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-e "postgres_major_version=${{ matrix.postgres_version }}" \
-e "arch=${{ matrix.arch.name }}" \
manifest-playbook.yml

- name: Upload nix flake revision to s3 prod
run: |
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/upgrade_bundle.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/20.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/24.04.tar.gz
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.arch_version.outputs.s3_version }}/upgrade_bundle.tar.gz

- name: GitHub OIDC Auth
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
Expand All @@ -197,13 +229,13 @@ jobs:
VERSION="${{ steps.process_release_version.outputs.version }}"
GIT_SHA="${{ github.sha }}"
PG_VERSION="${{ matrix.postgres_version }}"
SYSTEM="aarch64-linux"
SYSTEM="${{ matrix.arch.nix_system }}"

# Get store path for this build
STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath")

# Each postgres version gets its own catalog file (no race conditions)
CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}.json"
CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}-${SYSTEM}.json"

# Create catalog JSON for this version
jq -n \
Expand All @@ -225,12 +257,12 @@ jobs:
- name: Create release
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0
with:
name: ${{ steps.process_release_version.outputs.version }}
tag_name: ${{ steps.process_release_version.outputs.version }}
name: ${{ steps.arch_version.outputs.release_tag }}
tag_name: ${{ steps.arch_version.outputs.release_tag }}
target_commitish: ${{github.sha}}

- name: Create CLI tag for PG 17
if: matrix.postgres_version == '17' && github.event_name != 'workflow_dispatch'
if: matrix.postgres_version == '17' && matrix.arch.name == 'arm64' && github.event_name != 'workflow_dispatch'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand All @@ -241,6 +273,7 @@ jobs:
git push origin "${CLI_TAG}"

- name: Trigger pg_upgrade_scripts workflow
if: matrix.arch.name == 'arm64'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand All @@ -249,6 +282,7 @@ jobs:
-f postgresVersion="${{ steps.process_release_version.outputs.version }}"

- name: Trigger pg_upgrade_bin flake version workflow
if: matrix.arch.name == 'arm64'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand Down
Loading
Loading