From bc44f6e89656c7eeccf445b11fbe71ab404cdac0 Mon Sep 17 00:00:00 2001 From: Chris Munns Date: Mon, 6 Apr 2026 17:10:50 -0400 Subject: [PATCH 1/2] Add .gitignore and pgcopydb-templates README Add .gitignore covering OS files, env/secrets, Terraform state, pgcopydb migration artifacts, and editor files. Add top-level README for pgcopydb-templates to index the three provisioning options. --- .gitignore | 37 ++++++++++++++++++++++++++++++++++++ pgcopydb-templates/README.md | 24 +++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 .gitignore create mode 100644 pgcopydb-templates/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c7d75b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# OS +.DS_Store +Thumbs.db + +# AI assistant config +CLAUDE.md + +# Environment and secrets +.env +*.env +.env.* + +# Terraform +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +*.tfvars.json +crash.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraform.lock.hcl + +# pgcopydb migration artifacts +migration_*/ +filters.ini +*.log +core.* + +# Editor +*.swp +*.swo +*~ +.vscode/ +.idea/ diff --git a/pgcopydb-templates/README.md b/pgcopydb-templates/README.md new file mode 100644 index 0000000..f001247 --- /dev/null +++ b/pgcopydb-templates/README.md @@ -0,0 +1,24 @@ +# pgcopydb Migration Instance Templates + +Infrastructure-as-code templates for provisioning a migration instance pre-configured with [pgcopydb](https://github.com/planetscale/pgcopydb) and the [PlanetScale migration helper scripts](../pgcopydb-helpers/). Each template creates a compute instance running Ubuntu 24.04 that installs pgcopydb, PostgreSQL client tools, and the helper scripts at boot. + +## Available Templates + +| Template | Platform | Tool | README | +|----------|----------|------|--------| +| [AWS CloudFormation](./aws-cloudformation/) | AWS EC2 | CloudFormation | [README](./aws-cloudformation/README-pgcopydb-cfn.md) | +| [AWS Terraform](./aws-terraform/) | AWS EC2 | Terraform | [README](./aws-terraform/README-pgcopydb-aws-tf.md) | +| [GCP Terraform](./gcp-terraform/) | GCP Compute Engine | Terraform | [README](./gcp-terraform/README-pgcopydb-gcp.md) | + +All three templates produce an equivalent migration instance — choose based on your cloud provider and preferred provisioning tool. + +## What Gets Provisioned + +- A compute instance with pgcopydb and PostgreSQL 17 client tools +- An attached data volume for migration working data +- Network and access configuration (security group/firewall rule, IAM/SSH via SSM or IAP) +- Migration helper scripts from this repo deployed to `/home/ubuntu/` + +## After Provisioning + +Once the instance is running, connect to it and follow the [migration workflow](../pgcopydb-helpers/README.md#migration-workflow) starting with setting up `~/.env` and `~/filters.ini`. From c17bd07fd9f9e353b64b1e60c26b867aafdd9c4b Mon Sep 17 00:00:00 2001 From: Chris Munns Date: Mon, 6 Apr 2026 17:11:41 -0400 Subject: [PATCH 2/2] Add pgcopydb migration instance templates CloudFormation and Terraform templates for provisioning migration instances on AWS and GCP with pgcopydb pre-installed. --- .../aws-cloudformation/README-pgcopydb-cfn.md | 46 ++ .../pgcopydb-migration-instance.yaml | 418 ++++++++++++++++++ .../aws-terraform/README-pgcopydb-aws-tf.md | 55 +++ .../pgcopydb-migration-instance.tf | 275 ++++++++++++ pgcopydb-templates/aws-terraform/user-data.sh | 152 +++++++ .../gcp-terraform/README-pgcopydb-gcp.md | 58 +++ .../pgcopydb-migration-instance.tf | 342 ++++++++++++++ .../gcp-terraform/startup-script.sh | 127 ++++++ 8 files changed, 1473 insertions(+) create mode 100644 pgcopydb-templates/aws-cloudformation/README-pgcopydb-cfn.md create mode 100644 pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml create mode 100644 pgcopydb-templates/aws-terraform/README-pgcopydb-aws-tf.md create mode 100644 pgcopydb-templates/aws-terraform/pgcopydb-migration-instance.tf create mode 100644 pgcopydb-templates/aws-terraform/user-data.sh create mode 100644 pgcopydb-templates/gcp-terraform/README-pgcopydb-gcp.md create mode 100644 pgcopydb-templates/gcp-terraform/pgcopydb-migration-instance.tf create mode 100644 pgcopydb-templates/gcp-terraform/startup-script.sh diff --git a/pgcopydb-templates/aws-cloudformation/README-pgcopydb-cfn.md b/pgcopydb-templates/aws-cloudformation/README-pgcopydb-cfn.md new file mode 100644 index 0000000..d33e14d --- /dev/null +++ b/pgcopydb-templates/aws-cloudformation/README-pgcopydb-cfn.md @@ -0,0 +1,46 @@ +# PlanetScale Migration — pgcopydb Migration Instance (AWS CloudFormation) + +## What This Does + +This CloudFormation template creates an EC2 instance pre-configured with +[pgcopydb](https://pgcopydb.readthedocs.io/), the tool PlanetScale uses to migrate +your PostgreSQL data. The instance runs Ubuntu 24.04 and pulls the latest +[PlanetScale migration helper scripts](https://github.com/planetscale/migration-scripts) +at boot. + +## What Gets Created + +- An EC2 instance with pgcopydb and PostgreSQL client tools installed +- An EBS volume for migration working data +- A security group for network access +- An IAM instance profile with SSM Session Manager access +- Migration helper scripts from `github.com/planetscale/migration-scripts` in `/home/ubuntu/` + +## How to Deploy + +### Prerequisites +- The IAM role from Step 1 must be deployed first +- VPC ID and a subnet ID where the instance should run +- The restored database from Step 3 must be accessible from this subnet + +### Steps + +1. **Upload** the `pgcopydb-migration-instance.yaml` template to CloudFormation +2. **Fill in** VPC ID, Subnet ID, instance type, and volume size +3. **Deploy** and wait for completion (~10 minutes) +4. **Connect** via SSM Session Manager (no SSH key needed): + ```bash + aws ssm start-session --target INSTANCE_ID + ``` + +## How to Tear Down + +Delete the CloudFormation stack to remove the instance and all associated resources: + +```bash +aws cloudformation delete-stack --stack-name YOUR_STACK_NAME +``` + +## Questions? + +Contact your PlanetScale migration team representative. diff --git a/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml b/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml new file mode 100644 index 0000000..46a3c61 --- /dev/null +++ b/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml @@ -0,0 +1,418 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: 'EC2 instance with pgcopydb for PostgreSQL-to-PlanetScale migrations (Ubuntu 24.04 LTS). Requires dual connectivity: outbound internet access to PlanetScale and private network access to source databases. (Generated by Liftoff Migration Reviewer)' + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Resource Naming" + Parameters: + - ResourcePrefix + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - SubnetId + - Label: + default: "Instance Configuration" + Parameters: + - InstanceType + - VolumeSize + - Label: + default: "SSH Access (Optional - EC2 Instance Connect enabled by default)" + Parameters: + - KeyPairName + ParameterLabels: + ResourcePrefix: + default: "Resource Name Prefix" + VpcId: + default: "VPC ID" + SubnetId: + default: "Public Subnet ID" + InstanceType: + default: "EC2 Instance Type" + VolumeSize: + default: "EBS Volume Size (GB)" + KeyPairName: + default: "EC2 Key Pair (Optional)" + Generated: + Tool: 'PlanetScale Liftoff Migration Reviewer' + Date: '2026-04-06T19:28:48.774Z' + Purpose: 'pgcopydb EC2 instance for PostgreSQL to PlanetScale migrations' + +Parameters: + ResourcePrefix: + Type: String + Default: 'planetscale-migration' + Description: 'Prefix for all resource names (must match IAM role restriction)' + AllowedPattern: '[a-z0-9-]+' + ConstraintDescription: 'Must be lowercase alphanumeric with hyphens only' + + VpcId: + Type: AWS::EC2::VPC::Id + Description: VPC ID where the instance will be launched + + SubnetId: + Type: AWS::EC2::Subnet::Id + Description: Public subnet ID with internet gateway route (required for PlanetScale connectivity) + + InstanceType: + Type: String + Default: c7i.xlarge + AllowedValues: + - c7i.xlarge + - c7i.2xlarge + - c7i.4xlarge + - c7i.8xlarge + - c7i.12xlarge + - c7i.16xlarge + Description: Instance type (c7i.xlarge for <100GB, c7i.2xlarge for 100-500GB, c7i.4xlarge for >500GB) + + VolumeSize: + Type: String + Default: '1000' + AllowedValues: + - '500' + - '1000' + - '3000' + - '6000' + - '12000' + Description: 'EBS volume size in GB (io2 Block Express: 500GB=30K, 1TB=40K, 3TB=50K, 6TB=60K, 12TB=70K IOPS)' + + KeyPairName: + Type: String + Default: '' + Description: Optional - EC2 Key Pair for key-based SSH access. Leave empty to use EC2 Instance Connect only. + +Mappings: + VolumeConfig: + '500': + Iops: 30000 + '1000': + Iops: 40000 + '3000': + Iops: 50000 + '6000': + Iops: 60000 + '12000': + Iops: 70000 + +Conditions: + HasKeyPair: !Not [!Equals [!Ref KeyPairName, '']] + +Resources: + InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy + - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore + Tags: + - Key: Name + Value: !Sub '${ResourcePrefix}-ec2-role' + - Key: Purpose + Value: PostgreSQL to PlanetScale Migration + + InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref InstanceRole + + SecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Security group for pgcopydb migration instance + VpcId: !Ref VpcId + SecurityGroupEgress: + - IpProtocol: -1 + CidrIp: 0.0.0.0/0 + Description: Allow all outbound traffic + Tags: + - Key: Name + Value: !Sub '${ResourcePrefix}-sg' + - Key: Purpose + Value: PostgreSQL to PlanetScale Migration + + EC2InstanceConnectIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !Ref SecurityGroup + IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + Description: SSH access for EC2 Instance Connect and optional key-based SSH + + ICMPDestinationUnreachableIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !Ref SecurityGroup + IpProtocol: icmp + FromPort: 3 + ToPort: -1 + CidrIp: 0.0.0.0/0 + Description: ICMP Destination Unreachable for path MTU discovery + + Instance: + Type: AWS::EC2::Instance + CreationPolicy: + ResourceSignal: + Timeout: PT20M + Metadata: + AWS::CloudFormation::Init: + configSets: + default: + - setup_cfn + - install_packages + - configure_system + - create_directories + - verify_installation + setup_cfn: + files: + /etc/cfn/cfn-hup.conf: + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + mode: '000400' + owner: root + group: root + /etc/cfn/hooks.d/cfn-auto-reloader.conf: + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.Instance.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource Instance --region ${AWS::Region} + mode: '000400' + owner: root + group: root + install_packages: + commands: + 01_update_apt: + command: | + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + 02_install_prerequisites: + command: | + export DEBIAN_FRONTEND=noninteractive + apt-get install -y wget gnupg2 lsb-release curl unzip ca-certificates netcat-openbsd sqlite3 + 03_add_postgresql_repo: + command: | + # Add PostgreSQL APT repository (using modern method for Ubuntu 24.04) + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list + apt-get update + 04_install_postgresql: + command: | + export DEBIAN_FRONTEND=noninteractive + # Install PostgreSQL 17 client and build dependencies + apt-get install -y postgresql-client-17 postgresql-17 postgresql-server-dev-17 + 05_install_build_tools: + command: | + export DEBIAN_FRONTEND=noninteractive + # Install build tools and libraries needed for pgcopydb + apt-get install -y \ + build-essential \ + git \ + libssl-dev \ + libpq-dev \ + libgc-dev \ + liblz4-dev \ + libpam0g-dev \ + libxml2-dev \ + libxslt1-dev \ + libreadline-dev \ + zlib1g-dev \ + libncurses5-dev \ + libkrb5-dev \ + libselinux1-dev \ + libzstd-dev + 06_build_pgcopydb: + command: | + cd /tmp + git clone --branch v0.18.0 https://github.com/planetscale/pgcopydb.git + cd pgcopydb + export PATH=/usr/lib/postgresql/17/bin:$PATH + make clean || true + make + make install + ldconfig + test: test ! -f /usr/local/bin/pgcopydb + 07_install_aws_cli: + command: | + cd /tmp + curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + ./aws/install + rm -rf aws awscliv2.zip + test: test ! -f /usr/local/bin/aws + 08_install_cloudwatch: + command: | + cd /tmp + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i amazon-cloudwatch-agent.deb + rm -f amazon-cloudwatch-agent.deb + configure_system: + files: + /etc/security/limits.d/99-pgcopydb.conf: + content: | + * soft nofile 65536 + * hard nofile 65536 + mode: '000644' + owner: root + group: root + /etc/sysctl.d/99-pgcopydb.conf: + content: | + net.ipv4.tcp_keepalive_time = 60 + net.ipv4.tcp_keepalive_intvl = 10 + net.ipv4.tcp_keepalive_probes = 6 + net.core.rmem_max = 134217728 + net.core.wmem_max = 134217728 + net.ipv4.tcp_rmem = 4096 87380 67108864 + net.ipv4.tcp_wmem = 4096 65536 67108864 + mode: '000644' + owner: root + group: root + /etc/profile.d/pgcopydb.sh: + content: | + export PATH=/usr/lib/postgresql/17/bin:$PATH + alias pgcopydb-version='pgcopydb --version' + alias psql-version='psql --version' + alias check-planetscale='nc -zv app.connect.psdb.cloud 443 2>&1 | grep succeeded' + mode: '000644' + owner: root + group: root + /home/ubuntu/.env: + content: | + # PlanetScale Migration Environment Variables + # Edit these values before running the migration + + # Source Database + PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmode=require" + + # Target Database (PlanetScale) + PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" + mode: '000600' + owner: ubuntu + group: ubuntu + commands: + 02_clone_migration_scripts: + command: | + git clone --depth 1 https://github.com/planetscale/migration-scripts.git /tmp/migration-scripts + cp -r /tmp/migration-scripts/pgcopydb-helpers/* /home/ubuntu/ + rm -rf /tmp/migration-scripts + chown ubuntu:ubuntu /home/ubuntu/*.sh /home/ubuntu/*.md + chmod +x /home/ubuntu/*.sh + 01_apply_sysctl: + command: sysctl -p /etc/sysctl.d/99-pgcopydb.conf + create_directories: + commands: + 01_create_pgcopydb_dir: + command: mkdir -p /var/lib/pgcopydb && chmod 755 /var/lib/pgcopydb + verify_installation: + commands: + 01_log_versions: + command: | + echo "=== Installation verification ===" >> /var/log/cfn-init-verification.log + export PATH=/usr/lib/postgresql/17/bin:$PATH + pgcopydb --version >> /var/log/cfn-init-verification.log 2>&1 || echo "pgcopydb installation failed" >> /var/log/cfn-init-verification.log + psql --version >> /var/log/cfn-init-verification.log 2>&1 || echo "PostgreSQL client installation failed" >> /var/log/cfn-init-verification.log + aws --version >> /var/log/cfn-init-verification.log 2>&1 || echo "AWS CLI installation failed" >> /var/log/cfn-init-verification.log + echo "=== Installation complete at $(date) ===" >> /var/log/cfn-init-verification.log + Properties: + ImageId: '{{resolve:ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id}}' + InstanceType: !Ref InstanceType + KeyName: !If [HasKeyPair, !Ref KeyPairName, !Ref 'AWS::NoValue'] + IamInstanceProfile: !Ref InstanceProfile + SubnetId: !Ref SubnetId + SecurityGroupIds: + - !Ref SecurityGroup + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref VolumeSize + VolumeType: io2 + Iops: !FindInMap [VolumeConfig, !Ref VolumeSize, Iops] + DeleteOnTermination: true + Encrypted: true + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 1 + UserData: + Fn::Base64: !Sub | + #!/bin/bash -xe + exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + + # Update package lists + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + + # Install CloudFormation helper scripts + apt-get install -y python3-pip python3-setuptools + mkdir -p /opt/aws/bin + python3 -m pip install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz + + # Create symlinks for cfn tools + ln -sf /usr/local/bin/cfn-init /opt/aws/bin/cfn-init + ln -sf /usr/local/bin/cfn-signal /opt/aws/bin/cfn-signal + ln -sf /usr/local/bin/cfn-hup /opt/aws/bin/cfn-hup + + # Run cfn-init to execute the metadata configuration + /usr/local/bin/cfn-init -v --stack ${AWS::StackName} --resource Instance --region ${AWS::Region} --configsets default + + # Signal CloudFormation that the instance is ready + /usr/local/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Instance --region ${AWS::Region} + Tags: + - Key: Name + Value: !Sub '${ResourcePrefix}-ec2' + - Key: Purpose + Value: PostgreSQL to PlanetScale Migration + +Outputs: + InstanceId: + Description: EC2 Instance ID + Value: !Ref Instance + + PublicIP: + Description: Public IP address (for PlanetScale connectivity) + Value: !GetAtt Instance.PublicIp + + PrivateIP: + Description: Private IP address (for VPC database connectivity) + Value: !GetAtt Instance.PrivateIp + + SecurityGroupId: + Description: Security Group ID + Value: !Ref SecurityGroup + + SSHCommand: + Description: SSH connection command + Value: !Sub 'ssh -i /path/to/${KeyPairName}.pem ubuntu@${Instance.PublicIp}' + + EC2InstanceConnectCommand: + Description: EC2 Instance Connect command (no key required) + Value: !Sub 'aws ec2-instance-connect send-ssh-public-key --instance-id ${Instance} --instance-os-user ubuntu --ssh-public-key file://~/.ssh/id_rsa.pub --availability-zone ${Instance.AvailabilityZone}' + + SessionManagerCommand: + Description: AWS Session Manager connection command + Value: !Sub 'aws ssm start-session --target ${Instance}' + + PgcopydbVersion: + Description: Command to check pgcopydb version + Value: 'ssh to instance and run: pgcopydb --version' + + UsageGuide: + Description: Quick start guide + Value: '1. Connect via EC2 Instance Connect or SSH 2. Verify: pgcopydb --version 3. See ~/README.md for usage instructions' + + MigrationScriptsPath: + Description: Path to migration helper scripts + Value: '/home/ubuntu/' diff --git a/pgcopydb-templates/aws-terraform/README-pgcopydb-aws-tf.md b/pgcopydb-templates/aws-terraform/README-pgcopydb-aws-tf.md new file mode 100644 index 0000000..80f4a3d --- /dev/null +++ b/pgcopydb-templates/aws-terraform/README-pgcopydb-aws-tf.md @@ -0,0 +1,55 @@ +# PlanetScale Migration — pgcopydb Migration Instance (AWS Terraform) + +## What This Does + +This Terraform template creates an EC2 instance pre-configured with +[pgcopydb](https://pgcopydb.readthedocs.io/), the tool PlanetScale uses to migrate +your PostgreSQL data. The instance runs Ubuntu 24.04 and pulls the latest +[PlanetScale migration helper scripts](https://github.com/planetscale/migration-scripts) +at boot. + +## What Gets Created + +- An EC2 instance with pgcopydb and PostgreSQL client tools installed +- An EBS volume for migration working data +- A security group for network access +- An IAM instance profile with SSM Session Manager access +- Migration helper scripts from `github.com/planetscale/migration-scripts` in `/home/ubuntu/` + +## How to Deploy + +### Prerequisites +- [Terraform](https://developer.hashicorp.com/terraform/install) installed (v1.0+) +- VPC ID and a subnet ID +- `gcloud` or AWS CLI authenticated + +### Steps + +1. **Save both files** (`pgcopydb-migration-instance.tf` and `user-data.sh`) in the same directory + +2. **Deploy** + ```bash + terraform init + terraform apply \ + -var="region=us-east-1" \ + -var="vpc_id=vpc-xxx" \ + -var="subnet_id=subnet-xxx" + ``` + +3. **Connect** via SSM Session Manager: + ```bash + aws ssm start-session --target $(terraform output -raw instance_id) + ``` + +## How to Tear Down + +```bash +terraform destroy \ + -var="region=us-east-1" \ + -var="vpc_id=vpc-xxx" \ + -var="subnet_id=subnet-xxx" +``` + +## Questions? + +Contact your PlanetScale migration team representative. diff --git a/pgcopydb-templates/aws-terraform/pgcopydb-migration-instance.tf b/pgcopydb-templates/aws-terraform/pgcopydb-migration-instance.tf new file mode 100644 index 0000000..3f808ed --- /dev/null +++ b/pgcopydb-templates/aws-terraform/pgcopydb-migration-instance.tf @@ -0,0 +1,275 @@ +# AWS Terraform Template for pgcopydb Migration Instance +# Generated by PlanetScale Liftoff Migration Reviewer +# Generated on: 2026-04-06 +# +# This template creates an EC2 instance pre-configured with pgcopydb +# for PostgreSQL-to-PlanetScale migrations. +# +# Usage: +# 1. Save this file as main.tf and the user-data script as user-data.sh +# 2. Run: terraform init +# 3. Run: terraform plan -var="vpc_id=vpc-xxx" -var="subnet_id=subnet-xxx" +# 4. Run: terraform apply -var="vpc_id=vpc-xxx" -var="subnet_id=subnet-xxx" + +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region +} + +# ============================================================================= +# Variables +# ============================================================================= + +variable "resource_prefix" { + description = "Prefix for all resource names" + type = string + default = "planetscale-migration" + + validation { + condition = can(regex("^[a-z0-9-]+$", var.resource_prefix)) + error_message = "Must be lowercase alphanumeric with hyphens only." + } +} + +variable "vpc_id" { + description = "VPC ID where the instance will be launched" + type = string +} + +variable "subnet_id" { + description = "Public subnet ID with internet gateway route (required for PlanetScale connectivity)" + type = string +} + +variable "instance_type" { + description = "EC2 instance type (c7i.xlarge for <100GB, c7i.2xlarge for 100-500GB, c7i.4xlarge for >500GB)" + type = string + default = "c7i.xlarge" + + validation { + condition = can(regex("^c7i\\.", var.instance_type)) + error_message = "Instance type must be c7i family." + } +} + +variable "volume_size" { + description = "EBS volume size in GB (io2 Block Express)" + type = number + default = 1000 + + validation { + condition = contains([500, 1000, 3000, 6000, 12000], var.volume_size) + error_message = "Volume size must be one of: 500, 1000, 3000, 6000, 12000." + } +} + +variable "key_pair_name" { + description = "Optional EC2 Key Pair for key-based SSH access. Leave empty to use EC2 Instance Connect only." + type = string + default = "" +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-1" +} + +# ============================================================================= +# Local Values +# ============================================================================= + +locals { + # IOPS mapping matching CloudFormation VolumeConfig + volume_iops = { + 500 = 30000 + 1000 = 40000 + 3000 = 50000 + 6000 = 60000 + 12000 = 70000 + } + + tags = { + Name = "${var.resource_prefix}-ec2" + Purpose = "PostgreSQL to PlanetScale Migration" + } +} + +# ============================================================================= +# AMI Lookup - Ubuntu 24.04 LTS (same as CloudFormation SSM parameter) +# ============================================================================= + +data "aws_ssm_parameter" "ubuntu_ami" { + name = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id" +} + +# ============================================================================= +# IAM Role + Instance Profile +# ============================================================================= + +resource "aws_iam_role" "instance_role" { + name = "${var.resource_prefix}-ec2-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "ec2.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) + + tags = merge(local.tags, { + Name = "${var.resource_prefix}-ec2-role" + }) +} + +resource "aws_iam_role_policy_attachment" "cloudwatch" { + role = aws_iam_role.instance_role.name + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" +} + +resource "aws_iam_role_policy_attachment" "ssm" { + role = aws_iam_role.instance_role.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_instance_profile" "instance_profile" { + name = "${var.resource_prefix}-ec2-profile" + role = aws_iam_role.instance_role.name +} + +# ============================================================================= +# Security Group +# ============================================================================= + +resource "aws_security_group" "pgcopydb" { + name_prefix = "${var.resource_prefix}-sg" + description = "Security group for pgcopydb migration instance" + vpc_id = var.vpc_id + + tags = merge(local.tags, { + Name = "${var.resource_prefix}-sg" + }) +} + +resource "aws_vpc_security_group_egress_rule" "all_outbound" { + security_group_id = aws_security_group.pgcopydb.id + description = "Allow all outbound traffic" + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_ingress_rule" "ssh" { + security_group_id = aws_security_group.pgcopydb.id + description = "SSH access for EC2 Instance Connect and optional key-based SSH" + ip_protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_ingress_rule" "icmp_dest_unreachable" { + security_group_id = aws_security_group.pgcopydb.id + description = "ICMP Destination Unreachable for path MTU discovery" + ip_protocol = "icmp" + from_port = 3 + to_port = -1 + cidr_ipv4 = "0.0.0.0/0" +} + +# ============================================================================= +# EC2 Instance +# ============================================================================= + +resource "aws_instance" "pgcopydb" { + ami = data.aws_ssm_parameter.ubuntu_ami.value + instance_type = var.instance_type + key_name = var.key_pair_name != "" ? var.key_pair_name : null + iam_instance_profile = aws_iam_instance_profile.instance_profile.name + subnet_id = var.subnet_id + vpc_security_group_ids = [aws_security_group.pgcopydb.id] + + root_block_device { + volume_size = var.volume_size + volume_type = "io2" + iops = local.volume_iops[var.volume_size] + delete_on_termination = true + encrypted = true + } + + metadata_options { + http_tokens = "required" + http_put_response_hop_limit = 1 + } + + user_data = file("${path.module}/user-data.sh") + + tags = local.tags + + depends_on = [ + aws_iam_role_policy_attachment.cloudwatch, + aws_iam_role_policy_attachment.ssm, + ] +} + +# ============================================================================= +# Outputs +# ============================================================================= + +output "instance_id" { + description = "EC2 Instance ID" + value = aws_instance.pgcopydb.id +} + +output "public_ip" { + description = "Public IP address (for PlanetScale connectivity)" + value = aws_instance.pgcopydb.public_ip +} + +output "private_ip" { + description = "Private IP address (for VPC database connectivity)" + value = aws_instance.pgcopydb.private_ip +} + +output "security_group_id" { + description = "Security Group ID" + value = aws_security_group.pgcopydb.id +} + +output "ssh_command" { + description = "SSH connection command" + value = var.key_pair_name != "" ? "ssh -i /path/to/${var.key_pair_name}.pem ubuntu@${aws_instance.pgcopydb.public_ip}" : "Use EC2 Instance Connect or Session Manager" +} + +output "ec2_instance_connect_command" { + description = "EC2 Instance Connect command (no key required)" + value = "aws ec2-instance-connect send-ssh-public-key --instance-id ${aws_instance.pgcopydb.id} --instance-os-user ubuntu --ssh-public-key file://~/.ssh/id_rsa.pub" +} + +output "session_manager_command" { + description = "AWS Session Manager connection command" + value = "aws ssm start-session --target ${aws_instance.pgcopydb.id}" +} + +output "quick_start" { + description = "Quick start guide" + value = <<-EOT + 1. Connect via EC2 Instance Connect or SSH + 2. Verify: pgcopydb --version + 3. See ~/README.md for usage instructions + EOT +} diff --git a/pgcopydb-templates/aws-terraform/user-data.sh b/pgcopydb-templates/aws-terraform/user-data.sh new file mode 100644 index 0000000..2a14349 --- /dev/null +++ b/pgcopydb-templates/aws-terraform/user-data.sh @@ -0,0 +1,152 @@ +#!/bin/bash -xe +# User-data script for pgcopydb migration instance +# This script runs on first boot to install and configure pgcopydb +# Matches the CloudFormation cfn-init configuration + +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + +echo "==========================================" +echo "pgcopydb Migration Instance Setup" +echo "Started at: $(date)" +echo "==========================================" + +export DEBIAN_FRONTEND=noninteractive + +# ============================================================================= +# Install Prerequisites +# ============================================================================= +echo "Updating system packages..." +apt-get update -y +apt-get install -y wget gnupg2 lsb-release curl unzip ca-certificates netcat-openbsd sqlite3 + +# ============================================================================= +# Install PostgreSQL 17 +# ============================================================================= +echo "Installing PostgreSQL 17..." +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list +apt-get update +apt-get install -y postgresql-client-17 postgresql-17 postgresql-server-dev-17 + +# ============================================================================= +# Install Build Tools +# ============================================================================= +echo "Installing build dependencies..." +apt-get install -y \ + build-essential \ + git \ + libssl-dev \ + libpq-dev \ + libgc-dev \ + liblz4-dev \ + libpam0g-dev \ + libxml2-dev \ + libxslt1-dev \ + libreadline-dev \ + zlib1g-dev \ + libncurses5-dev \ + libkrb5-dev \ + libselinux1-dev \ + libzstd-dev + +# ============================================================================= +# Build pgcopydb from Source +# ============================================================================= +echo "Building pgcopydb from source..." +cd /tmp +git clone --branch v0.18.0 https://github.com/planetscale/pgcopydb.git +cd pgcopydb +export PATH=/usr/lib/postgresql/17/bin:$PATH +make clean || true +make +make install +ldconfig + +# ============================================================================= +# Install AWS CLI +# ============================================================================= +echo "Installing AWS CLI..." +cd /tmp +curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip -q awscliv2.zip +./aws/install +rm -rf aws awscliv2.zip + +# ============================================================================= +# Install CloudWatch Agent +# ============================================================================= +echo "Installing CloudWatch agent..." +cd /tmp +wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb +dpkg -i amazon-cloudwatch-agent.deb +rm -f amazon-cloudwatch-agent.deb + +# ============================================================================= +# System Configuration +# ============================================================================= + +# File descriptor limits +cat > /etc/security/limits.d/99-pgcopydb.conf << 'LIMITS_EOF' +* soft nofile 65536 +* hard nofile 65536 +LIMITS_EOF + +# Sysctl tuning for high-throughput migrations +cat > /etc/sysctl.d/99-pgcopydb.conf << 'SYSCTL_EOF' +net.ipv4.tcp_keepalive_time = 60 +net.ipv4.tcp_keepalive_intvl = 10 +net.ipv4.tcp_keepalive_probes = 6 +net.core.rmem_max = 134217728 +net.core.wmem_max = 134217728 +net.ipv4.tcp_rmem = 4096 87380 67108864 +net.ipv4.tcp_wmem = 4096 65536 67108864 +SYSCTL_EOF +sysctl -p /etc/sysctl.d/99-pgcopydb.conf + +# PATH configuration +cat > /etc/profile.d/pgcopydb.sh << 'PROFILE_EOF' +export PATH=/usr/lib/postgresql/17/bin:$PATH +alias pgcopydb-version='pgcopydb --version' +alias psql-version='psql --version' +alias check-planetscale='nc -zv app.connect.psdb.cloud 443 2>&1 | grep succeeded' +PROFILE_EOF + +# .env file +cat > /home/ubuntu/.env << 'ENV_EOF' +# PlanetScale Migration Environment Variables +# Edit these values before running the migration + +# Source Database +PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmode=require" + +# Target Database (PlanetScale) +PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" +ENV_EOF +chmod 600 /home/ubuntu/.env +chown ubuntu:ubuntu /home/ubuntu/.env + +# ============================================================================= +# Pull PlanetScale Migration Helper Scripts +# ============================================================================= +echo "Cloning PlanetScale migration helper scripts..." +git clone --depth 1 https://github.com/planetscale/migration-scripts.git /tmp/migration-scripts +cp -r /tmp/migration-scripts/pgcopydb-helpers/* /home/ubuntu/ +rm -rf /tmp/migration-scripts +chown ubuntu:ubuntu /home/ubuntu/*.sh /home/ubuntu/*.md +chmod +x /home/ubuntu/*.sh + +# ============================================================================= +# Create Directories & Verify +# ============================================================================= +mkdir -p /var/lib/pgcopydb && chmod 755 /var/lib/pgcopydb + +echo "=== Installation verification ===" >> /var/log/user-data-verification.log +export PATH=/usr/lib/postgresql/17/bin:$PATH +pgcopydb --version >> /var/log/user-data-verification.log 2>&1 || echo "pgcopydb installation failed" >> /var/log/user-data-verification.log +psql --version >> /var/log/user-data-verification.log 2>&1 || echo "PostgreSQL client installation failed" >> /var/log/user-data-verification.log +aws --version >> /var/log/user-data-verification.log 2>&1 || echo "AWS CLI installation failed" >> /var/log/user-data-verification.log + +echo "==========================================" +echo "Setup completed successfully!" +echo "Finished at: $(date)" +echo "==========================================" diff --git a/pgcopydb-templates/gcp-terraform/README-pgcopydb-gcp.md b/pgcopydb-templates/gcp-terraform/README-pgcopydb-gcp.md new file mode 100644 index 0000000..4dab90f --- /dev/null +++ b/pgcopydb-templates/gcp-terraform/README-pgcopydb-gcp.md @@ -0,0 +1,58 @@ +# PlanetScale Migration — pgcopydb Migration Instance (GCP Terraform) + +## What This Does + +This Terraform template creates a Compute Engine instance pre-configured with +[pgcopydb](https://pgcopydb.readthedocs.io/), the tool PlanetScale uses to migrate +your PostgreSQL data. The instance runs Ubuntu 24.04 and pulls the latest +[PlanetScale migration helper scripts](https://github.com/planetscale/migration-scripts) +at boot. + +## What Gets Created + +- A Compute Engine instance with pgcopydb and PostgreSQL client tools installed +- A persistent disk for migration working data +- A firewall rule for SSH access via IAP tunnel +- A startup script that installs all required tools +- Migration helper scripts from `github.com/planetscale/migration-scripts` in `/home/ubuntu/` + +## How to Deploy + +### Prerequisites +- [Terraform](https://developer.hashicorp.com/terraform/install) installed (v1.0+) +- A GCP project with a VPC and subnet +- `gcloud` CLI authenticated + +### Steps + +1. **Save both files** (`pgcopydb-migration-instance.tf` and `startup-script.sh`) in the same directory + +2. **Deploy** + ```bash + terraform init + terraform apply \ + -var="project_id=YOUR_PROJECT" \ + -var="vpc_name=YOUR_VPC" \ + -var="subnet_name=YOUR_SUBNET" + ``` + +3. **Connect** via IAP tunnel: + ```bash + gcloud compute ssh $(terraform output -raw instance_name) \ + --zone=$(terraform output -raw zone) \ + --project=YOUR_PROJECT \ + --tunnel-through-iap + ``` + +## How to Tear Down + +```bash +terraform destroy \ + -var="project_id=YOUR_PROJECT" \ + -var="vpc_name=YOUR_VPC" \ + -var="subnet_name=YOUR_SUBNET" +``` + +## Questions? + +Contact your PlanetScale migration team representative. diff --git a/pgcopydb-templates/gcp-terraform/pgcopydb-migration-instance.tf b/pgcopydb-templates/gcp-terraform/pgcopydb-migration-instance.tf new file mode 100644 index 0000000..39fdf2d --- /dev/null +++ b/pgcopydb-templates/gcp-terraform/pgcopydb-migration-instance.tf @@ -0,0 +1,342 @@ +# GCP Terraform Template for pgcopydb Migration Instance +# Generated by PlanetScale Liftoff Migration Reviewer +# Generated on: 2026-04-06 +# +# This template creates a compute instance pre-configured with pgcopydb +# for PostgreSQL-to-PlanetScale migrations. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region + + # Service account impersonation for PlanetScale-managed migrations + impersonate_service_account = var.impersonate_service_account + impersonate_service_account_delegates = var.impersonate_delegates +} + +# Variables +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "impersonate_service_account" { + description = "Service account to impersonate (the customer's migration SA)" + type = string + default = null +} + +variable "impersonate_delegates" { + description = "Delegation chain for impersonation (list of intermediate SAs)" + type = list(string) + default = null +} + +variable "region" { + description = "GCP region for resources" + type = string + default = "us-central1" +} + +variable "zone" { + description = "GCP zone for compute instance" + type = string + default = "us-central1-a" +} + +variable "vpc_name" { + description = "Name of existing VPC network" + type = string +} + +variable "subnet_name" { + description = "Name of existing subnet" + type = string +} + +variable "service_account_email" { + description = "Email of the migration service account (created by planetscale-gcp-iam template)" + type = string +} + +variable "ssh_public_key" { + description = "SSH public key for accessing the instance. If provided, OS Login is disabled and key-based SSH is used. If omitted, OS Login is enabled (requires direct IAM access)." + type = string + default = null +} + +variable "machine_type" { + description = "Compute instance machine type" + type = string + default = "n2-standard-8" + + validation { + condition = can(regex("^(n2-standard|c3d-standard)", var.machine_type)) + error_message = "Machine type must be n2-standard or c3d-standard family." + } +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 500 + + validation { + condition = var.disk_size_gb >= 100 && var.disk_size_gb <= 10000 + error_message = "Disk size must be between 100 and 10000 GB." + } +} + +variable "disk_type" { + description = "Persistent disk type (pd-ssd, pd-balanced, or pd-standard)" + type = string + default = "pd-ssd" + + validation { + condition = contains(["pd-ssd", "pd-balanced", "pd-standard"], var.disk_type) + error_message = "Disk type must be pd-ssd, pd-balanced, or pd-standard." + } +} + +variable "resource_prefix" { + description = "Prefix for resource names" + type = string + default = "planetscale-migration" + + validation { + condition = can(regex("^[a-z][-a-z0-9]*$", var.resource_prefix)) + error_message = "Resource prefix must start with lowercase letter and contain only lowercase letters, numbers, and hyphens." + } +} + +# Enable required APIs +resource "google_project_service" "compute" { + service = "compute.googleapis.com" + disable_on_destroy = false +} + +resource "google_project_service" "iap" { + service = "iap.googleapis.com" + disable_on_destroy = false +} + +# Data sources for existing network resources +data "google_compute_network" "vpc" { + name = var.vpc_name + + depends_on = [google_project_service.compute] +} + +data "google_compute_subnetwork" "subnet" { + name = var.subnet_name + region = var.region + + depends_on = [google_project_service.compute] +} + +# Firewall Rules +resource "google_compute_firewall" "allow_ssh" { + name = "${var.resource_prefix}-allow-ssh" + network = data.google_compute_network.vpc.id + + allow { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["${var.resource_prefix}-instance"] + + description = "Allow SSH access to migration instance" +} + +resource "google_compute_firewall" "allow_iap" { + name = "${var.resource_prefix}-allow-iap" + network = data.google_compute_network.vpc.id + + allow { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["35.235.240.0/20"] + target_tags = ["${var.resource_prefix}-instance"] + + description = "Allow IAP tunneling for SSH" +} + +resource "google_compute_firewall" "allow_internal" { + name = "${var.resource_prefix}-allow-internal" + network = data.google_compute_network.vpc.id + + allow { + protocol = "tcp" + } + + allow { + protocol = "udp" + } + + allow { + protocol = "icmp" + } + + source_ranges = [data.google_compute_subnetwork.subnet.ip_cidr_range] + target_tags = ["${var.resource_prefix}-instance"] + + description = "Allow internal VPC traffic" +} + +resource "google_compute_firewall" "allow_icmp" { + name = "${var.resource_prefix}-allow-icmp" + network = data.google_compute_network.vpc.id + + allow { + protocol = "icmp" + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["${var.resource_prefix}-instance"] + + description = "Allow ICMP for path MTU discovery" +} + +# Compute Instance +resource "google_compute_instance" "pgcopydb_instance" { + name = "${var.resource_prefix}-instance" + machine_type = var.machine_type + zone = var.zone + + tags = ["${var.resource_prefix}-instance"] + + labels = { + purpose = "postgresql-migration" + tool = "pgcopydb" + environment = "migration" + managed_by = "terraform" + } + + boot_disk { + initialize_params { + image = "ubuntu-os-cloud/ubuntu-2404-lts-amd64" + size = var.disk_size_gb + type = var.disk_type + } + } + + network_interface { + network = data.google_compute_network.vpc.id + subnetwork = data.google_compute_subnetwork.subnet.id + + access_config { + # Ephemeral external IP + } + } + + service_account { + email = var.service_account_email + scopes = ["cloud-platform"] + } + + metadata = var.ssh_public_key != null ? { + ssh-keys = "ubuntu:${var.ssh_public_key}" + } : { + enable-oslogin = "TRUE" + } + + metadata_startup_script = file("${path.module}/startup-script.sh") + + allow_stopping_for_update = true + + depends_on = [ + google_project_service.compute, + google_compute_firewall.allow_ssh, + google_compute_firewall.allow_iap, + google_compute_firewall.allow_internal + ] +} + +# Outputs +output "instance_name" { + description = "Name of the compute instance" + value = google_compute_instance.pgcopydb_instance.name +} + +output "instance_id" { + description = "ID of the compute instance" + value = google_compute_instance.pgcopydb_instance.id +} + +output "external_ip" { + description = "External IP address of the instance" + value = google_compute_instance.pgcopydb_instance.network_interface[0].access_config[0].nat_ip +} + +output "internal_ip" { + description = "Internal IP address of the instance" + value = google_compute_instance.pgcopydb_instance.network_interface[0].network_ip +} + +output "zone" { + description = "Zone where instance is deployed" + value = google_compute_instance.pgcopydb_instance.zone +} + +output "service_account_email" { + description = "Service account email" + value = var.service_account_email +} + +output "ssh_command" { + description = "Command to SSH into the instance" + value = "gcloud compute ssh ${google_compute_instance.pgcopydb_instance.name} --zone=${var.zone} --project=${var.project_id}" +} + +output "iap_tunnel_command" { + description = "Command to SSH via IAP tunnel (works without external IP)" + value = "gcloud compute ssh ${google_compute_instance.pgcopydb_instance.name} --zone=${var.zone} --project=${var.project_id} --tunnel-through-iap" +} + +output "console_url" { + description = "GCP Console URL for the instance" + value = "https://console.cloud.google.com/compute/instancesDetail/zones/${var.zone}/instances/${google_compute_instance.pgcopydb_instance.name}?project=${var.project_id}" +} + +output "migration_scripts_path" { + description = "Path to the migration helper scripts on the instance" + value = "/home/ubuntu/" +} + +output "quick_start" { + description = "Quick start instructions" + value = <<-EOT + 🚀 Quick Start Guide: + + 1. Wait 10-15 minutes for instance initialization to complete + + 2. Connect to the instance: + gcloud compute ssh ${google_compute_instance.pgcopydb_instance.name} --zone=${var.zone} --project=${var.project_id} + + 3. Check installation logs: + sudo tail -f /var/log/syslog | grep startup-script + + 4. Verify installations: + pgcopydb --version + psql --version + + 5. See ~/README.md for usage instructions + + 📖 Helper scripts location: /home/ubuntu/ + 🌐 Console: https://console.cloud.google.com/compute/instancesDetail/zones/${var.zone}/instances/${google_compute_instance.pgcopydb_instance.name}?project=${var.project_id} + EOT +} diff --git a/pgcopydb-templates/gcp-terraform/startup-script.sh b/pgcopydb-templates/gcp-terraform/startup-script.sh new file mode 100644 index 0000000..b71bef4 --- /dev/null +++ b/pgcopydb-templates/gcp-terraform/startup-script.sh @@ -0,0 +1,127 @@ +#!/bin/bash +# Startup script for pgcopydb migration instance +# This script runs on first boot to install and configure pgcopydb + +set -e + +# Logging +exec > >(tee -a /var/log/pgcopydb-setup.log) +exec 2>&1 + +echo "==========================================" +echo "pgcopydb Migration Instance Setup" +echo "Started at: $(date)" +echo "==========================================" + +export DEBIAN_FRONTEND=noninteractive + +# ============================================================================= +# Install Prerequisites +# ============================================================================= +echo "Updating system packages..." +apt-get update -y +apt-get install -y wget gnupg2 lsb-release curl unzip ca-certificates netcat-openbsd sqlite3 + +# ============================================================================= +# Install PostgreSQL 17 +# ============================================================================= +echo "Installing PostgreSQL 17..." +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list +apt-get update +apt-get install -y postgresql-client-17 postgresql-17 postgresql-server-dev-17 + +# ============================================================================= +# Install Build Tools +# ============================================================================= +echo "Installing build dependencies..." +apt-get install -y \ + build-essential \ + git \ + libssl-dev \ + libpq-dev \ + libgc-dev \ + liblz4-dev \ + libpam0g-dev \ + libxml2-dev \ + libxslt1-dev \ + libreadline-dev \ + zlib1g-dev \ + libncurses5-dev \ + libkrb5-dev \ + libselinux1-dev \ + libzstd-dev + +# ============================================================================= +# Build pgcopydb from Source +# ============================================================================= +echo "Building pgcopydb from source..." +cd /tmp +git clone --branch v0.18.0 https://github.com/planetscale/pgcopydb.git +cd pgcopydb +export PATH=/usr/lib/postgresql/17/bin:$PATH +make clean || true +make +make install +ldconfig + +# ============================================================================= +# System Configuration +# ============================================================================= + +# File descriptor limits +cat > /etc/security/limits.d/99-pgcopydb.conf << 'LIMITS_EOF' +* soft nofile 65536 +* hard nofile 65536 +LIMITS_EOF + +# Sysctl tuning for high-throughput migrations +cat > /etc/sysctl.d/99-pgcopydb.conf << 'SYSCTL_EOF' +net.ipv4.tcp_keepalive_time = 60 +net.ipv4.tcp_keepalive_intvl = 10 +net.ipv4.tcp_keepalive_probes = 6 +net.core.rmem_max = 134217728 +net.core.wmem_max = 134217728 +net.ipv4.tcp_rmem = 4096 87380 67108864 +net.ipv4.tcp_wmem = 4096 65536 67108864 +SYSCTL_EOF +sysctl -p /etc/sysctl.d/99-pgcopydb.conf + +# PATH configuration +cat > /etc/profile.d/pgcopydb.sh << 'PROFILE_EOF' +export PATH=/usr/lib/postgresql/17/bin:$PATH +alias pgcopydb-version='pgcopydb --version' +alias psql-version='psql --version' +alias check-planetscale='nc -zv app.connect.psdb.cloud 443 2>&1 | grep succeeded' +PROFILE_EOF + +# .env file +cat > /home/ubuntu/.env << 'ENV_EOF' +# PlanetScale Migration Environment Variables +# Edit these values before running the migration + +# Source Database +PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmode=require" + +# Target Database (PlanetScale) +PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" +ENV_EOF +chmod 600 /home/ubuntu/.env +chown ubuntu:ubuntu /home/ubuntu/.env + +# Pull PlanetScale migration helper scripts +echo "Cloning PlanetScale migration helper scripts..." +git clone --depth 1 https://github.com/planetscale/migration-scripts.git /tmp/migration-scripts +cp -r /tmp/migration-scripts/pgcopydb-helpers/* /home/ubuntu/ +rm -rf /tmp/migration-scripts +chown ubuntu:ubuntu /home/ubuntu/*.sh /home/ubuntu/*.md +chmod +x /home/ubuntu/*.sh + +echo "" +echo "==========================================" +echo "Setup completed successfully!" +echo "Finished at: $(date)" +echo "==========================================" +echo "" +echo "Migration helper scripts installed at: /home/ubuntu/" +echo ""