From 1711b9f826c3b9840e5973d073f77b82eeebcf67 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Oct 06 2023 16:44:59 +0000 Subject: [PATCH 1/3] Add Fedora 39 as a target Also, drop Fedora 36 and drop some code that doesn't apply to F37+. --- diff --git a/run-archive-repo-update b/run-archive-repo-update index 22b5b1b..f03010b 100755 --- a/run-archive-repo-update +++ b/run-archive-repo-update @@ -5,9 +5,9 @@ set -eu -o pipefail LOCAL_WORKDIR='/var/archive-repo-manager' PREVIOUS_RUNS_DATESTAMP_FILE=previous-run-datestamps.txt ARCHES="aarch64 ppc64le s390x x86_64" -declare -A SIGNINGKEYS=( [36]='38ab71f4' - [37]='5323552a' - [38]='eb10b464' ) +declare -A SIGNINGKEYS=( [37]='5323552a' + [38]='eb10b464' + [39]='18b8e74c' ) # A function to get the list of builds in the updates repo # at a given timestamp. @@ -22,11 +22,6 @@ main() { # Get the release to operate against release=$1 - # 32 bit ARM existed prior to F37 - if [ "${release}" -lt '37' ]; then - ARCHES+=" armhfp" - fi - # Check the repo structure to make sure it has been initialized # (the directories have been created): if [ ! -d "fedora/${release}" ]; then From 6a681c7ba4cd46a14df9d79ee39018e1b7e07628 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Oct 06 2023 16:47:15 +0000 Subject: [PATCH 2/3] Move to Fedora Linux 39 base --- diff --git a/Dockerfile b/Dockerfile index 0ac9c5a..9734091 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:38 +FROM registry.fedoraproject.org/fedora:39 # Get any latest updates since last container spin RUN dnf update -y From 95beae8b9ba169e99f8acf08948155f79b03a1ab Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Oct 06 2023 19:13:05 +0000 Subject: [PATCH 3/3] Switch provisioning to use tofu (open source terraform) --- diff --git a/.gitignore b/.gitignore index 2a036a6..5dfe310 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -config.ign.json +.terraform* +terraform.tfstate* diff --git a/README.md b/README.md index 987362f..0741e9a 100644 --- a/README.md +++ b/README.md @@ -67,55 +67,8 @@ the s3fs mount and run /usr/local/lib/archive_repo_manager.py directly. ## Rough notes for running archive-repo-manager on FCOS: -First set environment variables with the secrets so they get -substituted and create the Ignition configuration. - -``` -HISTCONTROL='ignoreboth' - export S3BUCKET=dustymabe-archive-repo-poc - export AWSACCESSKEYID= - export AWSSECRETACCESSKEY= - export AWS_ACCESS_KEY_ID= - export AWS_SECRET_ACCESS_KEY= - export SSH_KEY=$(cat id_rsa.pub) -cat archive-repo-manager.bu | envsubst | butane --pretty > archive-repo-manager.ign -``` - -Then launch the FCOS machine with the `archive-repo-manager.ign`. Here -is an example doing that in AWS: - -``` -# Add your credentials for EC2 to the environment -HISTCONTROL='ignoreboth' - export AWS_DEFAULT_REGION=us-east-1 - export AWS_ACCESS_KEY_ID=XXXX - export AWS_SECRET_ACCESS_KEY=YYYYYYYY - -# Bring the instance up with appropriate details -NAME="archive-repo-manager-$(date +%Y%m%d)" -AMI='ami-0560e1583a204b92b' -TYPE='t4g.medium' -DISK='20' -SUBNET='subnet-0732e4cda7466a2ae' -SECURITY_GROUPS='sg-7d0b4c05' -USERDATA="${PWD}/archive-repo-manager.ign" -aws ec2 run-instances \ - --output json \ - --image-id $AMI \ - --instance-type $TYPE \ - --subnet-id $SUBNET \ - --security-group-ids $SECURITY_GROUPS \ - --user-data "file://${USERDATA}" \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=${NAME}}]" \ - --block-device-mappings "VirtualName=/dev/xvda,DeviceName=/dev/xvda,Ebs={VolumeSize=${DISK},VolumeType=gp3}" \ - > out.json - -# Get IP and ssh in -INSTANCE=$(jq --raw-output .Instances[0].InstanceId out.json) -IP=$(aws ec2 describe-instances --instance-ids $INSTANCE --output json \ - | jq -r '.Reservations[0].Instances[0].PublicIpAddress') -ssh "core@${IP}" -``` +Follow the [README](provisioning/README.md) in the provisioning directory +to bring up the instance using `tofu` in AWS. After logging in you can switch to the `worker` user and monitor the systemd user units: diff --git a/archive-repo-manager.bu b/archive-repo-manager.bu index 158c86f..d1fda71 100644 --- a/archive-repo-manager.bu +++ b/archive-repo-manager.bu @@ -4,7 +4,7 @@ passwd: users: - name: core ssh_authorized_keys: - - $SSH_KEY + - ${core_user_ssh_pubkey_string} - name: worker kernel_arguments: should_exist: @@ -64,11 +64,11 @@ storage: name: worker contents: inline: | - S3BUCKET=$S3BUCKET - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - AWSACCESSKEYID=$AWSACCESSKEYID - AWSSECRETACCESSKEY=$AWSSECRETACCESSKEY + S3BUCKET=${s3_bucket} + AWS_ACCESS_KEY_ID=${aws_access_key_id} + AWS_SECRET_ACCESS_KEY=${aws_secret_access_key} + AWSACCESSKEYID=${aws_access_key_id} + AWSSECRETACCESSKEY=${aws_secret_access_key} - path: /home/worker/.config/containers/systemd/archive-repo-manager.container mode: 0644 user: diff --git a/provisioning/README.md b/provisioning/README.md new file mode 100644 index 0000000..3c052d0 --- /dev/null +++ b/provisioning/README.md @@ -0,0 +1,43 @@ + +If this is the first time being run then: + +``` +tofu init +``` + +Set the following environment variables to pass through to `tofu` +that will get substituted into the butane config: + +``` + export TF_VAR_core_user_ssh_pubkey_string= + export TF_VAR_s3_bucket= + export TF_VAR_aws_access_key_id= + export TF_VAR_aws_secret_access_key= +``` + +Configure AWS credentials in your environment (env variables or profile +files in expected locations). For example: + +``` + export AWS_REGION=us-east-1 + export AWS_ACCESS_KEY_ID=... + export AWS_SECRET_ACCESS_KEY=... +``` + +To bring up an instance and output the IP for you to SSH to: + +``` +tofu apply +``` + +To bring down just the instance: + +``` +tofu destroy -target aws_instance.archive-repo-manager +``` + +To bring down all resources: + +``` +tofu destroy +``` diff --git a/provisioning/main.tf b/provisioning/main.tf new file mode 100644 index 0000000..e10b903 --- /dev/null +++ b/provisioning/main.tf @@ -0,0 +1,81 @@ +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.13.0" + } + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + http = { + source = "hashicorp/http" + version = "2.1.0" + } + } +} + +provider "aws" {} +provider "ct" {} +provider "http" {} + +variable "core_user_ssh_pubkey_string" { + type = string +} + +variable "s3_bucket" { + type = string +} + +variable "aws_access_key_id" { + type = string +} + +variable "aws_secret_access_key" { + type = string +} + +data "aws_region" "aws_region" {} + +data "ct_config" "butane" { + content = templatefile("../archive-repo-manager.bu", { + core_user_ssh_pubkey_string = var.core_user_ssh_pubkey_string + s3_bucket=var.s3_bucket + aws_access_key_id=var.aws_access_key_id + aws_secret_access_key=var.aws_secret_access_key + }) + strict = true +} + +# Gather information about the AWS image for the current region +data "http" "stream_metadata" { + url = "https://builds.coreos.fedoraproject.org/streams/stable.json" + + request_headers = { + Accept = "application/json" + } +} +# Lookup the aarch64 AWS image for the current AWS region +locals { + ami = lookup(jsondecode(data.http.stream_metadata.body).architectures.aarch64.images.aws.regions, data.aws_region.aws_region.name).image +} + +resource "aws_instance" "archive-repo-manager" { + tags = { + Name = "archive-repo-manager-${formatdate("YYYYMMDD", timestamp())}" + } + ami = local.ami + instance_type = "t4g.medium" + user_data = data.ct_config.butane.rendered + associate_public_ip_address = "true" + vpc_security_group_ids = [aws_security_group.sg.id] + subnet_id = aws_subnet.private_subnets[0].id + root_block_device { + volume_size = "20" + volume_type = "gp3" + } +} + +output "instance_ip_addr" { + value = aws_instance.archive-repo-manager.public_ip +} diff --git a/provisioning/networks.tf b/provisioning/networks.tf new file mode 100644 index 0000000..2f96a96 --- /dev/null +++ b/provisioning/networks.tf @@ -0,0 +1,47 @@ +resource "aws_vpc" "vpc" { + cidr_block = "172.31.0.0/16" + tags = { + Name = "archive-repo-manager-vpc" + } +} + +resource "aws_internet_gateway" "gw" { + vpc_id = aws_vpc.vpc.id +} + +data "aws_availability_zones" "azs" { + state = "available" +} + +variable "private_subnet_cidrs" { + type = list(string) + description = "Private Subnet CIDR values" + default = ["172.31.1.0/24", "172.31.2.0/24", "172.31.3.0/24", "172.31.4.0/24", "172.31.5.0/24", "172.31.6.0/24", "172.31.7.0/24", "172.31.8.0/24"] +} + +resource "aws_subnet" "private_subnets" { + count = length(data.aws_availability_zones.azs.names) + vpc_id = aws_vpc.vpc.id + cidr_block = element(var.private_subnet_cidrs, count.index) + availability_zone = element(data.aws_availability_zones.azs.names, count.index) + tags = { + Name = "archive-repo-manager-private-subnet-${count.index + 1}" + } +} + + +resource "aws_route_table" "internet_route" { + vpc_id = aws_vpc.vpc.id + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gw.id + } + tags = { + Name = "archive-repo-manager-ig" + } +} + +resource "aws_main_route_table_association" "public-set-main-default-rt-assoc" { + vpc_id = aws_vpc.vpc.id + route_table_id = aws_route_table.internet_route.id +} diff --git a/provisioning/security-groups.tf b/provisioning/security-groups.tf new file mode 100644 index 0000000..3e4b37e --- /dev/null +++ b/provisioning/security-groups.tf @@ -0,0 +1,24 @@ +resource "aws_security_group" "sg" { + name = "archive-repo-manager-security-group" + description = "Allow SSH inbound traffic only" + vpc_id = aws_vpc.vpc.id + + ingress { + description = "SSH access" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "archive-repo-manager-security-group" + } +}