From 9def761076892f1bbcb825c9ff8d5667188f8f4c Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Feb 07 2025 14:52:05 +0000 Subject: aws: mark the imported snapshots as public This makes the snapshots we use to create the AMIs public and applies a set of tags to them to ensure they're cleaned up. Fixes #44 Signed-off-by: Jeremy Cline --- diff --git a/fedora-image-uploader/fedora_image_uploader/aws.py b/fedora-image-uploader/fedora_image_uploader/aws.py index 4fd9a31..05dd7bf 100644 --- a/fedora-image-uploader/fedora_image_uploader/aws.py +++ b/fedora-image-uploader/fedora_image_uploader/aws.py @@ -65,7 +65,7 @@ class Aws: try: try: user_bucket = self.aws_upload_image(image) - snapshot_id = self.aws_import_snapshot(image, user_bucket, ami_name) + snapshot_id = self.aws_import_snapshot(image, ffrel, user_bucket, ami_name) image_id = self.aws_register_image(image, ffrel, snapshot_id, ami_name) except boto_exceptions.WaiterError as e: if e.last_response["Error"].get("Code") == "ResourceLimitExceeded": @@ -134,7 +134,9 @@ class Aws: raise return {"S3Bucket": bucket, "S3Key": object_key} - def aws_import_snapshot(self, image: dict, user_bucket: dict, ami_name: str) -> str: + def aws_import_snapshot( + self, image: dict, ffrel: ff_release.Release, user_bucket: dict, ami_name: str + ) -> str: """ Import an image from S3 into an EC2 snapshot. @@ -161,9 +163,59 @@ class Aws: snapshot_imports = self.ec2_client.describe_import_snapshot_tasks( ImportTaskIds=[snapshot_import_task["ImportTaskId"]], ) - # TODO exception handling snapshot_import_task = snapshot_imports["ImportSnapshotTasks"][0] - return snapshot_import_task["SnapshotTaskDetail"]["SnapshotId"] + snapshot_id = snapshot_import_task["SnapshotTaskDetail"]["SnapshotId"] + _log.info("Successfully imported snapshot as %s", snapshot_id) + + eol = get_eol(ffrel) + delete_in_days = str((eol - datetime.datetime.now()).days + 7) if eol else "14" + eol = eol.strftime("%Y-%m-%d") if eol else "none" + self.ec2_client.create_tags( + Resources=[ + snapshot_id, + ], + Tags=[ + { + "Key": "fedora-compose-id", + "Value": ffrel.cid, + }, + { + "Key": "fedora-subvariant", + "Value": image["subvariant"], + }, + { + "Key": "fedora-release", + "Value": ffrel.release.lower(), + }, + { + "Key": "end-of-life", + "Value": eol, + }, + { + "Key": "Delete", + "Value": delete_in_days, + }, + { + "Key": "FedoraGroup", + "Value": "infra", + }, + ], + ) + _log.info( + "Tagged snapshot %s to be removed in %s days (EOL %s)", snapshot_id, delete_in_days, eol + ) + + self.ec2_client.modify_snapshot_attribute( + Attribute="createVolumePermission", + GroupNames=[ + "all", + ], + OperationType="add", + SnapshotId=snapshot_id, + ) + _log.info("Marked snapshot %s as public", snapshot_id) + + return snapshot_id def aws_register_image( self, image: dict, ffrel: ff_release.Release, snapshot_id: str, ami_name: str diff --git a/fedora-image-uploader/tests/test_aws.py b/fedora-image-uploader/tests/test_aws.py index c881532..1910f17 100644 --- a/fedora-image-uploader/tests/test_aws.py +++ b/fedora-image-uploader/tests/test_aws.py @@ -115,6 +115,31 @@ def test_aws_register_image_delayed_limit_exceeded(): {"ImportTaskIds": ["import-123"]}, ) ec2_stub.add_response( + "create_tags", + {"ResponseMetadata": {}}, + { + "Resources": ["snap-123"], + "Tags": [ + {"Key": "fedora-compose-id", "Value": "Compose-123"}, + {"Key": "fedora-subvariant", "Value": "Cloud_Base"}, + {"Key": "fedora-release", "Value": "40"}, + {"Key": "end-of-life", "Value": "2025-09-03"}, + {"Key": "Delete", "Value": "334"}, + {"Key": "FedoraGroup", "Value": "infra"}, + ], + }, + ) + ec2_stub.add_response( + "modify_snapshot_attribute", + {"ResponseMetadata": {}}, + { + "Attribute": "createVolumePermission", + "GroupNames": ["all"], + "OperationType": "add", + "SnapshotId": "snap-123", + }, + ) + ec2_stub.add_response( "register_image", {"ImageId": "ami-123"}, { @@ -228,6 +253,31 @@ def test_aws_register_image(): {"ImportTaskIds": ["import-123"]}, ) ec2_stub.add_response( + "create_tags", + {}, + { + "Resources": ["snap-123"], + "Tags": [ + {"Key": "fedora-compose-id", "Value": "Compose-123"}, + {"Key": "fedora-subvariant", "Value": "Cloud_Base"}, + {"Key": "fedora-release", "Value": "40"}, + {"Key": "end-of-life", "Value": "2025-09-03"}, + {"Key": "Delete", "Value": "335"}, + {"Key": "FedoraGroup", "Value": "infra"}, + ], + }, + ) + ec2_stub.add_response( + "modify_snapshot_attribute", + {}, + { + "Attribute": "createVolumePermission", + "GroupNames": ["all"], + "OperationType": "add", + "SnapshotId": "snap-123", + }, + ) + ec2_stub.add_response( "register_image", {"ImageId": "ami-123"}, { @@ -345,21 +395,206 @@ def test_aws_register_image_limit_exceeded(): }, {"ImportTaskIds": ["import-123"]}, ) + ec2_stub.add_response( + "create_tags", + {}, + { + "Resources": ["snap-123"], + "Tags": [ + {"Key": "fedora-compose-id", "Value": "Compose-123"}, + {"Key": "fedora-subvariant", "Value": "Cloud_Base"}, + {"Key": "fedora-release", "Value": "40"}, + {"Key": "end-of-life", "Value": "2025-09-03"}, + {"Key": "Delete", "Value": "218"}, + {"Key": "FedoraGroup", "Value": "infra"}, + ], + }, + ) + ec2_stub.add_response( + "modify_snapshot_attribute", + {}, + { + "Attribute": "createVolumePermission", + "GroupNames": ["all"], + "OperationType": "add", + "SnapshotId": "snap-123", + }, + ) ec2_stub.add_client_error("register_image", service_error_code="ResourceLimitExceeded") ec2_stub.activate() - consumer.handlers["aws"]( + with freezegun.freeze_time("2025-02-04"): + consumer.handlers["aws"]( + { + "type": "raw-xz", + "arch": "x86_64", + "subvariant": "Cloud_Base", + "path": "AmazonEC2/image.raw.xz", + "checksums": {"sha256": "abc123"}, + }, + mock.MagicMock( + relnum=40, label="Rc-1.14", release="40", cid="Compose-123", eol="2025-09-03" + ), + ) + + s3_stub.assert_no_pending_responses() + ec2_stub.assert_no_pending_responses() + + +@mock.patch.dict( + config.conf["consumer_config"], + { + "aws": { + "base_region": "us-east-1", + "ami_volume_type": "gp2", + "ami_volume_dev_name": "/dev/blah", + "s3_bucket_name": "bucket", + } + }, +) +def test_aws_tag_snapshot_failure(): + """ + When the snapshot can't be tagged, the message is nacked. + """ + aws_handler = aws.Aws() + aws_handler.aws_copy_image_to_regions = mock.Mock(return_value={"us-east-1": "ami-0123"}) + s3_stub = Stubber(aws_handler.s3_client) + s3_stub.add_response("head_object", {}, {"Bucket": "bucket", "Key": "abc123.raw"}) + s3_stub.activate() + + ec2_stub = Stubber(aws_handler.ec2_client) + ec2_stub.add_response( + "import_snapshot", + {"ImportTaskId": "import-123"}, { - "type": "raw-xz", - "arch": "x86_64", - "subvariant": "Cloud_Base", - "path": "AmazonEC2/image.raw.xz", - "checksums": {"sha256": "abc123"}, + "ClientToken": "abc123", + "Description": "Import Fedora-Cloud-Base-AmazonEC2.x86_64-40-1.14 from abc123.raw", + "DiskContainer": { + "Description": "Fedora-Cloud-Base-AmazonEC2.x86_64-40-1.14", + "Format": "raw", + "UserBucket": {"S3Bucket": "bucket", "S3Key": "abc123.raw"}, + }, }, - mock.MagicMock( - relnum=40, label="Rc-1.14", release="40", cid="Compose-123", eol="2025-09-03" - ), ) + ec2_stub.add_response( + "describe_import_snapshot_tasks", + {"ImportSnapshotTasks": [{"SnapshotTaskDetail": {"Status": "completed"}}]}, + {"ImportTaskIds": ["import-123"]}, + ) + ec2_stub.add_response( + "describe_import_snapshot_tasks", + { + "ImportSnapshotTasks": [ + {"SnapshotTaskDetail": {"Status": "completed", "SnapshotId": "snap-123"}} + ] + }, + {"ImportTaskIds": ["import-123"]}, + ) + ec2_stub.add_client_error("create_tags", service_error_code="InvalidCharacter") + ec2_stub.activate() + + with freezegun.freeze_time("2024-10-11"): + with pytest.raises(exceptions.Nack): + aws_handler( + { + "type": "raw-xz", + "arch": "x86_64", + "subvariant": "Cloud_Base", + "path": "AmazonEC2/image.raw.xz", + "checksums": {"sha256": "abc123"}, + }, + mock.MagicMock( + relnum=40, label="Rc-1.14", release="40", cid="Compose-123", eol="2025-09-03" + ), + ) + + s3_stub.assert_no_pending_responses() + ec2_stub.assert_no_pending_responses() + + +@mock.patch.dict( + config.conf["consumer_config"], + { + "aws": { + "base_region": "us-east-1", + "ami_volume_type": "gp2", + "ami_volume_dev_name": "/dev/blah", + "s3_bucket_name": "bucket", + } + }, +) +def test_aws_mark_snapshot_public_failure(): + """ + When the snapshot can't be tagged, the message is nacked. + """ + aws_handler = aws.Aws() + aws_handler.aws_copy_image_to_regions = mock.Mock(return_value={"us-east-1": "ami-0123"}) + s3_stub = Stubber(aws_handler.s3_client) + s3_stub.add_response("head_object", {}, {"Bucket": "bucket", "Key": "abc123.raw"}) + s3_stub.activate() + + ec2_stub = Stubber(aws_handler.ec2_client) + ec2_stub.add_response( + "import_snapshot", + {"ImportTaskId": "import-123"}, + { + "ClientToken": "abc123", + "Description": "Import Fedora-Cloud-Base-AmazonEC2.x86_64-40-1.14 from abc123.raw", + "DiskContainer": { + "Description": "Fedora-Cloud-Base-AmazonEC2.x86_64-40-1.14", + "Format": "raw", + "UserBucket": {"S3Bucket": "bucket", "S3Key": "abc123.raw"}, + }, + }, + ) + ec2_stub.add_response( + "describe_import_snapshot_tasks", + {"ImportSnapshotTasks": [{"SnapshotTaskDetail": {"Status": "completed"}}]}, + {"ImportTaskIds": ["import-123"]}, + ) + ec2_stub.add_response( + "describe_import_snapshot_tasks", + { + "ImportSnapshotTasks": [ + {"SnapshotTaskDetail": {"Status": "completed", "SnapshotId": "snap-123"}} + ] + }, + {"ImportTaskIds": ["import-123"]}, + ) + ec2_stub.add_response( + "create_tags", + {"ResponseMetadata": {}}, + { + "Resources": ["snap-123"], + "Tags": [ + {"Key": "fedora-compose-id", "Value": "Compose-123"}, + {"Key": "fedora-subvariant", "Value": "Cloud_Base"}, + {"Key": "fedora-release", "Value": "40"}, + {"Key": "end-of-life", "Value": "2025-09-03"}, + {"Key": "Delete", "Value": "334"}, + {"Key": "FedoraGroup", "Value": "infra"}, + ], + }, + ) + ec2_stub.add_client_error( + "modify_snapshot_attribute", service_error_code="UnauthorizedOperation" + ) + ec2_stub.activate() + + with freezegun.freeze_time("2024-10-11"): + with pytest.raises(exceptions.Nack): + aws_handler( + { + "type": "raw-xz", + "arch": "x86_64", + "subvariant": "Cloud_Base", + "path": "AmazonEC2/image.raw.xz", + "checksums": {"sha256": "abc123"}, + }, + mock.MagicMock( + relnum=40, label="Rc-1.14", release="40", cid="Compose-123", eol="2025-09-03" + ), + ) s3_stub.assert_no_pending_responses() ec2_stub.assert_no_pending_responses()