From 4776a239dc8519bacb78b0e7ce349b50eec0489b Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Mar 31 2016 15:09:45 +0000 Subject: add image diff to changelog this is a port of something currently done in my 'check-compose' tool which produces the 'compose check report' emails. I think it makes sense to do it here instead. It's pretty simple: we consider the 3-tuple (subvariant, type, arch) for an image to be a unique identifier for the image, and we compare the images in each compose and produce 'added_images' and 'dropped_images' from sets of the identifiers. We use the same identifier run through " ".join() as the human-readable identifier and include just a count of added/removed images in the summary, the id for each added and removed image in the brief log, and the id plus the path for each added and removed image in the verbose log. I tried to future proof it a bit, in case we need to change the identifier format at any point, the idea is that we can just change the `image_id` method and the rest of the code should keep working. At first I just got the tuples and stored those as added_images and dropped_images, but then it's a bit messy in future if we change the identifier format. It's also set up such that we can also add check-compose's 'expected images' feature quite easily using the `get_images` dict (planning that as a separate PR). --- diff --git a/compose_utils/changelog.py b/compose_utils/changelog.py index c855fb4..013bd89 100644 --- a/compose_utils/changelog.py +++ b/compose_utils/changelog.py @@ -205,12 +205,47 @@ class ComposeChangelog(object): ) return result + def image_id(self, img): + """Get a unique identifier for an image. Currently we use a + 3-tuple (subvariant, type, arch). Should produce a sensible + human-readable text when run through " ".join(). Should also + be hashable for use as a dict key. + """ + return (img['subvariant'], img['type'], img['arch']) + + def get_images(self, compose): + """Parse a compose's image list and return a dict whose keys + are image identifiers (whatever self.image_id gives) and whose + values are the original Image instances `serialize`d into + dicts. + """ + imgs = {} + imgdict = compose.images.images + for variant in imgdict: + for arch in imgdict[variant]: + for img in imgdict[variant][arch]: + # the way Image.serialize works is a bit odd... + parser = [] + img.serialize(parser) + img = parser[0] + imgs[self.image_id(img)] = img + return imgs + def get_changelog(self, old_compose, new_compose, max_logs=-1): result = {} result["old_compose"] = old_compose.rpms.compose.id result["new_compose"] = new_compose.rpms.compose.id + imgs_old = self.get_images(old_compose) + imgs_new = self.get_images(new_compose) + imgs_old_ids = set(imgs_old.keys()) + imgs_new_ids = set(imgs_new.keys()) + result["added_images"] = [imgs_new[imid] for imid in + imgs_new_ids - imgs_old_ids] + result["dropped_images"] = [imgs_old[imid] for imid in + imgs_old_ids - imgs_new_ids] + srpms_old = self.get_srpms(old_compose) srpms_new = self.get_srpms(new_compose) @@ -235,6 +270,8 @@ class ComposeChangelog(object): pkgs_changed = set(rpm_headers_old) & set(rpm_headers_new) result["summary"] = { + "added_images": len(result["added_images"]), + "dropped_images": len(result["dropped_images"]), "added_packages": 0, "dropped_packages": 0, "upgraded_packages": 0, @@ -321,6 +358,8 @@ class ComposeChangelog(object): def _get_summary(self, changelog_data): result = [] result.append("===== SUMMARY =====") + result.append("Added images: %s" % changelog_data["summary"]["added_images"]) + result.append("Dropped images: %s" % changelog_data["summary"]["dropped_images"]) result.append("Added packages: %s" % changelog_data["summary"]["added_packages"]) result.append("Dropped packages: %s" % changelog_data["summary"]["dropped_packages"]) result.append("Upgraded packages: %s" % changelog_data["summary"]["upgraded_packages"]) @@ -347,6 +386,16 @@ class ComposeChangelog(object): result.extend(self._get_summary(changelog_data)) + result.append("===== ADDED IMAGES =====") + for i in changelog_data["added_images"]: + result.append(" ".join(self.image_id(i))) + result.append("") + + result.append("===== DROPPED IMAGES =====") + for i in changelog_data["dropped_images"]: + result.append(" ".join(self.image_id(i))) + result.append("") + result.append("===== ADDED PACKAGES =====") for i in changelog_data["added_packages"]: result.append("%(name)s: %(nvr)s" % i) @@ -377,6 +426,18 @@ class ComposeChangelog(object): result.extend(self._get_summary(changelog_data)) + result.append("===== ADDED IMAGES =====") + for i in changelog_data["added_images"]: + result.append("Image: %s" % " ".join(self.image_id(i))) + result.append("Path: %s" % i['path']) + result.append("") + + result.append("===== DROPPED IMAGES =====") + for i in changelog_data["dropped_images"]: + result.append("Image: %s" % " ".join(self.image_id(i))) + result.append("Path: %s" % i['path']) + result.append("") + result.append("===== ADDED PACKAGES =====") for i in changelog_data["added_packages"]: result.append("Package: %s" % i["nvr"]) diff --git a/tests/composes/DP-1.0-20160315.t.0/compose/metadata/images.json b/tests/composes/DP-1.0-20160315.t.0/compose/metadata/images.json index 1cc2a9a..e81291a 100644 --- a/tests/composes/DP-1.0-20160315.t.0/compose/metadata/images.json +++ b/tests/composes/DP-1.0-20160315.t.0/compose/metadata/images.json @@ -28,7 +28,8 @@ "path": "Client/i386/iso/DP-1.0-20160315.t.0-Client-i386-dvd1.iso", "size": 507904, "type": "dvd", - "volume_id": "DP-1.0 Client.i386" + "volume_id": "DP-1.0 Client.i386", + "subvariant": "Client" } ], "src": [ @@ -48,7 +49,8 @@ "path": "Client/source/iso/DP-1.0-20160315.t.0-Client-source-dvd1.iso", "size": 473088, "type": "dvd", - "volume_id": "DP-1.0 Client.src" + "volume_id": "DP-1.0 Client.src", + "subvariant": "Client" } ], "x86_64": [ @@ -68,31 +70,12 @@ "path": "Client/x86_64/iso/DP-1.0-20160315.t.0-Client-x86_64-dvd1.iso", "size": 516096, "type": "dvd", - "volume_id": "DP-1.0 Client.x86_64" + "volume_id": "DP-1.0 Client.x86_64", + "subvariant": "Client" } ] }, "Server": { - "s390x": [ - { - "arch": "s390x", - "bootable": false, - "checksums": { - "md5": "bcc53aed2cd8ada9a5a346d7856cc23d", - "sha1": "d462e78474411c78ae569a940ba6eb2e9132fa69", - "sha256": "54f25d0d3bf06f2e752746d310271f42e6680cf9262feddf9eb3384bb7dcaf8f" - }, - "disc_count": 1, - "disc_number": 1, - "format": "iso", - "implant_md5": "f87289c658389c0457c6306d6aaaaf11", - "mtime": 1458031335, - "path": "Server/s390x/iso/DP-1.0-20160315.t.0-Server-s390x-dvd1.iso", - "size": 493568, - "type": "dvd", - "volume_id": "DP-1.0 Server.s390x" - } - ], "src": [ { "arch": "src", @@ -110,7 +93,8 @@ "path": "Server/source/iso/DP-1.0-20160315.t.0-Server-source-dvd1.iso", "size": 509952, "type": "dvd", - "volume_id": "DP-1.0 Server.src" + "volume_id": "DP-1.0 Server.src", + "subvariant": "Server" } ], "x86_64": [ @@ -130,10 +114,11 @@ "path": "Server/x86_64/iso/DP-1.0-20160315.t.0-Server-x86_64-dvd1.iso", "size": 577536, "type": "dvd", - "volume_id": "DP-1.0 Server.x86_64" + "volume_id": "DP-1.0 Server.x86_64", + "subvariant": "Server" } ] } } } -} \ No newline at end of file +} diff --git a/tests/composes/DP-1.0-20160315.t.1/compose/metadata/images.json b/tests/composes/DP-1.0-20160315.t.1/compose/metadata/images.json index 75c3710..7cdc9d7 100644 --- a/tests/composes/DP-1.0-20160315.t.1/compose/metadata/images.json +++ b/tests/composes/DP-1.0-20160315.t.1/compose/metadata/images.json @@ -28,7 +28,8 @@ "path": "Client/i386/iso/DP-1.0-20160315.t.1-Client-i386-dvd1.iso", "size": 507904, "type": "dvd", - "volume_id": "DP-1.0 Client.i386" + "volume_id": "DP-1.0 Client.i386", + "subvariant": "Client" } ], "src": [ @@ -48,27 +49,8 @@ "path": "Client/source/iso/DP-1.0-20160315.t.1-Client-source-dvd1.iso", "size": 475136, "type": "dvd", - "volume_id": "DP-1.0 Client.src" - } - ], - "x86_64": [ - { - "arch": "x86_64", - "bootable": false, - "checksums": { - "md5": "ca40a2f5ce5cf2e930dde4031b16428c", - "sha1": "69c0b27e3998a93c19f32fe77afcf547c70227fc", - "sha256": "2ec61698ed2f4fde41a5253ed0d0afaaad495ae4dba2fc974c1abfcd77dd7358" - }, - "disc_count": 1, - "disc_number": 1, - "format": "iso", - "implant_md5": "6dee6f24138cd7feb1223e78dc62e72d", - "mtime": 1458031712, - "path": "Client/x86_64/iso/DP-1.0-20160315.t.1-Client-x86_64-dvd1.iso", - "size": 516096, - "type": "dvd", - "volume_id": "DP-1.0 Client.x86_64" + "volume_id": "DP-1.0 Client.src", + "subvariant": "Client" } ] }, @@ -90,7 +72,8 @@ "path": "Server/s390x/iso/DP-1.0-20160315.t.1-Server-s390x-dvd1.iso", "size": 493568, "type": "dvd", - "volume_id": "DP-1.0 Server.s390x" + "volume_id": "DP-1.0 Server.s390x", + "subvariant": "Server" } ], "src": [ @@ -110,7 +93,8 @@ "path": "Server/source/iso/DP-1.0-20160315.t.1-Server-source-dvd1.iso", "size": 509952, "type": "dvd", - "volume_id": "DP-1.0 Server.src" + "volume_id": "DP-1.0 Server.src", + "subvariant": "Server" } ], "x86_64": [ @@ -130,10 +114,11 @@ "path": "Server/x86_64/iso/DP-1.0-20160315.t.1-Server-x86_64-dvd1.iso", "size": 577536, "type": "dvd", - "volume_id": "DP-1.0 Server.x86_64" + "volume_id": "DP-1.0 Server.x86_64", + "subvariant": "Server" } ] } } } -} \ No newline at end of file +} diff --git a/tests/test_changelog.py b/tests/test_changelog.py index 217ef95..a3a834a 100644 --- a/tests/test_changelog.py +++ b/tests/test_changelog.py @@ -78,7 +78,47 @@ class ChangelogTest(unittest.TestCase): self.assertItemsEqual(data['dropped_packages'], [DUMMY_TFTP]) self.assertItemsEqual(data['upgraded_packages'], [DUMMY_FIREFOX]) self.assertItemsEqual(data['downgraded_packages'], []) + self.assertDictEqual(data['added_images'][0], { + "arch": "s390x", + "bootable": False, + "checksums": { + "md5": "2d0a0e02d6a2225fba2c4cdf41d45ff7", + "sha1": "8788b0aa4974440f7ca12c5dd0cc6dfc78314fdd", + "sha256": "1b756cddb139e57fd6a07af72a7d52a9dc01c5fb0b9d37ef16fdf73a1a0a19af" + }, + "disc_count": 1, + "disc_number": 1, + "format": "iso", + "implant_md5": "0a2fc4f0e6f926c860e5df6c5ff0df32", + "mtime": 1458031712, + "path": "Server/s390x/iso/DP-1.0-20160315.t.1-Server-s390x-dvd1.iso", + "size": 493568, + "type": "dvd", + "volume_id": "DP-1.0 Server.s390x", + "subvariant": "Server" + }) + self.assertDictEqual(data['dropped_images'][0], { + "arch": "x86_64", + "bootable": False, + "checksums": { + "md5": "7ea0d315e4b51006eb7c1de3da763c55", + "sha1": "268bf671d5e62f35b70a2db5b1198b861d4b0eb8", + "sha256": "32fedd333df5a0c78f3dd8c2634f8a4ab2328c7e0c9904dd06ac865fa723134f" + }, + "disc_count": 1, + "disc_number": 1, + "format": "iso", + "implant_md5": "128abd11392887a8abb2bbf119889791", + "mtime": 1458031335, + "path": "Client/x86_64/iso/DP-1.0-20160315.t.0-Client-x86_64-dvd1.iso", + "size": 516096, + "type": "dvd", + "volume_id": "DP-1.0 Client.x86_64", + "subvariant": "Client" + }) self.assertDictEqual(data['summary'], { + 'added_images': 1, + 'dropped_images': 1, 'added_packages': 1, 'added_packages_size': DUMMY_ELINKS_SIZE, 'upgraded_packages': 1, @@ -90,3 +130,6 @@ class ChangelogTest(unittest.TestCase): 'downgraded_packages_size': 0, 'downgraded_packages_size_change': 0, }) + +if __name__ == '__main__': + unittest.main()