From adb4c1452f40a7c9cf2d6cbe4385dcbd1a5301f5 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Apr 01 2016 23:54:06 +0000 Subject: add a 'missing release blocking images' check This, again, is a port of a feature currently in check-compose, more or less. check-compose has an 'expected images' check, which covers rather more than the 'release blocking' images, but it's a fairly arbitrary list and I figured for something that isn't just my pet project, sticking to a properly defined list would be better. It's quite simple: it checks the images in the compose against a built-in list of 'release blocking' images, and logs any that are missing. Unlike the 'image diff' stuff from the last commit we cannot store productmd-style image dicts in the JSON because we don't have any - the images are *missing*, they have no dicts. All we can reasonably store is the image 'identifiers' we use to try and find them. In an ideal world, the list of release blocking images would be available in some easily parseable format that matches the productmd metadata (like, say, a simple set of image identifier tuples!), and either the code could check it live each time or (if you don't want a network dependency) we could have a simple utility for synchronizing this tool's list which could be run every checkin or something. In the real world, the 'official' list of release blocking deliverables is this ridiculous wiki page: https://fedoraproject.org/wiki/Fedora_Program_Management/ReleaseBlocking/Fedora24 The list in this commit is almost the same, only I left out the Cloud netinst because I'm 99% sure that being 'release blocking' is a mistake. I am quite good at parsing wikis and I *could* write a thing which would turn that page into this list. I am not going to because that would be incredibly dumb. I'd suggest we do two things to clean this up, but I'm easy on the order: i) merge this ii) produce an image list in a sane format (JSON or whatever) and a nice simple script for turning it into a static HTML page for human consumption iii) find an appropriate official server to host both things on, then kill the ridiculous wiki page and make this code use that list. --- diff --git a/compose_utils/changelog.py b/compose_utils/changelog.py index 013bd89..862bb36 100644 --- a/compose_utils/changelog.py +++ b/compose_utils/changelog.py @@ -17,6 +17,24 @@ from kobo.rpmlib import parse_nvra, make_nvr, make_nvra, get_changelogs_from_hea from kobo.threads import ThreadPool, WorkerThread +# The list of images (as 'image id' tuples of the format produced by +# the image_id function) considered 'release blocking'. The official +# list is: +# https://fedoraproject.org/wiki/Fedora_Program_Management/ReleaseBlocking/Fedora24 +# which is not sensibly parseable, so we just duplicate it. +BLOCKING_IMAGES = [ + ('Cloud_Base', 'qcow2', 'x86_64'), + ('Cloud_Base', 'raw-xz', 'x86_64'), + ('Server', 'dvd', 'x86_64'), + ('Server', 'boot', 'x86_64'), + ('Workstation', 'live', 'x86_64'), + ('Workstation', 'boot', 'x86_64'), + ('KDE', 'live', 'x86_64'), + ('Minimal', 'raw-xz', 'armhfp'), + ('Xfce', 'raw-xz', 'armhfp'), +] + + def formatsize(size): if size < 0: return '-' + formatsize(-size) @@ -246,6 +264,11 @@ class ComposeChangelog(object): result["dropped_images"] = [imgs_old[imid] for imid in imgs_old_ids - imgs_new_ids] + # we do this with a list comprehension rather than sets so the + # list will always be in the same order + result["missing_blocking_images"] = [img for img in BLOCKING_IMAGES if + img not in imgs_new_ids] + srpms_old = self.get_srpms(old_compose) srpms_new = self.get_srpms(new_compose) @@ -272,6 +295,7 @@ class ComposeChangelog(object): result["summary"] = { "added_images": len(result["added_images"]), "dropped_images": len(result["dropped_images"]), + "missing_blocking_images": len(result["missing_blocking_images"]), "added_packages": 0, "dropped_packages": 0, "upgraded_packages": 0, @@ -358,12 +382,13 @@ class ComposeChangelog(object): def _get_summary(self, changelog_data): result = [] result.append("===== SUMMARY =====") - result.append("Added images: %s" % changelog_data["summary"]["added_images"]) - result.append("Dropped images: %s" % changelog_data["summary"]["dropped_images"]) - result.append("Added packages: %s" % changelog_data["summary"]["added_packages"]) - result.append("Dropped packages: %s" % changelog_data["summary"]["dropped_packages"]) - result.append("Upgraded packages: %s" % changelog_data["summary"]["upgraded_packages"]) - result.append("Downgraded packages: %s" % changelog_data["summary"]["downgraded_packages"]) + result.append("Added images: %s" % changelog_data["summary"]["added_images"]) + result.append("Dropped images: %s" % changelog_data["summary"]["dropped_images"]) + result.append("Missing blocking images: %s" % changelog_data["summary"]["missing_blocking_images"]) + result.append("Added packages: %s" % changelog_data["summary"]["added_packages"]) + result.append("Dropped packages: %s" % changelog_data["summary"]["dropped_packages"]) + result.append("Upgraded packages: %s" % changelog_data["summary"]["upgraded_packages"]) + result.append("Downgraded packages: %s" % changelog_data["summary"]["downgraded_packages"]) result.append("") result.append("Size of added packages: %s" % (formatsize(changelog_data["summary"]["added_packages_size"]))) @@ -396,6 +421,11 @@ class ComposeChangelog(object): result.append(" ".join(self.image_id(i))) result.append("") + result.append("===== MISSING BLOCKING IMAGES =====") + for i in changelog_data["missing_blocking_images"]: + result.append(" ".join(i)) + result.append("") + result.append("===== ADDED PACKAGES =====") for i in changelog_data["added_packages"]: result.append("%(name)s: %(nvr)s" % i) @@ -438,6 +468,11 @@ class ComposeChangelog(object): result.append("Path: %s" % i['path']) result.append("") + result.append("===== MISSING BLOCKING IMAGES =====") + for i in changelog_data["missing_blocking_images"]: + result.append(" ".join(i)) + result.append("") + result.append("===== ADDED PACKAGES =====") for i in changelog_data["added_packages"]: result.append("Package: %s" % i["nvr"]) diff --git a/tests/test_changelog.py b/tests/test_changelog.py index 03d251e..522c684 100644 --- a/tests/test_changelog.py +++ b/tests/test_changelog.py @@ -4,7 +4,13 @@ import unittest import os import productmd.compose -from compose_utils.changelog import ComposeChangelog +import compose_utils.changelog + +# redefine this constant to something that makes sense for the test data +compose_utils.changelog.BLOCKING_IMAGES = [ + ('Client', 'dvd', 'x86_64'), + ('Server', 'dvd', 'x86_64'), +] def _get_compose_path(ident): @@ -55,7 +61,7 @@ class ChangelogTest(unittest.TestCase): def test_changelog(self): old_compose = productmd.compose.Compose(_get_compose_path('DP-1.0-20160315.t.0')) new_compose = productmd.compose.Compose(_get_compose_path('DP-1.0-20160315.t.1')) - changelog = ComposeChangelog() + changelog = compose_utils.changelog.ComposeChangelog() data = changelog.get_changelog(old_compose, new_compose) self.assertEqual(data['old_compose'], 'DP-1.0-20160315.t.0') @@ -64,6 +70,7 @@ class ChangelogTest(unittest.TestCase): self.assertItemsEqual(data['dropped_packages'], [DUMMY_TFTP]) self.assertItemsEqual(data['upgraded_packages'], [DUMMY_FIREFOX]) self.assertItemsEqual(data['downgraded_packages'], []) + self.assertItemsEqual(data['missing_blocking_images'], [('Client', 'dvd', 'x86_64')]) self.assertDictEqual(data['added_images'][0], { "arch": "s390x", "bootable": False, @@ -105,6 +112,7 @@ class ChangelogTest(unittest.TestCase): self.assertDictEqual(data['summary'], { 'added_images': 1, 'dropped_images': 1, + 'missing_blocking_images': 1, 'added_packages': 1, 'added_packages_size': DUMMY_ELINKS_SIZE, 'upgraded_packages': 1,