From de0125eb53a555411486afb5faf57e10af861d7a Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jul 11 2018 08:43:09 +0000 Subject: Add compose-diff-rpms script A bit more low level than the changelog. --- diff --git a/bin/compose-diff-rpms b/bin/compose-diff-rpms new file mode 100755 index 0000000..44537e6 --- /dev/null +++ b/bin/compose-diff-rpms @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import os +import sys + +here = sys.path[0] +if here != "/usr/bin": + # Git checkout + sys.path[0] = os.path.dirname(here) + +from compose_utils import diff + + +if __name__ == "__main__": + diff.main() diff --git a/compose_utils/diff.py b/compose_utils/diff.py new file mode 100644 index 0000000..6d5ff83 --- /dev/null +++ b/compose_utils/diff.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + + +import argparse +import copy +import json +import os + +import productmd +from kobo.rpmlib import parse_nvra + + +class ComposeRpmsDiff(object): + def get_srpms(self, rpmm): + """Get dictionary of all srpm paths""" + result = {} + for variant in rpmm.rpms: + for arch in rpmm.rpms[variant]: + for srpm_nevra in rpmm.rpms[variant][arch].keys(): + for data in rpmm.rpms[variant][arch][srpm_nevra].values(): + if data["category"] != "source": + continue + result[data["path"]] = (variant, srpm_nevra, data) + return result + + def get_rpms(self, rpmm): + """Get dictionary of all rpm paths""" + result = {} + for variant in rpmm.rpms: + for arch in rpmm.rpms[variant]: + if arch == "src": + continue + for srpm_nevra in rpmm.rpms[variant][arch]: + for rpm_nevra, data in rpmm.rpms[variant][arch][srpm_nevra].items(): + result[data["path"]] = ( + variant, + arch, + srpm_nevra, + rpm_nevra, + data, + ) + return result + + def get_diff(self, old_compose_path, new_compose_path): + """Compare composes and produce a dict with difference information""" + result = {} + + old_rm = productmd.compose.Compose(old_compose_path).rpms + result["old_compose"] = old_rm.compose.id + + new_rm = productmd.compose.Compose(new_compose_path).rpms + result["new_compose"] = new_rm.compose.id + + srpms_old = self.get_srpms(old_rm) + srpms_new = self.get_srpms(new_rm) + + rpms_old = self.get_rpms(old_rm) + rpms_new = self.get_rpms(new_rm) + + for i in ( + list(srpms_old.keys()) + + list(srpms_new.keys()) + + list(rpms_old.keys()) + + list(rpms_new.keys()) + ): + if i in srpms_old and i in srpms_new: + del srpms_old[i] + del srpms_new[i] + if i in rpms_old and i in rpms_new: + del rpms_old[i] + del rpms_new[i] + + result["summary"] = { + "added_srpms": len(srpms_new), + "dropped_srpms": len(srpms_old), + "added_rpms": len(rpms_new), + "dropped_rpms": len(rpms_old), + } + + # Build dicts of added/dropped rpms - rpm_manifest like format variant/arch/srpm_nvra/[rpm_nevra/]:data + added_rpms = {} + dropped_rpms = {} + + for srpms, dictionary in ((srpms_new, added_rpms), (srpms_old, dropped_rpms)): + for path, (variant, srpm_nevra, data) in srpms.items(): + dictionary.setdefault(variant, {}).setdefault("src", {})[ + srpm_nevra + ] = data + + for rpms, dictionary in ((rpms_new, added_rpms), (rpms_old, dropped_rpms)): + for path, (variant, arch, srpm_nevra, rpm_nevra, data) in rpms.items(): + dictionary.setdefault(variant, {}).setdefault(arch, {}).setdefault( + srpm_nevra, {} + )[rpm_nevra] = data + + result["manifest"] = {} + result["manifest"]["added"] = added_rpms + result["manifest"]["dropped"] = dropped_rpms + + # Build dicts of added/dropped rpms - by path path:data + added_rpms_by_path = {} + dropped_rpms_by_path = {} + + for srpms, dictionary in ( + (srpms_new, added_rpms_by_path), + (srpms_old, dropped_rpms_by_path), + ): + for path, (variant, srpm_nevra, data) in srpms.items(): + data = copy.copy(data) + data["variant"] = variant + data["arch"] = "src" + data["srpm_nevra"] = srpm_nevra + del data["path"] + dictionary[path] = data + + for rpms, dictionary in ( + (rpms_new, added_rpms_by_path), + (rpms_old, dropped_rpms_by_path), + ): + for path, (variant, arch, srpm_nevra, rpm_nevra, data) in rpms.items(): + data = copy.copy(data) + data["variant"] = variant + data["arch"] = arch + data["srpm_nevra"] = srpm_nevra + data["rpm_nevra"] = rpm_nevra + del data["path"] + dictionary[path] = data + + result["brief"] = {} + result["brief"]["added"] = added_rpms_by_path + result["brief"]["dropped"] = dropped_rpms_by_path + + # Chages by variants and name.arch + changes = {} # {"variant": {"added": {"pkgname.arch"}}} + for action, dictionary in ( + ("added", added_rpms_by_path), + ("dropped", dropped_rpms_by_path), + ): + for path, info in dictionary.items(): + variant = info.get("variant", "UnknownVariant") + arch = info.get("arch", "unknown_arch") + name = parse_nvra(os.path.basename(path))["name"] + changes.setdefault(variant, {}).setdefault(action, set()).add( + "%s.%s" % (name, arch) + ) + for variant in changes: + added = changes[variant].get("added", set()) + dropped = changes[variant].get("dropped", set()) + changes[variant]["added"] = sorted(added - dropped) + changes[variant]["dropped"] = sorted(dropped - added) + + result["diff"] = changes + + return result + + def _get_summary(self, data): + """Gen list of summary lines""" + result = [] + result.append("===== SUMMARY =====") + result.append("Added source rpms: %s" % data["summary"]["added_srpms"]) + result.append("Dropped source rpms: %s" % data["summary"]["dropped_srpms"]) + result.append("Added rpms: %s" % data["summary"]["added_rpms"]) + result.append("Dropped rpms: %s" % data["summary"]["dropped_rpms"]) + result.append("") + + return result + + def get_brief_log(self, data): + result = [] + result.append("OLD: %s" % data["old_compose"]) + result.append("NEW: %s" % data["new_compose"]) + result.append("") + + result.extend(self._get_summary(data)) + + brief_data = data.get("brief", {}) + + result.append("===== ADDED RPMS =====") + for i in sorted(brief_data["added"]): + result.append("+ %s" % i) + result.append("") + + result.append("===== DROPPED RPMS =====") + for i in sorted(brief_data["dropped"]): + result.append("- %s" % i) + result.append("") + + return "\n".join(result) + + def get_diff_log(self, data): + result = [] + result.append("OLD: %s" % data["old_compose"]) + result.append("NEW: %s" % data["new_compose"]) + result.append("") + + result.extend(self._get_summary(data)) + + for variant, data in data.get("diff", {}).items(): + added = sorted(data.get("added", set())) + dropped = sorted(data.get("dropped", set())) + result.append("===== %s =====" % variant) + for prefix, items in (("+ ", added), ("- ", dropped)): + for item in items: + result.append("%s%s" % (prefix, item)) + result.append("") + return "\n".join(result) + + def write(self, data, path=None, name=None): + """Write output files with results""" + if name: + name = "diff-rpms-%s" % name + else: + name = "diff-rpms" + name = name.replace(" ", "_") + name = name.replace("/", "_") + name = name.replace("\\", "_") + + path = path or "" + + # json + json_log = os.path.join(path, "%s.json" % name) + with open(json_log, "w") as f: + json.dump(data.get("manifest", {}), f, sort_keys=True, indent=4) + + json_log = os.path.join(path, "%s-brief.json" % name) + with open(json_log, "w") as f: + json.dump(data.get("brief", {}), f, sort_keys=True, indent=4) + + json_log = os.path.join(path, "%s-diff.json" % name) + with open(json_log, "w") as f: + json.dump(data.get("diff", {}), f, sort_keys=True, indent=4) + + # brief + brief_log = os.path.join(path, "%s-brief.log" % name) + with open(brief_log, "w") as f: + f.write(self.get_brief_log(data)) + + # diff + verbose_log = os.path.join(path, "%s-diff.log" % name) + with open(verbose_log, "w") as f: + f.write(self.get_diff_log(data)) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--old", metavar="PATH", help="old compose", required=True) + parser.add_argument("--new", metavar="PATH", help="new compose", required=True) + parser.add_argument("-o", "--outputdir", help="output dir") + parser.add_argument("--name", help="log name appended to file name") + + args = parser.parse_args(argv) + + cdiff = ComposeRpmsDiff() + data = cdiff.get_diff(args.old, args.new) + cdiff.write(data, args.outputdir, args.name) diff --git a/doc/compose-diff-rpms.1 b/doc/compose-diff-rpms.1 new file mode 100644 index 0000000..2160b19 --- /dev/null +++ b/doc/compose-diff-rpms.1 @@ -0,0 +1,30 @@ +.TH compose-diff-rpms 1 +.SH NAME +compose-diff-rpms \- show differences between package lists in two composes +.SH SYNOPSIS +.B compose-diff-rpms +[\fIOPTIONS\fR...] +--old=\fIOLD_COMPOSE\fR +--new=\fINEW_COMPOSE\fR +.SH DESCRIPTION +.B compose-diff-rpms +shows changes in package lists between two composes. It parses the metadata and +checks RPMs mentioned there. The results are saved in multiple files in current +directory (unless specified otherwise). +.P +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Print help and exit. +.TP +.BR \-o ", " \-\-outputdir =\fIDIR\fR +Set directory in which to create the files. +.TP +.BR \-\-name =\fINAME\fR +Set custom identifier for the generated files. +.SH EXIT CODE +This tool always exits with status code of \fB0\fR. +.SH BUGS +Please report bugs at +.br +https://pagure.io/compose-utils/issues diff --git a/tests/test_diff.py b/tests/test_diff.py new file mode 100644 index 0000000..fc10853 --- /dev/null +++ b/tests/test_diff.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- + +import locale + +try: + import unittest2 as unittest +except ImportError: + import unittest + +from .helpers import get_compose_path + +from compose_utils import diff + + +EXPECTED_DIFF = { + "Client": { + "added": [ + "dummy-elinks-debuginfo.i386", + "dummy-elinks-debuginfo.x86_64", + "dummy-elinks.i386", + "dummy-elinks.x86_64", + ], + "dropped": [ + "dummy-tftp-debuginfo.i386", + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.i386", + "dummy-tftp.x86_64", + ], + }, + "Server": { + "added": [ + "dummy-elinks-debuginfo.s390x", + "dummy-elinks-debuginfo.x86_64", + "dummy-elinks.s390x", + "dummy-elinks.x86_64", + ], + "dropped": [ + "dummy-tftp-debuginfo.s390x", + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.s390x", + "dummy-tftp.x86_64", + ], + }, +} + + +class DiffRpmsTest(unittest.TestCase): + def setUp(self): + locale.setlocale(locale.LC_TIME, "C") + + def test_diff(self): + old_compose = get_compose_path("DP-1.0-20160315.t.0") + new_compose = get_compose_path("DP-1.0-20160315.t.1") + cdiff = diff.ComposeRpmsDiff() + + data = cdiff.get_diff(old_compose, new_compose) + self.maxDiff = None + self.assertEqual(data["diff"], EXPECTED_DIFF)