From 7ef20ecad38673c8a037eaea345958e83a86d932 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jun 04 2019 09:17:13 +0000 Subject: Support modules in `compose_utils.diff`. The `compose_utils.diff` is currently not useful when creating diff of two composes with modules. The issue is that such composes often contains multiple RPMs with the same name comming from different modules (or even from non-modular part of the compose). Currently, the `compose_utils.diff` shows all changes in `$name.$arch` format. This PR changes this in a way that RPMs comming from modules are showed in the `$name.$module_name_stream.$arch` format. --- diff --git a/compose_utils/diff.py b/compose_utils/diff.py index 512a07f..8cf793c 100644 --- a/compose_utils/diff.py +++ b/compose_utils/diff.py @@ -41,16 +41,28 @@ class ComposeRpmsDiff(object): ) return result - def get_diff_rpms_file(self, rpms_file_path, new_compose_path): + def get_diff_rpms_file( + self, rpms_file_path, new_compose_path, modules_files_path=None): """ Diffs rpms from a new compose againts an rpms.json file """ old_rm = productmd.Rpms() old_rm.load(rpms_file_path) - new_rm = productmd.compose.Compose(new_compose_path).rpms + old_mm = productmd.Modules() + if modules_files_path: + old_mm.load(modules_files_path) + + new_compose = productmd.compose.Compose(new_compose_path) + new_rm = new_compose.rpms + + # Handle the situation when "modules.json" is not available. + try: + new_mm = new_compose.modules + except RuntimeError: + new_mm = productmd.Modules() - result = self._make_diff(old_rm, new_rm) + result = self._make_diff(old_rm, new_rm, old_mm, new_mm) return result @@ -59,14 +71,38 @@ class ComposeRpmsDiff(object): old_rm = productmd.compose.Compose(old_compose_path).rpms new_rm = productmd.compose.Compose(new_compose_path).rpms - result = self._make_diff(old_rm, new_rm) + # Handle the situation when "modules.json" is not available. + try: + old_mm = productmd.compose.Compose(old_compose_path).modules + except RuntimeError: + old_mm = productmd.Modules() + try: + new_mm = productmd.compose.Compose(new_compose_path).modules + except RuntimeError: + new_mm = productmd.Modules() + + result = self._make_diff(old_rm, new_rm, old_mm, new_mm) return result - def _make_diff(self, old_rm, new_rm): + def _get_modular_rpms(self, mm): + """ Returns the dict with RPM NEVRA as key and module name:stream as value. """ + rpms = {} + for per_arches_modules in mm.modules.values(): + for module in per_arches_modules.values(): + for nsvc, data in module.items(): + module_name_stream = ":".join(nsvc.split(":")[:2]) + for rpm in data["rpms"]: + rpms[rpm] = module_name_stream + return rpms + + def _make_diff(self, old_rm, new_rm, old_mm, new_mm): """ Creates a diff between two productmd Rpms objects. """ result = {} + modular_rpms = self._get_modular_rpms(old_mm) + modular_rpms.update(self._get_modular_rpms(new_mm)) + srpms_old = self.get_srpms(old_rm) srpms_new = self.get_srpms(new_rm) @@ -127,6 +163,7 @@ class ComposeRpmsDiff(object): data["variant"] = variant data["arch"] = "src" data["srpm_nevra"] = srpm_nevra + data["module_name_stream"] = modular_rpms.get(srpm_nevra, None) del data["path"] dictionary[path] = data @@ -140,6 +177,7 @@ class ComposeRpmsDiff(object): data["arch"] = arch data["srpm_nevra"] = srpm_nevra data["rpm_nevra"] = rpm_nevra + data["module_name_stream"] = modular_rpms.get(rpm_nevra, None) del data["path"] dictionary[path] = data @@ -157,8 +195,19 @@ class ComposeRpmsDiff(object): variant = info.get("variant", "UnknownVariant") arch = info.get("arch", "unknown_arch") name = parse_nvra(os.path.basename(path))["name"] + + # If the RPM comes from module, also add the module + # "name:stream" to change record, because there might + # be the RPM with the same name in other modules and we need + # a way to distinguish between them. + module_name_stream = info["module_name_stream"] + if module_name_stream: + change_record = "%s.%s.%s" % ( + name, module_name_stream, arch) + else: + change_record = "%s.%s" % (name, arch) changes.setdefault(variant, {}).setdefault(action, set()).add( - "%s.%s" % (name, arch) + change_record ) for variant in changes: added = changes[variant].get("added", set()) diff --git a/tests/composes/odcs-369-1-20190604.n.0/compose/metadata/modules.json b/tests/composes/odcs-369-1-20190604.n.0/compose/metadata/modules.json new file mode 100644 index 0000000..f404c28 --- /dev/null +++ b/tests/composes/odcs-369-1-20190604.n.0/compose/metadata/modules.json @@ -0,0 +1,80 @@ +{ + "header": { + "type": "productmd.modules", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20190604", + "id": "odcs-369-1-20190604.n.0", + "respin": 0, + "type": "nightly" + }, + "modules": { + "Temporary": { + "x86_64": { + "flatpak-common:f30:3020190603174434:548d4c8d": { + "metadata": { + "context": "548d4c8d", + "koji_tag": "module-flatpak-common-f30-3020190603174434-548d4c8d", + "name": "flatpak-common", + "stream": "f30", + "uid": "flatpak-common:f30:3020190603174434:548d4c8d", + "version": "3020190603174434" + }, + "modulemd_path": { + "binary": "Temporary/x86_64/os/repodata/49ee39613ff3bc6b0b2fed3c4ea24ef22db1827ffeae72051966d6f57ac77e58-modules.yaml.gz" + }, + "rpms": [ + "xorg-x11-font-utils-1:7.5-41.module_f30+4492+e9cd49e1.src", + "xorg-x11-font-utils-1:7.5-41.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-utils-debuginfo-1:7.5-41.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-utils-debugsource-1:7.5-41.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-only-in-old-1:7.5-41.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.src", + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-server-utils-debuginfo-0:7.7-28.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-server-utils-debugsource-0:7.7-28.module_f30+4492+e9cd49e1.x86_64" + ] + }, + "flatpak-runtime:f30:3020190603102507:a5b0195c": { + "metadata": { + "context": "a5b0195c", + "koji_tag": "module-flatpak-runtime-f30-3020190603102507-a5b0195c", + "name": "flatpak-runtime", + "stream": "f30", + "uid": "flatpak-runtime:f30:3020190603102507:a5b0195c", + "version": "3020190603102507" + }, + "modulemd_path": { + "binary": "Temporary/x86_64/os/repodata/49ee39613ff3bc6b0b2fed3c4ea24ef22db1827ffeae72051966d6f57ac77e58-modules.yaml.gz" + }, + "rpms": [ + "flatpak-rpm-macros-0:29-8.module_f30+3824+99ea9695.src", + "flatpak-rpm-macros-0:29-8.module_f30+3824+99ea9695.x86_64", + "flatpak-runtime-config-0:29-5.module_f30+3766+a8c7afc9.src", + "flatpak-runtime-config-0:29-5.module_f30+3766+a8c7afc9.x86_64" + ] + }, + "gnome-sound-recorder:master:3020190604074136:f1d02342": { + "metadata": { + "context": "f1d02342", + "koji_tag": "module-gnome-sound-recorder-master-3020190604074136-f1d02342", + "name": "gnome-sound-recorder", + "stream": "master", + "uid": "gnome-sound-recorder:master:3020190604074136:f1d02342", + "version": "3020190604074136" + }, + "modulemd_path": { + "binary": "Temporary/x86_64/os/repodata/49ee39613ff3bc6b0b2fed3c4ea24ef22db1827ffeae72051966d6f57ac77e58-modules.yaml.gz" + }, + "rpms": [ + "gnome-sound-recorder-0:3.32.0-1.module_f30+4502+31b46982.noarch", + "gnome-sound-recorder-0:3.32.0-1.module_f30+4502+31b46982.src" + ] + } + } + } + } + } +} diff --git a/tests/composes/odcs-369-1-20190604.n.0/compose/metadata/rpms.json b/tests/composes/odcs-369-1-20190604.n.0/compose/metadata/rpms.json new file mode 100644 index 0000000..52fe8db --- /dev/null +++ b/tests/composes/odcs-369-1-20190604.n.0/compose/metadata/rpms.json @@ -0,0 +1,86 @@ +{ + "header": { + "type": "productmd.rpms", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20190604", + "id": "odcs-369-1-20190604.n.0", + "respin": 0, + "type": "nightly" + }, + "rpms": { + "Temporary": { + "x86_64": { + "xorg-x11-font-utils-1:7.5-41.module_f30+4492+e9cd49e1.src": { + "xorg-x11-font-utils-1:7.5-41.module_f30+4492+e9cd49e1.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/x/xorg-x11-font-utils-7.5-41.module_f30+4492+e9cd49e1.src.rpm", + "sigkey": null + }, + "xorg-x11-font-utils-1:7.5-41.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-utils-7.5-41.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-only-in-old-1:7.5-41.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-only-in-old-7.5-41.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-utils-debuginfo-1:7.5-41.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-font-utils-debuginfo-7.5-41.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-utils-debugsource-1:7.5-41.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-font-utils-debugsource-7.5-41.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + } + }, + "xorg-x11-font-utils-1:7.3-41.src": { + "xorg-x11-font-utils-1:7.3-41.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/x/xorg-x11-font-utils-7.3-41.src.rpm", + "sigkey": null + }, + "xorg-x11-font-only-in-old-1:7.3-41.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-only-in-old-7.3-41.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-foobar-1:7.3-41.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-foobar-7.3-41.x86_64.rpm", + "sigkey": null + } + }, + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.src": { + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/x/xorg-x11-server-utils-7.7-28.module_f30+4492+e9cd49e1.src.rpm", + "sigkey": null + }, + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-server-utils-7.7-28.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-server-utils-debuginfo-0:7.7-28.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-server-utils-debuginfo-7.7-28.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-server-utils-debugsource-0:7.7-28.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-server-utils-debugsource-7.7-28.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + } + } + } + } + } + } +} diff --git a/tests/composes/odcs-370-1-20190604.n.0/compose/metadata/modules.json b/tests/composes/odcs-370-1-20190604.n.0/compose/metadata/modules.json new file mode 100644 index 0000000..b3f264f --- /dev/null +++ b/tests/composes/odcs-370-1-20190604.n.0/compose/metadata/modules.json @@ -0,0 +1,81 @@ +{ + "header": { + "type": "productmd.modules", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20190604", + "id": "odcs-369-1-20190604.n.0", + "respin": 0, + "type": "nightly" + }, + "modules": { + "Temporary": { + "x86_64": { + "flatpak-common:f30:3020190703174434:548d4c8d": { + "metadata": { + "context": "548d4c8d", + "koji_tag": "module-flatpak-common-f30-3020190603174434-548d4c8d", + "name": "flatpak-common", + "stream": "f30", + "uid": "flatpak-common:f30:3020190603174434:548d4c8d", + "version": "3020190603174434" + }, + "modulemd_path": { + "binary": "Temporary/x86_64/os/repodata/49ee39613ff3bc6b0b2fed3c4ea24ef22db1827ffeae72051966d6f57ac77e58-modules.yaml.gz" + }, + "rpms": [ + "xorg-x11-font-utils-1:7.5-42.module_f30+4492+e9cd49e1.src", + "xorg-x11-font-utils-1:7.5-42.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-utils-debuginfo-1:7.5-42.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-utils-debugsource-1:7.5-42.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-foobar-1:7.5-42.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-font-only-in-new-1:7.5-42.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.src", + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-server-utils-debuginfo-0:7.7-28.module_f30+4492+e9cd49e1.x86_64", + "xorg-x11-server-utils-debugsource-0:7.7-28.module_f30+4492+e9cd49e1.x86_64" + ] + }, + "flatpak-runtime:f30:3020190703102507:a5b0195c": { + "metadata": { + "context": "a5b0195c", + "koji_tag": "module-flatpak-runtime-f30-3020190603102507-a5b0195c", + "name": "flatpak-runtime", + "stream": "f30", + "uid": "flatpak-runtime:f30:3020190603102507:a5b0195c", + "version": "3020190603102507" + }, + "modulemd_path": { + "binary": "Temporary/x86_64/os/repodata/49ee39613ff3bc6b0b2fed3c4ea24ef22db1827ffeae72051966d6f57ac77e58-modules.yaml.gz" + }, + "rpms": [ + "flatpak-rpm-macros-0:29-8.module_f30+3824+99ea9695.src", + "flatpak-rpm-macros-0:29-8.module_f30+3824+99ea9695.x86_64", + "flatpak-runtime-config-0:29-5.module_f30+3766+a8c7afc9.src", + "flatpak-runtime-config-0:29-5.module_f30+3766+a8c7afc9.x86_64" + ] + }, + "gnome-sound-recorder:master:3020190704074136:f1d02342": { + "metadata": { + "context": "f1d02342", + "koji_tag": "module-gnome-sound-recorder-master-3020190604074136-f1d02342", + "name": "gnome-sound-recorder", + "stream": "master", + "uid": "gnome-sound-recorder:master:3020190604074136:f1d02342", + "version": "3020190604074136" + }, + "modulemd_path": { + "binary": "Temporary/x86_64/os/repodata/49ee39613ff3bc6b0b2fed3c4ea24ef22db1827ffeae72051966d6f57ac77e58-modules.yaml.gz" + }, + "rpms": [ + "gnome-sound-recorder-0:3.32.0-1.module_f30+4502+31b46982.noarch", + "gnome-sound-recorder-0:3.32.0-1.module_f30+4502+31b46982.src" + ] + } + } + } + } + } +} diff --git a/tests/composes/odcs-370-1-20190604.n.0/compose/metadata/rpms.json b/tests/composes/odcs-370-1-20190604.n.0/compose/metadata/rpms.json new file mode 100644 index 0000000..75d9201 --- /dev/null +++ b/tests/composes/odcs-370-1-20190604.n.0/compose/metadata/rpms.json @@ -0,0 +1,86 @@ +{ + "header": { + "type": "productmd.rpms", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20190604", + "id": "odcs-369-1-20190604.n.0", + "respin": 0, + "type": "nightly" + }, + "rpms": { + "Temporary": { + "x86_64": { + "xorg-x11-font-utils-1:7.5-42.module_f30+4492+e9cd49e1.src": { + "xorg-x11-font-utils-1:7.5-42.module_f30+4492+e9cd49e1.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/x/xorg-x11-font-utils-7.5-42.module_f30+4492+e9cd49e1.src.rpm", + "sigkey": null + }, + "xorg-x11-font-utils-1:7.5-42.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-utils-7.5-42.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-foobar-1:7.5-42.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-foobar-7.5-42.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-only-in-new-1:7.5-42.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-only-in-new-7.5-42.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-utils-debuginfo-1:7.5-42.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-font-utils-debuginfo-7.5-42.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-font-utils-debugsource-1:7.5-42.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-font-utils-debugsource-7.5-42.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + } + }, + "xorg-x11-font-utils-1:7.3-41.src": { + "xorg-x11-font-utils-1:7.3-41.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/x/xorg-x11-font-utils-7.3-41.src.rpm", + "sigkey": null + }, + "xorg-x11-font-foobar-1:7.3-41.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-font-foobar-7.3-41.x86_64.rpm", + "sigkey": null + } + }, + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.src": { + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/x/xorg-x11-server-utils-7.7-28.module_f30+4492+e9cd49e1.src.rpm", + "sigkey": null + }, + "xorg-x11-server-utils-0:7.7-28.module_f30+4492+e9cd49e1.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/x/xorg-x11-server-utils-7.7-28.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-server-utils-debuginfo-0:7.7-28.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-server-utils-debuginfo-7.7-28.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + }, + "xorg-x11-server-utils-debugsource-0:7.7-28.module_f30+4492+e9cd49e1.x86_64": { + "category": "debug", + "path": "Temporary/x86_64/debug/tree/Packages/x/xorg-x11-server-utils-debugsource-7.7-28.module_f30+4492+e9cd49e1.x86_64.rpm", + "sigkey": null + } + } + } + } + } + } +} diff --git a/tests/helpers.py b/tests/helpers.py index 19fe35c..7f294a1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -18,6 +18,13 @@ def get_rpms_json_path(ident): return pth +def get_modules_json_path(ident): + """Given a compose ID, return path to the rpms.json file of that compose.""" + base = os.path.dirname(__file__) + pth = os.path.join(base, 'composes', ident, 'compose', 'metadata', 'modules.json') + return pth + + def get_compose(ident, path=None): """Load a test compose and return the Compose instance.""" path = path or get_compose_path(ident) diff --git a/tests/test_diff.py b/tests/test_diff.py index b5deaf3..8ee6c1b 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -7,7 +7,8 @@ try: except ImportError: import unittest -from .helpers import get_compose_path, get_rpms_json_path +from .helpers import ( + get_compose_path, get_rpms_json_path, get_modules_json_path) from compose_utils import diff @@ -48,6 +49,20 @@ EXPECTED_DIFF = { } +EXPECTED_DIFF_MODULAR = { + "Temporary": { + "added": [ + "xorg-x11-font-foobar.flatpak-common:f30.x86_64", + "xorg-x11-font-only-in-new.flatpak-common:f30.x86_64" + ], + "dropped": [ + "xorg-x11-font-only-in-old.flatpak-common:f30.x86_64", + "xorg-x11-font-only-in-old.x86_64" + ] + } +} + + class DiffRpmsTest(unittest.TestCase): def setUp(self): locale.setlocale(locale.LC_TIME, "C") @@ -69,3 +84,23 @@ class DiffRpmsTest(unittest.TestCase): data = cdiff.get_diff_rpms_file(rpms_file_path, new_compose) self.maxDiff = None self.assertEqual(data["diff"], EXPECTED_DIFF) + + def test_diff_modules(self): + old_compose = get_compose_path("odcs-369-1-20190604.n.0") + new_compose = get_compose_path("odcs-370-1-20190604.n.0") + cdiff = diff.ComposeRpmsDiff() + + data = cdiff.get_diff(old_compose, new_compose) + self.maxDiff = None + self.assertEqual(data["diff"], EXPECTED_DIFF_MODULAR) + + def test_diff_with_rpms_json_file_modules(self): + rpms_file_path = get_rpms_json_path("odcs-369-1-20190604.n.0") + modules_file_path = get_modules_json_path("odcs-369-1-20190604.n.0") + new_compose = get_compose_path("odcs-370-1-20190604.n.0") + cdiff = diff.ComposeRpmsDiff() + + data = cdiff.get_diff_rpms_file( + rpms_file_path, new_compose, modules_file_path) + self.maxDiff = None + self.assertEqual(data["diff"], EXPECTED_DIFF_MODULAR)