From 820ce1ed606a3fd8e86eb10287779c644697b4c7 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Aug 13 2018 09:03:25 +0000 Subject: Add utility to dump package listings from a compose This can be fed back into Pungi as prepopulate file to ensure packages from compose don't go missing. --- diff --git a/README.rst b/README.rst index af2ced2..dfd1d2a 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,10 @@ Contents generate a difference in packages between two composes. This is a bit more low-level than the changelog. +**compose-dump-listings** + write listing of packages in a compose useful for tracking disappearing + packages in another compose + **compose-has-build** check if a build with a given NVR is present in the compose diff --git a/bin/compose-dump-listings b/bin/compose-dump-listings new file mode 100755 index 0000000..1718683 --- /dev/null +++ b/bin/compose-dump-listings @@ -0,0 +1,64 @@ +#!/usr/bin/env python2 +# -*- encoding: utf-8 -*- + +import argparse +import json +import os +import sys + +import productmd.compose + +here = sys.path[0] +if here != "/usr/bin": + sys.path[0] = os.path.dirname(here) + +import compose_utils + + +def run(opts): + if opts.compose.endswith("rpms.json"): + rpms = productmd.Rpms() + rpms.load(opts.compose) + else: + compose = productmd.compose.Compose(opts.compose) + rpms = compose.rpms + + listing = compose_utils.get_listing( + rpms, variants=opts.variant, arches=opts.arch, pkg_arches=opts.pkg_arch + ) + json.dump(listing, sys.stdout, sort_keys=True, indent=2) + + +DESCRIPTION = ( + "Create a product listing that can be passed to Pungi as prepopulate file." +) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument( + "compose", + metavar="COMPOSE", + help="path to compose that should be dumped (path to just rpms.json file is accepted as well)", + ) + parser.add_argument( + "--arch", + action="append", + default=[], + help="only dump this arch; can be used multiple times", + ) + parser.add_argument( + "--variant", + action="append", + default=[], + help="only dump this variant; can be used multiple times", + ) + parser.add_argument( + "--pkg-arch", + action="append", + default=[], + help="only include packages with this arch; can be used multiple times", + ) + + opts = parser.parse_args() + run(opts) diff --git a/compose_utils/__init__.py b/compose_utils/__init__.py index 8d8946d..c34ff70 100644 --- a/compose_utils/__init__.py +++ b/compose_utils/__init__.py @@ -8,7 +8,6 @@ import sys import productmd.compose import kobo.rpmlib - from .copy_compose import copy_compose # noqa @@ -45,3 +44,33 @@ def has_build(args=None): if not opts.quiet: print("Compose %s does not have build %s" % (compose_id, opts.NVR)) sys.exit(1) + + +def get_listing(rpms, variants=[], arches=[], pkg_arches=[]): + """Create product listing suitable for using as prepopulate file in Pungi.""" + listing = {} + + for variant in rpms.rpms: + if variants and variant not in variants: + continue + for arch in rpms.rpms[variant]: + if arches and arch not in arches: + continue + for srpm_nevra in rpms.rpms[variant][arch]: + pkgs = [] + srpm_data = rpms.rpms[variant][arch][srpm_nevra] + srpm_nvr = kobo.rpmlib.parse_nvra(srpm_nevra) + for nevra in srpm_data: + nvra = kobo.rpmlib.parse_nvra(nevra) + if pkg_arches and nvra["arch"] not in pkg_arches: + continue + if nvra["arch"] == "src": + continue + pkgs.append("%(name)s.%(arch)s" % nvra) + + if pkgs: + listing.setdefault(variant, {}).setdefault(arch, {})[ + srpm_nvr["name"] + ] = sorted(pkgs) + + return listing diff --git a/doc/compose-dump-listings.1 b/doc/compose-dump-listings.1 new file mode 100644 index 0000000..0c274f5 --- /dev/null +++ b/doc/compose-dump-listings.1 @@ -0,0 +1,36 @@ +.TH compose-dump-listings 1 +.SH NAME +compose-dump-listings \- dump package listings from a compose +.SH SYNOPSIS +.B compose-dump-listings +[\fIOPTIONS\fR...] +\fICOMPOSE\fR +.SH DESCRIPTION +.B compose-dump-listings +outputs JSON representation of package listings in a compose to stdout. You can +specify which variant and/or architectures you want dump. Also it's possible to +filter by package architecture. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Print help and exit. +.TP +.BR \-\-arch =\fIARCH\fR +Dump only this architecture. Can be used multiple times. + +This applies to tree architectures, so it is actually a basearch. +.TP +.BR \-\-variant =\fIVARIANT\fR +Dump only this variant. Can be used multiple times. +.TP +.BR \-\-package-arch =\fIARCH\fR +Only output packages with this architecture. This is actually the true +architecture for which the package was built. Source packages are never +included. +.SH EXIT CODE +This tool exits with status code of \fB0\fR on success and non-zero value on +failure. +.SH BUGS +Please report bugs at +.br +https://pagure.io/compose-utils/issues diff --git a/setup.py b/setup.py index c37da98..4e5f0e6 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ setup( 'bin/compose-changelog', 'bin/compose-create-legacy-composeinfo', 'bin/compose-diff-rpms', + 'bin/compose-dump-listings', 'bin/compose-has-build', 'bin/compose-latest-symlink', 'bin/compose-list', @@ -34,6 +35,7 @@ setup( 'doc/compose-changelog.1', 'doc/compose-create-legacy-composeinfo.1', 'doc/compose-diff-rpms.1', + 'doc/compose-dump-listings.1', 'doc/compose-has-build.1', 'doc/compose-list.1', 'doc/compose-partial-copy.1', diff --git a/tests/fixtures/full-listing.json b/tests/fixtures/full-listing.json new file mode 100644 index 0000000..aafc19d --- /dev/null +++ b/tests/fixtures/full-listing.json @@ -0,0 +1,138 @@ +{ + "Client": { + "i386": { + "Dummy-firefox": [ + "Dummy-firefox-debuginfo.i686", + "Dummy-firefox.i686" + ], + "Dummy-xulrunner": [ + "Dummy-xulrunner-debuginfo.i686", + "Dummy-xulrunner.i686" + ], + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.i686", + "dummy-bash.i686" + ], + "dummy-filesystem": [ + "dummy-filesystem.i686" + ], + "dummy-glibc": [ + "dummy-glibc-common.i686", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo.i686", + "dummy-glibc.i686" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.i686", + "dummy-lvm2-libs.i686", + "dummy-lvm2.i686" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.i686", + "dummy-tftp.i686" + ] + }, + "x86_64": { + "Dummy-firefox": [ + "Dummy-firefox-debuginfo.x86_64", + "Dummy-firefox.x86_64" + ], + "Dummy-xulrunner": [ + "Dummy-xulrunner-debuginfo.x86_64", + "Dummy-xulrunner.x86_64" + ], + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.x86_64", + "dummy-bash.x86_64" + ], + "dummy-filesystem": [ + "dummy-filesystem.x86_64" + ], + "dummy-glibc": [ + "dummy-glibc-common.x86_64", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo-common.x86_64", + "dummy-glibc-debuginfo.i686", + "dummy-glibc-debuginfo.x86_64", + "dummy-glibc.i686", + "dummy-glibc.x86_64" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.x86_64", + "dummy-lvm2-libs.x86_64", + "dummy-lvm2.x86_64" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.x86_64" + ] + } + }, + "Server": { + "s390x": { + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.s390x", + "dummy-bash.s390x" + ], + "dummy-filesystem": [ + "dummy-filesystem.s390x" + ], + "dummy-glibc": [ + "dummy-glibc-common.s390x", + "dummy-glibc-debuginfo-common.s390x", + "dummy-glibc-debuginfo.s390x", + "dummy-glibc.s390x" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.s390x", + "dummy-lvm2-libs.s390x", + "dummy-lvm2.s390x" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.s390x", + "dummy-tftp.s390x" + ] + }, + "x86_64": { + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.x86_64", + "dummy-bash.x86_64" + ], + "dummy-filesystem": [ + "dummy-filesystem.x86_64" + ], + "dummy-glibc": [ + "dummy-glibc-common.x86_64", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo-common.x86_64", + "dummy-glibc-debuginfo.i686", + "dummy-glibc-debuginfo.x86_64", + "dummy-glibc.i686", + "dummy-glibc.x86_64" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.x86_64", + "dummy-lvm2-devel.i686", + "dummy-lvm2-devel.x86_64", + "dummy-lvm2-libs.x86_64", + "dummy-lvm2.x86_64" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.x86_64" + ] + } + } +} diff --git a/tests/fixtures/listing-multilib.json b/tests/fixtures/listing-multilib.json new file mode 100644 index 0000000..cb4986f --- /dev/null +++ b/tests/fixtures/listing-multilib.json @@ -0,0 +1,23 @@ +{ + "Client": { + "x86_64": { + "dummy-glibc": [ + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo.i686", + "dummy-glibc.i686" + ] + } + }, + "Server": { + "x86_64": { + "dummy-glibc": [ + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo.i686", + "dummy-glibc.i686" + ], + "dummy-lvm2": [ + "dummy-lvm2-devel.i686" + ] + } + } +} diff --git a/tests/fixtures/listing-server-x86_64.json b/tests/fixtures/listing-server-x86_64.json new file mode 100644 index 0000000..ac11e1b --- /dev/null +++ b/tests/fixtures/listing-server-x86_64.json @@ -0,0 +1,36 @@ +{ + "Server": { + "x86_64": { + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.x86_64", + "dummy-bash.x86_64" + ], + "dummy-filesystem": [ + "dummy-filesystem.x86_64" + ], + "dummy-glibc": [ + "dummy-glibc-common.x86_64", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo-common.x86_64", + "dummy-glibc-debuginfo.i686", + "dummy-glibc-debuginfo.x86_64", + "dummy-glibc.i686", + "dummy-glibc.x86_64" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.x86_64", + "dummy-lvm2-devel.i686", + "dummy-lvm2-devel.x86_64", + "dummy-lvm2-libs.x86_64", + "dummy-lvm2.x86_64" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.x86_64" + ] + } + } +} diff --git a/tests/test_listings.py b/tests/test_listings.py new file mode 100644 index 0000000..192d147 --- /dev/null +++ b/tests/test_listings.py @@ -0,0 +1,60 @@ +# -*- encoding: utf-8 -*- + +import json + +try: + import unittest2 as unittest +except ImportError: + import unittest + +from .helpers import get_compose, get_fixture + +from compose_utils import get_listing + + +class ListingTest(unittest.TestCase): + + def test_get_listing(self): + compose = get_compose("DP-1.0-20160315.t.0") + with open(get_fixture("full-listing.json")) as fh: + expected = json.load(fh) + + listing = get_listing(compose.rpms) + + self.assertEqual(listing, expected) + + def test_get_listing_filter_but_keep_all_variants(self): + compose = get_compose("DP-1.0-20160315.t.0") + with open(get_fixture("full-listing.json")) as fh: + expected = json.load(fh) + + listing = get_listing(compose.rpms, variants=["Client", "Server"]) + + self.assertEqual(listing, expected) + + def test_get_listing_filter_but_keep_all_arches(self): + compose = get_compose("DP-1.0-20160315.t.0") + with open(get_fixture("full-listing.json")) as fh: + expected = json.load(fh) + + listing = get_listing(compose.rpms, arches=["x86_64", "i386", "s390x"]) + + self.assertEqual(listing, expected) + + def test_get_listing_filter_trees(self): + compose = get_compose("DP-1.0-20160315.t.0") + with open(get_fixture("listing-server-x86_64.json")) as fh: + expected = json.load(fh) + + listing = get_listing(compose.rpms, variants=["Server"], arches=["x86_64"]) + + self.assertEqual(listing, expected) + + def test_get_listing_filter_pkg_arch(self): + compose = get_compose("DP-1.0-20160315.t.0") + with open(get_fixture("listing-multilib.json")) as fh: + expected = json.load(fh) + + listing = get_listing(compose.rpms, pkg_arches=["i686"], arches=["x86_64"]) + + self.assertEqual(listing, expected)