From b4a3b9e215bd8d56e545e89d71326e5c516cb5b5 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 21 2019 12:02:05 +0000 Subject: Exclude modular packages from product listings It does not make much sense to include them. The listing is meant as input for Pungi's prepopulate list, and that will not handle modular packages nicely. Signed-off-by: Lubomír Sedlář --- diff --git a/bin/compose-dump-listings b/bin/compose-dump-listings index 1718683..207e71b 100755 --- a/bin/compose-dump-listings +++ b/bin/compose-dump-listings @@ -19,12 +19,18 @@ def run(opts): if opts.compose.endswith("rpms.json"): rpms = productmd.Rpms() rpms.load(opts.compose) + modular_packages = {} else: compose = productmd.compose.Compose(opts.compose) rpms = compose.rpms + modular_packages = compose_utils.get_modular_packages(compose) listing = compose_utils.get_listing( - rpms, variants=opts.variant, arches=opts.arch, pkg_arches=opts.pkg_arch + rpms, + variants=opts.variant, + arches=opts.arch, + pkg_arches=opts.pkg_arch, + skip_nevras=modular_packages, ) json.dump(listing, sys.stdout, sort_keys=True, indent=2) diff --git a/compose_utils/__init__.py b/compose_utils/__init__.py index f2f40d6..4bc4039 100644 --- a/compose_utils/__init__.py +++ b/compose_utils/__init__.py @@ -57,7 +57,7 @@ def has_build(args=None): sys.exit(1) -def get_listing(rpms, variants=[], arches=[], pkg_arches=[]): +def get_listing(rpms, variants=[], arches=[], pkg_arches=[], skip_nevras={}): """Create product listing suitable for using as prepopulate file in Pungi.""" listing = {} @@ -67,10 +67,15 @@ def get_listing(rpms, variants=[], arches=[], pkg_arches=[]): for arch in rpms.rpms[variant]: if arches and arch not in arches: continue + to_skip = skip_nevras.get(variant, {}).get(arch, set()) for srpm_nevra in rpms.rpms[variant][arch]: pkgs = [] srpm_data = rpms.rpms[variant][arch][srpm_nevra] srpm_nvr = kobo.rpmlib.parse_nvra(srpm_nevra) + if set(srpm_data.keys()) & to_skip: + # If something from the build should be skipped, whole + # build should be skipped. + continue for nevra in srpm_data: nvra = kobo.rpmlib.parse_nvra(nevra) if pkg_arches and nvra["arch"] not in pkg_arches: @@ -85,3 +90,14 @@ def get_listing(rpms, variants=[], arches=[], pkg_arches=[]): ] = sorted(pkgs) return listing + + +def get_modular_packages(compose): + """Return a mapping from variant and arch to a set of modular package NVRAs.""" + mapping = {} + for variant in compose.modules.modules: + for arch in compose.modules.modules[variant]: + for nsvc in compose.modules.modules[variant][arch]: + module = compose.modules.modules[variant][arch][nsvc] + mapping.setdefault(variant, {}).setdefault(arch, set()).update(module["rpms"]) + return mapping diff --git a/tests/composes/DP-1.0-20160315.t.0/compose/metadata/modules.json b/tests/composes/DP-1.0-20160315.t.0/compose/metadata/modules.json new file mode 100644 index 0000000..a1e2626 --- /dev/null +++ b/tests/composes/DP-1.0-20160315.t.0/compose/metadata/modules.json @@ -0,0 +1,72 @@ +{ + "header": { + "type": "productmd.modules", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20190117", + "id": "Fedora-Rawhide-20190117.n.0", + "respin": 0, + "type": "nightly" + }, + "modules": { + "Server": { + "aarch64": { + "fish:3:3020190116191836:602da195": { + "metadata": { + "context": "602da195", + "koji_tag": "module-fish-3-3020190116191836-602da195", + "name": "fish", + "stream": "3", + "uid": "fish:3:3020190116191836:602da195", + "version": "3020190116191836" + }, + "modulemd_path": { + "binary": "Modular/aarch64/os/repodata/895b54407639b9836d7d2d397d7e1962f5f6c9d92e056a2e4d8a102fe1163826-modules.yaml.gz" + }, + "rpms": [ + "fish-0:3.0.0-1.module_f30+2710+8dbb4023.aarch64" + ] + }, + "fish:master:3020190116160550:602da195": { + "metadata": { + "context": "602da195", + "koji_tag": "module-fish-master-3020190116160550-602da195", + "name": "fish", + "stream": "master", + "uid": "fish:master:3020190116160550:602da195", + "version": "3020190116160550" + }, + "modulemd_path": { + "binary": "Modular/aarch64/os/repodata/895b54407639b9836d7d2d397d7e1962f5f6c9d92e056a2e4d8a102fe1163826-modules.yaml.gz" + }, + "rpms": [ + "fish-0:3.0.0-1.module_f30+2703+82e92e7e.aarch64" + ] + } + }, + "x86_64": { + "testmodule:master:3020190115184641:a5b0195c": { + "metadata": { + "context": "a5b0195c", + "koji_tag": "module-testmodule-master-3020190115184641-a5b0195c", + "name": "testmodule", + "stream": "master", + "uid": "testmodule:master:3020190115184641:a5b0195c", + "version": "3020190115184641" + }, + "modulemd_path": { + "binary": "Modular/x86_64/os/repodata/5ff336ba62aa170e033efc2dc9e9b372b641900eb98cd9af53738e1838d3c27a-modules.yaml.gz" + }, + "rpms": [ + "perl-List-Compare-0:0.53-11.module_f30+2696+4b2456b0.noarch", + "dwm-user-0:6.1-8.module_f30+2696+4b2456b0.x86_64", + "dwm-0:6.1-8.module_f30+2696+4b2456b0.x86_64" + ] + } + } + } + } + } +} diff --git a/tests/fixtures/filtered-listing.json b/tests/fixtures/filtered-listing.json new file mode 100644 index 0000000..1c3d43d --- /dev/null +++ b/tests/fixtures/filtered-listing.json @@ -0,0 +1,131 @@ +{ + "Client": { + "i386": { + "Dummy-firefox": [ + "Dummy-firefox-debuginfo.i686", + "Dummy-firefox.i686" + ], + "Dummy-xulrunner": [ + "Dummy-xulrunner-debuginfo.i686", + "Dummy-xulrunner.i686" + ], + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.i686", + "dummy-bash.i686" + ], + "dummy-filesystem": [ + "dummy-filesystem.i686" + ], + "dummy-glibc": [ + "dummy-glibc-common.i686", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo.i686", + "dummy-glibc.i686" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.i686", + "dummy-lvm2-libs.i686", + "dummy-lvm2.i686" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.i686", + "dummy-tftp.i686" + ] + }, + "x86_64": { + "Dummy-xulrunner": [ + "Dummy-xulrunner-debuginfo.x86_64", + "Dummy-xulrunner.x86_64" + ], + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.x86_64", + "dummy-bash.x86_64" + ], + "dummy-filesystem": [ + "dummy-filesystem.x86_64" + ], + "dummy-glibc": [ + "dummy-glibc-common.x86_64", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo-common.x86_64", + "dummy-glibc-debuginfo.i686", + "dummy-glibc-debuginfo.x86_64", + "dummy-glibc.i686", + "dummy-glibc.x86_64" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.x86_64", + "dummy-lvm2-libs.x86_64", + "dummy-lvm2.x86_64" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.x86_64" + ] + } + }, + "Server": { + "s390x": { + "dummy-bash": [ + "dummy-bash-debuginfo.s390x", + "dummy-bash.s390x" + ], + "dummy-filesystem": [ + "dummy-filesystem.s390x" + ], + "dummy-glibc": [ + "dummy-glibc-common.s390x", + "dummy-glibc-debuginfo-common.s390x", + "dummy-glibc-debuginfo.s390x", + "dummy-glibc.s390x" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.s390x", + "dummy-lvm2-libs.s390x", + "dummy-lvm2.s390x" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.s390x", + "dummy-tftp.s390x" + ] + }, + "x86_64": { + "dummy-basesystem": [ + "dummy-basesystem.noarch" + ], + "dummy-bash": [ + "dummy-bash-debuginfo.x86_64", + "dummy-bash.x86_64" + ], + "dummy-filesystem": [ + "dummy-filesystem.x86_64" + ], + "dummy-glibc": [ + "dummy-glibc-common.x86_64", + "dummy-glibc-debuginfo-common.i686", + "dummy-glibc-debuginfo-common.x86_64", + "dummy-glibc-debuginfo.i686", + "dummy-glibc-debuginfo.x86_64", + "dummy-glibc.i686", + "dummy-glibc.x86_64" + ], + "dummy-lvm2": [ + "dummy-lvm2-debuginfo.x86_64", + "dummy-lvm2-devel.i686", + "dummy-lvm2-devel.x86_64", + "dummy-lvm2-libs.x86_64", + "dummy-lvm2.x86_64" + ], + "dummy-tftp": [ + "dummy-tftp-debuginfo.x86_64", + "dummy-tftp.x86_64" + ] + } + } +} diff --git a/tests/test_listings.py b/tests/test_listings.py index 192d147..ebb790e 100644 --- a/tests/test_listings.py +++ b/tests/test_listings.py @@ -9,7 +9,7 @@ except ImportError: from .helpers import get_compose, get_fixture -from compose_utils import get_listing +from compose_utils import get_listing, get_modular_packages class ListingTest(unittest.TestCase): @@ -58,3 +58,45 @@ class ListingTest(unittest.TestCase): listing = get_listing(compose.rpms, pkg_arches=["i686"], arches=["x86_64"]) self.assertEqual(listing, expected) + + def test_get_listing_skip_packages(self): + compose = get_compose("DP-1.0-20160315.t.0") + with open(get_fixture("filtered-listing.json")) as fh: + expected = json.load(fh) + + self.maxDiff = None + skip_nevras = { + "Server": { + "s390x": set(["dummy-basesystem-0:10.0-6.noarch"]), + }, + "Client": { + "x86_64": set(["Dummy-firefox-0:16.0.1-1.x86_64"]), + }, + } + + listing = get_listing(compose.rpms, skip_nevras=skip_nevras) + + self.assertEqual(listing, expected) + + +class GetModularPackagesTest(unittest.TestCase): + + def test_load(self): + compose = get_compose("DP-1.0-20160315.t.0") + + self.assertEqual( + get_modular_packages(compose), + { + "Server": { + "aarch64": set([ + "fish-0:3.0.0-1.module_f30+2710+8dbb4023.aarch64", + "fish-0:3.0.0-1.module_f30+2703+82e92e7e.aarch64", + ]), + "x86_64": set([ + "perl-List-Compare-0:0.53-11.module_f30+2696+4b2456b0.noarch", + "dwm-user-0:6.1-8.module_f30+2696+4b2456b0.x86_64", + "dwm-0:6.1-8.module_f30+2696+4b2456b0.x86_64" + ]), + } + } + )