From 7fa243cfc276f947d47f980fb00817973e01e20a Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 15 2024 09:52:06 +0000 Subject: [PATCH 1/5] Configure bandit scanner and make it pass There's one false positive about ignoring exception, plus a few valid points about using asserts to check invariants. --- diff --git a/compose_utils/find_latest.py b/compose_utils/find_latest.py index 2a89713..a3fef01 100644 --- a/compose_utils/find_latest.py +++ b/compose_utils/find_latest.py @@ -42,8 +42,9 @@ def find_composes(patterns): try: # Access the metadata to raise an error. compose.info - except Exception as ex: - continue + except Exception: + # It's a not a compose, we can ignore this entry. + continue # nosec result.append(compose) return result diff --git a/compose_utils/package_moves.py b/compose_utils/package_moves.py index b830344..de7c19a 100644 --- a/compose_utils/package_moves.py +++ b/compose_utils/package_moves.py @@ -150,7 +150,10 @@ class ComposePackageMoves(object): "items": dict((key, new_variant_pkgs[key]) for key in extra_pkgs_in_variant) } variant_key = "%s.%s" % (variant, arch) - assert variant_key not in result["new_items"] + if variant_key in result["new_items"]: + raise RuntimeError( + "Assertion failed: %r in %r" % (variant_key, result["new_items"]) + ) result["new_items"][variant_key] = variant_new_items # Add parent variant into new_items list @@ -163,7 +166,11 @@ class ComposePackageMoves(object): if variant_key not in result["new_items"]: result["new_items"][variant_key] = parent_variant_new_items else: - assert parent_variant_new_items == result["new_items"][variant_key] + if parent_variant_new_items != result["new_items"][variant_key]: + raise RuntimeError( + "Assertion failed: %r != %r" + % (parent_variant_new_items, result["new_items"][variant_key]) + ) # == Moved items == # (packages which were previously available in the other variant) @@ -229,7 +236,10 @@ class ComposePackageMoves(object): variant = val["variant"] arch = val["arch"] items = val["items"] - assert arch not in tmp.get(variant, {}) + if arch in tmp.get(variant, {}): + raise RuntimeError( + "Assertion failed: %r in %r" % (arch, tmp.get(variant, {})) + ) tmp.setdefault(variant, {})[arch] = items for variant in sorted(tmp.keys()): @@ -252,7 +262,10 @@ class ComposePackageMoves(object): tmp = {} # {"variant->variant": {"arch": {item}, ..}, ..} for item in data: key = "%s->%s" % (item["from"], item["to"]) - assert item["arch"] not in tmp.get(key, {}) + if item["arch"] in tmp.get(key, {}): + raise RuntimeError( + "Assertion failed: %r in %r" % (item["arch"], tmp.get(key, {})) + ) tmp.setdefault(key, {})[item["arch"]] = item for direction in sorted(tmp.keys()): diff --git a/tox.ini b/tox.ini index febbfba..4584514 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py36,py38,py39,py310,py311,coverage +envlist=py27,py36,py38,py39,py310,py311,coverage,bandit skip_missing_interpreters = true minversion=3.12.0 @@ -47,3 +47,7 @@ ignore = E501,E402,W503,E203 [run] omit = tests/* + +[testenv:bandit] +deps = bandit +commands = bandit -r bin compose_utils From 19dcd3590e5e9bffccba917c10b01a66d7d63b22 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 15 2024 09:52:08 +0000 Subject: [PATCH 2/5] Use rpm-shim instead of rpm-py-installer --- diff --git a/tox.ini b/tox.ini index 4584514..941e1a5 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ deps= freezegun koji kobo>=0.10.0 - rpm-py-installer + rpm setenv = COVERAGE_FILE={env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} commands= From 3a036f9114de5c727f59b6f6704884932194a0ce Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 15 2024 09:52:44 +0000 Subject: [PATCH 3/5] Update list of tested python version 3.6 and 3.9 are not supported in upstream anymore, but are in RHEL 8 and 9. --- diff --git a/tox.ini b/tox.ini index 941e1a5..1078b71 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py36,py38,py39,py310,py311,coverage,bandit +envlist=py27,py36,py39,py310,py311,py312,coverage,bandit skip_missing_interpreters = true minversion=3.12.0 @@ -37,7 +37,7 @@ commands = coverage report -m coverage xml -o {toxworkdir}/coverage.xml coverage html -d {toxworkdir}/htmlcov -depends = py27, py35, py36, py37 +depends = py27, py36, py39, py310, py311, py312 parallel_show_output = True [flake8] From f48fe536850cdd791a39f0cdc73ad4e08603a12f Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 15 2024 10:04:22 +0000 Subject: [PATCH 4/5] Stop using distutils.version.LegacyVersion It's deprecated in 3.12, and RPM bindings provide a similar and more accurate implementation anyway. --- diff --git a/compose_utils/changelog.py b/compose_utils/changelog.py index 5399cb1..2b53ad7 100644 --- a/compose_utils/changelog.py +++ b/compose_utils/changelog.py @@ -12,13 +12,14 @@ __all__ = ( import os import json import re -from distutils.version import LooseVersion import six import kobo.pkgset from kobo.rpmlib import parse_nvra, make_nvr, make_nvra, get_changelogs_from_header from kobo.threads import ThreadPool, WorkerThread +import rpm + def formatsize(size): @@ -115,10 +116,10 @@ def get_changelog_diff_from_headers(old, new, max_records=-1): result = [] try: old_time = old_changelog[0].time - old_nvr = LooseVersion(to_utf8(old_changelog[0].name).rsplit(None, 1)[-1]) + old_nvr = rpm.ver(to_utf8(old_changelog[0].name).rsplit(None, 1)[-1]) while new_changelog: entry = new_changelog.pop(0) - new_nvr = LooseVersion(to_utf8(entry.name).rsplit(None, 1)[-1]) + new_nvr = rpm.ver(to_utf8(entry.name).rsplit(None, 1)[-1]) if entry.time < old_time or ( entry.time == old_time and new_nvr <= old_nvr ): From 858a9da22e71878c01c58f9d37b8750c3d837dbc Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 15 2024 10:06:42 +0000 Subject: [PATCH 5/5] Drop compatibility with productmd 1.1 --- diff --git a/tests/composes/DP-1.0-20160720.t.8/compose/.productmd11.composeinfo b/tests/composes/DP-1.0-20160720.t.8/compose/.productmd11.composeinfo deleted file mode 100644 index fbed85d..0000000 --- a/tests/composes/DP-1.0-20160720.t.8/compose/.productmd11.composeinfo +++ /dev/null @@ -1,302 +0,0 @@ -[compose] -date = 20160720 -id = DP-1.0-20160720.t.8 -respin = 8 -type = test - -[product] -family = Dummy Product -name = DP-1.0-20160720.t.8 -short = DP -type = ga -variants = Client,Everything,Live,Server -version = 1.0 - -[product-Server-Gluster] -is_layered = true -name = Gluster -short = Gluster -type = ga -version = 2.3 - -[variant-Client] -arches = i386,x86_64 -id = Client -name = Client -type = variant -uid = Client -variants = Client-optional - -[variant-Client-optional] -arches = i386,x86_64 -id = optional -name = optional -type = optional -uid = Client-optional - -[variant-Client-optional.i386] -debug_dir = Client-optional/i386/debug/tree -debug_packages = Client-optional/i386/debug/tree/Packages -debug_repository = Client-optional/i386/debug/tree -debug_tree = Client-optional/i386/debug/tree -debuginfo = Client-optional/i386/debug/tree -os_dir = Client-optional/i386/os -os_tree = Client-optional/i386/os -packages = Client-optional/i386/os/Packages -repository = Client-optional/i386/os -source_dir = Client-optional/source/tree -source_packages = Client-optional/source/tree/Packages -source_repository = Client-optional/source/tree -source_tree = Client-optional/source/tree - -[variant-Client-optional.x86_64] -debug_dir = Client-optional/x86_64/debug/tree -debug_packages = Client-optional/x86_64/debug/tree/Packages -debug_repository = Client-optional/x86_64/debug/tree -debug_tree = Client-optional/x86_64/debug/tree -debuginfo = Client-optional/x86_64/debug/tree -os_dir = Client-optional/x86_64/os -os_tree = Client-optional/x86_64/os -packages = Client-optional/x86_64/os/Packages -repository = Client-optional/x86_64/os -source_dir = Client-optional/source/tree -source_packages = Client-optional/source/tree/Packages -source_repository = Client-optional/source/tree -source_tree = Client-optional/source/tree - -[variant-Client.i386] -debug_dir = Client/i386/debug/tree -debug_packages = Client/i386/debug/tree/Packages -debug_repository = Client/i386/debug/tree -debug_tree = Client/i386/debug/tree -debuginfo = Client/i386/debug/tree -iso_dir = Client/i386/iso -isos = Client/i386/iso -os_dir = Client/i386/os -os_tree = Client/i386/os -packages = Client/i386/os/Packages -repository = Client/i386/os -source_dir = Client/source/tree -source_iso_dir = Client/source/iso -source_isos = Client/source/iso -source_packages = Client/source/tree/Packages -source_repository = Client/source/tree -source_tree = Client/source/tree - -[variant-Client.x86_64] -debug_dir = Client/x86_64/debug/tree -debug_packages = Client/x86_64/debug/tree/Packages -debug_repository = Client/x86_64/debug/tree -debug_tree = Client/x86_64/debug/tree -debuginfo = Client/x86_64/debug/tree -iso_dir = Client/x86_64/iso -isos = Client/x86_64/iso -os_dir = Client/x86_64/os -os_tree = Client/x86_64/os -packages = Client/x86_64/os/Packages -repository = Client/x86_64/os -source_dir = Client/source/tree -source_iso_dir = Client/source/iso -source_isos = Client/source/iso -source_packages = Client/source/tree/Packages -source_repository = Client/source/tree -source_tree = Client/source/tree - -[variant-Everything] -arches = i386,x86_64 -id = Everything -name = Everything -type = variant -uid = Everything - -[variant-Everything.i386] -debug_dir = Everything/i386/debug/tree -debug_packages = Everything/i386/debug/tree/Packages -debug_repository = Everything/i386/debug/tree -debug_tree = Everything/i386/debug/tree -debuginfo = Everything/i386/debug/tree -iso_dir = Everything/i386/iso -isos = Everything/i386/iso -os_dir = Everything/i386/os -os_tree = Everything/i386/os -packages = Everything/i386/os/Packages -repository = Everything/i386/os -source_dir = Everything/source/tree -source_iso_dir = Everything/source/iso -source_isos = Everything/source/iso -source_packages = Everything/source/tree/Packages -source_repository = Everything/source/tree -source_tree = Everything/source/tree - -[variant-Everything.x86_64] -debug_dir = Everything/x86_64/debug/tree -debug_packages = Everything/x86_64/debug/tree/Packages -debug_repository = Everything/x86_64/debug/tree -debug_tree = Everything/x86_64/debug/tree -debuginfo = Everything/x86_64/debug/tree -iso_dir = Everything/x86_64/iso -isos = Everything/x86_64/iso -os_dir = Everything/x86_64/os -os_tree = Everything/x86_64/os -packages = Everything/x86_64/os/Packages -repository = Everything/x86_64/os -source_dir = Everything/source/tree -source_iso_dir = Everything/source/iso -source_isos = Everything/source/iso -source_packages = Everything/source/tree/Packages -source_repository = Everything/source/tree -source_tree = Everything/source/tree - -[variant-Live] -arches = x86_64 -id = Live -name = Live -type = variant -uid = Live - -[variant-Live.x86_64] -debug_dir = Live/x86_64/debug/tree -debug_packages = Live/x86_64/debug/tree/Packages -debug_repository = Live/x86_64/debug/tree -debug_tree = Live/x86_64/debug/tree -debuginfo = Live/x86_64/debug/tree -iso_dir = Live/x86_64/iso -isos = Live/x86_64/iso -os_dir = Live/x86_64/os -os_tree = Live/x86_64/os -packages = Live/x86_64/os/Packages -repository = Live/x86_64/os -source_dir = Live/source/tree -source_iso_dir = Live/source/iso -source_isos = Live/source/iso -source_packages = Live/source/tree/Packages -source_repository = Live/source/tree -source_tree = Live/source/tree - -[variant-Server] -arches = s390x,x86_64 -id = Server -name = Server -type = variant -uid = Server -variants = Server-Gluster,Server-ResilientStorage,Server-optional - -[variant-Server-Gluster] -arches = x86_64 -id = Gluster -name = Gluster Layered Product -type = layered-product -uid = Server-Gluster - -[variant-Server-Gluster.x86_64] -debug_dir = Server-Gluster/x86_64/debug/tree -debug_packages = Server-Gluster/x86_64/debug/tree/Packages -debug_repository = Server-Gluster/x86_64/debug/tree -debug_tree = Server-Gluster/x86_64/debug/tree -debuginfo = Server-Gluster/x86_64/debug/tree -os_dir = Server-Gluster/x86_64/os -os_tree = Server-Gluster/x86_64/os -packages = Server-Gluster/x86_64/os/Packages -repository = Server-Gluster/x86_64/os -source_dir = Server-Gluster/source/tree -source_packages = Server-Gluster/source/tree/Packages -source_repository = Server-Gluster/source/tree -source_tree = Server-Gluster/source/tree - -[variant-Server-ResilientStorage] -arches = x86_64 -id = ResilientStorage -name = Resilient Storage -type = addon -uid = Server-ResilientStorage - -[variant-Server-ResilientStorage.x86_64] -debug_dir = Server/x86_64/debug/tree -debug_packages = Server/x86_64/debug/tree/addons/ResilientStorage -debug_repository = Server/x86_64/debug/tree/addons/ResilientStorage -debug_tree = Server/x86_64/debug/tree -debuginfo = Server/x86_64/debug/tree/addons/ResilientStorage -os_dir = Server/x86_64/os -os_tree = Server/x86_64/os -packages = Server/x86_64/os/addons/ResilientStorage -repository = Server/x86_64/os/addons/ResilientStorage -source_dir = Server/source/tree -source_packages = Server/source/tree/addons/ResilientStorage -source_repository = Server/source/tree/addons/ResilientStorage -source_tree = Server/source/tree - -[variant-Server-optional] -arches = s390x,x86_64 -id = optional -name = optional -type = optional -uid = Server-optional - -[variant-Server-optional.s390x] -debug_dir = Server-optional/s390x/debug/tree -debug_packages = Server-optional/s390x/debug/tree/Packages -debug_repository = Server-optional/s390x/debug/tree -debug_tree = Server-optional/s390x/debug/tree -debuginfo = Server-optional/s390x/debug/tree -os_dir = Server-optional/s390x/os -os_tree = Server-optional/s390x/os -packages = Server-optional/s390x/os/Packages -repository = Server-optional/s390x/os -source_dir = Server-optional/source/tree -source_packages = Server-optional/source/tree/Packages -source_repository = Server-optional/source/tree -source_tree = Server-optional/source/tree - -[variant-Server-optional.x86_64] -debug_dir = Server-optional/x86_64/debug/tree -debug_packages = Server-optional/x86_64/debug/tree/Packages -debug_repository = Server-optional/x86_64/debug/tree -debug_tree = Server-optional/x86_64/debug/tree -debuginfo = Server-optional/x86_64/debug/tree -os_dir = Server-optional/x86_64/os -os_tree = Server-optional/x86_64/os -packages = Server-optional/x86_64/os/Packages -repository = Server-optional/x86_64/os -source_dir = Server-optional/source/tree -source_packages = Server-optional/source/tree/Packages -source_repository = Server-optional/source/tree -source_tree = Server-optional/source/tree - -[variant-Server.s390x] -debug_dir = Server/s390x/debug/tree -debug_packages = Server/s390x/debug/tree/Packages -debug_repository = Server/s390x/debug/tree -debug_tree = Server/s390x/debug/tree -debuginfo = Server/s390x/debug/tree -iso_dir = Server/s390x/iso -isos = Server/s390x/iso -os_dir = Server/s390x/os -os_tree = Server/s390x/os -packages = Server/s390x/os/Packages -repository = Server/s390x/os -source_dir = Server/source/tree -source_iso_dir = Server/source/iso -source_isos = Server/source/iso -source_packages = Server/source/tree/Packages -source_repository = Server/source/tree -source_tree = Server/source/tree - -[variant-Server.x86_64] -debug_dir = Server/x86_64/debug/tree -debug_packages = Server/x86_64/debug/tree/Packages -debug_repository = Server/x86_64/debug/tree -debug_tree = Server/x86_64/debug/tree -debuginfo = Server/x86_64/debug/tree -iso_dir = Server/x86_64/iso -isos = Server/x86_64/iso -os_dir = Server/x86_64/os -os_tree = Server/x86_64/os -packages = Server/x86_64/os/Packages -repository = Server/x86_64/os -source_dir = Server/source/tree -source_iso_dir = Server/source/iso -source_isos = Server/source/iso -source_packages = Server/source/tree/Packages -source_repository = Server/source/tree -source_tree = Server/source/tree - diff --git a/tests/test_legacy_composeinfo.py b/tests/test_legacy_composeinfo.py index 0be3369..3486f7b 100644 --- a/tests/test_legacy_composeinfo.py +++ b/tests/test_legacy_composeinfo.py @@ -16,14 +16,6 @@ BASE_PATH = os.path.join(os.path.dirname(__file__), 'composes', 'DP-1.0-20160720 def get_expected(pref=''): - import pkg_resources - from distutils.version import LooseVersion - try: - productmd_ver = LooseVersion(pkg_resources.get_distribution('productmd').version) - if productmd_ver == LooseVersion('1.1'): - return pref + '.productmd11.composeinfo' - except pkg_resources.DistributionNotFound: - pass return pref + '.composeinfo'