From 41e37c12dd69edc18fe4ca2312a8a0ca74bd997e Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jul 17 2019 06:01:58 +0000 Subject: Add tool to generate a repo file for a compose JIRA: COMPOSE-3613 --- diff --git a/README.rst b/README.rst index e8cd97e..28b6f95 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,9 @@ Contents **compose-latest-symlink** create a symbolic link with a well-known name to the given compose +**compose-write-repo-file** + create a `.repo` file pointing to repositories in a compose + Related tools ------------- diff --git a/bin/compose-write-repo-file b/bin/compose-write-repo-file new file mode 100755 index 0000000..4202da1 --- /dev/null +++ b/bin/compose-write-repo-file @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import os +import sys + +here = sys.path[0] +if here != "/usr/bin": + sys.path[0] = os.path.dirname(here) + +from compose_utils import repo_file + +if __name__ == '__main__': + repo_file.main() diff --git a/compose_utils/repo_file.py b/compose_utils/repo_file.py new file mode 100644 index 0000000..4edbc37 --- /dev/null +++ b/compose_utils/repo_file.py @@ -0,0 +1,157 @@ +# -*- encoding: utf-8 -*- + +from __future__ import print_function + +import argparse +import os +import sys + +import productmd + +REPO = """\ +[{name}] +named = {name} +baseurl = {baseurl} +enabled = {enabled} +gpgcheck = 0 +""" + +CONTENT_TYPES = { + "repository": "-rpms", + "debug_repository": "-debuginfo-rpms", + "source_repository": "-source-rpms", +} + + +def translate(path, strip_prefix, url_prefix): + """Remove prefix from path and replace it with url_prefix.""" + if not path.startswith(strip_prefix): + raise RuntimeError("{0} does not start with {1}.".format(path, strip_prefix)) + return url_prefix + path[len(strip_prefix) :] + + +def emit(compose, opts, variant, output, content_type="repository"): + """Print configuration for a single repository into output.""" + paths = getattr(variant.paths, content_type) + release = compose.info.release + bp = compose.info.base_product + name = opts.name_pattern.format( + release_id=compose.info.release_id, + release_short=compose.info.release.short, + release_name=release.name, + release_version=release.version, + release_type=release.type, + release_major_version=release.major_version, + base_product_short=bp.short if release.is_layered else None, + base_product_name=bp.name if release.is_layered else None, + base_product_version=bp.version if release.is_layered else None, + base_product_type=bp.type if release.is_layered else None, + compose_id=compose.info.compose.id, + compose_type=compose.info.compose.type, + compose_date=compose.info.compose.date, + compose_respin=compose.info.compose.respin, + label=compose.info.compose.label, + variant=variant.uid, + arch=opts.arch, + ) + name += CONTENT_TYPES[content_type] + enabled = "1" if not opts.disabled or variant.uid not in opts.disabled else "0" + + if opts.arch: + if opts.arch not in paths: + raise RuntimeError( + "Variant {0} does not have arch {1}.".format(variant.uid, opts.arch) + ) + url = paths[opts.arch] + elif paths: + arch, path = list(paths.items())[0] + url = path.replace(arch, "$basearch") + else: + # No paths... + return + + baseurl = os.path.join(compose.compose_path, url) + if opts.translate: + baseurl = translate(baseurl, *opts.translate) + else: + baseurl = "file://" + baseurl + + content = REPO.format(name=name, enabled=enabled, baseurl=baseurl) + print(content, file=output) + + +def run(opts, output): + compose = productmd.Compose(os.path.realpath(opts.COMPOSE)) + + def by_uid(variant): + return variant.uid + + for variant in sorted(compose.info.variants.variants.values(), key=by_uid): + if opts.variant and variant.uid not in opts.variant: + # Variant is excluded + continue + + emit(compose, opts, variant, output) + if opts.include_debuginfo: + emit(compose, opts, variant, output, content_type="debug_repository") + + if opts.include_source: + emit(compose, opts, variant, output, content_type="source_repository") + + +def parse_mapping(value): + return value.split(",", 1) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument("COMPOSE", help="Compose to work with.") + parser.add_argument( + "--arch", help="Hardcode given architecture instead of using $basearch." + ) + parser.add_argument( + "--include-debuginfo", + action="store_true", + help="Create entries for debuginfo repos as well.", + ) + parser.add_argument( + "--include-source", + action="store_true", + help="Create entries for source repos as well.", + ) + parser.add_argument( + "--variant", + action="append", + help="Include only these variants. May be used multiple times.", + ) + parser.add_argument( + "--disabled", + metavar="VARIANT", + action="append", + help="Disable this variant by default. May be used multiple times.", + ) + parser.add_argument( + "-o", + "--output", + default="/dev/stdout", + help="Path for output file. Defaults to stdout.", + ) + parser.add_argument( + "--translate", + metavar="PATH_PREFIX,URL_PREFIX", + help="How to generate URLs.", + type=parse_mapping, + ) + parser.add_argument( + "--name-pattern", + default="{release_short}-{release_version}-{variant}", + help="Pattern for repository names.", + ) + opts = parser.parse_args(args) + + try: + with open(opts.output, "w") as output: + run(opts, output) + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/doc/compose-write-repo-file.1 b/doc/compose-write-repo-file.1 new file mode 100644 index 0000000..3747876 --- /dev/null +++ b/doc/compose-write-repo-file.1 @@ -0,0 +1,66 @@ +.TH compose-write-repo-file 1 +.SH NAME +compose-write-repo-file \- write .repo file pointing to a compose +.SH SYNOPSIS +.B compose-write-repo-file +[\fIOPTIONS\fR...] +\fICOMPOSE_PATH\fR +.SH DESCRIPTION +.B compose-write-repo-file +loads metadata from a compose and creates a YUM/DNF configuration file with +entries for repositories in the compose. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Print help and exit. +.TP +.BR \-\-arch = \fIARCH\fR +Use URLs with this hardcoded architecture. By default \fI$basearch\fR is used. +.TP +.BR \-\-include\-debuginfo +Create entries for debuginfo repositories. +.TP +.BR \-\-include\-source +Create entries for source repositories. +.TP +.BR \-\-variant = \fIVARIANT\fR +Create entries only for listed variants. Can be used multiple times. Variants +that do not include repositories are skipped automatically. +.TP +.BR \-\-disabled = \fIVARIANT\fR +Make entries for this variant disabled. By default all repositories are created +as enabled. +.TP +.BR \-\-output = \fIFILE_PATH\fR +Write output to this file instead of to stdout. +.TP +.BR \-\-translate = \fIPATH_PREFIX,URL_PREFIX\fR +By default local filepaths are created. With this option it is possible to +generate URLs. The path prefix will be stripped from path, and replaced with +URL prefix. +.TP +.BR \-\-name\-pattern = \fIPATTERN\fR +Customize name for the repositories. These fragments are available: +.sp +.RS +- \fIrelease_id\fR, \fIrelease_short\fR, \fIrelease_name\fR, \fIrelease_version\fR, +\fIrelease_type\fR, \fIrelease_major_version\fR +.br +- \fIbase_product_short\fR, \fIbase_product_name\fR, \fIbase_product_version\fR, +\fIbase_product_type\fR (these are only available for layered product composes) +.br +- \fIcompose_id\fR, \fIcompose_type\fR, \fIcompose_date\fR, \fIcompose_respin\fR, +\fIlabel\fR +.br +- \fIvariant\fR (should always be used unless a single variant is processed) +.br +- \fIarch\fR (only makes sense if \fB\-\-arch\fR is used) +.sp +The default is \fI{release_short}-{release_version}-{variant}\fR +.SH EXIT CODE +Exit code is always 0 on success and non-zero if there was a problem loading +the compose. +.SH BUGS +Please report bugs at +.br +https://pagure.io/compose-utils/issues diff --git a/setup.py b/setup.py index 1b3cb80..10c0f6b 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ setup( 'bin/compose-partial-copy', 'bin/compose-print-essentials', 'bin/compose-report-package-moves', + 'bin/compose-write-repo-file', ], install_requires=[ 'productmd>=1.0', @@ -42,6 +43,7 @@ setup( 'doc/compose-partial-copy.1', 'doc/compose-print-essentials.1', 'doc/compose-report-package-moves.1', + 'doc/compose-write-repo-file.1', ]), ], include_package_data=True, diff --git a/tests/fixtures/client.repo b/tests/fixtures/client.repo new file mode 100644 index 0000000..9a705ef --- /dev/null +++ b/tests/fixtures/client.repo @@ -0,0 +1,6 @@ +[DP-1.0-Client-rpms] +named = DP-1.0-Client-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os +enabled = 1 +gpgcheck = 0 + diff --git a/tests/fixtures/debuginfo.repo b/tests/fixtures/debuginfo.repo new file mode 100644 index 0000000..668bf85 --- /dev/null +++ b/tests/fixtures/debuginfo.repo @@ -0,0 +1,24 @@ +[DP-1.0-Client-rpms] +named = DP-1.0-Client-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Client-debuginfo-rpms] +named = DP-1.0-Client-debuginfo-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/debug/tree +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-rpms] +named = DP-1.0-Server-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-debuginfo-rpms] +named = DP-1.0-Server-debuginfo-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/debug/tree +enabled = 1 +gpgcheck = 0 + diff --git a/tests/fixtures/default.repo b/tests/fixtures/default.repo new file mode 100644 index 0000000..6c739c0 --- /dev/null +++ b/tests/fixtures/default.repo @@ -0,0 +1,12 @@ +[DP-1.0-Client-rpms] +named = DP-1.0-Client-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-rpms] +named = DP-1.0-Server-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os +enabled = 1 +gpgcheck = 0 + diff --git a/tests/fixtures/empty.repo b/tests/fixtures/empty.repo new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/fixtures/empty.repo diff --git a/tests/fixtures/sources.repo b/tests/fixtures/sources.repo new file mode 100644 index 0000000..b4f10f5 --- /dev/null +++ b/tests/fixtures/sources.repo @@ -0,0 +1,24 @@ +[DP-1.0-Client-rpms] +named = DP-1.0-Client-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Client-source-rpms] +named = DP-1.0-Client-source-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/source/tree +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-rpms] +named = DP-1.0-Server-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-source-rpms] +named = DP-1.0-Server-source-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/source/tree +enabled = 1 +gpgcheck = 0 + diff --git a/tests/fixtures/translate.repo b/tests/fixtures/translate.repo new file mode 100644 index 0000000..53dc702 --- /dev/null +++ b/tests/fixtures/translate.repo @@ -0,0 +1,12 @@ +[DP-1.0-Client-rpms] +named = DP-1.0-Client-rpms +baseurl = http://example.com/composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-rpms] +named = DP-1.0-Server-rpms +baseurl = http://example.com/composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os +enabled = 1 +gpgcheck = 0 + diff --git a/tests/fixtures/x86_64.repo b/tests/fixtures/x86_64.repo new file mode 100644 index 0000000..1b80ba0 --- /dev/null +++ b/tests/fixtures/x86_64.repo @@ -0,0 +1,12 @@ +[DP-1.0-Client-rpms] +named = DP-1.0-Client-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/x86_64/os +enabled = 1 +gpgcheck = 0 + +[DP-1.0-Server-rpms] +named = DP-1.0-Server-rpms +baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/x86_64/os +enabled = 1 +gpgcheck = 0 + diff --git a/tests/test_repo_file.py b/tests/test_repo_file.py new file mode 100644 index 0000000..117780b --- /dev/null +++ b/tests/test_repo_file.py @@ -0,0 +1,102 @@ +# -*- encoding: utf-8 -*- + +import difflib +import os +import tempfile + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import mock +from six import StringIO + +from .helpers import get_compose_path, get_fixture + +from compose_utils import repo_file + + +class TestRepoFile(unittest.TestCase): + def setUp(self): + _, self.temp_file = tempfile.mkstemp(prefix="repo-file-") + + def tearDown(self): + os.remove(self.temp_file) + + def assertFilesEqual(self, fn1, fn2, strip=None): + with open(fn1, "r") as f1: + lines1 = f1.readlines() + with open(fn2, "r") as f2: + lines2 = f2.readlines() + if strip: + lines2 = [line.replace(strip, "") for line in lines2] + diff = "".join( + difflib.unified_diff(lines1, lines2, fromfile="EXPECTED", tofile="ACTUAL") + ) + self.assertEqual(diff, "", "Files differ:\n" + diff) + + def success_run(self, args, compose_id, expected): + repo_file.main( + args + ["--output", self.temp_file, get_compose_path(compose_id)] + ) + strip = os.path.dirname(__file__) if "--translate" not in args else None + self.assertFilesEqual(get_fixture(expected), self.temp_file, strip=strip) + + def test_no_filter(self): + self.success_run([], "DP-1.0-20160315.t.0", "default.repo") + + def test_translate(self): + self.success_run( + ["--translate", os.path.dirname(__file__) + ",http://example.com"], + "DP-1.0-20160315.t.0", + "translate.repo", + ) + + def test_with_source(self): + self.success_run(["--include-source"], "DP-1.0-20160315.t.0", "sources.repo") + + def test_with_debuginfo(self): + self.success_run( + ["--include-debuginfo"], "DP-1.0-20160315.t.0", "debuginfo.repo" + ) + + def test_skip_variant(self): + self.success_run(["--variant=Client"], "DP-1.0-20160315.t.0", "client.repo") + + def test_hardcode_arch(self): + self.success_run(["--arch=x86_64"], "DP-1.0-20160315.t.0", "x86_64.repo") + + def test_no_paths(self): + self.success_run([], "DP-1.0-20181012.t.0", "empty.repo") + + def test_bad_hardcode_arch(self): + with mock.patch("sys.stderr", new_callable=StringIO) as mock_err: + with self.assertRaises(SystemExit): + repo_file.main( + [ + "--output", + self.temp_file, + "--arch=ppc64le", + get_compose_path("DP-1.0-20160315.t.0"), + ] + ) + self.assertIn( + "Variant Client does not have arch ppc64le.\n", mock_err.getvalue() + ) + + def test_bad_translate(self): + with mock.patch("sys.stderr", new_callable=StringIO) as mock_err: + with self.assertRaises(SystemExit): + repo_file.main( + [ + "--output", + self.temp_file, + get_compose_path("DP-1.0-20160315.t.0"), + "--translate", + "/foo/bar,http://example.com", + ] + ) + self.assertIn( + "/Client/$basearch/os does not start with /foo/bar.\n", mock_err.getvalue() + ) diff --git a/tox.ini b/tox.ini index a9e28e3..4d332a4 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ commands= [flake8] exclude = doc,*.pyc,*.py~,*.in,*.spec,*.sh,*.rst filename = *.py -ignore = E501,E402,W503 +ignore = E501,E402,W503,E203 [run] omit = tests/*