From 08976580fc07124c781f7955cbb6c1a5a09c423f Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Jun 09 2020 18:47:46 +0000 Subject: [PATCH 1/3] Add support for comparing ODCS composes Signed-off-by: Merlin Mathesius --- diff --git a/.gitignore b/.gitignore index de5adb9..2a94dd2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc /.tox/ /cccc.egg-info/ +/.project +/.pydevproject diff --git a/cccc/cli.py b/cccc/cli.py index a42e4ca..1511e4d 100644 --- a/cccc/cli.py +++ b/cccc/cli.py @@ -187,6 +187,21 @@ def _create_composes(conf, default_commit, updated_commit): return default_compose, updated_compose +def _compare_composes(conf, default_compose, updated_compose): + """ + Call compose comparison. + + :param conf: Configuration parameters. + :type conf: dict + :param default_compose: URL location of the default configuration compose. + :type default_compose: str + :param updated_compose: URL location of the updated configuration compose. + :type updated_compose: str + :return: None + """ + cccc.compose.compare_composes(conf, default_compose, updated_compose) + + def run(args): conf = cccc.config.init_config(args) @@ -220,4 +235,6 @@ def run(args): default_compose, updated_compose = _create_composes( conf, merged_default_commit, merged_updated_commit) + _compare_composes(conf, default_compose, updated_compose) + return os.EX_OK diff --git a/cccc/compose.py b/cccc/compose.py index 24f1631..361dda8 100644 --- a/cccc/compose.py +++ b/cccc/compose.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: MIT +import cccc.utils import json import os +import tempfile import time import odcs.client.odcs @@ -25,7 +27,7 @@ def _get_authenticated_odcs_client(conf): ) elif conf["odcs_auth_method"] == "openidc": - with open(conf["odcs_auth_file"], 'r') as f: + with open(conf["odcs_auth_file"], "r") as f: token = f.readline().strip() odcs_client = odcs.client.odcs.ODCS( @@ -101,3 +103,82 @@ def generate_compose(conf, commit): compose_url = result["toplevel_url"] + "/compose" print("Compose results can be found at URL: %s" % compose_url) return compose_url + + +def compare_composes(conf, old, new): + """ + Compare compose data from old (default) to new (updated) composes. + + :param conf: Configuration parameters. + :type conf: dict + :param old: URL location of old (default) compose. + :type old: str + :param new: URL location of new (updated) compose. + :type new: str + :return: None + """ + if not old or not new: + print("Cannot compare composes.") + return + + old_compose = ComposeData(old) + new_compose = ComposeData(new) + + output_dirobj = tempfile.TemporaryDirectory() + output_dir = output_dirobj.name + + cmd = [ + "compose-diff-rpms", + "--old", + old_compose.dirname(), + "--new", + new_compose.dirname(), + "--outputdir", + output_dir, + ] + cccc.utils.execute_cmd(cmd) + + # send compare log to output + compare_file = os.path.join(output_dir, "diff-rpms-diff.log") + with open(compare_file, "r") as f: + for line in f: + print("%s" % line.rstrip()) + + +class ComposeData(object): + def __init__(self, compose_url): + self.download_data(compose_url) + + def download_data(self, compose_url): + """ + Download compose data from `compose_url`. A temporary directory will + be created in which the compose data will be stored. The temporary + directory will be automatically destroyed when the object is freed. + + :param compose_url: URL location of compose. + :type compose_url: str + """ + self.compose_data_dirobj = tempfile.TemporaryDirectory() + metadata_dirname = os.path.join(self.dirname(), "metadata") + os.mkdir(metadata_dirname) + + compose_metadata_files = [ + "composeinfo.json", + "extra_files.json", + "modules.json", + "rpms.json", + ] + for md_filename in compose_metadata_files: + download_url = compose_url + "/metadata/" + md_filename + download_filename = os.path.join(metadata_dirname, md_filename) + cccc.utils.download_file(download_url, download_filename) + + def dirname(self): + """ + Return the temporary directory path in which the downloaded compose + data is stored. + + :return: Directory path. + :rtype: str + """ + return self.compose_data_dirobj.name diff --git a/cccc/utils.py b/cccc/utils.py index 426b828..1e49762 100644 --- a/cccc/utils.py +++ b/cccc/utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: MIT +import requests import subprocess import sys @@ -51,3 +52,25 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None, timeout=None, env=None stdout.write(out.decode("utf-8")) if stderr: stderr.write(err.decode("utf-8")) + + +def download_file(url, filename): + """ + Downloads a file from remote URL `url` and saves it in `filename`. + + :param url: The URL to a file to be downloaded. + :type url: str + :param filename: The local file in which to save the downloaded `url`. + :type filename: str + :return: True if URL was successfully downloaded, otherwise False. + :rtype: bool + """ + r = requests.get(url, stream=True) + if not r: + return False + with open(filename, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + f.flush() + return True diff --git a/requirements.txt b/requirements.txt index 7121f1e..bf2f1aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ kobo odcs[client] openidc_client +# compose-utils diff --git a/setup.py b/setup.py index 7cc17e1..8cd60ef 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def read_requirements(filename): with open(filename, 'r') as f: for line in f: - if line.startswith('-r') or line.strip() == '': + if line.startswith(('-r', '#')) or line.strip() == '': continue if line.startswith('git+'): dep_links.append(line.strip()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..548d2d4 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/tests/data/composes/odcs-fake01/compose/metadata/composeinfo.json b/tests/data/composes/odcs-fake01/compose/metadata/composeinfo.json new file mode 100644 index 0000000..c3dfff2 --- /dev/null +++ b/tests/data/composes/odcs-fake01/compose/metadata/composeinfo.json @@ -0,0 +1,61 @@ +{ + "header": { + "type": "productmd.composeinfo", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20200601", + "id": "Fedora-ELN-Rawhide-20200601.t.0", + "respin": 0, + "type": "test" + }, + "release": { + "internal": false, + "name": "Fedora-ELN", + "short": "Fedora-ELN", + "type": "ga", + "version": "Rawhide" + }, + "variants": { + "Everything": { + "arches": [ + "aarch64" + ], + "id": "Everything", + "name": "Everything", + "paths": { + "debug_packages": { + "aarch64": "Everything/aarch64/debug/tree/Packages" + }, + "debug_repository": { + "aarch64": "Everything/aarch64/debug/tree" + }, + "debug_tree": { + "aarch64": "Everything/aarch64/debug/tree" + }, + "os_tree": { + "aarch64": "Everything/aarch64/os" + }, + "packages": { + "aarch64": "Everything/aarch64/os/Packages" + }, + "repository": { + "aarch64": "Everything/aarch64/os" + }, + "source_packages": { + "aarch64": "Everything/source/tree/Packages" + }, + "source_repository": { + "aarch64": "Everything/source/tree" + }, + "source_tree": { + "aarch64": "Everything/source/tree" + } + }, + "type": "variant", + "uid": "Everything" + } + } + } +} diff --git a/tests/data/composes/odcs-fake01/compose/metadata/modules.json b/tests/data/composes/odcs-fake01/compose/metadata/modules.json new file mode 100644 index 0000000..542af43 --- /dev/null +++ b/tests/data/composes/odcs-fake01/compose/metadata/modules.json @@ -0,0 +1,15 @@ +{ + "header": { + "type": "productmd.modules", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20200601", + "id": "Fedora-ELN-Rawhide-20200601.t.0", + "respin": 0, + "type": "test" + }, + "modules": {} + } +} \ No newline at end of file diff --git a/tests/data/composes/odcs-fake01/compose/metadata/rpms.json b/tests/data/composes/odcs-fake01/compose/metadata/rpms.json new file mode 100644 index 0000000..5616ee9 --- /dev/null +++ b/tests/data/composes/odcs-fake01/compose/metadata/rpms.json @@ -0,0 +1,312 @@ +{ + "header": { + "type": "productmd.rpms", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20200601", + "id": "Fedora-ELN-Rawhide-20200601.t.0", + "respin": 0, + "type": "test" + }, + "rpms": { + "Everything": { + "aarch64": { + "bash-0:5.0.11-2.eln100.0.src": { + "bash-0:5.0.11-2.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/b/bash-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-0:5.0.11-2.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/b/bash-5.0.11-2.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "bash-debuginfo-0:5.0.11-2.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/b/bash-debuginfo-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-debugsource-0:5.0.11-2.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/b/bash-debugsource-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-devel-0:5.0.11-2.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/b/bash-devel-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-doc-0:5.0.11-2.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/b/bash-doc-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + } + }, + "coreutils-0:8.32-5.eln100.0.src": { + "coreutils-0:8.32-5.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/c/coreutils-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-0:8.32-5.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/c/coreutils-8.32-5.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "coreutils-common-0:8.32-5.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/c/coreutils-common-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-debuginfo-0:8.32-5.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/c/coreutils-debuginfo-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-debugsource-0:8.32-5.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/c/coreutils-debugsource-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-single-0:8.32-5.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/c/coreutils-single-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-single-debuginfo-0:8.32-5.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/c/coreutils-single-debuginfo-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + } + }, + "fedora-logos-0:30.0.2-4.eln100.0.src": { + "fedora-logos-0:30.0.2-4.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-logos-30.0.2-4.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "fedora-logos-0:30.0.2-4.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/f/fedora-logos-30.0.2-4.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "fedora-logos-httpd-0:30.0.2-4.eln100.0.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-logos-httpd-30.0.2-4.eln100.0.noarch.rpm", + "sigkey": "9570ff31" + } + }, + "fedora-release-0:33-0.8.src": { + "fedora-release-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-0:33-0.8.src": { + "category": "source", + "path": "Everything/source/tree/Packages/f/fedora-release-33-0.8.src.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-cinnamon-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-cinnamon-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-cloud-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-cloud-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-common-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-common-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-container-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-container-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-coreos-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-coreos-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-basic-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-basic-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-cinnamon-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-cinnamon-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-cloud-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-cloud-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-container-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-container-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-coreos-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-coreos-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-iot-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-iot-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-kde-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-kde-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-matecompiz-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-matecompiz-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-server-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-server-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-silverblue-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-silverblue-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-snappy-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-snappy-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-soas-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-soas-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-workstation-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-workstation-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-xfce-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-xfce-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-iot-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-iot-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-kde-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-kde-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-matecompiz-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-matecompiz-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-server-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-server-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-silverblue-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-silverblue-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-snappy-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-snappy-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-soas-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-soas-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-workstation-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-workstation-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-xfce-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-xfce-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + } + }, + "fedora-repos-0:33-0.5.src": { + "fedora-gpg-keys-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-gpg-keys-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-repos-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-0:33-0.5.src": { + "category": "source", + "path": "Everything/source/tree/Packages/f/fedora-repos-33-0.5.src.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-ostree-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-repos-ostree-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-rawhide-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-repos-rawhide-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + } + }, + "zsh-0:5.8-1.eln100.0.src": { + "zsh-0:5.8-1.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/z/zsh-5.8-1.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "zsh-0:5.8-1.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/z/zsh-5.8-1.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "zsh-debuginfo-0:5.8-1.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/z/zsh-debuginfo-5.8-1.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "zsh-debugsource-0:5.8-1.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/z/zsh-debugsource-5.8-1.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "zsh-html-0:5.8-1.eln100.0.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/z/zsh-html-5.8-1.eln100.0.noarch.rpm", + "sigkey": "9570ff31" + } + } + } + } + } + } +} diff --git a/tests/data/composes/odcs-fake02/compose/metadata/composeinfo.json b/tests/data/composes/odcs-fake02/compose/metadata/composeinfo.json new file mode 100644 index 0000000..2c477d4 --- /dev/null +++ b/tests/data/composes/odcs-fake02/compose/metadata/composeinfo.json @@ -0,0 +1,61 @@ +{ + "header": { + "type": "productmd.composeinfo", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20200601", + "id": "Fedora-ELN-Rawhide-20200601.t.1", + "respin": 1, + "type": "test" + }, + "release": { + "internal": false, + "name": "Fedora-ELN", + "short": "Fedora-ELN", + "type": "ga", + "version": "Rawhide" + }, + "variants": { + "Everything": { + "arches": [ + "aarch64" + ], + "id": "Everything", + "name": "Everything", + "paths": { + "debug_packages": { + "aarch64": "Everything/aarch64/debug/tree/Packages" + }, + "debug_repository": { + "aarch64": "Everything/aarch64/debug/tree" + }, + "debug_tree": { + "aarch64": "Everything/aarch64/debug/tree" + }, + "os_tree": { + "aarch64": "Everything/aarch64/os" + }, + "packages": { + "aarch64": "Everything/aarch64/os/Packages" + }, + "repository": { + "aarch64": "Everything/aarch64/os" + }, + "source_packages": { + "aarch64": "Everything/source/tree/Packages" + }, + "source_repository": { + "aarch64": "Everything/source/tree" + }, + "source_tree": { + "aarch64": "Everything/source/tree" + } + }, + "type": "variant", + "uid": "Everything" + } + } + } +} diff --git a/tests/data/composes/odcs-fake02/compose/metadata/modules.json b/tests/data/composes/odcs-fake02/compose/metadata/modules.json new file mode 100644 index 0000000..96e555e --- /dev/null +++ b/tests/data/composes/odcs-fake02/compose/metadata/modules.json @@ -0,0 +1,15 @@ +{ + "header": { + "type": "productmd.modules", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20200601", + "id": "Fedora-ELN-Rawhide-20200601.t.1", + "respin": 1, + "type": "test" + }, + "modules": {} + } +} \ No newline at end of file diff --git a/tests/data/composes/odcs-fake02/compose/metadata/rpms.json b/tests/data/composes/odcs-fake02/compose/metadata/rpms.json new file mode 100644 index 0000000..f2db1d2 --- /dev/null +++ b/tests/data/composes/odcs-fake02/compose/metadata/rpms.json @@ -0,0 +1,322 @@ +{ + "header": { + "type": "productmd.rpms", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20200601", + "id": "Fedora-ELN-Rawhide-20200601.t.1", + "respin": 0, + "type": "test" + }, + "rpms": { + "Everything": { + "aarch64": { + "bash-0:5.0.11-2.eln100.0.src": { + "bash-0:5.0.11-2.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/b/bash-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-0:5.0.11-2.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/b/bash-5.0.11-2.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "bash-debuginfo-0:5.0.11-2.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/b/bash-debuginfo-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-debugsource-0:5.0.11-2.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/b/bash-debugsource-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-devel-0:5.0.11-2.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/b/bash-devel-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "bash-doc-0:5.0.11-2.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/b/bash-doc-5.0.11-2.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + } + }, + "coreutils-0:8.32-5.eln100.0.src": { + "coreutils-0:8.32-5.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/c/coreutils-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-0:8.32-5.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/c/coreutils-8.32-5.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "coreutils-common-0:8.32-5.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/c/coreutils-common-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-debuginfo-0:8.32-5.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/c/coreutils-debuginfo-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-debugsource-0:8.32-5.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/c/coreutils-debugsource-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-single-0:8.32-5.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/c/coreutils-single-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "coreutils-single-debuginfo-0:8.32-5.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/c/coreutils-single-debuginfo-8.32-5.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + } + }, + "fedora-logos-0:30.0.2-4.eln100.0.src": { + "fedora-logos-0:30.0.2-4.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-logos-30.0.2-4.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "fedora-logos-0:30.0.2-4.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/f/fedora-logos-30.0.2-4.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "fedora-logos-httpd-0:30.0.2-4.eln100.0.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-logos-httpd-30.0.2-4.eln100.0.noarch.rpm", + "sigkey": "9570ff31" + } + }, + "fedora-release-0:33-0.8.src": { + "fedora-release-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-0:33-0.8.src": { + "category": "source", + "path": "Everything/source/tree/Packages/f/fedora-release-33-0.8.src.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-cinnamon-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-cinnamon-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-cloud-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-cloud-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-common-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-common-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-container-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-container-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-coreos-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-coreos-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-basic-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-basic-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-cinnamon-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-cinnamon-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-cloud-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-cloud-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-container-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-container-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-coreos-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-coreos-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-iot-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-iot-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-kde-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-kde-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-matecompiz-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-matecompiz-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-server-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-server-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-silverblue-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-silverblue-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-snappy-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-snappy-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-soas-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-soas-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-workstation-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-workstation-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-identity-xfce-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-identity-xfce-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-iot-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-iot-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-kde-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-kde-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-matecompiz-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-matecompiz-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-server-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-server-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-silverblue-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-silverblue-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-snappy-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-snappy-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-soas-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-soas-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-workstation-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-workstation-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-release-xfce-0:33-0.8.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-release-xfce-33-0.8.noarch.rpm", + "sigkey": "9570ff31" + } + }, + "fedora-repos-0:33-0.5.src": { + "fedora-gpg-keys-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-gpg-keys-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-repos-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-0:33-0.5.src": { + "category": "source", + "path": "Everything/source/tree/Packages/f/fedora-repos-33-0.5.src.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-ostree-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-repos-ostree-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + }, + "fedora-repos-rawhide-0:33-0.5.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/f/fedora-repos-rawhide-33-0.5.noarch.rpm", + "sigkey": "9570ff31" + } + }, + "gawk-0:5.0.1-8.eln100.0.src": { + "gawk-0:5.0.1-8.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/g/gawk-5.0.1-8.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "gawk-0:5.0.1-8.eln100.0.src": { + "category": "source", + "path": "Everything/source/tree/Packages/g/gawk-5.0.1-8.eln100.0.src.rpm", + "sigkey": "9570ff31" + }, + "gawk-all-langpacks-0:5.0.1-8.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/g/gawk-all-langpacks-5.0.1-8.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "gawk-debuginfo-0:5.0.1-8.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/g/gawk-debuginfo-5.0.1-8.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "gawk-debugsource-0:5.0.1-8.eln100.0.aarch64": { + "category": "debug", + "path": "Everything/aarch64/debug/tree/Packages/g/gawk-debugsource-5.0.1-8.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "gawk-devel-0:5.0.1-8.eln100.0.aarch64": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/g/gawk-devel-5.0.1-8.eln100.0.aarch64.rpm", + "sigkey": "9570ff31" + }, + "gawk-doc-0:5.0.1-8.eln100.0.noarch": { + "category": "binary", + "path": "Everything/aarch64/os/Packages/g/gawk-doc-5.0.1-8.eln100.0.noarch.rpm", + "sigkey": "9570ff31" + } + } + } + } + } + } +} diff --git a/tests/data/utils/hello.txt b/tests/data/utils/hello.txt new file mode 100644 index 0000000..8ab686e --- /dev/null +++ b/tests/data/utils/hello.txt @@ -0,0 +1 @@ +Hello, World! diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..3082289 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: MIT + +import http.server +import os +import signal +import socket +import socketserver + + +class LocalWebServer: + def __init__(self, webrootdir="/"): + self.webrootdir = webrootdir + + # find a free port + s = socket.socket() + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.host, self.port = s.getsockname() + s.close() + + self.pid = os.fork() + if self.pid == 0: + os.chdir(self.webrootdir) + handler = http.server.SimpleHTTPRequestHandler + httpd = socketserver.TCPServer((self.host, self.port), handler) + httpd.serve_forever() + os._exit(os.EX_OK) + + def __del__(self): + if hasattr(self, "pid") and self.pid != 0: + os.kill(self.pid, signal.SIGTERM) + + def base_url(self): + return "http://%s:%s" % (self.host, self.port) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4937fb2..9c9adf1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -112,7 +112,7 @@ class TestCLI(unittest.TestCase): None, ), ( - "pungi+comps+mod_defaults", + "pungi+comps+mod_defaults+arches+phases", "https://example.com/pr-pungi-repo.git", "https://example.com/pr-comps-repo.git", "https://example.com/pr-module-defaults-repo.git", @@ -120,6 +120,7 @@ class TestCLI(unittest.TestCase): ["buildinstall"], ), ]) + @patch("cccc.compose.compare_composes") @patch("cccc.compose.generate_compose") @patch("cccc.pungi.PungiConfig") @patch("cccc.git.push") @@ -137,6 +138,7 @@ class TestCLI(unittest.TestCase): mock_git_push, mock_pungi_config, mock_compose_generate, + mock_compare_composes, ): args = Args( pr_pungi_repo=pr_pungi_repo, @@ -269,3 +271,6 @@ class TestCLI(unittest.TestCase): call(ANY, "commit2"), call(ANY, "commit3"), ]) + + mock_compare_composes.assert_called_once_with( + ANY, "https://odcs/compose1", "https://odcs/compose2") diff --git a/tests/test_compose.py b/tests/test_compose.py index b7e898f..687dfd7 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: MIT -import os - -from mock import patch - import cccc.compose import cccc.config +import os +import sys +from .helpers import LocalWebServer +from io import StringIO +from mock import patch try: import unittest2 as unittest @@ -14,18 +15,21 @@ except ImportError: import unittest -DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "config") +CONFIG_DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "config") +COMPOSES_DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "composes") class TestCompose(unittest.TestCase): @patch.dict(os.environ, { - "CCCC_CONFIG_FILE": os.path.join(DATA_DIR, "compose-test.cfg") + "CCCC_CONFIG_FILE": os.path.join(CONFIG_DATA_DIR, "compose-test.cfg") }) @patch("odcs.client.odcs.ODCS.request_compose") @patch("odcs.client.odcs.ODCS.wait_for_compose") def test_compose_noauth(self, mock_odcs_wait, mock_odcs_request): """ + Tests that a compose request with no authorization makes expected + calls and returns expected values. """ conf = cccc.config.init_config() @@ -38,3 +42,45 @@ class TestCompose(unittest.TestCase): compose_url = cccc.compose.generate_compose(conf, "c0mm17") self.assertEqual("https://odcs.example.com/composes/odcs-1234/compose", compose_url) + + @patch('sys.stdout', new=StringIO()) + def test_compare_composes(self): + """ + Tests that composes can be compared. + """ + conf = cccc.config.init_config() + + webserver = LocalWebServer(COMPOSES_DATA_DIR) + + default_compose = webserver.base_url() + "/odcs-fake01/compose" + updated_compose = webserver.base_url() + "/odcs-fake02/compose" + + cccc.compose.compare_composes(conf, default_compose, updated_compose) + + output = sys.stdout.getvalue().strip() + + expected_output = ( + "OLD: Fedora-ELN-Rawhide-20200601.t.0\n" + "NEW: Fedora-ELN-Rawhide-20200601.t.1\n" + "\n" + "===== SUMMARY =====\n" + "Added source rpms: 1\n" + "Dropped source rpms: 1\n" + "Added rpms: 6\n" + "Dropped rpms: 4\n" + "\n" + "===== Everything =====\n" + "+ gawk-all-langpacks.aarch64\n" + "+ gawk-debuginfo.aarch64\n" + "+ gawk-debugsource.aarch64\n" + "+ gawk-devel.aarch64\n" + "+ gawk-doc.aarch64\n" + "+ gawk.aarch64\n" + "+ gawk.src\n" + "- zsh-debuginfo.aarch64\n" + "- zsh-debugsource.aarch64\n" + "- zsh-html.aarch64\n" + "- zsh.aarch64\n" + "- zsh.src" + ) + self.assertEqual(expected_output, output) diff --git a/tests/test_utils.py b/tests/test_utils.py index 5811b09..fbfc8e1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: MIT +import os import sys +import tempfile import time +from .helpers import LocalWebServer +from cccc.utils import download_file, execute_cmd from io import StringIO -from mock import patch - -from cccc.utils import execute_cmd +from mock import Mock, patch try: import unittest2 as unittest @@ -14,6 +16,9 @@ except ImportError: import unittest +DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "utils") + + class TestUtilsExecuteCmd(unittest.TestCase): @patch("sys.stdout", new=StringIO()) @@ -50,3 +55,77 @@ class TestUtilsExecuteCmd(unittest.TestCase): with self.assertRaisesRegex( RuntimeError, "Command .* returned non-zero value .*"): execute_cmd(["/usr/bin/printenv", "mY_RanDoM_EnV_varblE"]) + + +class TestUtilsDownloadFile(unittest.TestCase): + + def test_download_file(self): + webserver = LocalWebServer(DATA_DIR) + + file_to_download = "hello.txt" + download_url = webserver.base_url() + "/" + file_to_download + + target_fileobj = tempfile.NamedTemporaryFile() + target_file = target_fileobj.name + os.remove(target_file) + + self.assertTrue(download_file(download_url, target_file)) + + self.assertTrue(os.path.isfile(target_file)) + self.assertEqual("Hello, World!\n", open(target_file, "r").read()) + + @patch("requests.get") + def test_download_file_mocked(self, mock_req_get): + file_to_download = "hello.txt" + path_to_download = os.path.join(DATA_DIR, file_to_download) + download_url = "http://localhost/" + file_to_download + + mock_rv = Mock() + mock_rv.ok = True + mock_rv.iter_content = Mock() + mock_rv.iter_content.return_value = [ + open(path_to_download, "rb").read() + ] + mock_req_get.return_value = mock_rv + + target_fileobj = tempfile.NamedTemporaryFile() + target_file = target_fileobj.name + os.remove(target_file) + + self.assertTrue(download_file(download_url, target_file)) + + mock_req_get.assert_called_once_with(download_url, stream=True) + + self.assertTrue(os.path.isfile(target_file)) + self.assertEqual("Hello, World!\n", open(target_file, "r").read()) + + @patch("requests.get") + def test_download_file_does_not_exist_mocked(self, mock_req_get): + file_to_download = "doesnotexist.txt" + download_url = "http://localhost/" + file_to_download + + mock_rv = Mock() + mock_rv.status_code = 404 + mock_rv.ok = False + mock_req_get.return_value = False + + _, target_file = tempfile.mkstemp() + os.remove(target_file) + + self.assertFalse(download_file(download_url, target_file)) + + mock_req_get.assert_called_once_with(download_url, stream=True) + self.assertFalse(os.path.isfile(target_file)) + + def test_download_file_does_not_exist(self): + webserver = LocalWebServer(DATA_DIR) + + file_to_download = "doesnotexist.txt" + download_url = webserver.base_url() + "/" + file_to_download + + _, target_file = tempfile.mkstemp() + os.remove(target_file) + + self.assertFalse(download_file(download_url, target_file)) + + self.assertFalse(os.path.isfile(target_file)) From 2be6842b1a1f2883c743af8bbdfb4eebafdd7da7 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Jun 09 2020 18:47:46 +0000 Subject: [PATCH 2/3] Optimization to generate ODCS composes in parallel Signed-off-by: Merlin Mathesius --- diff --git a/cccc/cli.py b/cccc/cli.py index 1511e4d..34aa321 100644 --- a/cccc/cli.py +++ b/cccc/cli.py @@ -170,18 +170,33 @@ def _create_composes(conf, default_commit, updated_commit): :rtype: str, str """ - odcs_default_config = "%s#%s" % (conf["odcs_raw_config_name"], default_commit) - odcs_updated_config = "%s#%s" % (conf["odcs_raw_config_name"], updated_commit) + print("odcs default raw_config = %s#%s" % (conf["odcs_raw_config_name"], default_commit)) + print("odcs updated raw_config = %s#%s" % (conf["odcs_raw_config_name"], updated_commit)) - print("odcs default raw_config = %s" % odcs_default_config) - print("odcs updated raw_config = %s" % odcs_updated_config) + # Make requests for both composes so they can run in parallel - print("Generating compose for default configuration") - default_compose = cccc.compose.generate_compose(conf, default_commit) + print("Requesting compose for default configuration.") + default_compose_id = cccc.compose.request_compose(conf, default_commit) + if not default_compose_id: + print("Request failed.") + return None, None + print("Compose ID: %d" % default_compose_id) + + print("Requesting compose for updated configuration.") + updated_compose_id = cccc.compose.request_compose(conf, updated_commit) + if not updated_compose_id: + print("Request failed.") + return None, None + print("Compose ID: %d" % updated_compose_id) + + # Collect the results of the composes + + print("Waiting for completion of default configuration compose.") + default_compose = cccc.compose.wait_for_compose(conf, default_compose_id) print("Results: %s" % default_compose) - print("Generating compose for updated configuration") - updated_compose = cccc.compose.generate_compose(conf, updated_commit) + print("Waiting for completion of updated configuration compose.") + updated_compose = cccc.compose.wait_for_compose(conf, updated_compose_id) print("Results: %s" % updated_compose) return default_compose, updated_compose diff --git a/cccc/compose.py b/cccc/compose.py index 361dda8..783a36a 100644 --- a/cccc/compose.py +++ b/cccc/compose.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: MIT import cccc.utils +import dateutil.parser import json import os import tempfile -import time import odcs.client.odcs @@ -52,16 +52,16 @@ def _get_authenticated_odcs_client(conf): return odcs_client -def generate_compose(conf, commit): +def request_compose(conf, commit): """ - Call ODCS to generate a compose. + Call ODCS to request a compose. :param conf: Configuration parameters. :type conf: dict :param commit: Commit hash of the configuration to compose. :type commit: str - :return: URL location of generated compose. - :rtype: str + :return: ODCS ID of requested compose, or None if an error occurred. + :rtype: int """ client = _get_authenticated_odcs_client(conf) if not client: @@ -75,16 +75,37 @@ def generate_compose(conf, commit): "compose_type": "test", } compose = client.request_compose(source, **request_args) - compose_id = compose["id"] + return compose["id"] + + +def wait_for_compose(conf, compose_id): + """ + Call ODCS to wait for a compose. + + :param conf: Configuration parameters. + :type conf: dict + :param compose_id: ODCS ID of compose to wait for. + :type compose_id: int + :return: URL location of generated compose, or None if an error occurred. + :rtype: str + """ + client = _get_authenticated_odcs_client(conf) + if not client: + print("Failed to authenticate ODCS client") + return None print("Waiting for compose %d to complete. Timeout = %d seconds." % ( compose_id, conf["odcs_compose_timeout"])) - t1 = time.time() result = client.wait_for_compose(compose_id, conf["odcs_compose_timeout"]) - t2 = time.time() - print("Compose completed in %d seconds. Result follows." % int(t2 - t1)) + if result.get("time_submitted") and result.get("time_done"): + t1 = dateutil.parser.parse(result["time_submitted"]) + t2 = dateutil.parser.parse(result["time_done"]) + elapsed = int((t2 - t1).total_seconds()) + print("Compose completed in %d seconds." % elapsed) + + print("Compose %d result follows." % compose_id) print(json.dumps(result, indent=4, sort_keys=True)) if result.get("state_name") != "done" or not result.get("toplevel_url"): @@ -101,7 +122,8 @@ def generate_compose(conf, commit): return None compose_url = result["toplevel_url"] + "/compose" - print("Compose results can be found at URL: %s" % compose_url) + print("Compose %d results can be found at URL: %s" % ( + compose_id, compose_url)) return compose_url diff --git a/cccc/pungi.py b/cccc/pungi.py index 199e5eb..b79d8f9 100644 --- a/cccc/pungi.py +++ b/cccc/pungi.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: MIT -import sys - from kobo.conf import PyConfigParser @@ -17,8 +15,6 @@ class PungiConfig(object): :param cfg_file: Full file path from which to read. :type cfg_file: str """ - print("Reading configuration from file '%s'." % cfg_file, - file=sys.stderr) self.conf = PyConfigParser() self.conf.load_from_file(cfg_file) @@ -42,8 +38,6 @@ class PungiConfig(object): :type cfg_file: str """ with open(cfg_file, "w") as f: - print("Writing configuration to file '%s'." % cfg_file, - file=sys.stderr) f.write(self.config_file_dumps()) def _get_scm_url(self, scm_attr): diff --git a/test-requirements.txt b/test-requirements.txt index bb67c5f..d69093d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ +python-dateutil mock pytest parameterized diff --git a/tests/test_cli.py b/tests/test_cli.py index 9c9adf1..9ab64b3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -121,7 +121,8 @@ class TestCLI(unittest.TestCase): ), ]) @patch("cccc.compose.compare_composes") - @patch("cccc.compose.generate_compose") + @patch("cccc.compose.wait_for_compose") + @patch("cccc.compose.request_compose") @patch("cccc.pungi.PungiConfig") @patch("cccc.git.push") @patch("cccc.git.merge_repo") @@ -137,7 +138,8 @@ class TestCLI(unittest.TestCase): mock_git_merge, mock_git_push, mock_pungi_config, - mock_compose_generate, + mock_compose_request, + mock_compose_wait, mock_compare_composes, ): args = Args( @@ -150,11 +152,15 @@ class TestCLI(unittest.TestCase): mock_git_commit.side_effect = [ "commit1", "commit2", - "commit3" + "commit3", ] - mock_compose_generate.side_effect = [ + mock_compose_request.side_effect = [ + 1, + 2, + ] + mock_compose_wait.side_effect = [ "https://odcs/compose1", - "https://odcs/compose2" + "https://odcs/compose2", ] self.assertEqual(cccc.cli.run(args), 0) @@ -266,11 +272,17 @@ class TestCLI(unittest.TestCase): except AssertionError: pass - self.assertEqual(mock_compose_generate.call_count, 2) - mock_compose_generate.assert_has_calls([ + self.assertEqual(mock_compose_request.call_count, 2) + mock_compose_request.assert_has_calls([ call(ANY, "commit2"), call(ANY, "commit3"), ]) + self.assertEqual(mock_compose_wait.call_count, 2) + mock_compose_wait.assert_has_calls([ + call(ANY, 1), + call(ANY, 2), + ]) + mock_compare_composes.assert_called_once_with( ANY, "https://odcs/compose1", "https://odcs/compose2") diff --git a/tests/test_compose.py b/tests/test_compose.py index 687dfd7..0c560b3 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -39,7 +39,10 @@ class TestCompose(unittest.TestCase): "toplevel_url": "https://odcs.example.com/composes/odcs-1234", } - compose_url = cccc.compose.generate_compose(conf, "c0mm17") + compose_id = cccc.compose.request_compose(conf, "c0mm17") + self.assertEqual(1234, compose_id) + + compose_url = cccc.compose.wait_for_compose(conf, 1234) self.assertEqual("https://odcs.example.com/composes/odcs-1234/compose", compose_url) From 52479b7fc24c6f2547db08d5d11a0448af053d41 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Jun 09 2020 18:47:46 +0000 Subject: [PATCH 3/3] Use an application-specific Kerberos credential cache to avoid any potential conflict with the default ccache. Signed-off-by: Merlin Mathesius --- diff --git a/cccc/compose.py b/cccc/compose.py index 783a36a..28b0580 100644 --- a/cccc/compose.py +++ b/cccc/compose.py @@ -38,6 +38,10 @@ def _get_authenticated_odcs_client(conf): elif conf["odcs_auth_method"] == "kerberos": os.environ["KRB5_CLIENT_KTNAME"] = conf["odcs_auth_file"] + # set an application-specific credential cache to avoid + # any potential conflict with the default ccache + os.environ["KRB5CCNAME"] = "FILE:%s/krb5cc_%d_cccc" % ( + tempfile.gettempdir(), os.getuid()) odcs_client = odcs.client.odcs.ODCS( odcs_url,