From 5b2f68f2144441b680e8ebbbc1c6f98f8aeed152 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 28 2020 22:03:16 +0000 Subject: Add support for generating the ODCS compose Signed-off-by: Merlin Mathesius --- diff --git a/README.md b/README.md index fbfb403..c0072af 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,23 @@ compose difference will be added as comment to the PR. It should therefore be clear if the most important parts of the compose still works with the PR applied and how the PR changed the compose. + + +## Development + +### Unit-testing + +Install packages required to build the python package: + +``` +$ sudo dnf install -y \ + git \ + make \ + krb5-devel +``` + +Run the tests: + +``` +$ make test +``` diff --git a/cccc/cli.py b/cccc/cli.py index cb85c76..2ed8c2a 100644 --- a/cccc/cli.py +++ b/cccc/cli.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: MIT +import cccc.compose import cccc.config import cccc.git import cccc.pungi @@ -150,9 +151,9 @@ def _push_merged_config(conf, merged_dir): cccc.git.push(merged_dir, id_file=conf["merged_repo_id_file"]) -def _odcs_placeholder(conf, default_commit, updated_commit): +def _create_composes(conf, default_commit, updated_commit): """ - Place holder for future calls to ODCS. + Call ODCS to generate composes. :param conf: Configuration parameters. :type conf: dict @@ -160,6 +161,10 @@ def _odcs_placeholder(conf, default_commit, updated_commit): :type default_commit: str :param updated_commit: Commit hash of the updated configuration. :type updated_commit: str + :return: URL locations of composes corresponding to default_commit, + updated_commit. + :rtype: str, str + """ odcs_default_config = "%s#%s" % (conf["odcs_raw_config_name"], default_commit) odcs_updated_config = "%s#%s" % (conf["odcs_raw_config_name"], updated_commit) @@ -167,6 +172,16 @@ def _odcs_placeholder(conf, default_commit, updated_commit): print("odcs default raw_config = %s" % odcs_default_config) print("odcs updated raw_config = %s" % odcs_updated_config) + print("Generating compose for default configuration") + default_compose = cccc.compose.generate_compose(conf, default_commit) + print("Results: %s" % default_compose) + + print("Generating compose for updated configuration") + updated_compose = cccc.compose.generate_compose(conf, updated_commit) + print("Results: %s" % updated_compose) + + return default_compose, updated_compose + def run(args): conf = cccc.config.init_config(args) @@ -189,7 +204,7 @@ def run(args): # push the branch back to the merged configuration repo _push_merged_config(conf, merged_dir) - # cleanup + # cleanup git scratch directories if conf["debug"]: print("Be sure to cleanup merged and scratch directories: " "rm -rf %s %s" % (merged_dir, scratch_dir)) @@ -197,7 +212,8 @@ def run(args): shutil.rmtree(merged_dir, ignore_errors=True) shutil.rmtree(scratch_dir, ignore_errors=True) - # future ODCS calls, etc. - _odcs_placeholder(conf, merged_default_commit, merged_updated_commit) + # call ODCS to generate composes + default_compose, updated_compose = _create_composes( + conf, merged_default_commit, merged_updated_commit) return os.EX_OK diff --git a/cccc/compose.py b/cccc/compose.py new file mode 100644 index 0000000..24f1631 --- /dev/null +++ b/cccc/compose.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: MIT + +import json +import os +import time + +import odcs.client.odcs + + +def _get_authenticated_odcs_client(conf): + """ + Instantiate authenticated ODCS client object. + + :param conf: Configuration parameters. + :type conf: dict + :return: Authenticated ODCS client object. + :rtype: odcs.client.odcs.ODCS + """ + odcs_url = conf["odcs_server_url"] + + if conf["odcs_auth_method"] == "anonymous": + odcs_client = odcs.client.odcs.ODCS( + odcs_url, + auth_mech=odcs.client.odcs.AuthMech.Anonymous, + ) + + elif conf["odcs_auth_method"] == "openidc": + with open(conf["odcs_auth_file"], 'r') as f: + token = f.readline().strip() + + odcs_client = odcs.client.odcs.ODCS( + odcs_url, + auth_mech=odcs.client.odcs.AuthMech.OpenIDC, + openidc_token=token, + ) + + elif conf["odcs_auth_method"] == "kerberos": + os.environ["KRB5_CLIENT_KTNAME"] = conf["odcs_auth_file"] + + odcs_client = odcs.client.odcs.ODCS( + odcs_url, + auth_mech=odcs.client.odcs.AuthMech.Kerberos, + ) + + else: + print("Unknown configuration 'odcs_auth_method' = '%s'" % + conf["odcs_auth_method"]) + return None + + return odcs_client + + +def generate_compose(conf, commit): + """ + Call ODCS to generate a compose. + + :param conf: Configuration parameters. + :type conf: dict + :param commit: Commit hash of the configuration to compose. + :type commit: str + :return: URL location of generated compose. + :rtype: str + """ + client = _get_authenticated_odcs_client(conf) + if not client: + print("Failed to authenticate ODCS client") + return None + + source = odcs.client.odcs.ComposeSourceRawConfig( + conf["odcs_raw_config_name"], commit, + ) + request_args = { + "compose_type": "test", + } + compose = client.request_compose(source, **request_args) + compose_id = compose["id"] + + print("Waiting for compose %d to complete. Timeout = %d seconds." % ( + compose_id, conf["odcs_compose_timeout"])) + + t1 = time.time() + result = client.wait_for_compose(compose_id, conf["odcs_compose_timeout"]) + t2 = time.time() + + print("Compose completed in %d seconds. Result follows." % int(t2 - t1)) + print(json.dumps(result, indent=4, sort_keys=True)) + + if result.get("state_name") != "done" or not result.get("toplevel_url"): + print("Compose failed.") + if result.get("state_reason"): + print("Reason: %s" % result["state_reason"]) + elif result.get("error"): + print("Reason: %s %s: %s" % ( + result.get("status"), + result.get("error"), + result.get("message"))) + else: + print("Reason: unknown") + return None + + compose_url = result["toplevel_url"] + "/compose" + print("Compose results can be found at URL: %s" % compose_url) + return compose_url diff --git a/cccc/config.py b/cccc/config.py index 449b731..a78ff0e 100644 --- a/cccc/config.py +++ b/cccc/config.py @@ -14,12 +14,16 @@ _DefaultConfig = { "merged_repo_id_file": os.path.join(CCCC_CONFIG_DIR, "id_file"), "merged_repo_pungi_config": "cccc.conf", "merged_repo_url": "https://example.com/cccc-merged-configs.git", + "odcs_auth_file": "/dev/null", + "odcs_auth_method": "anonymous", + "odcs_compose_timeout": 3600, "odcs_raw_config_name": "odcs_cccc", "odcs_server_url": "https://127.0.0.1/", "verbose": False, } _RHELConfiguration = { + "odcs_auth_method": "kerberos", "odcs_server_url": "https://odcs.engineering.redhat.com", } @@ -30,6 +34,7 @@ _CentOSConfig = { } _FedoraConfig = { + "odcs_auth_method": "openidc", "odcs_server_url": "https://odcs.fedoraproject.org", "pungi": "https://pagure.io/pungi-fedora", "comps": "https://pagure.io/fedora-comps", diff --git a/cccc/git.py b/cccc/git.py index 188adfe..443bc8f 100644 --- a/cccc/git.py +++ b/cccc/git.py @@ -98,10 +98,6 @@ def merge_repo(repodir, url, commit="proposed/master"): cmd = ["git", "fetch", "proposed"] execute_cmd(cmd, cwd=repodir) - # return to master branch of origin - cmd = ["git", "checkout", "origin/master"] - execute_cmd(cmd, cwd=repodir) - # finally, merge the proposed update cmd = ["git", "merge", "--no-ff", "-m", "Merge proposed", commit] execute_cmd(cmd, cwd=repodir) diff --git a/requirements.txt b/requirements.txt index c138ac6..7121f1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ kobo +odcs[client] +openidc_client diff --git a/tests/data/config/compose-test.cfg b/tests/data/config/compose-test.cfg new file mode 100644 index 0000000..0cfaa36 --- /dev/null +++ b/tests/data/config/compose-test.cfg @@ -0,0 +1,2 @@ +odcs_raw_config_name = "odcs_cccc" +odcs_server_url = "https://127.0.0.1/" diff --git a/tests/test_cli.py b/tests/test_cli.py index cbcb3c9..da0e4d9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,12 +41,6 @@ class Args(object): class TestCLI(unittest.TestCase): - # @pytest.mark.parametrize("pr_mod_defaults_repo", - # ["https://example.com/pr-module-defaults-repo.git"]) - # @pytest.mark.parametrize("pr_comps_repo", - # ["https://example.com/pr-comps-repo.git"]) - # @pytest.mark.parametrize("pr_pungi_repo", - # ["https://example.com/pr-pungi-repo.git"]) @parameterized.expand([ # (testcase_name, pr_pungi_repo, pr_comps_repo, pr_mod_defaults_repo) ( @@ -98,6 +92,7 @@ class TestCLI(unittest.TestCase): "https://example.com/pr-module-defaults-repo.git", ), ]) + @patch("cccc.compose.generate_compose") @patch("cccc.pungi.PungiConfig") @patch("cccc.git.push") @patch("cccc.git.merge_repo") @@ -112,13 +107,22 @@ class TestCLI(unittest.TestCase): mock_git_merge, mock_git_push, mock_pungi_config, + mock_compose_generate, ): args = Args( pr_pungi_repo=pr_pungi_repo, pr_comps_repo=pr_comps_repo, pr_mod_defaults_repo=pr_mod_defaults_repo, ) - mock_git_commit.side_effect = ["commit1", "commit2", "commit3"] + mock_git_commit.side_effect = [ + "commit1", + "commit2", + "commit3" + ] + mock_compose_generate.side_effect = [ + "https://odcs/compose1", + "https://odcs/compose2" + ] self.assertEqual(cccc.cli.run(args), 0) @@ -198,3 +202,9 @@ class TestCLI(unittest.TestCase): "was called but should NOT have been") except AssertionError: pass + + self.assertEqual(mock_compose_generate.call_count, 2) + mock_compose_generate.assert_has_calls([ + call(ANY, "commit2"), + call(ANY, "commit3"), + ]) diff --git a/tests/test_compose.py b/tests/test_compose.py new file mode 100644 index 0000000..b7e898f --- /dev/null +++ b/tests/test_compose.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: MIT + +import os + +from mock import patch + +import cccc.compose +import cccc.config + + +try: + import unittest2 as unittest +except ImportError: + import unittest + + +DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "config") + + +class TestCompose(unittest.TestCase): + + @patch.dict(os.environ, { + "CCCC_CONFIG_FILE": os.path.join(DATA_DIR, "compose-test.cfg") + }) + @patch("odcs.client.odcs.ODCS.request_compose") + @patch("odcs.client.odcs.ODCS.wait_for_compose") + def test_compose_noauth(self, mock_odcs_wait, mock_odcs_request): + """ + """ + conf = cccc.config.init_config() + + mock_odcs_request.return_value = {"id": 1234} + mock_odcs_wait.return_value = { + "state_name": "done", + "toplevel_url": "https://odcs.example.com/composes/odcs-1234", + } + + compose_url = cccc.compose.generate_compose(conf, "c0mm17") + self.assertEqual("https://odcs.example.com/composes/odcs-1234/compose", + compose_url) diff --git a/tests/test_config.py b/tests/test_config.py index 4f29788..a8d6280 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,20 @@ except ImportError: DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "config") +DEFAULT_DEFAULTS = { + "debug": False, + "git_user_email": "<>", + "git_user_name": "CCCC", + "merged_repo_id_file": "/etc/cccc/id_file", + "merged_repo_pungi_config": "cccc.conf", + "merged_repo_url": "https://example.com/cccc-merged-configs.git", + "odcs_auth_file": "/dev/null", + "odcs_auth_method": "anonymous", + "odcs_compose_timeout": 3600, + "odcs_raw_config_name": "odcs_cccc", + "odcs_server_url": "https://127.0.0.1/", + "verbose": False, +} class TestConfig(unittest.TestCase): @@ -23,17 +37,7 @@ class TestConfig(unittest.TestCase): Test that the config can be initialized to expected default values when a configuration file does not exist. """ - expected = { - "debug": False, - "git_user_email": "<>", - "git_user_name": "CCCC", - "merged_repo_id_file": "/etc/cccc/id_file", - "merged_repo_pungi_config": "cccc.conf", - "merged_repo_url": "https://example.com/cccc-merged-configs.git", - "odcs_raw_config_name": "odcs_cccc", - "odcs_server_url": "https://127.0.0.1/", - "verbose": False, - } + expected = DEFAULT_DEFAULTS.copy() conf = cccc.config.init_config() self.assertDictEqual(expected, conf) @@ -46,18 +50,8 @@ class TestConfig(unittest.TestCase): Test that the config can be initialized to expected values when loading a configuration file. """ - expected = { - "debug": False, - "git_user_email": "<>", - "git_user_name": "CCCC", - "merged_repo_id_file": "/etc/cccc/id_file", - "merged_repo_pungi_config": "cccc.conf", - "merged_repo_url": "https://example.com/cccc-merged-configs.git", - "odcs_raw_config_name": "odcs_cccc", - "odcs_server_url": "https://127.0.0.1/", - "verbose": False, - "config_file_loaded": True, - } + expected = DEFAULT_DEFAULTS.copy() + expected["config_file_loaded"] = True conf = cccc.config.init_config() self.assertDictEqual(expected, conf)