From aa50d89dd410331d0e0ef1df84f606c324dfbe7c Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Mar 23 2020 23:59:06 +0000 Subject: [PATCH 1/3] Add the ability to ping maintainers Signed-off-by: Mohan Boddu --- diff --git a/compose_tracker.py b/compose_tracker.py index e6df999..5b11d2d 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -24,6 +24,7 @@ import re import requests import sys import traceback +import toml import fedora_messaging.api import fedora_messaging.config @@ -134,6 +135,41 @@ class Consumer(object): kojitaskline = None return kojitaskline, text + def get_maintainers(self, line, ks_url): + # Read the toml file + req = requests.get(ks_url) + if not req.ok: + logger.info("Failed to retrieve maintainers info from ks repo.") + return None + maintainers_info = toml.loads(req.text) + arches = ['armhfp', 'aarch64', 'ppc64le', 'i686', 'x86_64', 's390x'] + variants = list(maintainers_info.keys()) + for variant in variants: + if variant.lower() in line.lower(): + subvariants = maintainers_info.get(variant) + for subvariant in subvariants: + if subvariant.lower() in line.lower(): + if 'arch *' in line.lower(): + maintainers_arches = [arch for arch in arches if arch in subvariants.get(subvariant).keys()] + if maintainers_arches: + maintainers = set() + for maintainers_arch in maintainers_arches: + maintainers.update(maintainers_info.get(variant).get(subvariant).get(maintainers_arch).get('fas')) + return [maintainers, variant, subvariant] + else: + maintainers = maintainers_info.get(variant).get(subvariant).get('fas') + return [maintainers, variant, subvariant] + else: + maintainers_arches = [arch for arch in arches if arch in subvariants.get(subvariant).keys()] + if maintainers_arches: + for maintainers_arch in maintainers_arches: + if maintainers_arch.lower() in line.lower(): + maintainers = maintainers_info.get(variant).get(subvariant).get(maintainers_arch).get('fas') + return [maintainers, variant, subvariant] + else: + maintainers = maintainers_info.get(variant).get(subvariant).get('fas') + return [maintainers, variant, subvariant] + def __call__(self, message: fedora_messaging.api.Message): # Catch any exceptions and don't raise them further because # it will cause /usr/bin/fedora-messaging to crash and we'll @@ -221,11 +257,32 @@ class Consumer(object): # next line and add them in markdown format. Also grab # the taskid if we can and print a hyperlink to koji if re.search('\[FAIL\]', line): + # Find the maintainers in kickstarts repo under + # mainatiners.toml file and ping them in the ticket + # Find the branch in kickstarts repo + ks_branch = None + if 'rawhide'.lower() == msg['release_version'].lower(): + ks_branch = 'master' + elif 'epel' not in msg['release_name'].lower(): + ks_branch = 'f' + msg['release_version'] + ks_url = 'https://pagure.io/fedora-kickstarts/raw/' + ks_branch + '/f/maintainers.toml' + kojitaskline, text = self.get_supporting_text(lines[x-1:]) content+=f'{text}\n' content+= "```\n%s\n%s\n```\n\n" % \ (line, kojitaskline or nextline) + # Ping the maintainers + pinging_list = self.get_maintainers(line, ks_url) + if not pinging_list: + logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") + else: + ping_line = 'Variant: {}, subvariant: {} task failed. Pinging maintainers: '.format(pinging_list[1], pinging_list[2]) + for maintainer in pinging_list[0]: + ping_line += '@{} '.format(maintainer) + content += ping_line + '\n' + logger.info(ping_line) + # If this is the Compose run failed line, then add it # to the description too if re.search('.*Compose run failed.*', line): diff --git a/test_consumer.py b/test_consumer.py index 1c2bf6e..fe6eef9 100644 --- a/test_consumer.py +++ b/test_consumer.py @@ -3,6 +3,7 @@ import json import logging import pytest +import toml import fedora_messaging.api import fedora_messaging.config @@ -45,6 +46,22 @@ EXAMPLE_PUNGI_LOG_INCOMPLETE = """2020-02-04 05:59:24 [INFO ] [BEGIN] ------- 2020-02-04 06:04:44 [INFO ] [DONE ] ---------- PHASE: OSTREE ---------- """ +EXAMPLE_PUNGI_LOG_PING_MAINTAINERS = """2020-02-04 05:59:24 [INFO ] [BEGIN] ---------- PHASE: LIVE MEDIA ---------- +2020-02-04 06:03:08 [LIVE_MEDIA ] [ERROR ] [FAIL] Live media (variant Spins, arch *, subvariant SoaS) failed, but going on anyway. +2020-02-04 06:03:08 [LIVE_MEDIA ] [ERROR ] [FAIL] Live media (variant Spins, arch *, subvariant kde) failed, but going on anyway. +2020-02-04 06:03:40 [LIVE_MEDIA ] [ERROR ] [FAIL] Live media (variant Spins, arch x86_64, subvariant xfce) failed, but going on anyway. +2020-02-04 06:03:40 [LIVE_MEDIA ] [ERROR ] [FAIL] Live media (variant Spins, arch x86_64, subvariant lxqt) failed, but going on anyway. +2020-02-04 06:04:44 [INFO ] [DONE ] ---------- PHASE: LIVE MEDIA ---------- +""" + +EXAMPLE_KS_MAINTAINERS = """ +[spins] +[spins.soas] +[spins.soas.x86_64] +fas = [ "pbrobinson" ] +maintainers = [ "Peter Robinson" ] +""" + def test_consumer_message_process(mocker, caplog): "Test that we can successfully process a message" @@ -152,12 +169,19 @@ def test_consumer_logfile_parsing_failures(mocker, caplog): caplog.set_level(logging.DEBUG) mocker.patch("compose_tracker.logger.level", logging.DEBUG) + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + {"consumer_config": {"composes_to_skip": ["IoT"]}}, + ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) text_mock = mocker.MagicMock() text_mock.text.splitlines.return_value = EXAMPLE_PUNGI_LOG_INCOMPLETE.splitlines() req.return_value = text_mock + # mock maintainers.toml file + ks_toml = mocker.patch("compose_tracker.toml.loads", return_value = toml.loads(EXAMPLE_KS_MAINTAINERS)) + con = Consumer() msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, @@ -175,14 +199,50 @@ def test_adding_labels(mocker, caplog): caplog.set_level(logging.DEBUG) mocker.patch("compose_tracker.logger.level", logging.DEBUG) + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + {"consumer_config": {"composes_to_skip": ["IoT"]}}, + ) req = mocker.patch("compose_tracker.requests.get", autospec=True) text_mock = mocker.MagicMock() text_mock.text.splitlines.return_value = EXAMPLE_PUNGI_LOG_INCOMPLETE.splitlines() req.return_value = text_mock + # mock maintainers.toml file + ks_toml = mocker.patch("compose_tracker.toml.loads", return_value = toml.loads(EXAMPLE_KS_MAINTAINERS)) + con = Consumer() msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) con.process(msg) assert "Adding Labels ['Fedora', 'Rawhide', 'DOOMED']" in caplog.text + +def test_consumer_maintainer_pings(mocker, caplog): + "Test that the consumer correctly pings the right maintainers" + + mocker.patch("compose_tracker.PagureService") + + # set logs at DEBUG and capture this level + caplog.set_level(logging.DEBUG) + mocker.patch("compose_tracker.logger.level", logging.DEBUG) + + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + {"consumer_config": {"composes_to_skip": ["IoT"]}}, + ) + # mock the pungi.global.log file + req = mocker.patch("compose_tracker.requests.get", autospec=True) + text_mock = mocker.MagicMock() + text_mock.text.splitlines.return_value = EXAMPLE_PUNGI_LOG_PING_MAINTAINERS.splitlines() + req.return_value = text_mock + + # mock maintainers.toml file + ks_toml = mocker.patch("compose_tracker.toml.loads", return_value = toml.loads(EXAMPLE_KS_MAINTAINERS)) + + con = Consumer() + msg = fedora_messaging.api.Message( + topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, + ) + con.process(msg) + assert "Variant: spins, subvariant: soas task failed. Pinging maintainers: @pbrobinson" in caplog.text \ No newline at end of file From 359658c0dc03b1813745897fc2d20fef4a8db985 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Mar 24 2020 00:03:03 +0000 Subject: [PATCH 2/3] RegEx pattern as raw string Removing some deprecated stuff Signed-off-by: Mohan Boddu --- diff --git a/compose_tracker.py b/compose_tracker.py index 5b11d2d..1582656 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -104,7 +104,7 @@ class Consumer(object): # then we'll get: # [FAIL] Image build (variant AtomicHost, arch *, subvariant AtomicHost) failed, but going on anyway. # ImageBuild task failed: 35659757. See /mnt/koji/compose/updates/Fedora-29-updates-testing-20190620.1/ - r = re.search('.*failed: (\d{8}).*', line) + r = re.search(r'.*failed: (\d{8}).*', line) if r: taskid = r.group(1) text = "- [%s](%s%s)" % (taskid, KOJI_TASK_URL, taskid) @@ -117,7 +117,7 @@ class Consumer(object): # [IMAGE_BUILD ] [INFO ] Hardlinking /mnt/koji/packages/Fedora-AtomicHost/29_Update/20190620.1/... # [IMAGE_BUILD ] [INFO ] Hardlinking /mnt/koji/packages/Fedora-AtomicHost/29_Update/20190620.1/... # [IMAGE_BUILD ] [INFO ] [DONE ] Creating image (formats: qcow2-raw-xz, arches: aarch64-ppc64le-x86_64, variant: AtomicHost, subvariant: AtomicHost) (task id: 35659753) - r = re.search('.*\[DONE \].*task id: (\d{8}).*', line) + r = re.search(r'.*\[DONE \].*task id: (\d{8}).*', line) if r: taskid = r.group(1) text = "- [%s](%s%s)" % (taskid, KOJI_TASK_URL, taskid) @@ -125,7 +125,7 @@ class Consumer(object): # If we get to an end of a phase then stop searching # [INFO ] [DONE ] ---------- PHASE: IMAGE_BUILD ---------- - r = re.search('.*\[DONE \] ---------- PHASE.*', line) + r = re.search(r'.*\[DONE \] ---------- PHASE.*', line) if r: break @@ -242,12 +242,12 @@ class Consumer(object): line = lines[x-1][20:] # trim date off log lines nextline = lines[x][20:] # trim date off log lines - r = re.search('.*\[BEGIN\] ---------- PHASE: (\w+) .*', line) + r = re.search(r'.*\[BEGIN\] ---------- PHASE: (\w+) .*', line) if r: name = r.group(1) phases[name] = [dt.datetime.fromisoformat(linedate)] - r = re.search('.*\[DONE \] ---------- PHASE: (\w+) .*', line) + r = re.search(r'.*\[DONE \] ---------- PHASE: (\w+) .*', line) if r: name = r.group(1) phase = phases.get(name) @@ -256,7 +256,7 @@ class Consumer(object): # If this is a [FAIL] line then we take it and the # next line and add them in markdown format. Also grab # the taskid if we can and print a hyperlink to koji - if re.search('\[FAIL\]', line): + if re.search(r'\[FAIL\]', line): # Find the maintainers in kickstarts repo under # mainatiners.toml file and ping them in the ticket # Find the branch in kickstarts repo From 7452a477a2327c52966777f18ec01671939bba59 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Mar 25 2020 15:03:58 +0000 Subject: [PATCH 3/3] Improvements to maintainer pings Signed-off-by: Mohan Boddu --- diff --git a/compose-tracker.toml b/compose-tracker.toml index 1f8b4ab..759ffae 100644 --- a/compose-tracker.toml +++ b/compose-tracker.toml @@ -40,6 +40,7 @@ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"] [consumer_config] composes_to_skip = ["IoT"] +ks_repo = "https://pagure.io/fedora-kickstarts" [qos] prefetch_size = 0 diff --git a/compose_tracker.py b/compose_tracker.py index 1582656..1fe42ee 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -25,6 +25,7 @@ import requests import sys import traceback import toml +import itertools import fedora_messaging.api import fedora_messaging.config @@ -142,33 +143,31 @@ class Consumer(object): logger.info("Failed to retrieve maintainers info from ks repo.") return None maintainers_info = toml.loads(req.text) - arches = ['armhfp', 'aarch64', 'ppc64le', 'i686', 'x86_64', 's390x'] - variants = list(maintainers_info.keys()) - for variant in variants: - if variant.lower() in line.lower(): - subvariants = maintainers_info.get(variant) - for subvariant in subvariants: - if subvariant.lower() in line.lower(): - if 'arch *' in line.lower(): - maintainers_arches = [arch for arch in arches if arch in subvariants.get(subvariant).keys()] - if maintainers_arches: - maintainers = set() - for maintainers_arch in maintainers_arches: - maintainers.update(maintainers_info.get(variant).get(subvariant).get(maintainers_arch).get('fas')) - return [maintainers, variant, subvariant] - else: - maintainers = maintainers_info.get(variant).get(subvariant).get('fas') - return [maintainers, variant, subvariant] - else: - maintainers_arches = [arch for arch in arches if arch in subvariants.get(subvariant).keys()] - if maintainers_arches: - for maintainers_arch in maintainers_arches: - if maintainers_arch.lower() in line.lower(): - maintainers = maintainers_info.get(variant).get(subvariant).get(maintainers_arch).get('fas') - return [maintainers, variant, subvariant] - else: - maintainers = maintainers_info.get(variant).get(subvariant).get('fas') - return [maintainers, variant, subvariant] + try: + matching_lists = re.findall(r'variant (\w+), arch (\S+), subvariant (\w+)', line) + if matching_lists: + for ml in matching_lists: + # For all arches + if ml[1] == '*': + # Get the list of all maintainers for all arches for that variant and sub variant + maintainers_list = list(maintainers_info.get(ml[0].lower()).get(ml[2].lower()).values()) + # Get the list of all fas usernames from the above list + maintainer_fas_list = [maintainer.get('fas') for maintainer in maintainers_list] + # Since there can be repetitions of fas usernames, create a set of usernames + maintainers = set(itertools.chain(*maintainer_fas_list)) + # return the list of maintainers, variant and sub variant + return maintainers, ml[0], ml[2] + else: + # Get the list of all maintainers for the variant, sub variant and arch + maintainers_list = maintainers_info.get(ml[0].lower()).get(ml[2].lower()).get(ml[1].lower()) + # Get a set of fas usernames from the above list + maintainers = set(maintainers_list.get('fas')) + # return the list of maintainers, variant and sub variant + return maintainers, ml[0], ml[2] + except Exception: + logger.info("No maintainer info found, skipping") + return None + def __call__(self, message: fedora_messaging.api.Message): # Catch any exceptions and don't raise them further because @@ -260,28 +259,29 @@ class Consumer(object): # Find the maintainers in kickstarts repo under # mainatiners.toml file and ping them in the ticket # Find the branch in kickstarts repo - ks_branch = None - if 'rawhide'.lower() == msg['release_version'].lower(): - ks_branch = 'master' - elif 'epel' not in msg['release_name'].lower(): - ks_branch = 'f' + msg['release_version'] - ks_url = 'https://pagure.io/fedora-kickstarts/raw/' + ks_branch + '/f/maintainers.toml' + if self.config["ks_repo"]: + if 'rawhide' == msg['release_version'].lower(): + ks_branch = 'master' + elif 'epel' not in msg['release_name'].lower(): + ks_branch = 'f' + msg['release_version'] + ks_url = self.config["ks_repo"] + '/raw/' + ks_branch + '/f/maintainers.toml' kojitaskline, text = self.get_supporting_text(lines[x-1:]) content+=f'{text}\n' content+= "```\n%s\n%s\n```\n\n" % \ (line, kojitaskline or nextline) - # Ping the maintainers - pinging_list = self.get_maintainers(line, ks_url) - if not pinging_list: - logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") - else: - ping_line = 'Variant: {}, subvariant: {} task failed. Pinging maintainers: '.format(pinging_list[1], pinging_list[2]) - for maintainer in pinging_list[0]: - ping_line += '@{} '.format(maintainer) - content += ping_line + '\n' - logger.info(ping_line) + if self.config["ks_repo"]: + # Ping the maintainers + maintainers, variant, sub_variant = self.get_maintainers(line, ks_url) or (None, None, None) + if not maintainers: + logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") + else: + ping_line = f'Variant: {variant}, subvariant: {sub_variant} task failed. Pinging maintainers: ' + for maintainer in maintainers: + ping_line += f'@{maintainer} ' + content += ping_line + '\n' + logger.info(ping_line) # If this is the Compose run failed line, then add it # to the description too diff --git a/test_consumer.py b/test_consumer.py index fe6eef9..25b86f5 100644 --- a/test_consumer.py +++ b/test_consumer.py @@ -55,11 +55,13 @@ EXAMPLE_PUNGI_LOG_PING_MAINTAINERS = """2020-02-04 05:59:24 [INFO ] [BEGIN] - """ EXAMPLE_KS_MAINTAINERS = """ -[spins] -[spins.soas] [spins.soas.x86_64] fas = [ "pbrobinson" ] maintainers = [ "Peter Robinson" ] + +[spins.xfce.x86_64] +fas = [ "nonamedotc", "kevin", "maxamillion" ] +maintainers = [ "Mukundan Ragavan", "Kevin Fenzi", "Adam Miller" ] """ @@ -88,7 +90,7 @@ def test_consumer_settings_ignore_compose(mocker, caplog): mocker.patch("compose_tracker.requests") mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) con = Consumer() body = copy.copy(EXAMPLE_MESSAGE_BODY) @@ -106,7 +108,7 @@ def test_consumer_settings_ignore_compose_2_values(mocker, caplog): mocker.patch("compose_tracker.requests") mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"]}}, + {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) con = Consumer() msg = fedora_messaging.api.Message( @@ -171,7 +173,7 @@ def test_consumer_logfile_parsing_failures(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -201,7 +203,7 @@ def test_adding_labels(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) req = mocker.patch("compose_tracker.requests.get", autospec=True) text_mock = mocker.MagicMock() @@ -229,7 +231,7 @@ def test_consumer_maintainer_pings(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -245,4 +247,6 @@ def test_consumer_maintainer_pings(mocker, caplog): topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) con.process(msg) - assert "Variant: spins, subvariant: soas task failed. Pinging maintainers: @pbrobinson" in caplog.text \ No newline at end of file + assert "Variant: Spins, subvariant: SoaS task failed. Pinging maintainers: @pbrobinson" in caplog.text + # Cannot assert particular maintainers since the set order is random + assert "Variant: Spins, subvariant: xfce task failed. Pinging maintainers" in caplog.text and "@kevin" in caplog.text and '@nonamedotc' in caplog.text and '@maxamillion' in caplog.text \ No newline at end of file