From f86e0e8ce927458d1ac38b0fa6550c59e64f0105 Mon Sep 17 00:00:00 2001 From: Bala Konda Reddy M Date: Mar 04 2026 00:14:28 +0000 Subject: Add composes.json manifest for test results dashboard Maintain composes.json along with junit and html results to the azure blob sotrage. This enables to query test results programmatically for the dashboard Signed-off-by: Bala Konda Reddy M --- diff --git a/fedora-image-tester/fedora_image_tester/azure.py b/fedora-image-tester/fedora_image_tester/azure.py index 7cf8d1e..9538157 100644 --- a/fedora-image-tester/fedora_image_tester/azure.py +++ b/fedora-image-tester/fedora_image_tester/azure.py @@ -7,6 +7,7 @@ image with the results. """ import asyncio +import json import logging import os import xml.etree.ElementTree as ET @@ -18,6 +19,7 @@ from tempfile import TemporaryDirectory import lisa import yaml from azure import identity as az_identity +from fedfind import release as ff_release from azure.core.exceptions import AzureError from azure.mgmt.resource import ResourceManagementClient from azure.storage.blob import BlobServiceClient, ContentSettings @@ -185,21 +187,29 @@ class TestRunner: try: self.upload_results( - log_path, run_name, message.body["compose_id"], message.body["architecture"] + log_path, run_name, message.body["compose_id"], + message.body["architecture"], ) except AzureError as e: _log.warning("Failed to upload results to storage account: %s", str(e)) except OSError as e: _log.exception("Failed to trigger LISA: %s", str(e)) - def upload_results(self, log_path: str, run_name: str, compose_id: str, arch: str) -> None: + def upload_results( + self, log_path: str, run_name: str, compose_id: str, arch: str, + ) -> None: html_results_path = os.path.join(log_path, run_name, "lisa.html") junit_results_path = os.path.join(log_path, run_name, "lisa.junit.xml") + + # Parse compose ID to get structured path components + meta = self._parse_compose_id(compose_id) + base_path = f"{meta['distro']}/{meta['version']}/{meta['date']}/{meta['build']}/{arch}" + html_blob = self.azure_blob_client.get_blob_client( - container="$web", blob=f"{compose_id}/{arch}/index.html" + container="$web", blob=f"{base_path}/index.html" ) junit_blob = self.azure_blob_client.get_blob_client( - container="$web", blob=f"{compose_id}/{arch}/junit.xml" + container="$web", blob=f"{base_path}/junit.xml" ) with open(html_results_path, "rb") as data: content_settings = ContentSettings(content_type="text/html; charset=utf-8") @@ -235,6 +245,11 @@ class TestRunner: overwrite=True, ) + try: + self._rebuild_composes_manifest() + except AzureError as e: + _log.warning("Failed to update composes.json manifest: %s", str(e)) + def _build_result_message_body(self, original_message, test_results): """ Build the message body for test results publication. @@ -409,6 +424,95 @@ class TestRunner: _log.warning("No XML file with suffix 'lisa.junit.xml' found in %s", xml_path) return None + def _parse_compose_id(self, compose_id): + """Parse a compose ID into structured metadata using fedfind. + + Args: + compose_id (str): Compose identifier string + + Returns: + dict: Metadata with keys distro, version, date, build + + Examples: + Fedora-Cloud-43-20260212.0 + -> distro=Fedora, version=43, date=2026-02-12, build=0 + Fedora-Rawhide-20260212.n.0 + -> distro=Rawhide, version=Rawhide, date=2026-02-12, build=n.0 + Fedora-eln-20260212.n.0 + -> distro=ELN, version=eln, date=2026-02-12, build=n.0 + """ + ffrel = ff_release.get_release(cid=compose_id) + release_lower = ffrel.release.lower() + is_nightly = ffrel.type == "nightly" + build = f"n.{ffrel.respin}" if is_nightly else str(ffrel.respin) + + # Map release names to display names + if release_lower == "rawhide": + distro = "Rawhide" + elif release_lower == "eln": + distro = "ELN" + else: + distro = "Fedora" + + # Format date from YYYYMMDD to YYYY-MM-DD + date_str = ffrel.compose + formatted_date = f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}" + + return { + "distro": distro, + "version": ffrel.release, + "date": formatted_date, + "build": build, + } + + def _rebuild_composes_manifest(self): + """List blobs in storage and build composes.json manifest from scratch. + + This approach is resilient to reorganization or pruning of old results, + as it always reflects the current state of the storage container. + """ + container_client = self.azure_blob_client.get_container_client("$web") + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + + # Discover compose/arch pairs from blob paths + # Pattern: {distro}/{version}/{date}/{build}/{arch}/junit.xml + composes = {} + for blob in container_client.list_blobs(): + parts = blob.name.split("/") + if len(parts) == 6 and parts[5] == "junit.xml": + distro, version, date, build, arch = parts[:5] + compose_key = f"{distro}/{version}/{date}/{build}" + if compose_key not in composes: + composes[compose_key] = { + "distro": distro, + "version": version, + "date": date, + "build": build, + "results": {}, + } + base_path = f"{distro}/{version}/{date}/{build}/{arch}" + composes[compose_key]["results"][arch] = { + "junit_xml": f"{base_path}/junit.xml", + "html_report": f"{base_path}/index.html", + } + + # Build manifest sorted by date descending + manifest = { + "last_updated": timestamp, + "composes": sorted(composes.values(), key=lambda c: c.get("date", ""), reverse=True), + } + + # Upload + manifest_blob = self.azure_blob_client.get_blob_client( + container="$web", blob="composes.json" + ) + manifest_blob.upload_blob( + data=json.dumps(manifest, indent=2), + content_settings=ContentSettings(content_type="application/json"), + overwrite=True, + ) + _log.info("Rebuilt composes.json manifest with %d composes", len(composes)) + def cleanup_old_rgs(self) -> None: """In the event there's leftover resources from previous runs, do some tidying.""" resource_groups = self.resource_client.resource_groups.list() diff --git a/fedora-image-tester/pyproject.toml b/fedora-image-tester/pyproject.toml index 524c373..753d990 100644 --- a/fedora-image-tester/pyproject.toml +++ b/fedora-image-tester/pyproject.toml @@ -13,6 +13,7 @@ requires-python = ">=3.10" dependencies = [ "azure-mgmt-resource", + "fedfind", "fedora-messaging", "fedora-image-uploader-messages", "mslisa[azure,aws]",