diff --git a/test/renderdiff/src/compare.py b/test/renderdiff/src/compare.py index 70b3e8e22d..4756233302 100644 --- a/test/renderdiff/src/compare.py +++ b/test/renderdiff/src/compare.py @@ -14,15 +14,12 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None): for f in all_files) all_results = [] for test_dir in test_dirs: - results_meta = {} results = [] - output_test_dir = None if not out_dir else os.path.join(out_dir, test_dir) + output_test_dir = None if not out_dir else os.path.abspath(os.path.join(out_dir, test_dir)) if output_test_dir: mkdir_p(output_test_dir) base_test_dir = os.path.abspath(os.path.join(base_dir, test_dir)) comp_test_dir = os.path.abspath(os.path.join(comparison_dir, test_dir)) - results_meta['base_dir'] = base_test_dir - results_meta['comparison_dir'] = comp_test_dir for golden_file in \ glob.glob(os.path.join(base_test_dir, "*.tif")): base_fname = os.path.abspath(golden_file) @@ -45,8 +42,12 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None): result['result'] = RESULT_OK results.append(result) if output_test_dir: - results_meta['results'] = results output_fname = os.path.join(output_test_dir, "compare_results.json") + results_meta = { + 'results': results, + 'base_dir': os.path.relpath(output_fname, base_test_dir), + 'comparison_dir': os.path.relpath(output_fname, comp_test_dir), + } with open(output_fname, 'w') as f: f.write(json.dumps(results_meta, indent=2)) important_print(f'Written comparison results for {test_dir} to \n {output_fname}') @@ -64,7 +65,7 @@ if __name__ == '__main__': dest = args.dest if not dest: print('Assume the default renderdiff output folder') - dest = os.path.join(os.getcwd(), './out/renderdiff_tests') + dest = os.path.join(os.getcwd(), './out/renderdiff') assert os.path.exists(dest), f"Destination folder={dest} does not exist." results = _compare_goldens(args.src, dest, out_dir=args.out) diff --git a/test/renderdiff/src/viewer.py b/test/renderdiff/src/viewer.py index 30f9ac1042..ec1d743e26 100644 --- a/test/renderdiff/src/viewer.py +++ b/test/renderdiff/src/viewer.py @@ -17,6 +17,9 @@ import sys import flask import pathlib import json +import requests +import io +import zipfile from utils import ArgParseImpl @@ -25,14 +28,163 @@ from flask import Flask, request, make_response, send_from_directory DIR = pathlib.Path(__file__).parent.absolute() HTML_DIR = os.path.join(DIR, "viewer_html") +# Generated by gemini +def _download_github_artifacts(pr_number, github_token, output_dir= ".") -> None: + """ + Downloads artifacts associated with a specific GitHub Pull Request. + + This function performs the following steps: + 1. Fetches the details of the Pull Request to get its head commit SHA. + 2. Searches for GitHub Actions workflow runs triggered by that specific commit. + 3. Iterates through successful workflow runs to find and list all associated artifacts. + 4. Downloads each artifact (which comes as a ZIP file). + 5. Extracts the contents of each downloaded ZIP file into a unique subdirectory + within the specified output directory. + + Args: + owner (str): The GitHub repository owner (e.g., "octocat"). + repo (str): The GitHub repository name (e.g., "Spoon-Knife"). + pr_number (int): The Pull Request number. + output_dir (str): The local directory where downloaded artifacts will be saved. + Defaults to the current directory. + """ + + # Prepare HTTP headers for GitHub API requests + headers = {"Accept": "application/vnd.github.v3+json"} + if github_token: + headers["Authorization"] = f"token {github_token}" + + OWNER_REPO = 'google/filament' + + # --- Step 1: Get PR details to find the head commit SHA --- + print(f"Fetching details for PR #{pr_number} in {OWNER_REPO}...") + pr_url = f"https://api.github.com/repos/{OWNER_REPO}/pulls/{pr_number}" + try: + response = requests.get(pr_url, headers=headers) + response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx) + pr_data = response.json() + commit_sha = pr_data["head"]["sha"] + print(f"PR #{pr_number} is associated with commit SHA: {commit_sha}") + except requests.exceptions.HTTPError as e: + if e.response.status_code == 404: + print(f"Error: PR #{pr_number} not found in {OWNER_REPO}. Please check the PR number, owner, and repository name.") + elif e.response.status_code == 403: + print(f"Error: Access forbidden to PR #{pr_number}. You might be hitting API rate limits or need a valid GitHub Token.") + else: + print(f"An HTTP error occurred while fetching PR details: {e}") + return # Exit function on error + except requests.exceptions.RequestException as e: + print(f"A network error occurred while fetching PR details: {e}") + return # Exit function on error + + # --- Step 2: Find workflow runs associated with the commit SHA --- + print(f"Searching for workflow runs for commit SHA: {commit_sha}...") + workflow_runs_url = f"https://api.github.com/repos/{OWNER_REPO}/actions/runs" + # Filter by head_sha and event='pull_request' for precision + params = {"head_sha": commit_sha, "event": "pull_request"} + try: + response = requests.get(workflow_runs_url, headers=headers, params=params) + response.raise_for_status() + runs_data = response.json() + workflow_runs = runs_data.get("workflow_runs", []) + + if not workflow_runs: + print(f"No workflow runs found directly associated with PR #{pr_number} (commit SHA: {commit_sha}).") + print("This might happen if the workflow was triggered by a push after the PR was opened,") + print("or if the PR head branch was updated without triggering a new workflow run with this exact SHA.") + print("Consider checking GitHub Actions runs manually for this PR's branch on GitHub.") + return None + + # Filter for runs that completed successfully + successful_runs = [run for run in workflow_runs if run.get("conclusion") == "success" and run.get("status") == "completed"] + if not successful_runs: + print(f"No *successful and completed* workflow runs found for PR #{pr_number} with commit SHA {commit_sha}. Exiting.") + return None + + except requests.exceptions.HTTPError as e: + print(f"An HTTP error occurred while searching for workflow runs: {e}") + return None + except requests.exceptions.RequestException as e: + print(f"A network error occurred while searching for workflow runs: {e}") + return None + + # Create the main output directory if it doesn't exist + os.makedirs(output_dir, exist_ok=True) + print(f"Ensuring output directory exists: {os.path.abspath(output_dir)}") + + downloaded_any_artifact = False # Flag to track if any artifact was downloaded + + # --- Step 3 & 4: List and Download Artifacts for each successful run --- + for run in successful_runs: + run_id = run["id"] + run_name = run["name"] + print(f"\nProcessing workflow run '{run_name}' (ID: {run_id})...") + + artifacts_url = f"https://api.github.com/repos/{OWNER_REPO}/actions/runs/{run_id}/artifacts" + try: + response = requests.get(artifacts_url, headers=headers) + response.raise_for_status() + artifacts_data = response.json() + artifacts = artifacts_data.get("artifacts", []) + + if not artifacts: + print(f" No artifacts found for workflow run ID {run_id}.") + continue # Move to the next workflow run + + for artifact in artifacts: + artifact_id = artifact["id"] + artifact_name = artifact["name"] + archive_download_url = artifact["archive_download_url"] + + print(f" Found artifact: '{artifact_name}' (ID: {artifact_id})") + + # Perform the download request + print(f" Downloading '{artifact_name}'...") + # Use a copy of headers and specific Accept for ZIP download + download_headers = headers.copy() + download_headers["Accept"] = "application/vnd.github.v3+zip" + download_response = requests.get(archive_download_url, headers=download_headers, stream=True) + download_response.raise_for_status() # Check for errors in download + + # --- Step 5: Extract the contents --- + # Use BytesIO to handle the zip file content in memory without saving to a temporary file + with io.BytesIO(download_response.content) as zip_buffer: + try: + with zipfile.ZipFile(zip_buffer, 'r') as zip_ref: + # Create a unique subdirectory for each artifact to avoid file name conflicts + extract_path = os.path.join(output_dir, f"{artifact_name}_{artifact_id}") + os.makedirs(extract_path, exist_ok=True) + zip_ref.extractall(extract_path) + print(f" Successfully extracted '{artifact_name}' to '{extract_path}/'") + downloaded_any_artifact = True + except zipfile.BadZipFile: + print(f" Error: Downloaded file for '{artifact_name}' is not a valid zip file. Skipping extraction.") + except Exception as e: + print(f" An error occurred during extraction of '{artifact_name}': {e}") + except requests.exceptions.HTTPError as e: + print(f" An HTTP error occurred while fetching artifacts for run {run_id}: {e}") + if e.response.status_code == 403: + print(" This often means you need a GitHub Personal Access Token with 'repo' scope (even for public repos for artifact downloads).") + continue # Continue processing the next workflow run + except requests.exceptions.RequestException as e: + print(f" A network error occurred while fetching artifacts for run {run_id}: {e}") + continue # Continue processing the next workflow run + + if not downloaded_any_artifact: + print("\nNo artifacts were downloaded for the specified PR.") + else: + print("\nAll available artifacts have been processed.") + return 'Done' + def _create_app(config): app = Flask(__name__) client_config = config.copy() - base_dir = client_config['base_dir'] - comparison_dir = client_config['comparison_dir'] diff_dir = client_config['diff_dir'] + base_dir = os.path.join(diff_dir, client_config['base_dir']) + comparison_dir = os.path.join(diff_dir, client_config['comparison_dir']) + del client_config['base_dir'] del client_config['comparison_dir'] del client_config['diff_dir'] @@ -67,12 +219,37 @@ def _create_app(config): if __name__ == '__main__': PORT = 8901 parser = ArgParseImpl() - parser.add_argument('--diff', help='Diff directory', required=True) + parser.add_argument('--diff', type=str, help='Diff result directory') + parser.add_argument('--pr_number', type=str, help='Pull request artifacts to examine') + parser.add_argument('--github_token', type=str, help='Necessary for pull PR artifacts') args, _ = parser.parse_known_args(sys.argv[1:]) - with open(os.path.join(args.diff, 'compare_results.json'), 'r') as f: + if not args.diff and not args.pr_number: + print('Need to specify either a diff result directory or a Pull Request number') + exit(1) + + if args.diff and args.pr_number: + print('Cannot specify both a diff result directory and a Pull Request number') + exit(1) + + fdir = args.diff + if args.pr_number: + if not args.github_token: + print('Must provide --github_token to be able to download artifacts') + exit(1) + output_dir = f'/tmp/filament-pr{args.pr_number}-rdiff-result' + res = _download_github_artifacts(args.pr_number, args.github_token, output_dir) + if not res: + print('Failed to retrieve PR artifacts') + exit(1) + + # TODO: Clean up the following so that we're not so specific on the paths diffs/presubmit + directory_name = list(os.listdir(output_dir))[0] + fdir = os.path.join(os.path.join(output_dir, directory_name), 'diffs/presubmit') + + with open(os.path.join(fdir, 'compare_results.json'), 'r') as f: config = json.loads(f.read()) - config['diff_dir'] = os.path.abspath(args.diff) + config['diff_dir'] = os.path.abspath(fdir) app = _create_app(config) from waitress import serve