From bbc4747bec06e838150803f4bfb5558ea855384a Mon Sep 17 00:00:00 2001 From: Kevin Burke Date: Mon, 15 Jun 2026 11:49:46 -0700 Subject: [PATCH] util: fix GNU test result artifacts GNU tar emits Autotest result lines with dotted test numbers and records failures in the detailed failure section. The previous extractor only matched an older colon-based format, so CI uploaded an empty gnu-full-result.json while still exiting successfully. The aggregate step also called analyze-gnu-results.py with -o and a single input file, but the script only wrote the requested output in the multi-input path. Parse GNU tar's current result lines, fail loudly when extraction finds no results, and always write the requested aggregate output. --- util/analyze-gnu-results.py | 61 +++++++----- util/gnu-json-result.py | 161 +++++++++++++++++++++---------- util/test_analyze_gnu_results.py | 39 ++++++++ util/test_gnu_json_result.py | 61 ++++++++++++ 4 files changed, 247 insertions(+), 75 deletions(-) create mode 100644 util/test_analyze_gnu_results.py create mode 100644 util/test_gnu_json_result.py diff --git a/util/analyze-gnu-results.py b/util/analyze-gnu-results.py index b9e8534..1775903 100755 --- a/util/analyze-gnu-results.py +++ b/util/analyze-gnu-results.py @@ -127,6 +127,31 @@ def analyze_test_results(json_data): } +def load_or_aggregate_results(json_files): + if not json_files: + raise ValueError("no JSON input files provided") + + if len(json_files) == 1: + try: + with open(json_files[0], "r") as file: + return json.load(file) + except FileNotFoundError: + print(f"Error: File '{json_files[0]}' not found.", file=sys.stderr) + sys.exit(1) + except json.JSONDecodeError: + print( + f"Error: '{json_files[0]}' is not a valid JSON file.", file=sys.stderr + ) + sys.exit(1) + + return aggregate_results(json_files) + + +def write_results(output_file, json_data): + with open(output_file, "w") as f: + json.dump(json_data, f, indent=2) + + def main(): """ Main function to process JSON files and export variables. @@ -147,30 +172,18 @@ def main(): output_file = json_files[0][3:] json_files = json_files[1:] - # Process the files - if len(json_files) == 1: - # Single file analysis - try: - with open(json_files[0], "r") as file: - json_data = json.load(file) - results = analyze_test_results(json_data) - except FileNotFoundError: - print(f"Error: File '{json_files[0]}' not found.", file=sys.stderr) - sys.exit(1) - except json.JSONDecodeError: - print( - f"Error: '{json_files[0]}' is not a valid JSON file.", file=sys.stderr - ) - sys.exit(1) - else: - # Multiple files - aggregate them - json_data = aggregate_results(json_files) - results = analyze_test_results(json_data) - - # Save aggregated data if output file is specified - if output_file: - with open(output_file, "w") as f: - json.dump(json_data, f, indent=2) + try: + json_data = load_or_aggregate_results(json_files) + except ValueError as e: + print(f"Error: {e}.", file=sys.stderr) + sys.exit(1) + + results = analyze_test_results(json_data) + + # Save aggregated data if output file is specified. For a single input, this + # writes the normalized input so downstream workflow steps always have it. + if output_file: + write_results(output_file, json_data) # Print export statements for shell evaluation print(f"export TOTAL={results['TOTAL']}") diff --git a/util/gnu-json-result.py b/util/gnu-json-result.py index 37622b4..69b9513 100755 --- a/util/gnu-json-result.py +++ b/util/gnu-json-result.py @@ -7,56 +7,115 @@ import sys from pathlib import Path -out = {} - -if len(sys.argv) != 2: - print("Usage: python gnu-json-result.py ") - sys.exit(1) - -test_dir = Path(sys.argv[1]) -if not test_dir.is_dir(): - print(f"Directory {test_dir} does not exist.") - sys.exit(1) - -# Test all the logs from the test execution -for filepath in test_dir.glob("**/*.log"): - path = Path(filepath) - if path.name == "testsuite.log": - # Handle Autotest testsuite.log +STATUS_MAP = {"ok": "PASS", "FAILED": "FAIL", "skipped": "SKIP"} + +# GNU tar's Autotest log emits successful/skipped tests as: +# 4. interspersed options (options02.at:26): ok (0m0.000s 0m0.004s) +# and failed tests in the detailed section as: +# 1. version.at:19: 1. tar version (version.at:19): FAILED (version.at:21) +AUTOTEST_RESULT_RE = re.compile( + r"^\s*(\d+)\.\s+" + r"(?:(?:\S+\.at:\d+):\s+)?" + r"(?:\d+\.\s+)?" + r"(.+?)\s+\(\S+\.at:\d+\):\s+" + r"(ok|FAILED|skipped)\b" +) + +# Older Autotest output uses a colon after the test number. +AUTOTEST_COLON_RESULT_RE = re.compile( + r"^\s*(\d+):\s+(.*?)\s+(ok|FAILED|skipped)(?:\s+\(.*\))?$" +) + +AUTOMAKE_RESULT_RE = re.compile( + r"(PASS|FAIL|SKIP|ERROR) [^ ]+ \(exit status: \d+\)$" +) + + +def parse_autotest_line(line): + match = AUTOTEST_RESULT_RE.match(line) + if not match: + match = AUTOTEST_COLON_RESULT_RE.match(line) + if not match: + return None + + num, name, status = match.groups() + return f"test {num}", name.strip(), STATUS_MAP.get(status, status) + + +def parse_autotest_log(path): + results = {} + with open(path, "r", errors="ignore") as f: + for line in f: + parsed = parse_autotest_line(line) + if parsed: + test_group, name, status = parsed + results[test_group] = {name: status} + return results + + +def parse_automake_log(path): + with open(path, "r", errors="ignore") as f: + # Only read the end of the file where the result is usually located. + f.seek(0, 2) + size = f.tell() + f.seek(max(0, size - 1000), 0) + content = f.read() + result = AUTOMAKE_RESULT_RE.search(content) + if result: + return result.group(1) + return None + + +def extract_results(test_dir): + out = {} + + # Test all the logs from the test execution. + for filepath in test_dir.glob("**/*.log"): + path = Path(filepath) + if path == test_dir / "testsuite.log": + try: + out.update(parse_autotest_log(path)) + except Exception as e: + print(f"Error processing testsuite.log {path}: {e}", file=sys.stderr) + continue + try: - with open(path, "r", errors="ignore") as f: - for line in f: - # Look for lines like: " 1: basic functionality ok" - # or " 10: ... FAILED (basic.at:123)" - match = re.match(r"^\s*(\d+):\s+(.*?)\s+(ok|FAILED|skipped)(?:\s+\(.*\))?$", line) - if match: - num, name, status = match.groups() - # Map Autotest status to Automake-style status - status_map = {"ok": "PASS", "FAILED": "FAIL", "skipped": "SKIP"} - out[f"test {num}"] = {name.strip(): status_map.get(status, status)} + result = parse_automake_log(path) except Exception as e: - print(f"Error processing testsuite.log {path}: {e}", file=sys.stderr) - continue - - # Handle individual Automake-style .log files - current = out - for key in path.parent.relative_to(test_dir).parts: - if key not in current: - current[key] = {} - current = current[key] - try: - with open(path, "r", errors="ignore") as f: - # Only read the end of the file where the result is usually located - f.seek(0, 2) - size = f.tell() - f.seek(max(0, size - 1000), 0) - content = f.read() - result = re.search( - r"(PASS|FAIL|SKIP|ERROR) [^ ]+ \(exit status: \d+\)$", content - ) - if result: - current[path.name] = result.group(1) - except Exception as e: - print(f"Error processing file {path}: {e}", file=sys.stderr) - -print(json.dumps(out, indent=2, sort_keys=True)) + print(f"Error processing file {path}: {e}", file=sys.stderr) + continue + if not result: + continue + + # Handle individual Automake-style .log files. + current = out + for key in path.parent.relative_to(test_dir).parts: + if key not in current: + current[key] = {} + current = current[key] + current[path.name] = result + + return out + + +def main(): + if len(sys.argv) != 2: + print("Usage: python gnu-json-result.py ") + return 1 + + test_dir = Path(sys.argv[1]) + if not test_dir.is_dir(): + print(f"Directory {test_dir} does not exist.", file=sys.stderr) + return 1 + + out = extract_results(test_dir) + if not out: + print(f"No GNU test results found in {test_dir}", file=sys.stderr) + return 1 + + print(json.dumps(out, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/util/test_analyze_gnu_results.py b/util/test_analyze_gnu_results.py new file mode 100644 index 0000000..bddb815 --- /dev/null +++ b/util/test_analyze_gnu_results.py @@ -0,0 +1,39 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).with_name("analyze-gnu-results.py") +SPEC = importlib.util.spec_from_file_location("analyze_gnu_results", SCRIPT_PATH) +analyze_gnu_results = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(analyze_gnu_results) + + +class AnalyzeGnuResultsTests(unittest.TestCase): + def test_single_input_can_be_written_as_aggregated_output(self): + data = { + "test 1": {"tar version": "FAIL"}, + "test 4": {"interspersed options": "PASS"}, + "test 11": {"--pax-option compatibility": "SKIP"}, + } + + with tempfile.TemporaryDirectory() as temp: + temp_dir = Path(temp) + input_file = temp_dir / "gnu-full-result.json" + output_file = temp_dir / "aggregated-result.json" + input_file.write_text(json.dumps(data)) + + results = analyze_gnu_results.load_or_aggregate_results([input_file]) + analyze_gnu_results.write_results(output_file, results) + + self.assertEqual(json.loads(output_file.read_text()), data) + + def test_load_or_aggregate_requires_input_files(self): + with self.assertRaisesRegex(ValueError, "no JSON input files provided"): + analyze_gnu_results.load_or_aggregate_results([]) + + +if __name__ == "__main__": + unittest.main() diff --git a/util/test_gnu_json_result.py b/util/test_gnu_json_result.py new file mode 100644 index 0000000..a6a4c01 --- /dev/null +++ b/util/test_gnu_json_result.py @@ -0,0 +1,61 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).with_name("gnu-json-result.py") +SPEC = importlib.util.spec_from_file_location("gnu_json_result", SCRIPT_PATH) +gnu_json_result = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gnu_json_result) + + +class GnuJsonResultTests(unittest.TestCase): + def test_parse_gnu_tar_autotest_results(self): + with tempfile.TemporaryDirectory() as temp: + test_dir = Path(temp) + (test_dir / "testsuite.log").write_text( + "\n".join( + [ + "4. interspersed options (options02.at:26): ok (0m0.000s 0m0.004s)", + "11. --pax-option compatibility (opcomp06.at:21): skipped (opcomp06.at:24)", + "1. version.at:19: 1. tar version (version.at:19): FAILED (version.at:21)", + ] + ) + ) + + self.assertEqual( + gnu_json_result.extract_results(test_dir), + { + "test 1": {"tar version": "FAIL"}, + "test 4": {"interspersed options": "PASS"}, + "test 11": {"--pax-option compatibility": "SKIP"}, + }, + ) + + def test_parse_legacy_autotest_result(self): + self.assertEqual( + gnu_json_result.parse_autotest_line( + " 1: basic functionality ok" + ), + ("test 1", "basic functionality", "PASS"), + ) + + def test_ignores_nested_testsuite_log_without_result(self): + with tempfile.TemporaryDirectory() as temp: + test_dir = Path(temp) + (test_dir / "testsuite.log").write_text( + "4. interspersed options (options02.at:26): ok (0m0.000s 0m0.004s)" + ) + nested = test_dir / "testsuite.dir" / "004" + nested.mkdir(parents=True) + (nested / "testsuite.log").write_text("test detail without trailer\n") + + self.assertEqual( + gnu_json_result.extract_results(test_dir), + {"test 4": {"interspersed options": "PASS"}}, + ) + + +if __name__ == "__main__": + unittest.main()