[rtems-central commit] runtests: Augment test reports here

[Sebastian Huber]

Module:    rtems-central
Branch:    master
Commit:    180a5bd5078a282ad952039e525a448a7e450679
Changeset: http://git.rtems.org/rtems-central/commit/?id=180a5bd5078a282ad952039e525a448a7e450679

Author:    Sebastian Huber <sebastian.huber at embedded-brains.de>
Date:      Mon Nov 27 20:15:30 2023 +0100

runtests: Augment test reports here

---

 rtemsspec/runtests.py                |  3 +++
 rtemsspec/testrunner.py              |  2 --
 rtemsspec/tests/test_packagebuild.py | 39 ++++++++++++++++++++----------------
 3 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/rtemsspec/runtests.py b/rtemsspec/runtests.py
index 2d8f75f9..14fbecfc 100644
--- a/rtemsspec/runtests.py
+++ b/rtemsspec/runtests.py
@@ -34,6 +34,7 @@ from typing import Dict, List
 from rtemsspec.directorystate import DirectoryState
 from rtemsspec.items import Item
 from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
+from rtemsspec.testoutputparser import augment_report
 from rtemsspec.testrunner import Executable, Report, TestRunner
 
 
@@ -119,6 +120,8 @@ class RunTests(BuildItem):
         reports_by_path: Dict[str, Report] = {}
         while executables and max_run_count:
             for new_report in runner.run_tests(executables):
+                augment_report(new_report, new_report["output"])
+                logging.warning("%s: report: %s", self.uid, new_report)
                 reports_by_path[new_report["executable"]] = new_report
             next_executables: List[Executable] = []
             for executable in executables:
diff --git a/rtemsspec/testrunner.py b/rtemsspec/testrunner.py
index 63d92e32..f6a1a61e 100644
--- a/rtemsspec/testrunner.py
+++ b/rtemsspec/testrunner.py
@@ -38,7 +38,6 @@ from typing import Any, Dict, List, NamedTuple
 
 from rtemsspec.items import Item, ItemGetValueContext
 from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
-from rtemsspec.testoutputparser import augment_report
 
 Report = Dict[str, Any]
 
@@ -151,7 +150,6 @@ def _worker(work_queue: queue.Queue, item: BuildItem):
             except Exception:  # pylint: disable=broad-exception-caught
                 stdout = ""
             output = stdout.rstrip().replace("\r\n", "\n").split("\n")
-            augment_report(job.report, output)
             job.report["output"] = output
             job.report["duration"] = time.monotonic() - begin
             logging.debug("%s: done: %s", item.uid, job.report["executable"])
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index 3c002493..e0680f71 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -107,21 +107,34 @@ class _TestRunner(TestRunner):
             return []
         if self.run_count == 2:
             return [{
-                "executable": executables[0].path,
-                "gcov-info-hash": "a",
-                "gcov-info-hash-calculated": "b"
+                "executable":
+                executables[0].path,
+                "output": [
+                    "*** BEGIN OF GCOV INFO BASE64 ***", "foobar",
+                    "*** END OF GCOV INFO BASE64 ***"
+                ]
             }]
         if self.run_count == 3:
             return [{
-                "executable": executables[0].path,
-                "test-suite": {
-                    "report-hash": "c",
-                    "report-hash-calculated": "d"
-                }
+                "executable":
+                executables[0].path,
+                "output": [
+                    "*** BEGIN OF TEST TS ***", "*** TEST VERSION: V",
+                    "*** TEST STATE: EXPECTED_PASS", "*** TEST BUILD:",
+                    "*** TEST TOOLS: C", "A:TS", "S:Platform:RTEMS",
+                    "S:Compiler:C", "S:Version:V", "S:BSP:bsp",
+                    "S:BuildLabel:DEFAULT",
+                    "S:TargetHash:SHA256:qYOFDHUGg5--JyB28V7llk_t6WYeA3VAogeqwGLZeCM=",
+                    "S:RTEMS_DEBUG:0", "S:RTEMS_MULTIPROCESSING:0",
+                    "S:RTEMS_POSIX_API:0", "S:RTEMS_PROFILING:0",
+                    "S:RTEMS_SMP:0", "Z:TS:C:0:N:0:F:0:D:0.014590",
+                    "Y:ReportHash:SHA256:JlS-9kM8jYqTjFvRbuUDzHpfph6PznxFxCLx30NkcoI="
+                ]
             }]
         return [{
             "executable": executable.path,
-            "executable-sha512": executable.digest
+            "executable-sha512": executable.digest,
+            "output": []
         } for executable in executables]
 
 
@@ -437,7 +450,6 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
     reports[3]["duration"] = 5.
     assert reports == [{
         "command-line": ["foo", "bar", "a.exe"],
-        "data-ranges": [],
         "duration":
         2.0,
         "executable":
@@ -445,13 +457,11 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
         "executable-sha512":
         "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA_"
         "l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==",
-        "info": {},
         "output": [""],
         "start-time":
         "c"
     }, {
         "command-line": ["foo", "bar", "b.exe"],
-        "data-ranges": [],
         "duration":
         3.,
         "executable":
@@ -459,13 +469,11 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
         "executable-sha512":
         "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAcA"
         "u5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==",
-        "info": {},
         "output": [""],
         "start-time":
         "d"
     }, {
         "command-line": ["foo", "bar", "c.exe"],
-        "data-ranges": [],
         "duration":
         4.,
         "executable":
@@ -473,13 +481,11 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
         "executable-sha512":
         "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W"
         "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==",
-        "info": {},
         "output": [""],
         "start-time":
         "e"
     }, {
         "command-line": ["foo", "bar", "d.exe"],
-        "data-ranges": [],
         "duration":
         5.,
         "executable":
@@ -487,7 +493,6 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
         "executable-sha512":
         "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W"
         "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==",
-        "info": {},
         "output": ["u", "v", "w"],
         "start-time":
         "f"