[rtems-central commit] runtests: Retry run in case of corruption
Sebastian Huber
sebh at rtems.org
Tue Nov 28 13:05:16 UTC 2023
Module: rtems-central
Branch: master
Commit: 1151535c7756e890360b19bd5693f05edb93a9ab
Changeset: http://git.rtems.org/rtems-central/commit/?id=1151535c7756e890360b19bd5693f05edb93a9ab
Author: Sebastian Huber <sebastian.huber at embedded-brains.de>
Date: Mon Nov 27 19:45:24 2023 +0100
runtests: Retry run in case of corruption
---
rtemsspec/runtests.py | 38 ++++++++++++++++++++--
rtemsspec/testrunner.py | 4 +--
.../spec-packagebuild/qdp/test-runner/dummy.yml | 1 +
.../qdp/test-runner/grmon-manual.yml | 1 +
.../qdp/test-runner/subprocess.yml | 1 +
.../spec-packagebuild/qdp/test-runner/test.yml | 1 +
rtemsspec/tests/test_packagebuild.py | 24 ++++++++++++++
spec-qdp/qdp/test-runner/board-dummy.yml | 1 +
spec-qdp/qdp/test-runner/board-grmon-manual.yml | 1 +
spec-qdp/qdp/test-runner/simulator-dummy.yml | 1 +
spec-qdp/qdp/test-runner/simulator-sis.yml | 1 +
spec-qdp/spec/qdp-test-runner.yml | 7 ++++
12 files changed, 76 insertions(+), 5 deletions(-)
diff --git a/rtemsspec/runtests.py b/rtemsspec/runtests.py
index a0de436d..2d8f75f9 100644
--- a/rtemsspec/runtests.py
+++ b/rtemsspec/runtests.py
@@ -99,9 +99,7 @@ class RunTests(BuildItem):
# Run the tests with changed executables
if executables:
- runner = self.input("runner")
- assert isinstance(runner, TestRunner)
- reports.extend(runner.run_tests(executables))
+ reports.extend(self._run_tests(executables))
# Save the reports
os.makedirs(os.path.dirname(log.file), exist_ok=True)
@@ -113,3 +111,37 @@ class RunTests(BuildItem):
"start-time": start_time
}
json.dump(data, dst, sort_keys=True, indent=2)
+
+ def _run_tests(self, executables: List[Executable]) -> List[Report]:
+ runner = self.input("runner")
+ assert isinstance(runner, TestRunner)
+ max_run_count = runner["max-retry-count-per-executable"] + 1
+ reports_by_path: Dict[str, Report] = {}
+ while executables and max_run_count:
+ for new_report in runner.run_tests(executables):
+ reports_by_path[new_report["executable"]] = new_report
+ next_executables: List[Executable] = []
+ for executable in executables:
+ report = reports_by_path.get(executable.path, None)
+ if report is None:
+ logging.warning("%s: no report for: %s", self.uid,
+ executable.path)
+ next_executables.append(executable)
+ continue
+ if report.get("gcov-info-hash",
+ "") != report.get("gcov-info-hash-calculated",
+ ""):
+ next_executables.append(executable)
+ logging.warning("%s: gcov info is corrupt for: %s",
+ self.uid, executable.path)
+ continue
+ test_suite = report.get("test-suite", {})
+ if test_suite.get("report-hash", "") != test_suite.get(
+ "report-hash-calculated", ""):
+ next_executables.append(executable)
+ logging.warning("%s: test suite report is corrupt for: %s",
+ self.uid, executable.path)
+ continue
+ executables = next_executables
+ max_run_count -= 1
+ return list(reports_by_path.values())
diff --git a/rtemsspec/testrunner.py b/rtemsspec/testrunner.py
index 94bad5a4..63d92e32 100644
--- a/rtemsspec/testrunner.py
+++ b/rtemsspec/testrunner.py
@@ -34,13 +34,13 @@ from subprocess import run as subprocess_run
import tarfile
import time
import threading
-from typing import Any, Dict, List, NamedTuple, Union
+from typing import Any, Dict, List, NamedTuple
from rtemsspec.items import Item, ItemGetValueContext
from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
from rtemsspec.testoutputparser import augment_report
-Report = Dict[str, Union[str, List[str]]]
+Report = Dict[str, Any]
class Executable(NamedTuple):
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
index 3dfc14e5..218f68bf 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
@@ -4,6 +4,7 @@ copyrights:
description: Description.
enabled-by: true
links: []
+max-retry-count-per-executable: 0
params: {}
qdp-type: test-runner
test-runner-type: dummy
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
index 0325b127..7686556c 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
@@ -9,6 +9,7 @@ copyrights:
description: Description.
enabled-by: true
links: []
+max-retry-count-per-executable: 0
params:
x: abc
qdp-type: test-runner
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
index ffcd549e..13960d48 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
@@ -8,6 +8,7 @@ copyrights:
description: Description.
enabled-by: true
links: []
+max-retry-count-per-executable: 0
params: {}
qdp-type: test-runner
test-runner-type: subprocess
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
index 53dbaa06..46d47a9f 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
@@ -4,6 +4,7 @@ copyrights:
description: Description.
enabled-by: true
links: []
+max-retry-count-per-executable: 3
params: {}
qdp-type: test-runner
test-runner-type: test
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index eff36428..3c002493 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -95,9 +95,30 @@ class _TestItem(BuildItem):
class _TestRunner(TestRunner):
+ def __init__(self, director: PackageBuildDirector, item: Item):
+ super().__init__(director, item)
+ self.run_count = 0
+
def run_tests(self, executables: List[Executable]) -> List[Report]:
logging.info("executables: %s", executables)
super().run_tests(executables)
+ self.run_count += 1
+ if self.run_count == 1:
+ return []
+ if self.run_count == 2:
+ return [{
+ "executable": executables[0].path,
+ "gcov-info-hash": "a",
+ "gcov-info-hash-calculated": "b"
+ }]
+ if self.run_count == 3:
+ return [{
+ "executable": executables[0].path,
+ "test-suite": {
+ "report-hash": "c",
+ "report-hash-calculated": "d"
+ }
+ }]
return [{
"executable": executable.path,
"executable-sha512": executable.digest
@@ -479,6 +500,9 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
build_bsp.load()
director.build_package(None, None)
log = get_and_clear_log(caplog)
+ assert "no report for" in log
+ assert "gcov info is corrupt for" in log
+ assert "test suite report is corrupt for" in log
assert (f"executables: [Executable(path='{build_bsp.directory}"
"/a.exe', digest='z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg_SpIdNs6c5H0NE8"
"XYXysP-DGNKHfuwvY7kxvUdBeoGlODJ6-SfaPg==', timeout=1800), "
diff --git a/spec-qdp/qdp/test-runner/board-dummy.yml b/spec-qdp/qdp/test-runner/board-dummy.yml
index 961d31ee..548ddfbf 100644
--- a/spec-qdp/qdp/test-runner/board-dummy.yml
+++ b/spec-qdp/qdp/test-runner/board-dummy.yml
@@ -7,6 +7,7 @@ enabled-by: true
links:
- role: proxy-member
uid: board
+max-retry-count-per-executable: 0
params: {}
qdp-type: test-runner
test-runner-type: dummy
diff --git a/spec-qdp/qdp/test-runner/board-grmon-manual.yml b/spec-qdp/qdp/test-runner/board-grmon-manual.yml
index 9a8e4446..7e58cf47 100644
--- a/spec-qdp/qdp/test-runner/board-grmon-manual.yml
+++ b/spec-qdp/qdp/test-runner/board-grmon-manual.yml
@@ -24,6 +24,7 @@ links:
name: variant
role: input
uid: ../variant
+max-retry-count-per-executable: 0
params: {}
qdp-type: test-runner
script-base-path: ${../variant:/build-directory}/grmon
diff --git a/spec-qdp/qdp/test-runner/simulator-dummy.yml b/spec-qdp/qdp/test-runner/simulator-dummy.yml
index a490621b..daa60e0c 100644
--- a/spec-qdp/qdp/test-runner/simulator-dummy.yml
+++ b/spec-qdp/qdp/test-runner/simulator-dummy.yml
@@ -8,6 +8,7 @@ enabled-by:
links:
- role: proxy-member
uid: simulator
+max-retry-count-per-executable: 0
params: {}
qdp-type: test-runner
test-runner-type: dummy
diff --git a/spec-qdp/qdp/test-runner/simulator-sis.yml b/spec-qdp/qdp/test-runner/simulator-sis.yml
index 2a1839fe..b20f5b7b 100644
--- a/spec-qdp/qdp/test-runner/simulator-sis.yml
+++ b/spec-qdp/qdp/test-runner/simulator-sis.yml
@@ -23,6 +23,7 @@ links:
name: variant
role: input
uid: ../variant
+max-retry-count-per-executable: 0
params: {}
qdp-type: test-runner
test-runner-type: subprocess
diff --git a/spec-qdp/spec/qdp-test-runner.yml b/spec-qdp/spec/qdp-test-runner.yml
index 0fd7de54..4ac0da7e 100644
--- a/spec-qdp/spec/qdp-test-runner.yml
+++ b/spec-qdp/spec/qdp-test-runner.yml
@@ -18,6 +18,13 @@ spec-info:
description: |
It shall be the test runner description.
spec-type: str
+ max-retry-count-per-executable:
+ description: |
+ It shall be the maximum count of retries per executable. With a
+ value of zero exactly one run per executable is performed. An
+ executable run is retried if no report was produced, the gcov
+ information was corrupt, or the test suite report was corrupt.
+ spec-type: int
params:
description: |
It shall be an optional set of parameters which may be used for
More information about the vc
mailing list