[rtems-central commit] testrunner: New
Sebastian Huber
sebh at rtems.org
Tue Nov 21 13:35:41 UTC 2023
Module: rtems-central
Branch: master
Commit: 3bf2f4c13bd334bb4e9e2b05336b88f60fdf687c
Changeset: http://git.rtems.org/rtems-central/commit/?id=3bf2f4c13bd334bb4e9e2b05336b88f60fdf687c
Author: Sebastian Huber <sebastian.huber at embedded-brains.de>
Date: Tue Nov 21 11:13:16 2023 +0100
testrunner: New
---
rtemsspec/packagebuildfactory.py | 6 +
rtemsspec/testrunner.py | 179 +++++++++++++++++++++
.../spec-packagebuild/qdp/test-runner/dummy.yml | 10 ++
.../qdp/test-runner/grmon-manual.yml | 16 ++
.../qdp/test-runner/subprocess.yml | 14 ++
rtemsspec/tests/test_packagebuild.py | 130 ++++++++++++++-
spec-qdp/spec/qdp-test-runner-dummy.yml | 22 +++
spec-qdp/spec/qdp-test-runner-grmon-manual.yml | 34 ++++
spec-qdp/spec/qdp-test-runner-subprocess.yml | 26 +++
spec-qdp/spec/qdp-test-runner.yml | 35 ++++
10 files changed, 471 insertions(+), 1 deletion(-)
diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py
index 22ba8ded..f62fa78d 100644
--- a/rtemsspec/packagebuildfactory.py
+++ b/rtemsspec/packagebuildfactory.py
@@ -29,6 +29,8 @@ from rtemsspec.directorystate import DirectoryState
from rtemsspec.packagebuild import BuildItemFactory, PackageVariant
from rtemsspec.reposubset import RepositorySubset
from rtemsspec.runactions import RunActions
+from rtemsspec.testrunner import DummyTestRunner, GRMONManualTestRunner, \
+ SubprocessTestRunner
def create_build_item_factory() -> BuildItemFactory:
@@ -42,5 +44,9 @@ def create_build_item_factory() -> BuildItemFactory:
factory.add_constructor("qdp/directory-state/repository", DirectoryState)
factory.add_constructor("qdp/directory-state/unpacked-archive",
DirectoryState)
+ factory.add_constructor("qdp/test-runner/dummy", DummyTestRunner)
+ factory.add_constructor("qdp/test-runner/grmon-manual",
+ GRMONManualTestRunner)
+ factory.add_constructor("qdp/test-runner/subprocess", SubprocessTestRunner)
factory.add_constructor("qdp/variant", PackageVariant)
return factory
diff --git a/rtemsspec/testrunner.py b/rtemsspec/testrunner.py
new file mode 100644
index 00000000..94bad5a4
--- /dev/null
+++ b/rtemsspec/testrunner.py
@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: BSD-2-Clause
+""" This module provides a build item to run tests. """
+
+# Copyright (C) 2022, 2023 embedded brains GmbH & Co. KG
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+import logging
+import multiprocessing
+import os
+import queue
+import subprocess
+from subprocess import run as subprocess_run
+import tarfile
+import time
+import threading
+from typing import Any, Dict, List, NamedTuple, Union
+
+from rtemsspec.items import Item, ItemGetValueContext
+from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
+from rtemsspec.testoutputparser import augment_report
+
+Report = Dict[str, Union[str, List[str]]]
+
+
+class Executable(NamedTuple):
+ """ This data class represents a test executable. """
+ path: str
+ digest: str
+ timeout: int
+
+
+class TestRunner(BuildItem):
+ """ Runs the tests. """
+
+ def __init__(self, director: PackageBuildDirector, item: Item):
+ super().__init__(director, item)
+ self._executable = "/dev/null"
+ self._executables: List[Executable] = []
+ self.mapper.add_get_value(f"{self.item.type}:/test-executable",
+ self._get_test_executable)
+ self.mapper.add_get_value(f"{self.item.type}:/test-executables-grmon",
+ self._get_test_executables_grmon)
+
+ def _get_test_executable(self, _ctx: ItemGetValueContext) -> Any:
+ return self._executable
+
+ def _get_test_executables_grmon(self, _ctx: ItemGetValueContext) -> Any:
+ return " \\\n".join(
+ os.path.basename(executable.path)
+ for executable in self._executables)
+
+ def run_tests(self, executables: List[Executable]) -> List[Report]:
+ """
+ Runs the test executables and produces a log file of the test run.
+ """
+ self._executables = executables
+ return []
+
+
+class DummyTestRunner(TestRunner):
+ """ Cannot run the tests. """
+
+ def run_tests(self, _executables: List[Executable]) -> List[Report]:
+ """ Raises an exception. """
+ raise IOError("this test runner cannot run tests")
+
+
+class GRMONManualTestRunner(TestRunner):
+ """ Provides scripts to run the tests using GRMON. """
+
+ def run_tests(self, executables: List[Executable]) -> List[Report]:
+ super().run_tests(executables)
+ base = self["script-base-path"]
+ dir_name = os.path.basename(base)
+ grmon_name = f"{base}.grmon"
+ shell_name = f"{base}.sh"
+ tar_name = f"{base}.tar.xz"
+ os.makedirs(os.path.dirname(base), exist_ok=True)
+ with tarfile.open(tar_name, "w:xz") as tar_file:
+ with open(grmon_name, "w", encoding="utf-8") as grmon_file:
+ grmon_file.write(self["grmon-script"])
+ tar_file.add(grmon_name, os.path.join(dir_name, "run.grmon"))
+ with open(shell_name, "w", encoding="utf-8") as shell_file:
+ shell_file.write(self["shell-script"])
+ tar_file.add(shell_name, os.path.join(dir_name, "run.sh"))
+ for executable in executables:
+ tar_file.add(
+ executable.path,
+ os.path.join(dir_name, os.path.basename(executable.path)))
+ raise IOError(f"Run the tests provided by {tar_name}")
+
+
+def _now_utc() -> str:
+ return datetime.datetime.utcnow().isoformat()
+
+
+class _Job:
+ # pylint: disable=too-few-public-methods
+ def __init__(self, executable: Executable, command: List[str]):
+ self.report: Report = {
+ "executable": executable.path,
+ "executable-sha512": executable.digest,
+ "command-line": command
+ }
+ self.timeout = executable.timeout
+
+
+def _worker(work_queue: queue.Queue, item: BuildItem):
+ with open(os.devnull, "rb") as devnull:
+ while True:
+ try:
+ job = work_queue.get_nowait()
+ except queue.Empty:
+ return
+ logging.info("%s: run: %s", item.uid, job.report["command-line"])
+ job.report["start-time"] = _now_utc()
+ begin = time.monotonic()
+ try:
+ process = subprocess_run(job.report["command-line"],
+ check=False,
+ stdin=devnull,
+ stdout=subprocess.PIPE,
+ timeout=job.timeout)
+ stdout = process.stdout.decode("utf-8")
+ except subprocess.TimeoutExpired as timeout:
+ if timeout.stdout is not None:
+ stdout = timeout.stdout.decode("utf-8")
+ else:
+ stdout = ""
+ except Exception: # pylint: disable=broad-exception-caught
+ stdout = ""
+ output = stdout.rstrip().replace("\r\n", "\n").split("\n")
+ augment_report(job.report, output)
+ job.report["output"] = output
+ job.report["duration"] = time.monotonic() - begin
+ logging.debug("%s: done: %s", item.uid, job.report["executable"])
+ work_queue.task_done()
+
+
+class SubprocessTestRunner(TestRunner):
+ """ Runs the tests in subprocesses. """
+
+ def run_tests(self, executables: List[Executable]) -> List[Report]:
+ super().run_tests(executables)
+ work_queue: queue.Queue[_Job] = \
+ queue.Queue() # pylint: disable=unsubscriptable-object
+ jobs: List[_Job] = []
+ for executable in executables:
+ self._executable = executable.path
+ job = _Job(executable, self["command"])
+ jobs.append(job)
+ work_queue.put(job)
+ for _ in range(min(multiprocessing.cpu_count(), len(executables))):
+ threading.Thread(target=_worker,
+ args=(work_queue, self),
+ daemon=True).start()
+ work_queue.join()
+ return [job.report for job in jobs]
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
new file mode 100644
index 00000000..3dfc14e5
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml
@@ -0,0 +1,10 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params: {}
+qdp-type: test-runner
+test-runner-type: dummy
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
new file mode 100644
index 00000000..0325b127
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml
@@ -0,0 +1,16 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+script-base-path: ${../variant:/prefix-directory}/tests
+grmon-script: |
+ ${.:/test-executables-grmon}
+shell-script: |
+ ${.:/params/x}
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params:
+ x: abc
+qdp-type: test-runner
+test-runner-type: grmon-manual
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
new file mode 100644
index 00000000..ffcd549e
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml
@@ -0,0 +1,14 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+command:
+- foo
+- bar
+- ${.:/test-executable}
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params: {}
+qdp-type: test-runner
+test-runner-type: subprocess
+type: qdp
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index aad59c63..2aa54228 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -24,18 +24,23 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
+import json
import logging
import os
import pytest
from pathlib import Path
import shutil
+import subprocess
import tarfile
+from typing import NamedTuple
from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext
from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \
build_item_input, PackageBuildDirector
from rtemsspec.packagebuildfactory import create_build_item_factory
from rtemsspec.specverify import verify
+import rtemsspec.testrunner
+from rtemsspec.testrunner import Executable
from rtemsspec.tests.util import get_and_clear_log
from rtemsspec.util import run_command
@@ -80,7 +85,21 @@ class _TestItem(BuildItem):
super().__init__(director, item, BuildItemMapper(item, recursive=True))
-def test_packagebuild(caplog, tmpdir):
+class _Subprocess(NamedTuple):
+ stdout: bytes
+
+
+def _test_runner_subprocess(command, check, stdin, stdout, timeout):
+ if command[2] == "a.exe":
+ raise Exception
+ if command[2] == "b.exe":
+ raise subprocess.TimeoutExpired(command[2], timeout, b"")
+ if command[2] == "c.exe":
+ raise subprocess.TimeoutExpired(command[2], timeout, None)
+ return _Subprocess(b"u\r\nv\nw\n")
+
+
+def test_packagebuild(caplog, tmpdir, monkeypatch):
tmp_dir = Path(tmpdir)
item_cache = _create_item_cache(tmp_dir, Path("spec-packagebuild"))
@@ -221,3 +240,112 @@ def test_packagebuild(caplog, tmpdir):
assert not os.path.exists(os.path.join(tmpdir, "pkg", "sub-repo", "bsp.c"))
director.build_package(None, None)
assert os.path.exists(os.path.join(tmpdir, "pkg", "sub-repo", "bsp.c"))
+
+ # Test DummyTestRunner
+ dummy_runner = director["/qdp/test-runner/dummy"]
+ with pytest.raises(IOError):
+ dummy_runner.run_tests([])
+
+ # Test GRMONManualTestRunner
+ grmon_manual_runner = director["/qdp/test-runner/grmon-manual"]
+ exe = tmp_dir / "a.exe"
+ exe.touch()
+ with pytest.raises(IOError):
+ grmon_manual_runner.run_tests([
+ Executable(
+ str(exe), "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA"
+ "_l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", 456)
+ ])
+ with tarfile.open(tmp_dir / "tests.tar.xz", "r:*") as archive:
+ assert archive.getnames() == [
+ "tests/run.grmon", "tests/run.sh", "tests/a.exe"
+ ]
+ with archive.extractfile("tests/run.grmon") as src:
+ assert src.read() == b"a.exe\n"
+ with archive.extractfile("tests/run.sh") as src:
+ assert src.read() == b"abc\n"
+
+ # Test SubprocessTestRunner
+ subprocess_runner = director["/qdp/test-runner/subprocess"]
+ monkeypatch.setattr(rtemsspec.testrunner, "subprocess_run",
+ _test_runner_subprocess)
+ reports = subprocess_runner.run_tests([
+ Executable(
+ "a.exe", "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA"
+ "_l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", 1),
+ Executable(
+ "b.exe", "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAc"
+ "Au5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==", 2),
+ Executable(
+ "c.exe", "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_"
+ "W_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", 3),
+ Executable(
+ "d.exe", "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_"
+ "W_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", 4)
+ ])
+ monkeypatch.undo()
+ reports[0]["start-time"] = "c"
+ reports[0]["duration"] = 2.
+ reports[1]["start-time"] = "d"
+ reports[1]["duration"] = 3.
+ reports[2]["start-time"] = "e"
+ reports[2]["duration"] = 4.
+ reports[3]["start-time"] = "f"
+ reports[3]["duration"] = 5.
+ assert reports == [{
+ "command-line": ["foo", "bar", "a.exe"],
+ "data-ranges": [],
+ "duration":
+ 2.0,
+ "executable":
+ "a.exe",
+ "executable-sha512":
+ "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA_"
+ "l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==",
+ "info": {},
+ "output": [""],
+ "start-time":
+ "c"
+ }, {
+ "command-line": ["foo", "bar", "b.exe"],
+ "data-ranges": [],
+ "duration":
+ 3.,
+ "executable":
+ "b.exe",
+ "executable-sha512":
+ "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAcA"
+ "u5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==",
+ "info": {},
+ "output": [""],
+ "start-time":
+ "d"
+ }, {
+ "command-line": ["foo", "bar", "c.exe"],
+ "data-ranges": [],
+ "duration":
+ 4.,
+ "executable":
+ "c.exe",
+ "executable-sha512":
+ "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W"
+ "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==",
+ "info": {},
+ "output": [""],
+ "start-time":
+ "e"
+ }, {
+ "command-line": ["foo", "bar", "d.exe"],
+ "data-ranges": [],
+ "duration":
+ 5.,
+ "executable":
+ "d.exe",
+ "executable-sha512":
+ "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W"
+ "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==",
+ "info": {},
+ "output": ["u", "v", "w"],
+ "start-time":
+ "f"
+ }]
diff --git a/spec-qdp/spec/qdp-test-runner-dummy.yml b/spec-qdp/spec/qdp-test-runner-dummy.yml
new file mode 100644
index 00000000..6f5c5834
--- /dev/null
+++ b/spec-qdp/spec/qdp-test-runner-dummy.yml
@@ -0,0 +1,22 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: test-runner-type
+ spec-value: dummy
+ uid: qdp-test-runner
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes: {}
+ description: |
+ This set of attributes specifies a test runner which cannot run tests.
+ mandatory-attributes: all
+spec-name: Dummy Test Runner Item Type
+spec-type: qdp-test-runner-dummy
+type: spec
diff --git a/spec-qdp/spec/qdp-test-runner-grmon-manual.yml b/spec-qdp/spec/qdp-test-runner-grmon-manual.yml
new file mode 100644
index 00000000..26d89669
--- /dev/null
+++ b/spec-qdp/spec/qdp-test-runner-grmon-manual.yml
@@ -0,0 +1,34 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: test-runner-type
+ spec-value: grmon-manual
+ uid: qdp-test-runner
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes:
+ script-base-path:
+ description: |
+ It shall be the base path for scripts.
+ spec-type: str
+ grmon-script:
+ description: |
+ It shall be the GRMON script to run the tests.
+ spec-type: str
+ shell-script:
+ description: |
+ It shall be the shell script to run the GRMON script.
+ spec-type: str
+ description: |
+ This set of attributes specifies a GRMON manual test runner.
+ mandatory-attributes: all
+spec-name: GRMON Manual Test Runner Item Type
+spec-type: qdp-test-runner-grmon-manual
+type: spec
diff --git a/spec-qdp/spec/qdp-test-runner-subprocess.yml b/spec-qdp/spec/qdp-test-runner-subprocess.yml
new file mode 100644
index 00000000..a8846c5b
--- /dev/null
+++ b/spec-qdp/spec/qdp-test-runner-subprocess.yml
@@ -0,0 +1,26 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: test-runner-type
+ spec-value: subprocess
+ uid: qdp-test-runner
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes:
+ command:
+ description: |
+ It shall be the test runner subprocess command.
+ spec-type: list-str
+ description: |
+ This set of attributes specifies a subprocess test runner.
+ mandatory-attributes: all
+spec-name: Subprocess Test Runner Item Type
+spec-type: qdp-test-runner-subprocess
+type: spec
diff --git a/spec-qdp/spec/qdp-test-runner.yml b/spec-qdp/spec/qdp-test-runner.yml
new file mode 100644
index 00000000..0fd7de54
--- /dev/null
+++ b/spec-qdp/spec/qdp-test-runner.yml
@@ -0,0 +1,35 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2022 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: qdp-type
+ spec-value: test-runner
+ uid: qdp-root
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes:
+ description:
+ description: |
+ It shall be the test runner description.
+ spec-type: str
+ params:
+ description: |
+ It shall be an optional set of parameters which may be used for
+ variable subsitution.
+ spec-type: any
+ test-runner-type:
+ description: |
+ It shall be the test runner type.
+ spec-type: name
+ description: |
+ This set of attributes specifies a test runner.
+ mandatory-attributes: all
+spec-name: Test Runner Item Type
+spec-type: qdp-test-runner
+type: spec
More information about the vc
mailing list