[rtems-central commit] runtests: New
Sebastian Huber
sebh at rtems.org
Tue Nov 21 13:35:41 UTC 2023
Module: rtems-central
Branch: master
Commit: 61a4f93ed2e6f8353cac2cbd43a52b39cb8ef1c9
Changeset: http://git.rtems.org/rtems-central/commit/?id=61a4f93ed2e6f8353cac2cbd43a52b39cb8ef1c9
Author: Sebastian Huber <sebastian.huber at embedded-brains.de>
Date: Tue Nov 21 11:13:16 2023 +0100
runtests: New
---
rtemsspec/packagebuildfactory.py | 3 +
rtemsspec/runtests.py | 115 +++++++++++++++++++++
.../tests/spec-packagebuild/qdp/build/bsp.yml | 19 ++++
.../tests/spec-packagebuild/qdp/package-build.yml | 2 +
.../spec-packagebuild/qdp/steps/run-tests.yml | 23 +++++
.../qdp/test-logs/bsp-previous.yml | 15 +++
.../tests/spec-packagebuild/qdp/test-logs/bsp.yml | 15 +++
.../spec-packagebuild/qdp/test-runner/test.yml | 10 ++
.../spec/qdp-test-runner-test.yml | 22 ++++
rtemsspec/tests/test-files/pkg/build/bsp/a.exe | 0
rtemsspec/tests/test-files/pkg/build/bsp/b.exe | 1 +
.../tests/test-files/pkg/build/bsp/b.norun.exe | 0
rtemsspec/tests/test_packagebuild.py | 34 +++++-
spec-qdp/spec/qdp-input-generic-role.yml | 4 +
spec-qdp/spec/qdp-run-tests.yml | 22 ++++
spec-qdp/spec/qdp-test-log.yml | 22 ++++
spec-qdp/spec/qdp-test-procedure-role.yml | 23 +++++
17 files changed, 328 insertions(+), 2 deletions(-)
diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py
index f62fa78d..9b3e7944 100644
--- a/rtemsspec/packagebuildfactory.py
+++ b/rtemsspec/packagebuildfactory.py
@@ -29,6 +29,7 @@ from rtemsspec.directorystate import DirectoryState
from rtemsspec.packagebuild import BuildItemFactory, PackageVariant
from rtemsspec.reposubset import RepositorySubset
from rtemsspec.runactions import RunActions
+from rtemsspec.runtests import RunTests, TestLog
from rtemsspec.testrunner import DummyTestRunner, GRMONManualTestRunner, \
SubprocessTestRunner
@@ -40,8 +41,10 @@ def create_build_item_factory() -> BuildItemFactory:
factory.add_constructor("qdp/build-step/repository-subset",
RepositorySubset)
factory.add_constructor("qdp/build-step/run-actions", RunActions)
+ factory.add_constructor("qdp/build-step/run-tests", RunTests)
factory.add_constructor("qdp/directory-state/generic", DirectoryState)
factory.add_constructor("qdp/directory-state/repository", DirectoryState)
+ factory.add_constructor("qdp/directory-state/test-log", TestLog)
factory.add_constructor("qdp/directory-state/unpacked-archive",
DirectoryState)
factory.add_constructor("qdp/test-runner/dummy", DummyTestRunner)
diff --git a/rtemsspec/runtests.py b/rtemsspec/runtests.py
new file mode 100644
index 00000000..a0de436d
--- /dev/null
+++ b/rtemsspec/runtests.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: BSD-2-Clause
+""" This module provides a build step to run the RTEMS Tester. """
+
+# Copyright (C) 2022 embedded brains GmbH (http://www.embedded-brains.de)
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+import json
+import os
+import logging
+import time
+from typing import Dict, List
+
+from rtemsspec.directorystate import DirectoryState
+from rtemsspec.items import Item
+from rtemsspec.packagebuild import BuildItem, PackageBuildDirector
+from rtemsspec.testrunner import Executable, Report, TestRunner
+
+
+def _now_utc() -> str:
+ return datetime.datetime.utcnow().isoformat()
+
+
+class TestLog(DirectoryState):
+ """ Maintains a test log. """
+
+ def __init__(self, director: PackageBuildDirector, item: Item):
+ super().__init__(director, item)
+ self.reports: List[Report] = []
+
+ def discard(self) -> None:
+ try:
+ with open(self.file, "r", encoding="utf-8") as src:
+ self.reports = json.load(src)["reports"]
+ logging.info("%s: loaded test log: %s", self.uid, self.file)
+ except FileNotFoundError:
+ self.reports = []
+ super().discard()
+
+ def get_reports_by_hash(self) -> Dict[str, Report]:
+ """ Gets the reports by executable hash. """
+ reports_by_hash: Dict[str, Report] = {}
+ for report in self.reports:
+ digest = report["executable-sha512"]
+ assert digest not in reports_by_hash
+ assert isinstance(digest, str)
+ reports_by_hash[digest] = report
+ return reports_by_hash
+
+
+class RunTests(BuildItem):
+ """ Runs the tests. """
+
+ def run(self) -> None:
+ start_time = _now_utc()
+ begin = time.monotonic()
+ log = self.output("log")
+ assert isinstance(log, TestLog)
+ previous_reports_by_hash = log.get_reports_by_hash()
+
+ # Use previous report if the executable hash did not change
+ source = self.input("source")
+ assert isinstance(source, DirectoryState)
+ reports: List[Report] = []
+ executables: List[Executable] = []
+ for path, digest in source.files_and_hashes():
+ if not path.endswith(".exe") or path.endswith(".norun.exe"):
+ continue
+ assert digest
+ report = previous_reports_by_hash.get(digest, None)
+ if report is None:
+ logging.debug("%s: run: %s", self.uid, path)
+ executables.append(Executable(path, digest, 1800))
+ else:
+ logging.debug("%s: use previous report for: %s", self.uid,
+ path)
+ report["executable"] = path
+ reports.append(report)
+
+ # Run the tests with changed executables
+ if executables:
+ runner = self.input("runner")
+ assert isinstance(runner, TestRunner)
+ reports.extend(runner.run_tests(executables))
+
+ # Save the reports
+ os.makedirs(os.path.dirname(log.file), exist_ok=True)
+ with open(log.file, "w", encoding="utf-8") as dst:
+ data = {
+ "duration": time.monotonic() - begin,
+ "end-time": _now_utc(),
+ "reports": sorted(reports, key=lambda x: x["executable"]),
+ "start-time": start_time
+ }
+ json.dump(data, dst, sort_keys=True, indent=2)
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml b/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml
new file mode 100644
index 00000000..109b8ed8
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml
@@ -0,0 +1,19 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/build-directory}/bsp
+directory-state-type: generic
+enabled-by: true
+files:
+- file: a.exe
+ hash: null
+- file: b.exe
+ hash: null
+- file: b.norun.exe
+ hash: null
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml
index 02c4b721..2af718ef 100644
--- a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml
+++ b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml
@@ -13,6 +13,8 @@ links:
uid: steps/repository-subset
- role: build-step
uid: steps/run-actions
+- role: build-step
+ uid: steps/run-tests
- role: build-step
uid: steps/archive
qdp-type: package-build
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml b/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml
new file mode 100644
index 00000000..5e165bdf
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml
@@ -0,0 +1,23 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+build-step-type: run-tests
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: |
+ Run the tests.
+enabled-by: run-tests
+links:
+- role: test-procedure
+ uid: ../variant
+- hash: null
+ name: source
+ role: input
+ uid: ../build/bsp
+- hash: null
+ name: runner
+ role: input
+ uid: ../test-runner/test
+- name: log
+ role: output
+ uid: ../test-logs/bsp
+qdp-type: build-step
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml
new file mode 100644
index 00000000..e679c62a
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml
@@ -0,0 +1,15 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/deployment-directory}
+directory-state-type: generic
+enabled-by: true
+files:
+- file: log-empty.json
+ hash: null
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml
new file mode 100644
index 00000000..16838238
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml
@@ -0,0 +1,15 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/deployment-directory}
+directory-state-type: test-log
+enabled-by: true
+files:
+- file: test-log-bsp.json
+ hash: null
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
new file mode 100644
index 00000000..53dbaa06
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml
@@ -0,0 +1,10 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+description: Description.
+enabled-by: true
+links: []
+params: {}
+qdp-type: test-runner
+test-runner-type: test
+type: qdp
diff --git a/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml
new file mode 100644
index 00000000..23e75a04
--- /dev/null
+++ b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml
@@ -0,0 +1,22 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: test-runner-type
+ spec-value: test
+ uid: qdp-test-runner
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes: {}
+ description: |
+ This set of attributes specifies a test test runner.
+ mandatory-attributes: all
+spec-name: Subprocess Test Runner Item Type
+spec-type: qdp-test-runner-test
+type: spec
diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/a.exe b/rtemsspec/tests/test-files/pkg/build/bsp/a.exe
new file mode 100644
index 00000000..e69de29b
diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/b.exe b/rtemsspec/tests/test-files/pkg/build/bsp/b.exe
new file mode 100644
index 00000000..61780798
--- /dev/null
+++ b/rtemsspec/tests/test-files/pkg/build/bsp/b.exe
@@ -0,0 +1 @@
+b
diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe b/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe
new file mode 100644
index 00000000..e69de29b
diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py
index 2aa54228..fa0309d9 100644
--- a/rtemsspec/tests/test_packagebuild.py
+++ b/rtemsspec/tests/test_packagebuild.py
@@ -32,7 +32,7 @@ from pathlib import Path
import shutil
import subprocess
import tarfile
-from typing import NamedTuple
+from typing import List, NamedTuple
from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext
from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \
@@ -40,10 +40,12 @@ from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \
from rtemsspec.packagebuildfactory import create_build_item_factory
from rtemsspec.specverify import verify
import rtemsspec.testrunner
-from rtemsspec.testrunner import Executable
+from rtemsspec.testrunner import Executable, Report, TestRunner
from rtemsspec.tests.util import get_and_clear_log
from rtemsspec.util import run_command
+TestRunner.__test__ = False
+
def _copy_dir(src, dst):
dst.mkdir(parents=True, exist_ok=True)
@@ -85,6 +87,17 @@ class _TestItem(BuildItem):
super().__init__(director, item, BuildItemMapper(item, recursive=True))
+class _TestRunner(TestRunner):
+
+ def run_tests(self, executables: List[Executable]) -> List[Report]:
+ logging.info("executables: %s", executables)
+ super().run_tests(executables)
+ return [{
+ "executable": executable.path,
+ "executable-sha512": executable.digest
+ } for executable in executables]
+
+
class _Subprocess(NamedTuple):
stdout: bytes
@@ -349,3 +362,20 @@ def test_packagebuild(caplog, tmpdir, monkeypatch):
"start-time":
"f"
}]
+
+ # Test RunTests
+ variant["enabled"] = ["run-tests"]
+ factory.add_constructor("qdp/test-runner/test", _TestRunner)
+ build_bsp = director["/qdp/build/bsp"]
+ build_bsp.load()
+ director.build_package(None, None)
+ log = get_and_clear_log(caplog)
+ assert (f"executables: [Executable(path='{build_bsp.directory}"
+ "/a.exe', digest='z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg_SpIdNs6c5H0NE8"
+ "XYXysP-DGNKHfuwvY7kxvUdBeoGlODJ6-SfaPg==', timeout=1800), "
+ f"Executable(path='{build_bsp.directory}/b.exe', "
+ "digest='hopqxuHQKT10-tB_bZWVKz4B09MVPbZ3p12Ad5g_1OMNtr_Im3YIqT-yZ"
+ "GkjOp8aCVctaHqcXaeLID6xUQQKFQ==', timeout=1800)]") in log
+ director.build_package(None, ["/qdp/steps/run-tests"])
+ log = get_and_clear_log(caplog)
+ assert f"use previous report for: {build_bsp.directory}/a.exe"
diff --git a/spec-qdp/spec/qdp-input-generic-role.yml b/spec-qdp/spec/qdp-input-generic-role.yml
index c69157d1..ff5ab32d 100644
--- a/spec-qdp/spec/qdp-input-generic-role.yml
+++ b/spec-qdp/spec/qdp-input-generic-role.yml
@@ -29,6 +29,10 @@ links:
spec-key: name
spec-value: member
uid: qdp-input-role
+- role: spec-refinement
+ spec-key: name
+ spec-value: runner
+ uid: qdp-input-role
- role: spec-refinement
spec-key: name
spec-value: section
diff --git a/spec-qdp/spec/qdp-run-tests.yml b/spec-qdp/spec/qdp-run-tests.yml
new file mode 100644
index 00000000..4a08043a
--- /dev/null
+++ b/spec-qdp/spec/qdp-run-tests.yml
@@ -0,0 +1,22 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2022 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: build-step-type
+ spec-value: run-tests
+ uid: qdp-build-step
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes: {}
+ description: |
+ This set of attributes specifies a run of tests.
+ mandatory-attributes: all
+spec-name: Run Tests Item Type
+spec-type: qdp-run-tests
+type: spec
diff --git a/spec-qdp/spec/qdp-test-log.yml b/spec-qdp/spec/qdp-test-log.yml
new file mode 100644
index 00000000..368a3c7c
--- /dev/null
+++ b/spec-qdp/spec/qdp-test-log.yml
@@ -0,0 +1,22 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: directory-state-type
+ spec-value: test-log
+ uid: qdp-directory-state
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes: {}
+ description: |
+ This set of attributes specifies a test log.
+ mandatory-attributes: all
+spec-name: Test Log Item Type
+spec-type: qdp-test-log
+type: spec
diff --git a/spec-qdp/spec/qdp-test-procedure-role.yml b/spec-qdp/spec/qdp-test-procedure-role.yml
new file mode 100644
index 00000000..84a3b88c
--- /dev/null
+++ b/spec-qdp/spec/qdp-test-procedure-role.yml
@@ -0,0 +1,23 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: role
+ spec-value: test-procedure
+ uid: link
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes: {}
+ description: |
+ It defines the test procedure role of links and is used to associate a
+ test procedure with a variant.
+ mandatory-attributes: all
+spec-name: Test Procedure Link Role
+spec-type: qdp-test-procedure-role
+type: spec
More information about the vc
mailing list