[rtems commit] testsuite: Fix rtems-test-check not excluding tests.
Chris Johns
chrisj at rtems.org
Tue Apr 18 02:28:42 UTC 2017
Module: rtems
Branch: master
Commit: 18f63c0004cc3348bc785a642da82d2a3d46db5d
Changeset: http://git.rtems.org/rtems/commit/?id=18f63c0004cc3348bc785a642da82d2a3d46db5d
Author: Chris Johns <chrisj at rtems.org>
Date: Sun Apr 16 09:51:57 2017 +1000
testsuite: Fix rtems-test-check not excluding tests.
The include file handling was broken.
Add a test configuration data README.
Closes #2981.
---
testsuites/README.testdata | 72 +++++++++++++++++++++++++++++++++++++++++
testsuites/automake/subdirs.am | 7 ++--
tools/build/rtems-test-check-py | 60 +++++++++++++++++++++++++++-------
3 files changed, 125 insertions(+), 14 deletions(-)
diff --git a/testsuites/README.testdata b/testsuites/README.testdata
new file mode 100644
index 0000000..74a64e6
--- /dev/null
+++ b/testsuites/README.testdata
@@ -0,0 +1,72 @@
+RTEMS Testsuite Configuration
+=============================
+
+The RTEMS Testsuite lets you configure the tests for a BSP. Every BSP can have
+a test configuration data file and this file is read when building the
+tests. The test configuration data can control what tests are build and now the
+tests are built.
+
+The test configuration data files have a `.tcfg` file extension. You can
+include other test configuration data files reducing repeated test
+sequences. There is also a global test configuration data file for global
+configurations.
+
+Command
+-------
+
+The build system invokes the test check tool to determine what it does.
+
+ rtems-test-check mode bsp testconfig includepaths test[s]
+
+Mode (mode)
+~~~~~~~~~~~
+
+The check command modes are:
+
+ 1. `exclude`: The input list of tests is checked against the excluded tests
+ and the tests that are not excluded are printed.
+
+ 2. `flags': The test build flags are returned. These flags can enumerate a
+ test in a specific way.
+
+BSP (bsp)
+~~~~~~~~~
+
+The name of the BSP.
+
+Test Configuration (testconfig)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The path to the BSP's test configration file. If the file does not exist the
+input list of tests is returned and there are no special build flags.
+
+Include Paths (inputpaths)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A colon (`:`) separated list of paths test configuration files are search
+for.
+
+Tests (test[s])
+~~~~~~~~~~~~~~~
+
+If the mode is `exclude` this argument is a list of tests to be checked. If the
+mode is `flags` a single test is required and the flags for the test are returned.
+
+File Format
+-----------
+
+The file is an ASCII text file of lines. And text after and including the `#`
+character is removed. Empty lines are ignored.
+
+A line is either the `include` directive or a test state. The states are:
+
+ exclude : Exclude the test from being built.
+ expected-fail : The test is built but expected to fail.
+ user-input : The test requires user input and may be aborted when running
+ the tests.
+ indeterminate : The test result is indeterminate. This means the test may pass
+ or may fail therefore it cannot be included in the regression
+ results.
+ benchmark : The test is a benchmark and maybe is aborted when running the
+ tests because of the load and time the test may take to run.
+ Performance changes are currently not viewed as a regression.
diff --git a/testsuites/automake/subdirs.am b/testsuites/automake/subdirs.am
index 70a7ecb..63b1da1 100644
--- a/testsuites/automake/subdirs.am
+++ b/testsuites/automake/subdirs.am
@@ -15,7 +15,8 @@ all-local:
target="all"; \
fi; \
tcheck="$(top_srcdir)/../../tools/build/rtems-test-check-py"; \
- tdata="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \
+ tdata="$(RTEMS_BSP)-testsuite.tcfg"; \
+ tincludes="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom:$(top_srcdir)/.."; \
if test -f "$$tdata"; then \
vtdata="$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \
else \
@@ -23,7 +24,7 @@ all-local:
fi; \
echo "BSP Testsuite Data: $$vtdata"; \
if test -f $$tcheck; then \
- list=`$$tcheck exclude $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $(_SUBDIRS)`; \
+ list=`$$tcheck exclude $(RTEMS_BSP) $$tdata $$tincludes $(_SUBDIRS)`; \
else \
list=$(_SUBDIRS); \
fi; \
@@ -31,7 +32,7 @@ all-local:
echo "Making $$target in $$subdir"; \
if test "$$subdir" != "."; then \
if test -f $$tcheck; then \
- test_FLAGS=`$$tcheck flags $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $$subdir`; \
+ test_FLAGS=`$$tcheck flags $(RTEMS_BSP) $$tdata $$tincludes $$subdir`; \
fi; \
local_target="$$target"; \
if test -z "$$test_FLAGS"; then \
diff --git a/tools/build/rtems-test-check-py b/tools/build/rtems-test-check-py
index 04630d2..5cf9570 100755
--- a/tools/build/rtems-test-check-py
+++ b/tools/build/rtems-test-check-py
@@ -16,6 +16,16 @@ def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
#
+# Search the include paths for a file.
+#
+def find_testdata(paths, name):
+ for p in paths:
+ fn = os.path.join(p, name)
+ if os.path.exists(fn):
+ return fn
+ return None
+
+#
# Arguments. Keep it simple.
#
if len(sys.argv) < 4:
@@ -23,11 +33,18 @@ if len(sys.argv) < 4:
print('INVALID-TEST-DATA')
sys.exit(2)
-mode = sys.argv[1]
-testconfig = [sys.argv[2]]
-includepath = sys.argv[3]
-bsp = sys.argv[4]
-tests = sys.argv[5:]
+verbose = False
+args = 0
+
+if sys.argv[1] == '-v':
+ verbose = True
+ args = 1
+
+mode = sys.argv[args + 1]
+bsp = sys.argv[args + 2]
+includepaths = sys.argv[args + 4].split(':')
+testconfig = [find_testdata(includepaths, sys.argv[args + 3])]
+tests = sys.argv[args + 5:]
#
# Handle the modes.
@@ -47,8 +64,8 @@ else:
#
# Common RTEMS testsuite configuration. Load first.
#
-rtems_testdata = os.path.join(includepath, 'testdata', 'rtems.tcfg')
-if os.path.exists(rtems_testdata):
+rtems_testdata = find_testdata(includepaths, os.path.join('testdata', 'rtems.tcfg'))
+if rtems_testdata is not None:
testconfig.insert(0, rtems_testdata)
states = ['exclude',
@@ -63,6 +80,13 @@ defines = { 'expected-fail' : '-DTEST_STATE_EXPECTED_FAIL=1',
output = []
testdata = {}
+if verbose:
+ eprint('mode: %s' % (mode))
+ eprint('testconfig: %s' % (', '.join(testconfig)))
+ eprint('includepaths: %s' % (includepaths))
+ eprint('bsp: %s' % (bsp))
+ eprint('tests: %s' % (', '.join(tests)))
+
def clean(line):
line = line[0:-1]
b = line.find('#')
@@ -73,10 +97,16 @@ def clean(line):
#
# Load the test data.
#
-for tc in range(0, len(testconfig)):
- if not os.path.exists(testconfig[tc]):
+while len(testconfig):
+ tc = testconfig[0]
+ testconfig.remove(tc)
+ if verbose:
+ eprint('reading: %s' % (tc))
+ if not os.path.exists(tc):
+ if verbose:
+ eprint('%s: not found' % (tc))
continue
- with open(testconfig[tc]) as f:
+ with open(tc) as f:
tdata = [clean(l) for l in f.readlines()]
lc = 0
for line in tdata:
@@ -84,6 +114,8 @@ for tc in range(0, len(testconfig)):
ls = [s.strip() for s in line.split(':')]
if len(line) == 0:
continue
+ if verbose:
+ eprint('%4d: %s' % (lc, line))
if len(ls) != 2:
eprint('error: syntax error: %s:%d' % (tc, lc))
print('INVALID-TEST-DATA')
@@ -91,7 +123,13 @@ for tc in range(0, len(testconfig)):
state = ls[0]
test = ls[1]
if state == 'include':
- testconfig.insert(tc, test)
+ td = find_testdata(includepaths, test)
+ if td is None:
+ eprint('error: include not found: %s:%d' % (tc, lc))
+ print('INVALID-TEST-DATA')
+ testconfig.insert(0, td)
+ if verbose:
+ eprint('include: %s' % (', '.join(testconfig)))
elif state in states:
if state not in testdata:
testdata[state] = [test]
More information about the vc
mailing list