[PATCH] testsuite: Fix rtems-test-check not excluding tests.
Gedare Bloom
gedare at rtems.org
Sun Apr 16 01:54:01 UTC 2017
Thanks for adding some documentation. I spotted a couple minor
typographic issues.
On Sat, Apr 15, 2017 at 7:57 PM, Chris Johns <chrisj at rtems.org> wrote:
> The include file handling was broken.
>
> Add a test configuration data README.
> ---
> testsuites/README.testdata | 72 +++++++++++++++++++++++++++++++++++++++++
> testsuites/automake/subdirs.am | 7 ++--
> tools/build/rtems-test-check-py | 60 +++++++++++++++++++++++++++-------
> 3 files changed, 125 insertions(+), 14 deletions(-)
> create mode 100644 testsuites/README.testdata
>
> diff --git a/testsuites/README.testdata b/testsuites/README.testdata
> new file mode 100644
> index 0000000000..7210feab93
> --- /dev/null
> +++ b/testsuites/README.testdata
> @@ -0,0 +1,72 @@
> +RTEMS Testsuite Configuration
> +=============================
> +
> +The RTEMS Testsuite lets you configure the tests for a BSP. Every BSP can have
> +a test configuration data file and this file is read when building the
> +tests. The test configuration data can control what tests are build and now the
> +tests are built.
> +
> +The test configuration data files have a `.tcfg` file extension. You can
> +include other test configuration data files reducing repeated test
> +sequences. There is also a global test configuration data file for global
> +configurations.
> +
> +Command
> +-------
> +
> +The biuld system invokes the test check tool to determine what it does.
build
> +
> + rtems-test-check mode bsp testconfig includepaths test[s]
> +
> +Mode (mode)
> +~~~~~~~~~~~
> +
> +The check command modes are:
> +
> + 1. `exclude`: The input list of tests is checked against the exluded tests and
excluded
> + the tests that are not excluded are printed.
> +
> + 2. `flags': The test provide is check for any specical build flags. These
provided is checked
special
> + flags can enumerate a test in a specific way.
> +
> +BSP (bsp)
> +~~~~~~~~~
> +
> +The name of the BSP.
> +
> +Test Configuration (testconfig)
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +The path to the BSP's test configration file. The the file does not exist the
configuration
If the file?
> +input list of tests is returned and not flags a reported.
I'm not sure what "not flags a reported" means.
> +
> +Include Paths (inputpaths)
> +~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +A colon (`:`) separated list of paths test configuration files are search
> +for.
> +
> +Tests (test[s])
> +~~~~~~~~~~~~~~~
> +
> +If the mode is `exclude` this argument is a list of tests to be checked. If the
> +mode is `flags` a single test is required and the flags for the test are returned.
> +
> +File Format
> +-----------
> +
> +The file is an ASCII text file of lines. And text after and including the `#`
And -> Any?
Also, it probably should say "Any text on a line after ..."? Or just
say "# starts a comment that ends at the new line."
> +character is removed. Empty lines are ignored.
> +
> +A line is either the `include` directive or a test state. The states are:
> +
> + exclude : Exclude the test from being built.
> + expected-fail : The test is built but expected to fail.
> + user-input : The test requires user input and may be aborted when running
> + the tests
Add a period after tests for consistency.
> + indeterminate : The test result is indeterminate. This means the test may pass
> + or may fail therefore it cannot be included in the requression
regression
> + results.
> + benchmark : The test is a benchmark and maybe is aborted when running the
> + tests because of the load and time the test may take to run.
> + Performance changes are currently not viewed as a regression.
> diff --git a/testsuites/automake/subdirs.am b/testsuites/automake/subdirs.am
> index 70a7ecb78c..63b1da1049 100644
> --- a/testsuites/automake/subdirs.am
> +++ b/testsuites/automake/subdirs.am
> @@ -15,7 +15,8 @@ all-local:
> target="all"; \
> fi; \
> tcheck="$(top_srcdir)/../../tools/build/rtems-test-check-py"; \
> - tdata="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \
> + tdata="$(RTEMS_BSP)-testsuite.tcfg"; \
> + tincludes="$(top_srcdir)/../../c/src/lib/libbsp/$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom:$(top_srcdir)/.."; \
> if test -f "$$tdata"; then \
> vtdata="$(RTEMS_CPU)/$(RTEMS_BSP_FAMILY)/make/custom/$(RTEMS_BSP)-testsuite.tcfg"; \
> else \
> @@ -23,7 +24,7 @@ all-local:
> fi; \
> echo "BSP Testsuite Data: $$vtdata"; \
> if test -f $$tcheck; then \
> - list=`$$tcheck exclude $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $(_SUBDIRS)`; \
> + list=`$$tcheck exclude $(RTEMS_BSP) $$tdata $$tincludes $(_SUBDIRS)`; \
> else \
> list=$(_SUBDIRS); \
> fi; \
> @@ -31,7 +32,7 @@ all-local:
> echo "Making $$target in $$subdir"; \
> if test "$$subdir" != "."; then \
> if test -f $$tcheck; then \
> - test_FLAGS=`$$tcheck flags $$tdata $(top_srcdir)/.. $(RTEMS_BSP) $$subdir`; \
> + test_FLAGS=`$$tcheck flags $(RTEMS_BSP) $$tdata $$tincludes $$subdir`; \
> fi; \
> local_target="$$target"; \
> if test -z "$$test_FLAGS"; then \
> diff --git a/tools/build/rtems-test-check-py b/tools/build/rtems-test-check-py
> index 04630d25f7..5cf957049f 100755
> --- a/tools/build/rtems-test-check-py
> +++ b/tools/build/rtems-test-check-py
> @@ -16,6 +16,16 @@ def eprint(*args, **kwargs):
> print(*args, file=sys.stderr, **kwargs)
>
> #
> +# Search the include paths for a file.
> +#
> +def find_testdata(paths, name):
> + for p in paths:
> + fn = os.path.join(p, name)
> + if os.path.exists(fn):
> + return fn
> + return None
> +
> +#
> # Arguments. Keep it simple.
> #
> if len(sys.argv) < 4:
> @@ -23,11 +33,18 @@ if len(sys.argv) < 4:
> print('INVALID-TEST-DATA')
> sys.exit(2)
>
> -mode = sys.argv[1]
> -testconfig = [sys.argv[2]]
> -includepath = sys.argv[3]
> -bsp = sys.argv[4]
> -tests = sys.argv[5:]
> +verbose = False
> +args = 0
> +
> +if sys.argv[1] == '-v':
> + verbose = True
> + args = 1
> +
> +mode = sys.argv[args + 1]
> +bsp = sys.argv[args + 2]
> +includepaths = sys.argv[args + 4].split(':')
> +testconfig = [find_testdata(includepaths, sys.argv[args + 3])]
> +tests = sys.argv[args + 5:]
>
> #
> # Handle the modes.
> @@ -47,8 +64,8 @@ else:
> #
> # Common RTEMS testsuite configuration. Load first.
> #
> -rtems_testdata = os.path.join(includepath, 'testdata', 'rtems.tcfg')
> -if os.path.exists(rtems_testdata):
> +rtems_testdata = find_testdata(includepaths, os.path.join('testdata', 'rtems.tcfg'))
> +if rtems_testdata is not None:
> testconfig.insert(0, rtems_testdata)
>
> states = ['exclude',
> @@ -63,6 +80,13 @@ defines = { 'expected-fail' : '-DTEST_STATE_EXPECTED_FAIL=1',
> output = []
> testdata = {}
>
> +if verbose:
> + eprint('mode: %s' % (mode))
> + eprint('testconfig: %s' % (', '.join(testconfig)))
> + eprint('includepaths: %s' % (includepaths))
> + eprint('bsp: %s' % (bsp))
> + eprint('tests: %s' % (', '.join(tests)))
> +
> def clean(line):
> line = line[0:-1]
> b = line.find('#')
> @@ -73,10 +97,16 @@ def clean(line):
> #
> # Load the test data.
> #
> -for tc in range(0, len(testconfig)):
> - if not os.path.exists(testconfig[tc]):
> +while len(testconfig):
> + tc = testconfig[0]
> + testconfig.remove(tc)
> + if verbose:
> + eprint('reading: %s' % (tc))
> + if not os.path.exists(tc):
> + if verbose:
> + eprint('%s: not found' % (tc))
> continue
> - with open(testconfig[tc]) as f:
> + with open(tc) as f:
> tdata = [clean(l) for l in f.readlines()]
> lc = 0
> for line in tdata:
> @@ -84,6 +114,8 @@ for tc in range(0, len(testconfig)):
> ls = [s.strip() for s in line.split(':')]
> if len(line) == 0:
> continue
> + if verbose:
> + eprint('%4d: %s' % (lc, line))
> if len(ls) != 2:
> eprint('error: syntax error: %s:%d' % (tc, lc))
> print('INVALID-TEST-DATA')
> @@ -91,7 +123,13 @@ for tc in range(0, len(testconfig)):
> state = ls[0]
> test = ls[1]
> if state == 'include':
> - testconfig.insert(tc, test)
> + td = find_testdata(includepaths, test)
> + if td is None:
> + eprint('error: include not found: %s:%d' % (tc, lc))
> + print('INVALID-TEST-DATA')
> + testconfig.insert(0, td)
> + if verbose:
> + eprint('include: %s' % (', '.join(testconfig)))
> elif state in states:
> if state not in testdata:
> testdata[state] = [test]
> --
> 2.11.0
>
> _______________________________________________
> devel mailing list
> devel at rtems.org
> http://lists.rtems.org/mailman/listinfo/devel
More information about the devel
mailing list