[PATCH] tester: Add reporting the RTEMS version, build and tools.
Chris Johns
chrisj at rtems.org
Sun Nov 5 07:49:18 UTC 2017
Update #3217.
---
tester/rt/config.py | 19 +++++++----
tester/rt/report.py | 98 +++++++++++++++++++++++++++++++++++++++++++----------
tester/rt/run.py | 4 ++-
tester/rt/test.py | 18 +++++++---
4 files changed, 109 insertions(+), 30 deletions(-)
diff --git a/tester/rt/config.py b/tester/rt/config.py
index eb11740..1c2670a 100644
--- a/tester/rt/config.py
+++ b/tester/rt/config.py
@@ -67,6 +67,7 @@ class file(config.file):
self.realtime_trace = self.exe_trace('output')
self.console_trace = self.exe_trace('console')
self.console_prefix = console_prefix
+ self.show_header = not self.defined('test_disable_header')
self.process = None
self.console = None
self.output = None
@@ -301,8 +302,8 @@ class file(config.file):
bsp_arch = self.expand('%{arch}')
bsp = self.expand('%{bsp}')
fexe = self._target_exe_filter(exe)
- if self.report is not None:
- self.report.start(index, total, exe, fexe, bsp_arch, bsp)
+ self.report.start(index, total, exe, fexe,
+ bsp_arch, bsp, self.show_header)
if self.index == 1:
self._target_command('on', bsp_arch, bsp, exe, fexe)
self._target_command('pretest', bsp_arch, bsp, exe, fexe)
@@ -321,10 +322,14 @@ class file(config.file):
self._target_command('off', bsp_arch, bsp, exe, fexe)
self._target_command('posttest', bsp_arch, bsp, exe, fexe)
try:
- status = ''
- if self.report is not None:
- status = self.report.end(exe, self.output)
- self._capture_console('test result: %s' % (status))
+ status = self.report.end(exe, self.output, self.console_prefix)
+ version = self.report.get_config('version', not_found = 'n/p')
+ build = self.report.get_config('build', not_found = 'n/p')
+ tools = self.report.get_config('tools', not_found = 'n/p')
+ self._capture_console('test result: %s' % (status))
+ self._capture_console('test version: %s' % (version))
+ self._capture_console('test build: %s' % (build))
+ self._capture_console('test tools: %s' % (tools))
if status == 'timeout':
if self.index != self.total:
self._target_command('reset', bsp_arch, bsp, exe, fexe)
@@ -336,7 +341,7 @@ class file(config.file):
def _realtime_trace(self, text):
for l in text:
- print(l)
+ print(''.join(l))
def run(self):
self.target_start_regx = self._target_regex('target_start_regex')
diff --git a/tester/rt/report.py b/tester/rt/report.py
index 7c906de..4900c2d 100644
--- a/tester/rt/report.py
+++ b/tester/rt/report.py
@@ -55,14 +55,18 @@ class report(object):
self.total = total
self.total_len = len(str(total))
self.passed = 0
- self.user_input = 0
self.failed = 0
+ self.user_input = 0
self.expected_fail = 0
self.indeterminate = 0
self.benchmark = 0
self.timeouts = 0
self.invalids = 0
+ self.wrong_version = 0
+ self.wrong_build = 0
+ self.wrong_tools = 0
self.results = {}
+ self.config = {}
self.name_max_len = 0
def __str__(self):
@@ -74,10 +78,14 @@ class report(object):
msg += 'Benchmark: %*d%s' % (self.total_len, self.self.benchmark, os.linesep)
msg += 'Timeout: %*d%s' % (self.total_len, self.timeouts, os.linesep)
msg += 'Invalid: %*d%s' % (self.total_len, self.invalids, os.linesep)
+ msg += 'Wrong Version %*d%s' % (self.total_len, self.wrong_version, os.linesep)
+ msg += 'Wrong Build %*d%s' % (self.total_len, self.wrong_build, os.linesep)
+ msg += 'Wrong Tools %*d%s' % (self.total_len, self.wrong_tools, os.linesep)
return msg
- def start(self, index, total, name, executable, bsp_arch, bsp):
- header = '[%*d/%*d] p:%-*d f:%-*d u:%-*d e:%-*d I:%-*d B:%-*d t:%-*d i:%-*d | %s/%s: %s' % \
+ def start(self, index, total, name, executable, bsp_arch, bsp, show_header):
+ header = '[%*d/%*d] p:%-*d f:%-*d u:%-*d e:%-*d I:%-*d B:%-*d ' \
+ 't:%-*d i:%-*d W:%-*d | %s/%s: %s' % \
(len(str(total)), index,
len(str(total)), total,
len(str(total)), self.passed,
@@ -88,6 +96,7 @@ class report(object):
len(str(total)), self.benchmark,
len(str(total)), self.timeouts,
len(str(total)), self.invalids,
+ len(str(total)), self.wrong_version + self.wrong_build + self.wrong_tools,
bsp_arch,
bsp,
path.basename(name))
@@ -106,25 +115,36 @@ class report(object):
'header': header }
self.lock.release()
- log.notice(header, stdout_only = True)
+ if show_header:
+ log.notice(header, stdout_only = True)
- def end(self, name, output):
+ def end(self, name, output, output_prefix):
start = False
end = False
state = None
+ version = None
+ build = None
+ tools = None
timeout = False
prefixed_output = []
for line in output:
- if line[0] == ']':
+ if line[0] == output_prefix:
if line[1].startswith('*** '):
- if line[1][4:].startswith('BEGIN OF '):
+ banner = line[1][4:]
+ if banner.startswith('BEGIN OF '):
start = True
- if line[1][4:].startswith('END OF '):
+ elif line[1][4:].startswith('END OF '):
end = True
- if line[1][4:].startswith('TEST STATE:'):
- state = line[1][15:].strip()
- if line[1][4:].startswith('TIMEOUT TIMEOUT'):
+ elif banner.startswith('TIMEOUT TIMEOUT'):
timeout = True
+ elif banner.startswith('TEST VERSION:'):
+ version = banner[13:].strip()
+ elif banner.startswith('TEST STATE:'):
+ state = banner[11:].strip()
+ elif banner.startswith('TEST BUILD:'):
+ build = ','.join(banner[11:].strip().split(' '))
+ elif banner.startswith('TEST TOOLS:'):
+ tools = banner[11:].strip()
prefixed_output += [line[0] + ' ' + line[1]]
self.lock.acquire()
if name not in self.results:
@@ -134,9 +154,27 @@ class report(object):
self.lock.release()
raise error.general('test already finished: %s' % (name))
self.results[name]['end'] = datetime.datetime.now()
- if state is None:
+ if version:
+ if 'version' not in self.config:
+ self.config['version'] = version
+ else:
+ if version != self.config['version']:
+ state = 'WRONG-VERSION'
+ if build:
+ if 'build' not in self.config:
+ self.config['build'] = build
+ else:
+ if build != self.config['build']:
+ state = 'WRONG-BUILD'
+ if tools:
+ if 'tools' not in self.config:
+ self.config['tools'] = tools
+ else:
+ if tools != self.config['tools']:
+ state = 'WRONG-TOOLS'
+ if state is None or state == 'EXPECTED-PASS':
if start and end:
- if state is None:
+ if state is None or state == 'EXPECTED-PASS':
status = 'passed'
self.passed += 1
elif timeout:
@@ -175,6 +213,15 @@ class report(object):
elif state == 'BENCHMARK':
status = 'benchmark'
self.benchmark += 1
+ elif state == 'WRONG-VERSION':
+ status = 'wrong-version'
+ self.wrong_version += 1
+ elif state == 'WRONG-BUILD':
+ status = 'wrong-build'
+ self.wrong_build += 1
+ elif state == 'WRONG-TOOLS':
+ status = 'wrong-tools'
+ self.wrong_tools += 1
else:
raise error.general('invalid test state: %s: %s' % (name, state))
self.results[name]['result'] = status
@@ -208,12 +255,16 @@ class report(object):
log.output(result)
log.output(output)
+ def get_config(self, config, not_found = None):
+ if config in self.config:
+ return self.config[config]
+ return not_found
+
def score_card(self, mode = 'full'):
if mode == 'short':
- return 'Passed:%d Failed:%d Timeout:%d Invalid:%d' % (self.passed,
- self.failed,
- self.timeouts,
- self.invalids)
+ wrongs = self.wrong_version + self.wrong_build + self.wrong_tools
+ return 'Passed:%d Failed:%d Timeout:%d Invalid:%d Wrong:%d' % \
+ (self.passed, self.failed, self.timeouts, self.invalids, wrongs)
elif mode == 'full':
l = []
l += ['Passed: %*d' % (self.total_len, self.passed)]
@@ -224,6 +275,9 @@ class report(object):
l += ['Benchmark: %*d' % (self.total_len, self.benchmark)]
l += ['Timeout: %*d' % (self.total_len, self.timeouts)]
l += ['Invalid: %*d' % (self.total_len, self.invalids)]
+ l += ['Wrong Version: %*d' % (self.total_len, self.wrong_version)]
+ l += ['Wrong Build: %*d' % (self.total_len, self.wrong_build)]
+ l += ['Wrong Tools: %*d' % (self.total_len, self.wrong_tools)]
l += ['---------------%s' % ('-' * self.total_len)]
l += ['Total: %*d' % (self.total_len, self.total)]
return os.linesep.join(l)
@@ -258,10 +312,18 @@ class report(object):
if self.invalids:
l += ['Invalid:']
l += show_state(self.results, 'invalid', self.name_max_len)
+ if self.wrong_version:
+ l += ['Wrong Version:']
+ l += show_state(self.results, 'wrong-version', self.name_max_len)
+ if self.wrong_build:
+ l += ['Wrong Build:']
+ l += show_state(self.results, 'wrong-build', self.name_max_len)
+ if self.wrong_tools:
+ l += ['Wrong Tools:']
+ l += show_state(self.results, 'wrong-tools', self.name_max_len)
return os.linesep.join(l)
def summary(self):
log.output()
log.notice(self.score_card())
- log.output()
log.output(self.failures())
diff --git a/tester/rt/run.py b/tester/rt/run.py
index bcd0697..cf8c4e5 100644
--- a/tester/rt/run.py
+++ b/tester/rt/run.py
@@ -143,9 +143,11 @@ def run(command_path = None):
executables = find_executables(opts.params())
if len(executables) != 1:
raise error.general('one executable required, found %d' % (len(executables)))
+ opts.defaults['test_disable_header'] = '1'
+ reports = report.report(1)
start_time = datetime.datetime.now()
opts.defaults['exe_trace'] = debug_trace
- tst = test(1, 1, None, executables[0], rtems_tools, bsp, bsp_config, opts)
+ tst = test(1, 1, reports, executables[0], rtems_tools, bsp, bsp_config, opts)
tst.run()
end_time = datetime.datetime.now()
total_time = 'Run time : %s' % (str(end_time - start_time))
diff --git a/tester/rt/test.py b/tester/rt/test.py
index 32ff56f..1a91763 100644
--- a/tester/rt/test.py
+++ b/tester/rt/test.py
@@ -351,15 +351,25 @@ def run(command_path = None):
if mail is not None and output is not None:
m_arch = opts.defaults.expand('%{arch}')
m_bsp = opts.defaults.expand('%{bsp}')
- subject = '[rtems-test] %s/%s: %s' % (m_arch,
- m_bsp,
- reports.score_card('short'))
+ build = ' %s:' % (reports.get_config('build', not_found = ''))
+ subject = '[rtems-test] %s/%s:%s %s' % (m_arch,
+ m_bsp,
+ build,
+ reports.score_card('short'))
+ np = 'Not present in test'
+ ver = reports.get_config('version', not_found = np)
+ build = reports.get_config('build', not_found = np)
+ tools = reports.get_config('tools', not_found = np)
body = [total_time, average_time,
'', 'Host', '====', host.label(mode = 'all'),
+ '', 'Configuration', '=============',
+ 'Version: %s' % (ver),
+ 'Build : %s' % (build),
+ 'Tools : %s' % (tools),
'', 'Summary', '=======', '',
reports.score_card(), '',
reports.failures(),
- '', 'Log', '===', ''] + output.get()
+ 'Log', '===', ''] + output.get()
mail.send(to_addr, subject, os.linesep.join(body))
except error.general as gerr:
--
2.13.2
More information about the devel
mailing list