[tools v2] tester: Normalize JSON and YAML reports
Chris Johns
chrisj at rtems.org
Wed Jul 27 02:45:02 UTC 2022
OK and thanks.
Chris
On 26/7/2022 10:10 pm, Sebastian Huber wrote:
> Report the same data in JSON and YAML reports. Do not report redundant
> information.
>
> Update 4671.
> ---
> v2:
>
> * Add 'report-version' attribute with a value of 1.
>
> * Add 'tester-version' attribute.
>
> * Use a list for both 'command-line' attributes.
>
> tester/rt/test.py | 193 ++++++++++------------------------------------
> 1 file changed, 42 insertions(+), 151 deletions(-)
>
> diff --git a/tester/rt/test.py b/tester/rt/test.py
> index 113936c..a13b40a 100644
> --- a/tester/rt/test.py
> +++ b/tester/rt/test.py
> @@ -218,76 +218,48 @@ def killall(tests):
> test.kill()
>
>
> +def results_to_data(args, reports, start_time, end_time):
> + data = {}
> + data['report-version'] = 1
> + data['tester-version'] = version.string()
> + data['command-line'] = args
> + data['host'] = host.label(mode='all')
> + data['python'] = sys.version.replace('\n', '')
> + data['start-time'] = start_time.isoformat()
> + data['end-time'] = start_time.isoformat()
> + reports_data = []
> +
> + for name, run in reports.results.items():
> + run_data = {}
> + result = run['result']
> + run_data['result'] = result
> + output = []
> + for line in run['output']:
> + if line.startswith('] '):
> + output.append(line[2:])
> + elif line.startswith('=> exe:'):
> + run_data['command-line'] = line[9:].split()
> + run_data['output'] = output
> + run_data['executable'] = name
> + run_data['executable-sha512'] = get_hash512(name)
> + run_data['start-time'] = run['start'].isoformat()
> + run_data['end-time'] = run['end'].isoformat()
> + run_data['bsp'] = run['bsp']
> + run_data['arch'] = run['bsp_arch']
> + reports_data.append(run_data)
> +
> + data['reports'] = reports_data
> + return data
> +
> +
> def generate_json_report(args, reports, start_time, end_time,
> - total, json_file):
> + _total, json_file):
> import json
> - import sys
> - json_log = {}
> - json_log['Command Line'] = " ".join(args)
> - json_log['Python'] = sys.version.replace('\n', '')
> - json_log['test_groups'] = []
> - json_log['Host'] = host.label(mode='all')
> - json_log['summary'] = {}
> - json_log['summary']['passed_count'] = reports.passed
> - json_log['summary']['failed_count'] = reports.failed
> - json_log['summary']['user-input_count'] = reports.user_input
> - json_log['summary']['expected-fail_count'] = reports.expected_fail
> - json_log['summary']['indeterminate_count'] = reports.indeterminate
> - json_log['summary']['benchmark_count'] = reports.benchmark
> - json_log['summary']['timeout_count'] = reports.timeouts
> - json_log['summary']['test-too-long_count'] = reports.test_too_long
> - json_log['summary']['invalid_count'] = reports.invalids
> - json_log['summary']['wrong-version_count'] = reports.wrong_version
> - json_log['summary']['wrong-build_count'] = reports.wrong_build
> - json_log['summary']['wrong-tools_count'] = reports.wrong_tools
> - json_log['summary']['invalid-header_count'] = reports.wrong_header
> - json_log['summary']['total_count'] = reports.total
> - time_delta = end_time - start_time
> - json_log['summary']['average_test_time'] = str(time_delta / total)
> - json_log['summary']['testing_time'] = str(time_delta)
> -
> - result_types = [
> - 'failed', 'user-input', 'expected-fail', 'indeterminate',
> - 'benchmark', 'timeout', 'test-too-long', 'invalid', 'wrong-version',
> - 'wrong-build', 'wrong-tools', 'wrong-header'
> - ]
> - json_results = {}
> - for result_type in result_types:
> - json_log['summary'][result_type] = []
> -
> - # collate results for JSON log
> - for name in reports.results:
> - result_type = reports.results[name]['result']
> - # map fatal-error on to failed since the report adds both to the failed count
> - if result_type == "fatal-error":
> - result_type = "failed"
> - test_parts = name.split("/")
> - test_category = test_parts[-2]
> - test_name = test_parts[-1]
> - if result_type != 'passed':
> - json_log['summary'][result_type].append(test_name)
> - if test_category not in json_results:
> - json_results[test_category] = []
> - json_result = {}
> - # remove the file extension
> - json_result["name"] = test_name.split('.')[0]
> - json_result["result"] = result_type
> - if result_type == "failed" or result_type == "timeout":
> - json_result["output"] = reports.results[name]["output"]
> - json_results[test_category].append(json_result)
> -
> - # convert results to a better format for report generation
> - sorted_keys = sorted(json_results.keys())
> - for i in range(len(sorted_keys)):
> - results_log = {}
> - results_log["index"] = i + 1
> - results_log["name"] = sorted_keys[i]
> - results_log["results"] = json_results[sorted_keys[i]]
> - json_log["test_groups"].append(results_log)
> -
> - # write out JSON log
> +
> + data = results_to_data(args, reports, start_time, end_time);
> +
> with open(json_file, 'w') as outfile:
> - json.dump(json_log, outfile, sort_keys=True, indent=4)
> + json.dump(data, outfile, sort_keys=True, indent=4)
>
>
> def generate_junit_report(args, reports, start_time, end_time,
> @@ -343,97 +315,16 @@ def generate_junit_report(args, reports, start_time, end_time,
>
>
> def generate_yaml_report(args, reports, start_time, end_time,
> - total, yaml_file):
> + _total, yaml_file):
> """ Generates a YAML file containing information about the test run,
> including all test outputs """
>
> import yaml
>
> - def format_output(output_list):
> - return "\n".join(output_list).replace("] ", '').replace('=> ', '')
> -
> - yaml_log = {}
> - yaml_log['command-line'] = args
> - yaml_log['host'] = host.label(mode='all')
> - yaml_log['python'] = sys.version.replace('\n', '')
> - yaml_log['summary'] = {}
> - yaml_log['summary']['passed-count'] = reports.passed
> - yaml_log['summary']['failed-count'] = reports.failed
> - yaml_log['summary']['user-input-count'] = reports.user_input
> - yaml_log['summary']['expected-fail-count'] = reports.expected_fail
> - yaml_log['summary']['indeterminate-count'] = reports.indeterminate
> - yaml_log['summary']['benchmark-count'] = reports.benchmark
> - yaml_log['summary']['timeout-count'] = reports.timeouts
> - yaml_log['summary']['test-too-long_count'] = reports.test_too_long
> - yaml_log['summary']['invalid-count'] = reports.invalids
> - yaml_log['summary']['wrong-version-count'] = reports.wrong_version
> - yaml_log['summary']['wrong-build-count'] = reports.wrong_build
> - yaml_log['summary']['wrong-tools-count'] = reports.wrong_tools
> - yaml_log['summary']['wrong-header-count'] = reports.wrong_header
> - yaml_log['summary']['total-count'] = reports.total
> - time_delta = end_time - start_time
> - yaml_log['summary']['average-test-time'] = str(time_delta / total)
> - yaml_log['summary']['testing-time'] = str(time_delta)
> -
> - result_types = [
> - 'failed', 'user-input', 'expected-fail', 'indeterminate',
> - 'benchmark', 'timeout', 'test-too-long', 'invalid', 'wrong-version',
> - 'wrong-build', 'wrong-tools', 'wrong-header'
> - ]
> - for result_type in result_types:
> - yaml_log['summary'][result_type] = []
> -
> - result_element = {}
> - yaml_log['outputs'] = []
> -
> - # process output of each test
> - for exe_name in reports.results:
> - test_parts = exe_name.split("/")
> - test_name = test_parts[-1]
> - result_element['executable-name'] = test_name
> - result_element['executable-sha512'] = get_hash512(exe_name)
> - result_element['execution-start'] = reports.results[exe_name]['start'].isoformat()
> - result_element['execution-end'] = reports.results[exe_name]['end'].isoformat()
> - date_diff = reports.results[exe_name]['end'] - reports.results[exe_name]['start']
> - result_element['execution-duration'] = str(date_diff)
> - result_element['execution-result'] = reports.results[exe_name]['result']
> - result_element['bsp'] = reports.results[exe_name]['bsp']
> - result_element['bsp-arch'] = reports.results[exe_name]['bsp_arch']
> - result_output = reports.results[exe_name]['output']
> -
> - dbg_output = []
> - test_output = []
> - idxs_output = [] # store indices of given substrings
> - for elem in result_output:
> - if '=> ' in elem:
> - idxs_output.append(result_output.index(elem))
> - if '*** END' in elem:
> - idxs_output.append(result_output.index(elem))
> -
> - if len(idxs_output) == 3: # test executed and has result
> - dbg_output = result_output[idxs_output[0]:idxs_output[1]]
> - dbg_output.append("=== Executed Test ===")
> - dbg_output = dbg_output + result_output[idxs_output[2]+1:len(result_output)]
> - test_output = result_output[idxs_output[1]:idxs_output[2]+1]
> - else:
> - dbg_output = result_output
> -
> - result_element['debugger-output'] = format_output(dbg_output)
> - result_element['console-output'] = format_output(test_output)
> - yaml_log['outputs'].append(result_element)
> -
> - result_type = reports.results[exe_name]['result']
> - # map "fatal-error" on to "failed"
> - if result_type == "fatal-error":
> - result_type = "failed"
> -
> - if result_type != 'passed':
> - yaml_log['summary'][result_type].append(test_name)
> -
> - result_element = {}
> + data = results_to_data(args, reports, start_time, end_time);
>
> with open(yaml_file, 'w') as outfile:
> - yaml.dump(yaml_log, outfile, default_flow_style=False, allow_unicode=True)
> + yaml.dump(data, outfile, default_flow_style=False, allow_unicode=True)
>
>
> def get_hash512(exe):
More information about the devel
mailing list