[PATCH 1/1] Add yaml format to the supported report formats

[Cláudio Maia]

On 02/12/20 20:41, Chris Johns wrote:
> On 3/12/20 3:05 am, clrrm at isep.ipp.pt wrote:
>> From: Cláudio Maia <clrrm at isep.ipp.pt>
> Thanks. Does this work on python2?
I had to do a small change in v2 due to the unicode string returned by path.basename() which was not being handled correctly by PyYAML.
After that change, I tested with Python 2.7 and 3.6.9 and it works as expected.
 
>
>> ---
>>  tester/rt/test.py | 104 +++++++++++++++++++++++++++++++++++++++++++++-
>>  1 file changed, 102 insertions(+), 2 deletions(-)
>>
>> diff --git a/tester/rt/test.py b/tester/rt/test.py
>> index 9b157e9..0178a8d 100644
>> --- a/tester/rt/test.py
>> +++ b/tester/rt/test.py
>> @@ -339,9 +339,109 @@ def generate_junit_report(args, reports, start_time, end_time,
>>      with open(junit_file, 'w') as f:
>>          TestSuite.to_file(f, [ts], prettyprint = True)
>>  
>> +def generate_yaml_report(args, reports, start_time, end_time,
>> +                         total, yaml_file):
>> +    import yaml
> I would catch an import error and generate a nice error message for the user
> suggesting installing YAML from pip or where ever we recommend.

Added your suggestion to version 2 of the patch.

Cláudio

>
> Chris
>
>> +
>> +    def format_output(output_list):
>> +        return "\n".join(output_list).replace("] ", '').replace('=>  ', '')
>> +
>> +    yaml_log = {}
>> +    yaml_log['command-line'] = args
>> +    yaml_log['host'] = host.label(mode='all')
>> +    yaml_log['python'] = sys.version.replace('\n', '')
>> +    yaml_log['summary'] = {}
>> +    yaml_log['summary']['passed-count'] = reports.passed
>> +    yaml_log['summary']['failed-count'] = reports.failed
>> +    yaml_log['summary']['user-input-count'] = reports.user_input
>> +    yaml_log['summary']['expected-fail-count'] = reports.expected_fail
>> +    yaml_log['summary']['indeterminate-count'] = reports.indeterminate
>> +    yaml_log['summary']['benchmark-count'] = reports.benchmark
>> +    yaml_log['summary']['timeout-count'] = reports.timeouts
>> +    yaml_log['summary']['test-too-long_count'] = reports.test_too_long
>> +    yaml_log['summary']['invalid-count'] = reports.invalids
>> +    yaml_log['summary']['wrong-version-count'] = reports.wrong_version
>> +    yaml_log['summary']['wrong-build-count'] = reports.wrong_build
>> +    yaml_log['summary']['wrong-tools-count'] = reports.wrong_tools
>> +    yaml_log['summary']['total-count'] = reports.total
>> +    time_delta = end_time - start_time
>> +    yaml_log['summary']['average-test-time'] = str(time_delta / total)
>> +    yaml_log['summary']['testing-time'] = str(time_delta)
>> +
>> +    result_types = [
>> +            'failed', 'user-input', 'expected-fail', 'indeterminate',
>> +            'benchmark', 'timeout', 'test-too-long', 'invalid', 'wrong-version',
>> +            'wrong-build', 'wrong-tools'
>> +    ]
>> +    for result_type in result_types:
>> +        yaml_log['summary'][result_type] = []
>> +
>> +    result_element = {}
>> +    yaml_log['outputs'] = []
>> +
>> +    # process output of each test
>> +    for exe_name in reports.results:
>> +        result_element['executable-name'] = path.basename(exe_name)
>> +        result_element['executable-sha512'] = get_hash512(exe_name)
>> +        result_element['execution-start'] = reports.results[exe_name]['start'].isoformat()
>> +        result_element['execution-end'] = reports.results[exe_name]['end'].isoformat()
>> +        date_diff = reports.results[exe_name]['end'] - reports.results[exe_name]['start']
>> +        result_element['execution-duration'] = str(date_diff)
>> +        result_element['execution-result'] = reports.results[exe_name]['result']
>> +        result_element['bsp'] = reports.results[exe_name]['bsp']
>> +        result_element['bsp-arch'] = reports.results[exe_name]['bsp_arch']
>> +        result_output = reports.results[exe_name]['output']
>> +
>> +        dbg_output = []
>> +        test_output = []
>> +        idxs_output = []  # store indices of given substrings
>> +        for elem in result_output:
>> +            if '=> ' in elem:
>> +                idxs_output.append(result_output.index(elem))
>> +            if '*** END' in elem:
>> +                idxs_output.append(result_output.index(elem))
>> +
>> +        if len(idxs_output) == 3:  # test executed and has result
>> +            dbg_output = result_output[idxs_output[0]:idxs_output[1]]
>> +            dbg_output.append("=== Executed Test ===")
>> +            dbg_output = dbg_output + result_output[idxs_output[2]+1:len(result_output)]
>> +            test_output = result_output[idxs_output[1]:idxs_output[2]+1]
>> +        else:
>> +            dbg_output = result_output
>> +
>> +        result_element['debugger-output'] = format_output(dbg_output)
>> +        result_element['console-output'] = format_output(test_output)
>> +        yaml_log['outputs'].append(result_element)
>> +
>> +        result_type = reports.results[exe_name]['result']
>> +        # map "fatal-error" on to "failed"
>> +        if result_type == "fatal-error":
>> +            result_type = "failed"
>> +
>> +        if result_type != 'passed':
>> +            yaml_log['summary'][result_type].append(path.basename(exe_name))
>> +
>> +        result_element = {}
>> +
>> +    with open(yaml_file, 'w') as outfile:
>> +        yaml.dump(yaml_log, outfile, default_flow_style=False, allow_unicode=True)
>> +
>> +
>> +def get_hash512(exe):
>> +    """ returns SHA512 hash string of a given binary file passed as argument """
>> +    import hashlib
>> +
>> +    hash = hashlib.sha512()
>> +    with open(exe, "rb") as f:
>> +        for byte_block in iter(lambda: f.read(4096), b""):
>> +            hash.update(byte_block)
>> +    return hash.hexdigest()
>> +
>> +
>>  report_formatters = {
>>          'json': generate_json_report,
>> -        'junit': generate_junit_report
>> +        'junit': generate_junit_report,
>> +        'yaml': generate_yaml_report
>>  }
>>  
>>  
>> @@ -365,7 +465,7 @@ def run(args):
>>                      '--rtems-bsp':      'The RTEMS BSP to run the test on',
>>                      '--user-config':    'Path to your local user configuration INI file',
>>                      '--report-path':    'Report output base path (file extension will be added)',
>> -                    '--report-format':  'Formats in which to report test results in addition to txt: json',
>> +                    '--report-format':  'Formats in which to report test results in addition to txt: json, yaml',
>>                      '--log-mode':       'Reporting modes, failures (default),all,none',
>>                      '--list-bsps':      'List the supported BSPs',
>>                      '--debug-trace':    'Debug trace based on specific flags (console,gdb,output,cov)',
>>