[rtems-tools commit] bsb-builder: Add email support, and config report types.
Chris Johns
chrisj at rtems.org
Sun Oct 22 07:56:35 UTC 2017
Module: rtems-tools
Branch: master
Commit: 7051ba5b1958b3926cb69b98d0d9f1db384f754f
Changeset: http://git.rtems.org/rtems-tools/commit/?id=7051ba5b1958b3926cb69b98d0d9f1db384f754f
Author: Chris Johns <chrisj at rtems.org>
Date: Sun Oct 22 18:09:39 2017 +1100
bsb-builder: Add email support, and config report types.
---
rtemstoolkit/mailer.py | 48 ++++++++---
tester/rt/check.py | 163 +++++++++++++++++++++++++++++---------
tester/rtems/rtems-bsps-tiers.ini | 79 ++++++++++++++----
3 files changed, 229 insertions(+), 61 deletions(-)
diff --git a/rtemstoolkit/mailer.py b/rtemstoolkit/mailer.py
index 81a5c70..b5a0a1e 100644
--- a/rtemstoolkit/mailer.py
+++ b/rtemstoolkit/mailer.py
@@ -51,16 +51,42 @@ except (ValueError, SystemError):
import options
import path
+_options = {
+ '--mail' : 'Send email report or results.',
+ '--smtp-host': 'SMTP host to send via.',
+ '--mail-to' : 'Email address to send the email too.',
+ '--mail-from': 'Email address the report is from.'
+}
+
def append_options(opts):
- opts['--mail'] = 'Send email report or results.'
- opts['--smtp-host'] = 'SMTP host to send via.'
- opts['--mail-to'] = 'Email address to send the email too.'
- opts['--mail-from'] = 'Email address the report is from.'
+ for o in _options:
+ opts[o] = _options[o]
+
+def add_arguments(argsp):
+ argsp.add_argument('--mail', help = _options['--mail'], action = 'store_true')
+ for o in ['--smtp-host', '--mail-to', '--mail-from']:
+ argsp.add_argument(o, help = _options[o], type = str)
class mail:
def __init__(self, opts):
self.opts = opts
+ def _args_are_macros(self):
+ return type(self.opts) is 'command_line'
+
+ def _get_arg(self, arg):
+ if self._args_are_macros():
+ value = self.opts.find_arg(arg)[1]
+ else:
+ if arg.startswith('--'):
+ arg = arg[2:]
+ arg = arg.replace('-', '_')
+ if arg in vars(self.opts):
+ value = vars(self.opts)[arg]
+ else:
+ value = None
+ return value
+
def from_address(self):
def _clean(l):
@@ -72,9 +98,9 @@ class mail:
l = l[:l.index('\n')]
return l.strip()
- addr = self.opts.find_arg('--mail-from')
+ addr = self._get_arg('--mail-from')
if addr is not None:
- return addr[1]
+ return addr
mailrc = None
if 'MAILRC' in os.environ:
mailrc = os.environ['MAILRC']
@@ -95,14 +121,18 @@ class mail:
addr = fa[fa.index('=') + 1:].replace('"', ' ').strip()
if addr is not None:
return addr
- addr = self.opts.defaults.get_value('%{_sbgit_mail}')
+ if self._args_are_macros():
+ addr = self.opts.defaults.get_value('%{_sbgit_mail}')
+ else:
+ raise error.general('no valid from address for mail')
return addr
def smtp_host(self):
- host = self.opts.find_arg('--smtp-host')
+ host = self._get_arg('--smtp-host')
if host is not None:
return host[1]
- host = self.opts.defaults.get_value('%{_mail_smtp_host}')
+ if self._args_are_macros():
+ host = self.opts.defaults.get_value('%{_mail_smtp_host}')
if host is not None:
return host
return 'localhost'
diff --git a/tester/rt/check.py b/tester/rt/check.py
index af1d3e9..7e81d30 100755
--- a/tester/rt/check.py
+++ b/tester/rt/check.py
@@ -49,6 +49,7 @@ from rtemstoolkit import execute
from rtemstoolkit import error
from rtemstoolkit import host
from rtemstoolkit import log
+from rtemstoolkit import mailer
from rtemstoolkit import path
from rtemstoolkit import textbox
from rtemstoolkit import version
@@ -267,7 +268,7 @@ class warnings_errors:
data[category]['common'] = common
return data
- def _report_category(self, label, warnings, group_counts):
+ def _report_category(self, label, warnings, group_counts, summary):
width = 70
cols_1 = [width]
cols_2 = [8, width - 8]
@@ -311,15 +312,16 @@ class warnings_errors:
d = gs[row:+4]
s += textbox.row(cols_4, d, indent = 1)
s += textbox.line(cols_2_4, marker = '+', indent = 1)
- vw = sorted([(w, warnings[build][w]) for w in build_warnings],
- key = operator.itemgetter(1),
- reverse = True)
- for w in vw:
- c1 = '%6d' % w[1]
- for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
- s += textbox.row(cols_2, [c1, l], indent = 1)
- c1 = ' ' * 6
- s += textbox.line(cols_2, marker = '+', indent = 1)
+ if not summary:
+ vw = sorted([(w, warnings[build][w]) for w in build_warnings],
+ key = operator.itemgetter(1),
+ reverse = True)
+ for w in vw:
+ c1 = '%6d' % w[1]
+ for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
+ s += textbox.row(cols_2, [c1, l], indent = 1)
+ c1 = ' ' * 6
+ s += textbox.line(cols_2, marker = '+', indent = 1)
return s
def _report_warning_map(self):
@@ -341,9 +343,9 @@ class warnings_errors:
s += textbox.line(cols_1, marker = '+', indent = 1)
return s
- def warnings_report(self):
+ def warnings_report(self, summary):
self.lock.acquire()
- s = ' No warnings'
+ s = ' No warnings' + os.linesep
try:
total = 0
for build in self.warnings:
@@ -351,16 +353,20 @@ class warnings_errors:
if total != 0:
data = self._analyze(self.warnings, self.groups['exclude'])
s = self._report_category('By Architecture (total : %d)' % (total),
- data['arch'], data['groups']['arch'])
+ data['arch'], data['groups']['arch'],
+ summary)
s += os.linesep
s += self._report_category('By BSP (total : %d)' % (total),
- data['arch_bsp'], data['groups']['arch_bsp'])
+ data['arch_bsp'], data['groups']['arch_bsp'],
+ summary)
s += os.linesep
s += self._report_category('By Build (total : %d)' % (total),
- data['build'], data['groups']['build'])
- s += os.linesep
- s += self._report_warning_map()
+ data['build'], data['groups']['build'],
+ summary)
s += os.linesep
+ if not summary:
+ s += self._report_warning_map()
+ s += os.linesep
finally:
self.lock.release()
return s
@@ -631,11 +637,11 @@ class results:
if build_fails > 0:
s += bs + os.linesep
if count == 0:
- s = ' No failures'
+ s = ' No failures' + os.linesep
return s
- def warnings_report(self):
- return self.warnings_errors.warnings_report()
+ def warnings_report(self, summary = False):
+ return self.warnings_errors.warnings_report(summary)
def report(self):
self.lock.acquire()
@@ -923,7 +929,7 @@ class configuration_:
for p in self.profiles['profiles']:
profile = {}
profile['name'] = p
- profile['archs'] = self.config.comma_list(profile['name'], 'archs')
+ profile['archs'] = self.config.comma_list(profile['name'], 'archs', err = False)
archs += profile['archs']
for arch in profile['archs']:
bsps = 'bsps_%s' % (arch)
@@ -1281,6 +1287,17 @@ class builder:
f.write(command_line() + os.linesep)
f.write(self.results.warnings_errors.report())
+ def _failures_report(self):
+ if self.options['failures-report'] is not None:
+ with open(self.options['failures-report'], 'w') as f:
+ f.write(title() + os.linesep)
+ f.write(os.linesep)
+ f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
+ os.linesep))
+ f.write(os.linesep)
+ f.write(command_line() + os.linesep)
+ f.write(self.results.failures_report())
+
def _finished(self):
log.notice('Total: Warnings:%d exes:%d objs:%d libs:%d' % \
(self.results.get_warning_count(), self.counts['exes'],
@@ -1292,19 +1309,23 @@ class builder:
log.notice('Failures:')
log.notice(self.results.failures_report())
self._warnings_report()
+ self._failures_report()
def run_jobs(self, jobs):
if path.exists(self.build_dir):
log.notice('Cleaning: %s' % (self.build_dir))
path.removeall(self.build_dir)
- start = datetime.datetime.now()
+ self.start = datetime.datetime.now()
+ self.end = datetime.datetime.now()
+ self.duration = self.end - self.start
+ self.average = self.duration
env_path = os.environ['PATH']
os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
os.pathsep + os.environ['PATH']
job_count, build_job_count = jobs_option_parse(self.options['jobs'])
builds = self._create_build_jobs(jobs, build_job_count)
active_jobs = []
- jobs_completed = 0
+ self.jobs_completed = 0
try:
while len(builds) > 0 or len(active_jobs) > 0:
new_jobs = job_count - len(active_jobs)
@@ -1323,7 +1344,7 @@ class builder:
job.log_output()
job.clean()
active_jobs.remove(job)
- jobs_completed += 1
+ self.jobs_completed += 1
time.sleep(0.250)
except:
for job in active_jobs:
@@ -1332,15 +1353,15 @@ class builder:
except:
pass
raise
- end = datetime.datetime.now()
+ self.end = datetime.datetime.now()
os.environ['PATH'] = env_path
- duration = end - start
- if jobs_completed == 0:
- jobs_completed = 1
+ self.duration = self.end - self.start
+ if self.jobs_completed == 0:
+ self.jobs_completed = 1
self._finished()
- log.notice('Average BSP Build Time: %s' % \
- (str(duration / jobs_completed)))
- log.notice('Total Time %s' % (str(duration)))
+ self.average = self.duration / self.jobs_completed
+ log.notice('Average BSP Build Time: %s' % (str(self.average)))
+ log.notice('Total Time %s' % (str(self.duration)))
def arch_bsp_jobs(self, arch, bsps):
jobs = []
@@ -1401,6 +1422,7 @@ def run_args(args):
if 'msys' in cspath[0] and cspath[0].endswith('bin'):
os.environ['PATH'] = os.pathsep.join(cspath[1:])
+ start = datetime.datetime.now()
top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
prefix = '/opt/rtems/%s' % (rtems_version())
tools = prefix
@@ -1423,9 +1445,12 @@ def run_args(args):
type = str)
argsp.add_argument('--log', help = 'Log file.', type = str)
argsp.add_argument('--config-report', help = 'Report the configuration.',
- action = 'store_true')
+ type = str, default = None,
+ choices = ['all', 'profiles', 'builds', 'archs'])
argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
type = str, default = None)
+ argsp.add_argument('--failures-report', help = 'Report the failures to a file.',
+ type = str, default = None)
argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
action = 'store_true')
argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
@@ -1442,20 +1467,48 @@ def run_args(args):
type = str, default = '1/%d' % (host.cpus()))
argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
action = 'store_true')
+ mailer.add_arguments(argsp)
opts = argsp.parse_args(args[1:])
+ mail = None
+ if opts.mail:
+ mail = mailer.mail(opts)
+ # Request these now to generate any errors.
+ from_addr = mail.from_address()
+ smtp_host = mail.smtp_host()
+ if 'mail_to' in opts and opts.mail_to is not None:
+ to_addr = opts.mail_to
+ else:
+ to_addr = 'build at rtems.org'
if opts.log is not None:
logf = opts.log
log.default = log.log([logf])
log.notice(title())
log.output(command_line())
+ if mail:
+ log.notice('Mail: from:%s to:%s smtp:%s' % (from_addr,
+ to_addr,
+ smtp_host))
config = configuration_()
config.load(config_file, opts.build)
if opts.config_report:
- log.notice('Configuration Report:')
- log.notice(config.report())
+ log.notice('Configuration Report: %s' % (opts.config_report))
+ c_profiles = False
+ c_builds = False
+ c_archs = False
+ if opts.config_report == 'all':
+ c_profiles = True
+ c_builds = True
+ c_archs = True
+ elif opts.config_report == 'profiles':
+ c_profiles = True
+ elif opts.config_report == 'builds':
+ c_builds = True
+ elif opts.config_report == 'archs':
+ c_archs = True
+ log.notice(config.report(c_profiles, c_builds, c_archs))
sys.exit(0)
if opts.rtems is None:
@@ -1471,7 +1524,8 @@ def run_args(args):
'no-clean' : opts.no_clean,
'dry-run' : opts.dry_run,
'jobs' : opts.jobs,
- 'warnings-report' : opts.warnings_report }
+ 'warnings-report' : opts.warnings_report,
+ 'failures-report' : opts.failures_report }
b = builder(config, rtems_version(), prefix, tools,
path.shell(opts.rtems), build_dir, options)
@@ -1485,12 +1539,49 @@ def run_args(args):
#
if bsps is not None:
if archs is not None:
- raise error.general('--arch supplied with --bsp; use --bsp=arch/bsp,arch/bsp,..')
+ raise error.general('--arch supplied with --bsp;' \
+ ' use --bsp=arch/bsp,arch/bsp,..')
+ what = 'BSPs: %s' % (' '.join(bsps))
b.build_bsps(bsps)
elif archs is not None:
+ what = 'Archs: %s' % (' '.join(archs))
b.build_archs(archs)
else:
+ what = 'Profile(s): %s' % (' '.join(profiles))
b.build_profiles(profiles)
+ end = datetime.datetime.now()
+
+ #
+ # Email the results of the build.
+ #
+ if mail is not None:
+ subject = '[rtems-bsp-builder] %s: %s' % (str(start).split('.')[0],
+ what)
+ t = title()
+ body = t + os.linesep
+ body += '=' * len(t) + os.linesep
+ body += os.linesep
+ body += 'Host: %s' % (os.uname()[3]) + os.linesep
+ body += os.linesep
+ body += command_line()
+ body += os.linesep
+ body += 'Total Time : %s for %d completed job(s)' % \
+ (str(b.duration), b.jobs_completed)
+ body += os.linesep
+ body += 'Average BSP Build Time: %s' % (str(b.average))
+ body += os.linesep + os.linesep
+ body += 'Builds' + os.linesep
+ body += '======' + os.linesep
+ body += os.linesep.join([' ' + cb for cb in config.builds()])
+ body += os.linesep + os.linesep
+ body += 'Failures Report' + os.linesep
+ body += '===============' + os.linesep
+ body += b.results.failures_report()
+ body += os.linesep
+ body += 'Warnings Report' + os.linesep
+ body += '===============' + os.linesep
+ body += b.results.warnings_report(summary = True)
+ mail.send(to_addr, subject, body)
except error.general as gerr:
print(gerr)
diff --git a/tester/rtems/rtems-bsps-tiers.ini b/tester/rtems/rtems-bsps-tiers.ini
index c6bba14..cc611cb 100644
--- a/tester/rtems/rtems-bsps-tiers.ini
+++ b/tester/rtems/rtems-bsps-tiers.ini
@@ -21,36 +21,31 @@
# Tier 1: no build errors and no unexpected tests failures on hardware.
#
[tier-1]
-archs = arm, i386, sparc
-bsps_arm = altcycv_devkit,
- altcycv_devkit_smp,
- xilinx_zynq_zc702, xilinx_zynq_zc706, xilinx_zynq_zedboard
+archs = arm, i386
+bsps_arm = beagleboneblack, xilinx_zynq_zedboard
bsps_i386 = pc686
-bsps_sparc = leon2, leon3
#
# Tier 2: no build errors and no unexpected tests failures on hardware and
# simulators.
#
[tier-2]
-archs = arm, sparc
-bsps_arm = lm3s6965_qemu,
- realview_pbx_a9_qemu, realview_pbx_a9_qemu_smp,
- xilinx_zynq_a9_qemu
-bsps_sparc = erc32
+
#
# Tier 3: no build errors, no tests run.
#
[tier-3]
-archs = arm, moxie
-bsps_arm = arm1136jfs,
- arm1136js, arm7tdmi, arm920, armcortexa9, atsamv,
- beagleboardorig, beagleboardxm, beagleboneblack, beaglebonewhite,
+archs = arm, bfin, i386, lm32, m32c, m68k, mips, moxie,
+ nios2, or1k, powerpc, sh, sparc, sparc64, v850
+bsps_arm = altcycv_devkit, altcycv_devkit_smp,
+ arm1136jfs, arm1136js, arm7tdmi, arm920, armcortexa9, atsamv,
+ beagleboardorig, beagleboardxm, beaglebonewhite,
csb336, csb337, csb637,
edb7312,
kit637_v6,
gumstix,
- lm3s3749, lm3s6965, lm4f120,
+ imx7,
+ lm3s3749, lm3s6965, lm3s6965_qemu, lm4f120,
lpc1768_mbed, lpc1768_mbed_ahb_ram, lpc1768_mbed_ahb_ram_eth,
lpc17xx_ea_ram, lpc17xx_ea_rom_int, lpc17xx_plx800_ram,
lpc17xx_plx800_rom_int, lpc2362, lpc23xx_tli800, lpc24xx_ea,
@@ -59,13 +54,65 @@ bsps_arm = arm1136jfs,
lpc40xx_ea_rom_int, lpc32xx_mzx, lpc32xx_mzx_stage_1,
lpc32xx_mzx_stage_2, lpc32xx_phycore,
raspberrypi, raspberrypi2,
+ realview_pbx_a9_qemu, realview_pbx_a9_qemu_smp,
rtl22xx, rtl22xx_t,
smdk2410,
stm32f105rc, stm32f4,
tms570ls3137_hdk, tms570ls3137_hdk_intram,
tms570ls3137_hdk_sdram,
- tms570ls3137_hdk_with_loader
+ tms570ls3137_hdk_with_loader,
+ xilinx_zynq_a9_qemu, xilinx_zynq_zc702, xilinx_zynq_zc706
+bsps_bfin = TLL6527M, bf537Stamp, eZKit533
+bsps_i386 = pc386, pc486, pc586-sse, pc586, pcp4
+bsps_lm32 = lm32_evr, lm32_evr_gdbsim, milkymist
+bsps_m32c = m32csim
+bsps_m68k = av5282,
+ csb360,
+ gen68340, gen68360, gen68360_040,
+ pgh360,
+ COBRA5475,
+ m5484FireEngine,
+ mcf5206elite,
+ mcf52235, mcf5225x,
+ mcf5235,
+ mcf5329,
+ mrm332,
+ mvme147, mvme147s, mvme162, mvme162lx, mvme167,
+ uC5282
+bsps_mips = csb350, hurricane, jmr3904, malta, rbtx4925, rbtx4938
bsps_moxie = moxiesim
+bsps_nios2 = nios2_iss
+bsps_or1k = generic_or1k
+bsps_powerpc = beatnik,
+ br_uid, brs5l, brs6l,
+ dp2,
+ gwlcfm,
+ haleakala,
+ hsc_cm01,
+ icecube,
+ mcp750,
+ mpc5566evb, mpc5566evb_spe, phycore_mpc5554,
+ mpc5643l_dpu, mpc5643l_evb, mpc5668g,
+ mpc5674f_ecu508_app, mpc5674f_ecu508_boot, mpc5674f_rsm6, mpc5674fevb, mpc5674fevb_spe,
+ mpc8260ads,
+ mpc8309som,
+ mpc8313erdb,
+ mpc8349eamds,
+ mtx603e,
+ mvme2100, mvme2307, mvme3100, mvme5500,
+ pghplus,
+ pm520_cr825, pm520_ze30,
+ psim,
+ qemuppc, qemuprep, qemuprep-altivec,
+ qoriq_core_0, qoriq_core_1, qoriq_e500, qoriq_e6500_32, qoriq_e6500_64
+ ss555,
+ t32mppc,
+ tqm8xx_stk8xx,
+ virtex, virtex4, virtex5
+bsps_sh = gensh1, gensh2, gensh4, simsh1, simsh2, simsh2e, simsh4
+bsps_sparc = erc32, leon2, leon3, ngmp
+bsps_sparc64 = niagara, usiii
+bsps_v850 = v850e1sim, v850e2sim, v850e2v3sim, v850esim, v850essim, v850sim
#
# Tier 4: nothing expected.
More information about the vc
mailing list