[PATCH 6/7] sb/setbuilder: Support staging of build set build before installing.
chrisj at rtems.org
chrisj at rtems.org
Mon Jul 22 00:10:08 UTC 2019
From: Chris Johns <chrisj at rtems.org>
- Staging lets build sets use previously built build sets.
- Fix the rtems-bsp builds to support staging.
---
rtems/config/5/rtems-libbsd.bset | 2 +-
rtems/config/rtems-bsp.cfg | 28 +++-
rtems/config/tools/rtems-libbsd-common.cfg | 2 +-
source-builder/defaults.mc | 16 +-
source-builder/sb/setbuilder.py | 163 +++++++++++++++------
source-builder/sb/shell.py | 75 ++++++++++
6 files changed, 232 insertions(+), 54 deletions(-)
create mode 100644 source-builder/sb/shell.py
diff --git a/rtems/config/5/rtems-libbsd.bset b/rtems/config/5/rtems-libbsd.bset
index ff9c2e7..2b84cf8 100644
--- a/rtems/config/5/rtems-libbsd.bset
+++ b/rtems/config/5/rtems-libbsd.bset
@@ -3,6 +3,6 @@
#
#
-# Build the kernel.
+# Build the BSD library for RTEMS.
#
tools/rtems-libbsd-5
diff --git a/rtems/config/rtems-bsp.cfg b/rtems/config/rtems-bsp.cfg
index 404ad6a..bf1f42b 100644
--- a/rtems/config/rtems-bsp.cfg
+++ b/rtems/config/rtems-bsp.cfg
@@ -24,7 +24,7 @@
# If a dry-run and with download ignore errors and correct setting for tools
# and BSPs. Only after the source to download.
#
-%if %{_dry_run} && %{defined with_download}
+%if %{_dry_run}
%log BSP configuration errors ignored
%define rtems_bsp_error 0
%else
@@ -36,7 +36,7 @@
# version. It can be defined in rtems_host.
#
%if %{defined rtems_host}
- %define _host %{rtems_host}
+ %{triplet _host %{rtems_host}}
%endif
%if %{rtems_bsp_error}
%if %{_host} == %{nil} || %{_host} == %{_build}
@@ -55,18 +55,29 @@
%endif
#
-# If no tools or RTEMS provided use the prefix.
+# If no tools or RTEMS provided use the prefix. If staging use the staging
+# root.
#
%ifn %{defined with_tools}
- %define with_tools %{_prefix}
- %define rtems_waf_tools %{nil}
+ %if %{install_mode} == staging
+ %define with_tools %{stagingroot}
+ %define rtems_waf_tools --rtems-tools=%{with_tools}
+ %else
+ %define with_tools %{_prefix}
+ %define rtems_waf_tools %{nil}
+ %endif
%else
%define rtems_waf_tools --rtems-tools=%{with_tools}
%endif
%ifn %{defined with_rtems}
- %define with_rtems %{_prefix}
- %define rtems_waf_rtems %{nil}
+ %if %{install_mode} == staging
+ %define with_rtems %{stagingroot}
+ %define rtems_waf_rtems --rtems=%{with_rtems}
+ %else
+ %define with_rtems %{_prefix}
+ %define rtems_waf_rtems %{nil}
+ %endif
%else
%define rtems_waf_rtems --rtems=%{with_rtems}
%endif
@@ -111,6 +122,7 @@
# RTEMS BSP paths
#
%define rtems_bsp_cc %{_host}-gcc
+%define rtems_bsp_cxx %{_host}-g++
%define rtems_bsp_incpath %{with_rtems}/%{_host}/%{rtems_bsp}/include
%define rtems_bsp_libpath %{with_rtems}/%{_host}/%{rtems_bsp}/lib
@@ -160,6 +172,8 @@
#
# Map to names used for cross compiling.
#
+%define host_cc %{rtems_bsp_cc}
+%define host_cxx %{rtems_bsp_cxx}
%define host_includes -I%{_includedir}
%define host_cflags %{rtems_bsp_cflags}
%define host_cxxflags %{rtems_bsp_ccflags}
diff --git a/rtems/config/tools/rtems-libbsd-common.cfg b/rtems/config/tools/rtems-libbsd-common.cfg
index 4503e1b..f5f78f2 100644
--- a/rtems/config/tools/rtems-libbsd-common.cfg
+++ b/rtems/config/tools/rtems-libbsd-common.cfg
@@ -62,7 +62,7 @@ URL: https://git.rtems.org/rtems-libbsd.git/
./waf distclean configure \
--prefix=%{_prefix} \
%{rtems_waf_tools} \
- %{rtems_waf_rtems}
+ %{rtems_waf_rtems} \
--rtems-bsp=%{rtems_bsp_arch_bsp}
./waf build
diff --git a/source-builder/defaults.mc b/source-builder/defaults.mc
index 1feaca1..ce5d2b0 100644
--- a/source-builder/defaults.mc
+++ b/source-builder/defaults.mc
@@ -75,6 +75,19 @@ build_ldflags: none, convert, ''
build_includes: none, convert, ''
build_libs: none, convert, ''
+#
+# Build and staging paths.
+#
+buildroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}'
+buildcxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-cxc'
+buildxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-xx'
+stagingroot: dir, none, '%{_tmppath}/sb-%{_uid}-staging'
+
+#
+# Install mode can be installing or staging. Defaults to installing.
+#
+install_mode: none, none, 'installing'
+
# Extra path a platform can override.
_extra_path: none, none, '%{_sbdir}'
_ld_library_path: none, none, 'LD_LIBRARY_PATH'
@@ -96,9 +109,6 @@ _docdir: dir, none, '%{_defaultdocdir}'
_tmppath: dir, none, '%{_topdir}/build/tmp'
_tmproot: dir, none, '%{_tmppath}/sb-%{_uid}/%{_bset_tmp}'
_tmpcxcroot: dir, none, '%{_tmppath}/sb-%{_uid}-cxc/%{_bset_tmp}'
-buildroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}'
-buildcxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-cxc'
-buildxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-xx'
_datadir: dir, none, '%{_prefix}/share'
_defaultdocdir: dir, none, '%{_prefix}/share/doc'
_exeext: none, none, ''
diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py
index 159a673..d0a119f 100644
--- a/source-builder/sb/setbuilder.py
+++ b/source-builder/sb/setbuilder.py
@@ -41,6 +41,7 @@ try:
import options
import path
import reports
+ import shell
import sources
import version
except KeyboardInterrupt:
@@ -50,6 +51,14 @@ except:
print('error: unknown application load error', file = sys.stderr)
sys.exit(1)
+def macro_expand(macros, _str):
+ cstr = None
+ while cstr != _str:
+ cstr = _str
+ _str = macros.expand(_str)
+ _str = shell.expand(macros, _str)
+ return _str
+
class log_capture(object):
def __init__(self):
self.log = []
@@ -71,21 +80,21 @@ class buildset:
"""Build a set builds a set of packages."""
def __init__(self, bset, _configs, opts, macros = None):
- log.trace('_bset: %s: init' % (bset))
+ log.trace('_bset: : %s: init' % (bset))
self.configs = _configs
self.opts = opts
if macros is None:
self.macros = copy.copy(opts.defaults)
else:
self.macros = copy.copy(macros)
- log.trace('_bset: %s: macro defaults' % (bset))
+ log.trace('_bset: : %s: macro defaults' % (bset))
log.trace(str(self.macros))
self.bset = bset
- _target = self.macros.expand('%{_target}')
+ _target = macro_expand(self.macros, '%{_target}')
if len(_target):
pkg_prefix = _target
else:
- pkg_prefix = self.macros.expand('%{_host}')
+ pkg_prefix = macro_expand(self.macros, '%{_host}')
self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
self.mail_header = ''
self.mail_report = ''
@@ -152,7 +161,7 @@ class buildset:
else:
raise error.general('invalid report format: %s' % (format))
buildroot = _build.config.abspath('%{buildroot}')
- prefix = _build.macros.expand('%{_prefix}')
+ prefix = macro_expand(_build.macros, '%{_prefix}')
name = _build.main_package().name() + ext
log.notice('reporting: %s -> %s' % (_config, name))
if not _build.opts.get_arg('--no-report'):
@@ -182,17 +191,24 @@ class buildset:
def root_copy(self, src, dst):
what = '%s -> %s' % \
(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
- log.trace('_bset: %s: collecting: %s' % (self.bset, what))
+ log.trace('_bset: : %s: collecting: %s' % (self.bset, what))
self.copy(src, dst)
- def install(self, name, buildroot, prefix):
- dst = prefix
- src = path.join(buildroot, prefix)
- log.notice('installing: %s -> %s' % (name, path.host(dst)))
+ def install(self, mode, name, src, dst):
+ log.notice('%s: %s -> %s' % (mode, name, path.host(dst)))
self.copy(src, dst)
+ def install_mode(self):
+ return macro_expand(self.macros, '%{install_mode}')
+
+ def installing(self):
+ return self.install_mode() == 'installing'
+
+ def staging(self):
+ return not self.installing()
+
def canadian_cross(self, _build):
- log.trace('_bset: Cxc for build machine: _build => _host')
+ log.trace('_bset: : Cxc for build machine: _build => _host')
macros_to_copy = [('%{_host}', '%{_build}'),
('%{_host_alias}', '%{_build_alias}'),
('%{_host_arch}', '%{_build_arch}'),
@@ -204,7 +220,7 @@ class buildset:
('%{_builddir}', '%{_buildcxcdir}')]
cxc_macros = _build.copy_init_macros()
for m in macros_to_copy:
- log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
+ log.trace('_bset: : Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
cxc_macros[m[0]] = cxc_macros[m[1]]
_build.set_macros(cxc_macros)
_build.reload()
@@ -229,12 +245,15 @@ class buildset:
if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
and not _build.macros.get('%{_disable_packaging}'):
path.mkdir(tardir)
- tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
+ tar = path.join(tardir,
+ _build.config.expand('%s.tar.bz2' % \
+ (_build.main_package().name())))
log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
if not self.opts.dry_run():
tmproot = _build.config.expand('%{_tmproot}')
cmd = _build.config.expand('"cd ' + tmproot + \
- ' && %{__tar} -cf - . | %{__bzip2} > ' + tar + '"')
+ ' && %{__tar} -cf - . | %{__bzip2} > ' + \
+ tar + '"')
_build.run(cmd, shell_opts = '-c', cwd = tmproot)
def parse(self, bset):
@@ -249,7 +268,7 @@ class buildset:
bsetname = bset
if not path.exists(bsetname):
- for cp in self.macros.expand('%{_configdir}').split(':'):
+ for cp in macro_expand(self.macros, '%{_configdir}').split(':'):
configdir = path.abspath(cp)
bsetname = path.join(configdir, bset)
if path.exists(bsetname):
@@ -258,7 +277,7 @@ class buildset:
if bsetname is None:
raise error.general('no build set file found: %s' % (bset))
try:
- log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
+ log.trace('_bset: : %s: open: %s' % (self.bset, bsetname))
bset = open(path.host(bsetname), 'r')
except IOError as err:
raise error.general('error opening bset file: %s' % (bsetname))
@@ -272,7 +291,7 @@ class buildset:
l = _clean(l)
if len(l) == 0:
continue
- log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
+ log.trace('_bset: : %s: %03d: %s' % (self.bset, lc, l))
ls = l.split()
if ls[0][-1] == ':' and ls[0][:-1] == 'package':
self.bset_pkg = ls[1].strip()
@@ -288,8 +307,8 @@ class buildset:
self.macros.define(ls[1].strip())
elif ls[0] == '%undefine':
if len(ls) > 2:
- raise error.general('%s:%d: %undefine requires just the name' % \
- (self.bset, lc))
+ raise error.general('%s:%d: %undefine requires ' \
+ 'just the name' % (self.bset, lc))
self.macros.undefine(ls[1].strip())
elif ls[0] == '%include':
configs += self.parse(ls[1].strip())
@@ -301,7 +320,8 @@ class buildset:
l = l.strip()
c = build.find_config(l, self.configs)
if c is None:
- raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
+ raise error.general('%s:%d: cannot find file: %s' % (self.bset,
+ lc, l))
configs += [c]
except:
bset.close()
@@ -320,9 +340,12 @@ class buildset:
if self.bset.endswith('.cfg'):
configs = [self.bset]
else:
- exbset = self.macros.expand(self.bset)
+ exbset = macro_expand(self.macros, self.bset)
self.macros['_bset'] = exbset
- self.macros['_bset_tmp'] = build.short_name(exbset)
+ bset_tmp = build.short_name(exbset)
+ if bset_tmp.endswith('.bset'):
+ bset_tmp = bset_tmp[:-5]
+ self.macros['_bset_tmp'] = bset_tmp
root, ext = path.splitext(exbset)
if exbset.endswith('.bset'):
bset = exbset
@@ -340,7 +363,7 @@ class buildset:
if mail:
mail['output'].clear()
- log.trace('_bset: %s: make' % (self.bset))
+ log.trace('_bset: %2d: %s: make' % (nesting_count, self.bset))
log.notice('Build Set: %s' % (self.bset))
mail_subject = '%s on %s' % (self.bset,
@@ -350,16 +373,44 @@ class buildset:
start = datetime.datetime.now()
- mail_report = False
- have_errors = False
+ mail_report = False
+ have_errors = False
if mail:
mail['output'].clear()
+ #
+ # If this is the outter most buildset it's files installed. Nested
+ # build sets staged their installed file. The staged files are install
+ # when the outtter most build finishes.
+ #
+ if nesting_count != 1:
+ if self.installing():
+ self.macros['install_mode'] = 'staging'
+ #
+ # Prepend staging areas, bin directory tothe
+ # path. Lets the later package depend on the eailier
+ # ones.
+ #
+ pathprepend = ['%{stagingroot}/bin'] + \
+ macro_expand(self.macros, '%{_pathprepend}').split(':')
+ pathprepend = [pp for pp in pathprepend if len(pp)]
+ if len(pathprepend) == 1:
+ self.macros['_pathprepend'] = pathprepend[0]
+ else:
+ self.macros['_pathprepend'] = ':'.join(pathprepend)
+
+ #
+ # Only the outter build set can have staging to install. Get the staging
+ # root via the config because it could require a valid config.
+ #
+ have_staging = False
+
try:
configs = self.load()
- log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
+ log.trace('_bset: %2d: %s: configs: %s' % (nesting_count,
+ self.bset, ', '.join(configs)))
sizes_valid = False
builds = []
@@ -374,14 +425,20 @@ class buildset:
opts = copy.copy(self.opts)
macros = copy.copy(self.macros)
if configs[s].endswith('.bset'):
- log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
+ log.trace('_bset: %2d: %s %s' % (nesting_count,
+ configs[s],
+ '=' * (74 - len(configs[s]))))
bs = buildset(configs[s], self.configs, opts, macros)
bs.build(deps, nesting_count, mail)
+ if self.installing():
+ have_staging = True
del bs
elif configs[s].endswith('.cfg'):
if mail:
mail_report = True
- log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
+ log.trace('_bset: %2d: %s %s' % (nesting_count,
+ configs[s],
+ '=' * (74 - len(configs[s]))))
try:
b = build.build(configs[s],
self.opts.get_arg('--pkg-tar-files'),
@@ -412,7 +469,7 @@ class buildset:
#
# Dump post build macros.
#
- log.trace('_bset: macros post-build')
+ log.trace('_bset: : macros post-build')
log.trace(str(b.macros))
else:
raise error.general('invalid config type: %s' % (configs[s]))
@@ -436,19 +493,25 @@ class buildset:
else:
raise
#
- # Installing ...
+ # Installing or staging ...
#
- log.trace('_bset: installing: deps:%r no-install:%r' % \
- (deps is None, self.opts.no_install()))
- if deps is None \
- and not self.opts.no_install() \
- and not have_errors:
+ log.trace('_bset: %2d: %s: deps:%r no-install:%r' % \
+ (nesting_count, self.install_mode(),
+ deps is None, self.opts.no_install()))
+ log.trace('_bset: %2d: %s: builds: %s' % \
+ (nesting_count, self.install_mode(),
+ ', '.join([b.name() for b in builds])))
+ if deps is None and not have_errors:
for b in builds:
- log.trace('_bset: installing: %r' % b.installable())
+ log.trace('_bset: : %s: %r' % (self.install_mode(),
+ b.installable()))
if b.installable():
- self.install(b.name(),
- b.config.expand('%{buildroot}'),
- b.config.expand('%{_prefix}'))
+ prefix = b.config.expand('%{_prefix}')
+ buildroot = path.join(b.config.expand('%{buildroot}'), prefix)
+ if self.staging():
+ prefix = b.config.expand('%{stagingroot}')
+ self.install(self.install_mode(), b.name(), buildroot, prefix)
+
#
# Sizes ...
#
@@ -469,7 +532,8 @@ class buildset:
for p in builds[0].config.expand('%{_patchdir}').split(':'):
size_patches += path.get_size(p)
size_total = size_sources + size_patches + size_installed
- build_max_size_human = build.humanize_number(size_build_max + size_installed, 'B')
+ build_max_size_human = build.humanize_number(size_build_max +
+ size_installed, 'B')
build_total_size_human = build.humanize_number(size_total, 'B')
build_sources_size_human = build.humanize_number(size_sources, 'B')
build_patches_size_human = build.humanize_number(size_patches, 'B')
@@ -499,6 +563,21 @@ class buildset:
#
for b in builds:
del b
+
+ #
+ # If builds have been staged install into the finaly prefix.
+ #
+ if have_staging and not self.opts.no_install() and not have_errors:
+ log.trace('_bset: %2d: install staging' % (nesting_count))
+ stagingroot = macro_expand(self.macros, '%{stagingroot}')
+ prefix = macro_expand(self.macros, '%{_prefix}')
+ self.install(self.install_mode(), self.bset, stagingroot, prefix)
+ staging_size = path.get_size(stagingroot)
+ if not self.opts.no_clean() or self.opts.always_clean():
+ log.notice('clean staging: %s' % (self.bset))
+ log.trace('cleanup: %s' % (stagingroot))
+ self.rmdir(stagingroot)
+ log.notice('Staging Size: %s' % (build.humanize_number(staging_size)))
except error.general as gerr:
if not build_error:
log.stderr(str(gerr))
@@ -586,7 +665,7 @@ def run():
if to_addr is not None:
mail['to'] = to_addr[1]
else:
- mail['to'] = opts.defaults.expand('%{_mail_tools_to}')
+ mail['to'] = macro_expand(opts.defaults, '%{_mail_tools_to}')
mail['from'] = mail['mail'].from_address()
log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
opts.log_info()
@@ -606,7 +685,7 @@ def run():
else:
deps = None
if not list_bset_cfg_files(opts, configs):
- prefix = opts.defaults.expand('%{_prefix}')
+ prefix = macro_expand(opts.defaults, '%{_prefix}')
if opts.canadian_cross():
opts.disable_install()
diff --git a/source-builder/sb/shell.py b/source-builder/sb/shell.py
new file mode 100644
index 0000000..bebbe8a
--- /dev/null
+++ b/source-builder/sb/shell.py
@@ -0,0 +1,75 @@
+#
+# RTEMS Tools Project (http://www.rtems.org/)
+# Copyright 2019 Chris Johns (chrisj at rtems.org)
+# All rights reserved.
+#
+# This file is part of the RTEMS Tools package in 'rtems-tools'.
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+#
+# This code builds a package compiler tool suite given a tool set. A tool
+# set lists the various tools. These are specific tool configurations.
+#
+
+from __future__ import print_function
+
+try:
+ import error
+ import execute
+ import log
+ import options
+except KeyboardInterrupt:
+ print('abort: user terminated', file = sys.stderr)
+ sys.exit(1)
+except:
+ print('error: unknown application load error', file = sys.stderr)
+ sys.exit(1)
+
+def expand(macros, line):
+ #
+ # Parse the line and handle nesting '()' pairs.
+ #
+ def _exec(shell_macro):
+ output = ''
+ if len(shell_macro) > 3:
+ e = execute.capture_execution()
+ if options.host_windows:
+ cmd = '%s -c "%s"' % (macros.expand('%{__sh}'), shell_macro[2:-1])
+ else:
+ cmd = shell_macro[2:-1]
+ exit_code, proc, output = e.shell(cmd)
+ log.trace('shell-output: %d %s' % (exit_code, output))
+ if exit_code != 0:
+ raise error.general('shell macro failed: %s: %d: %s' % (cmd,
+ exit_code,
+ output))
+ return output
+
+ updating = True
+ while updating:
+ updating = False
+ pos = line.find('%(')
+ if pos >= 0:
+ braces = 0
+ for p in range(pos + 2, len(line)):
+ if line[p] == '(':
+ braces += 1
+ elif line[p] == ')':
+ if braces > 0:
+ braces -= 1
+ else:
+ line = line[:pos] + _exec(line[pos:p + 1]) + line[p + 1:]
+ updating = True
+ break
+ return line
--
2.19.1
More information about the devel
mailing list