[rtems-source-builder commit] sb/get-sources: Fix getting sources with archs
Chris Johns
chrisj at rtems.org
Wed Aug 17 04:55:32 UTC 2022
Module: rtems-source-builder
Branch: master
Commit: cbcf8698a8bf9f70f0c696b8783f3ad2371914dc
Changeset: http://git.rtems.org/rtems-source-builder/commit/?id=cbcf8698a8bf9f70f0c696b8783f3ad2371914dc
Author: Chris Johns <chrisj at rtems.org>
Date: Sun Aug 14 22:04:40 2022 +1000
sb/get-sources: Fix getting sources with archs
- Add a stop on error option
- Add listing root or toplevel buildset files
- Add used and unused file output to track what is not used
for clean ups
- Update to handle macro expanded includes in buildset files
---
source-builder/sb/getsources.py | 30 +++++++++-
source-builder/sb/simhost.py | 128 ++++++++++++++++++++++++++++++----------
2 files changed, 123 insertions(+), 35 deletions(-)
diff --git a/source-builder/sb/getsources.py b/source-builder/sb/getsources.py
index d348da4..eb5c855 100644
--- a/source-builder/sb/getsources.py
+++ b/source-builder/sb/getsources.py
@@ -60,7 +60,9 @@ def run(args = sys.argv):
default = version.version())
argsp.add_argument('--list-hosts', help = 'List the hosts.',
action = 'store_true')
- argsp.add_argument('--list-bsets', help = 'List the hosts.',
+ argsp.add_argument('--list-bsets', help = 'List the buildsets.',
+ action = 'store_true')
+ argsp.add_argument('--list-root-bsets', help = 'List the toplevel or root buildsets.',
action = 'store_true')
argsp.add_argument('--download-dir', help = 'Download directory.',
type = str)
@@ -71,8 +73,14 @@ def run(args = sys.argv):
argsp.add_argument('--log', help = 'Log file.',
type = str,
default = simhost.log_default('getsource'))
+ argsp.add_argument('--stop-on-error', help = 'Stop on error.',
+ action = 'store_true')
argsp.add_argument('--trace', help = 'Enable trace logging for debugging.',
action = 'store_true')
+ argsp.add_argument('--used', help = 'Save the used buildset and config files.',
+ type = str, default = None)
+ argsp.add_argument('--unused', help = 'Save the unused buildset and config files.',
+ type = str, default = None)
argsp.add_argument('bsets', nargs='*', help = 'Build sets.')
argopts = argsp.parse_args(args[1:])
@@ -84,10 +92,14 @@ def run(args = sys.argv):
opts = simhost.load_options(args, argopts, extras = ['--with-download'])
configs = build.get_configs(opts)
+ stop_on_error = argopts.stop_on_error
+
if argopts.list_hosts:
simhost.list_hosts()
elif argopts.list_bsets:
simhost.list_bset_files(opts, configs)
+ elif argopts.list_root_bsets:
+ simhost.list_root_bset_files(opts, configs)
else:
if argopts.clean:
if argopts.download_dir is None:
@@ -95,11 +107,11 @@ def run(args = sys.argv):
if path.exists(argopts.download_dir):
log.notice('Cleaning source directory: %s' % (argopts.download_dir))
path.removeall(argopts.download_dir)
- all_bsets = simhost.get_bset_files(configs)
if len(argopts.bsets) == 0:
- bsets = all_bsets
+ bsets = simhost.get_root_bset_files(opts, configs)
else:
bsets = argopts.bsets
+ deps = copy.copy(simhost.strip_common_prefix(bsets))
for bset in bsets:
b = None
try:
@@ -108,11 +120,23 @@ def run(args = sys.argv):
b = simhost.buildset(bset, configs, opts)
get_sources_error = False
b.build(host)
+ deps += b.deps()
del b
except error.general as gerr:
+ if stop_on_error:
+ raise
log.stderr(str(gerr))
log.stderr('Build FAILED')
b = None
+ deps = sorted(list(set(deps)))
+ if argopts.used:
+ with open(argopts.used, 'w') as o:
+ o.write(os.linesep.join(deps))
+ if argopts.unused:
+ cfgs_bsets = \
+ [cb for cb in simhost.get_config_bset_files(opts, configs) if not cb in deps]
+ with open(argopts.unused, 'w') as o:
+ o.write(os.linesep.join(cfgs_bsets))
except error.general as gerr:
if get_sources_error:
log.stderr(str(gerr))
diff --git a/source-builder/sb/simhost.py b/source-builder/sb/simhost.py
index 1ff98e8..f58a66d 100644
--- a/source-builder/sb/simhost.py
+++ b/source-builder/sb/simhost.py
@@ -36,6 +36,7 @@ try:
from . import log
from . import macros
from . import path
+ from . import shell
from . import sources
from . import version
except KeyboardInterrupt:
@@ -122,7 +123,6 @@ profiles = {
'_var': ('dir', 'optional', '/usr/local/var') },
}
-
class log_capture(object):
def __init__(self):
self.log = []
@@ -154,6 +154,18 @@ def find_bset_config(bset_config, macros):
raise error.general('no build set file found: %s' % (bset_config))
return name
+def macro_expand(macros, _str):
+ cstr = None
+ while cstr != _str:
+ cstr = _str
+ _str = macros.expand(_str)
+ _str = shell.expand(macros, _str)
+ return _str
+
+def strip_common_prefix(files):
+ commonprefix = os.path.commonprefix(files)
+ return sorted(list(set([f[len(commonprefix):] for f in files])))
+
#
# A skinny options command line class to get the configs to load.
#
@@ -252,10 +264,10 @@ class options(object):
a += 1
return None
- def rtems_bsp(self):
+ def rtems_bsp(self, arch='arch'):
self.defaults['rtems_version'] = str(version.version())
- self.defaults['_target'] = 'arch-rtems'
- self.defaults['rtems_host'] = 'rtems-arch'
+ self.defaults['_target'] = arch + '-rtems'
+ self.defaults['rtems_host'] = 'rtems-' + arch
self.defaults['with_rtems_bsp'] = 'rtems-bsp'
def sb_git(self):
@@ -383,8 +395,19 @@ class buildset:
rebased += [i]
return rebased
+ def root(self):
+ for i in self._includes:
+ si = i.split(':')
+ if len(si) == 2:
+ if si[1] == 'root':
+ return si[0]
+ return None
+
def includes(self):
- return sorted(list(set(self._includes)))
+ return [i for i in self._includes if not i.endswith(':root')]
+
+ def deps(self):
+ return strip_common_prefix([i.split(':')[0] for i in self.includes()])
def errors(self):
return sorted(list(set(self._errors)))
@@ -393,7 +416,7 @@ class buildset:
if not _build.disabled():
_build.make()
- def parse(self, bset):
+ def parse(self, bset, expand=True):
#
# Ouch, this is a copy of the setbuilder.py code.
@@ -409,7 +432,7 @@ class buildset:
bsetname = find_bset_config(bset, self.macros)
try:
- log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
+ log.trace('_bset: %s: open: %s %s' % (self.bset, bsetname, expand))
bsetf = open(path.host(bsetname), 'r')
except IOError as err:
raise error.general('error opening bset file: %s' % (bsetname))
@@ -432,19 +455,26 @@ class buildset:
if ls[0][-1] == ':' and ls[0][:-1] == 'package':
self.bset_pkg = ls[1].strip()
self.macros['package'] = self.bset_pkg
- elif ls[0][0] == '%':
+ elif ls[0][0] == '%' and (len(ls[0]) > 1 and ls[0][1] != '{'):
def err(msg):
raise error.general('%s:%d: %s' % (self.bset, lc, msg))
- if ls[0] == '%define':
+ if ls[0] == '%define' or ls[0] == '%defineifnot' :
+ name = ls[1].strip()
+ value = None
if len(ls) > 2:
- self.macros.define(ls[1].strip(),
- ' '.join([f.strip() for f in ls[2:]]))
- else:
- self.macros.define(ls[1].strip())
+ value = ' '.join([f.strip() for f in ls[2:]])
+ if ls[0] == '%defineifnot':
+ if self.macros.defined(name):
+ name = None
+ if name is not None:
+ if value is not None:
+ self.macros.define(name, value)
+ else:
+ self.macros.define(name)
elif ls[0] == '%undefine':
if len(ls) > 2:
- raise error.general('%s:%d: %undefine requires just the name' \
- % (self.bset, lc))
+ raise error.general('%s:%d: %undefine requires ' \
+ 'just the name' % (self.bset, lc))
self.macros.undefine(ls[1].strip())
elif ls[0] == '%include':
configs += self.parse(ls[1].strip())
@@ -453,12 +483,18 @@ class buildset:
elif ls[0] == '%hash':
sources.hash(ls[1:], self.macros, err)
else:
- l = l.strip()
- c = build.find_config(l, self.configs)
- if c is None:
- raise error.general('%s:%d: cannot find file: %s'
- % (self.bset, lc, l))
- configs += [c + ':' + self.parent]
+ try:
+ l = macro_expand(self.macros, l.strip())
+ except:
+ if expand:
+ raise
+ l = None
+ if l is not None:
+ c = build.find_config(l, self.configs)
+ if c is None:
+ raise error.general('%s:%d: cannot find file: %s'
+ % (self.bset, lc, l))
+ configs += [c + ':' + self.parent]
finally:
bsetf.close()
self.parent = parent
@@ -523,7 +559,7 @@ class buildset:
nesting_count += 1
- log.trace('_bset: %s for %s: make' % (self.bset, host))
+ log.trace('_bset: %2d: %s for %s: make' % (nesting_count, self.bset, host))
log.notice('Build Set: %s for %s' % (self.bset, host))
mail_subject = '%s on %s' % (self.bset,
@@ -538,7 +574,7 @@ class buildset:
try:
configs = self.load()
- log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
+ log.trace('_bset: %2d: %s: configs: %s' % (nesting_count, self.bset, ','.join(configs)))
sizes_valid = False
builds = []
@@ -556,14 +592,14 @@ class buildset:
self.set_host_details(host, opts, macros)
config, parent = configs[s].split(':', 2)
if config.endswith('.bset'):
- log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
+ log.trace('_bset: %2d: %s' % (nesting_count + 1, '=' * 75))
bs = buildset(config, self.configs, opts, macros)
bs.build(host, nesting_count)
self._includes += \
self._rebase_includes(bs.includes(), parent)
del bs
elif config.endswith('.cfg'):
- log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
+ log.trace('_bset: %2d: %s' % (nesting_count + 1, '-' * 75))
try:
b = build.build(config,
False,
@@ -579,7 +615,7 @@ class buildset:
#
# Dump post build macros.
#
- log.trace('_bset: macros post-build')
+ log.trace('_bset: %2d: macros post-build' % (nesting_count))
log.trace(str(macros))
else:
raise error.general('invalid config type: %s' % (config))
@@ -589,13 +625,16 @@ class buildset:
if self.build_failure is None:
self.build_failure = b.name()
self._includes += b.includes()
- self._errors += [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
+ self._errors += \
+ [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
raise
#
# Clear out the builds ...
#
for b in builds:
del b
+ self._includes += \
+ [find_bset_config(c.split(':')[0], self.macros) + ':' + self.bset for c in configs]
except error.general as gerr:
if not build_error:
log.stderr(str(gerr))
@@ -616,9 +655,9 @@ def list_hosts():
max_os_len = max(len(h) for h in hosts)
max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts)
for h in hosts:
- print('%*s: %-*s %s' % (max_os_len, h, max_host_len,
- profiles[h]['_host'][2],
- profiles[h]['_host'][2]))
+ log.notice('%*s: %-*s %s' % (max_os_len, h, max_host_len,
+ profiles[h]['_host'][2],
+ profiles[h]['_host'][2]))
def get_files(configs, ext, localpath):
files = []
@@ -635,14 +674,39 @@ def get_config_files(configs, localpath = False):
def get_bset_files(configs, localpath = False):
return get_files(configs, '.bset', localpath)
+def get_config_bset_files(opts, configs):
+ cbs = get_config_files(configs) + get_bset_files(configs)
+ return strip_common_prefix([find_bset_config(cb, opts.defaults) for cb in cbs])
+
+def get_root_bset_files(opts, configs, localpath = False):
+ bsets = get_bset_files(configs, localpath)
+ incs = {}
+ for bs in bsets:
+ bset = buildset(bs, configs, opts)
+ cfgs = [find_bset_config(c.split(':')[0], bset.macros) for c in bset.parse(bs, False)]
+ incs[bset.root()] = bset.includes() + cfgs
+ roots = sorted(incs.keys())
+ for inc in incs:
+ for i in incs[inc]:
+ si = i.split(':')
+ if len(si) > 0 and si[0] in roots:
+ roots.remove(si[0])
+ return roots
+
def get_root(configs):
return configs['root']
+def list_root_bset_files(opts, configs):
+ for p in configs['paths']:
+ log.notice('Examining: %s' % (os.path.relpath(p)))
+ for r in strip_common_prefix(get_root_bset_files(opts, configs)):
+ log.notice(' %s' % (r))
+
def list_bset_files(opts, configs):
for p in configs['paths']:
- print('Examining: %s' % (os.path.relpath(p)))
+ log.notice('Examining: %s' % (os.path.relpath(p)))
for b in get_bset_files(configs):
- print(' %s' % (b[:b.rfind('.')]))
+ log.notice(' %s' % (b[:b.rfind('.')]))
def load_log(logfile):
log.default = log.log(streams = [logfile])
More information about the vc
mailing list