[rtems-source-builder commit] sb: Monitor the build disk usage. Report the usage, total and various sizes
Chris Johns
chrisj at rtems.org
Thu Sep 27 21:37:37 UTC 2018
Module: rtems-source-builder
Branch: master
Commit: 38fd56c8a896a52bcab4f8921f67c483e36cf728
Changeset: http://git.rtems.org/rtems-source-builder/commit/?id=38fd56c8a896a52bcab4f8921f67c483e36cf728
Author: Chris Johns <chrisj at rtems.org>
Date: Fri Sep 28 07:27:57 2018 +1000
sb: Monitor the build disk usage. Report the usage, total and various sizes
- Track the size of a build of a package in a build set to determine the
maximum amout of disk space used. This can be used as a guide to
documenting how much space a user needs to set aside to build a specific
set of tools.
- The `%clean` stage of a build is now split into a separate script.
I do not think this is an issue because I could not find any `%clean`
sections in any build configs we have. In time support for the
`%clean` section will be removed, the package builder cleans up.
Closes #3516
---
source-builder/sb/build.py | 125 +++++++++++++++++++++++++++-------------
source-builder/sb/config.py | 13 ++++-
source-builder/sb/path.py | 68 ++++++++++++++++++++--
source-builder/sb/setbuilder.py | 42 +++++++++++++-
4 files changed, 199 insertions(+), 49 deletions(-)
diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py
index e28a831..88446bb 100644
--- a/source-builder/sb/build.py
+++ b/source-builder/sb/build.py
@@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2010-2013 Chris Johns (chrisj at rtems.org)
+# Copyright 2010-2018 Chris Johns (chrisj at rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@@ -51,6 +51,13 @@ except:
print('error: unknown application load error')
sys.exit(1)
+def humanize_number(num, suffix):
+ for unit in ['','K','M','G','T','P','E','Z']:
+ if abs(num) < 1024.0:
+ return "%5.3f%s%s" % (num, unit, suffix)
+ num /= 1024.0
+ return "%.3f%s%s" % (size, 'Y', suffix)
+
class script:
"""Create and manage a shell script."""
@@ -124,7 +131,8 @@ class build:
log.notice('config: ' + name)
self.set_macros(macros)
self.config = config.file(name, opts, self.macros)
- self.script = script()
+ self.script_build = script()
+ self.script_clean = script()
self.macros['buildname'] = self._name_(self.macros['name'])
except error.general as gerr:
log.notice(str(gerr))
@@ -288,21 +296,21 @@ class build:
raise error.general('setup source tag not found: %d' % (source_tag))
else:
name = opt_name
- self.script.append(self.config.expand('cd %{_builddir}'))
+ self.script_build.append(self.config.expand('cd %{_builddir}'))
if not deleted_dir and delete_before_unpack:
- self.script.append(self.config.expand('%{__rm} -rf ' + name))
+ self.script_build.append(self.config.expand('%{__rm} -rf ' + name))
deleted_dir = True
if not created_dir and create_dir:
- self.script.append(self.config.expand('%{__mkdir_p} ' + name))
+ self.script_build.append(self.config.expand('%{__mkdir_p} ' + name))
created_dir = True
if not changed_dir and (not unpack_before_chdir or create_dir):
- self.script.append(self.config.expand('cd ' + name))
+ self.script_build.append(self.config.expand('cd ' + name))
changed_dir = True
- self.script.append(self.config.expand(source['script']))
+ self.script_build.append(self.config.expand(source['script']))
if not changed_dir and (unpack_before_chdir and not create_dir):
- self.script.append(self.config.expand('cd ' + name))
+ self.script_build.append(self.config.expand('cd ' + name))
changed_dir = True
- self.script.append(self.config.expand('%{__setup_post}'))
+ self.script_build.append(self.config.expand('%{__setup_post}'))
def patch_setup(self, package, args):
name = args[1]
@@ -360,7 +368,7 @@ class build:
else:
patch['script'] = '%{__cat} ' + patch['local']
patch['script'] += ' | %%{__patch} %s' % (opts)
- self.script.append(self.config.expand(patch['script']))
+ self.script_build.append(self.config.expand(patch['script']))
def run(self, command, shell_opts = '', cwd = None):
e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
@@ -378,7 +386,7 @@ class build:
self.mkdir(builddir)
def prep(self, package):
- self.script.append('echo "==> %prep:"')
+ self.script_build.append('echo "==> %prep:"')
_prep = package.prep()
if _prep:
for l in _prep:
@@ -400,59 +408,78 @@ class build:
sources.hash(args[1:], self.macros, err)
self.hash(package, args)
else:
- self.script.append(' '.join(args))
+ self.script_build.append(' '.join(args))
def build(self, package):
- self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
- self.script.append('%s ${SB_BUILD_ROOT}' %
- (self.config.expand('%{__rmdir}')))
- self.script.append('%s ${SB_BUILD_ROOT}' %
- (self.config.expand('%{__mkdir_p}')))
- self.script.append('echo "==> %build:"')
+ self.script_build.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
+ self.script_build.append('%s ${SB_BUILD_ROOT}' %
+ (self.config.expand('%{__rmdir}')))
+ self.script_build.append('%s ${SB_BUILD_ROOT}' %
+ (self.config.expand('%{__mkdir_p}')))
+ self.script_build.append('echo "==> %build:"')
_build = package.build()
if _build:
for l in _build:
- self.script.append(l)
+ self.script_build.append(l)
def install(self, package):
- self.script.append('echo "==> %install:"')
+ self.script_build.append('echo "==> %install:"')
_install = package.install()
if _install:
for l in _install:
args = l.split()
- self.script.append(' '.join(args))
+ self.script_build.append(' '.join(args))
def files(self, package):
if self.create_tar_files \
and not self.macros.get('%{_disable_packaging'):
- self.script.append('echo "==> %files:"')
+ self.script_build.append('echo "==> %files:"')
inpath = path.abspath(self.config.expand('%{buildroot}'))
tardir = path.abspath(self.config.expand('%{_tardir}'))
- self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
- self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
- self.script.append(self.config.expand(' cd ' + inpath))
+ self.script_build.append(self.config.expand('if test -d %s; then' % (inpath)))
+ self.script_build.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
+ self.script_build.append(self.config.expand(' cd ' + inpath))
tar = path.join(tardir, package.long_name() + '.tar.bz2')
cmd = self.config.expand(' %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
- self.script.append(cmd)
- self.script.append(self.config.expand(' cd %{_builddir}'))
- self.script.append('fi')
+ self.script_build.append(cmd)
+ self.script_build.append(self.config.expand(' cd %{_builddir}'))
+ self.script_build.append('fi')
def clean(self, package):
- self.script.append('echo "==> %clean:"')
+ self.script_clean.reset()
+ self.script_clean.append(self.config.expand('%{___build_template}'))
+ self.script_clean.append('echo "=> ' + package.name() + ': CLEAN"')
+ self.script_clean.append('echo "==> %clean:"')
_clean = package.clean()
if _clean is not None:
for l in _clean:
args = l.split()
- self.script.append(' '.join(args))
+ self.script_clean.append(' '.join(args))
+
+ def sizes(self, package):
+ def _sizes(package, what, path):
+ package.set_size(what, path)
+ s = humanize_number(package.get_size(what), 'B')
+ log.trace('size: %s (%s): %s (%d)' % (what, path, s, package.get_size(what)))
+ return s
+ s = {}
+ for p in [('build', '%{_builddir}'),
+ ('build', '%{buildroot}'),
+ ('installed', '%{buildroot}')]:
+ hs = _sizes(package, p[0], self.config.expand(p[1]))
+ s[p[0]] = hs
+ log.notice('sizes: %s: %s (installed: %s)' % (package.name(),
+ s['build'],
+ s['installed']))
def build_package(self, package):
if self.canadian_cross():
if not self.config.defined('%{allow_cxc}'):
raise error.general('Canadian Cross is not allowed')
- self.script.append('echo "==> Candian-cross build/target:"')
- self.script.append('SB_CXC="yes"')
+ self.script_build.append('echo "==> Candian-cross build/target:"')
+ self.script_build.append('SB_CXC="yes"')
else:
- self.script.append('SB_CXC="no"')
+ self.script_build.append('SB_CXC="no"')
self.build(package)
self.install(package)
self.files(package)
@@ -498,18 +525,24 @@ class build:
log.trace('---- macro maps %s' % ('-' * 55))
log.trace('%s' % (str(self.config.macros)))
log.trace('-' * 70)
- self.script.reset()
- self.script.append(self.config.expand('%{___build_template}'))
- self.script.append('echo "=> ' + name + ':"')
+ self.script_build.reset()
+ self.script_build.append(self.config.expand('%{___build_template}'))
+ self.script_build.append('echo "=> ' + name + ': BUILD"')
self.prep(package)
self.build_package(package)
if not self.opts.dry_run():
self.builddir()
- sn = path.join(self.config.expand('%{_builddir}'), 'doit')
- log.output('write script: ' + sn)
- self.script.write(sn)
+ build_sn = path.join(self.config.expand('%{_builddir}'), 'do-build')
+ log.output('write script: ' + build_sn)
+ self.script_build.write(build_sn)
+ clean_sn = path.join(self.config.expand('%{_builddir}'), 'do-clean')
+ log.output('write script: ' + clean_sn)
+ self.script_clean.write(clean_sn)
log.notice('building: %s%s' % (cxc_label, name))
- self.run(sn)
+ self.run(build_sn)
+ self.sizes(package)
+ log.notice('cleaning: %s%s' % (cxc_label, name))
+ self.run(clean_sn)
except error.general as gerr:
log.notice(str(gerr))
log.stderr('Build FAILED')
@@ -536,6 +569,18 @@ class build:
package = packages['main']
return package.disabled()
+ def get_build_size(self):
+ package = self.main_package()
+ if package.disabled():
+ return 0
+ return package.get_size('build')
+
+ def get_installed_size(self):
+ package = self.main_package()
+ if package.disabled():
+ return 0
+ return package.get_size('installed')
+
def get_configs(opts):
def _scan(_path, ext):
diff --git a/source-builder/sb/config.py b/source-builder/sb/config.py
index 74c002e..a901038 100644
--- a/source-builder/sb/config.py
+++ b/source-builder/sb/config.py
@@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2010-2016 Chris Johns (chrisj at rtems.org)
+# Copyright 2010-2018 Chris Johns (chrisj at rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@@ -73,6 +73,7 @@ class package:
self.config = config
self.directives = {}
self.infos = {}
+ self.sizes = {}
def __str__(self):
@@ -218,6 +219,16 @@ class package:
def disabled(self):
return len(self.name()) == 0
+ def set_size(self, what, path_):
+ if what not in self.sizes:
+ self.sizes[what] = 0
+ self.sizes[what] += path.get_size(path_)
+
+ def get_size(self, what):
+ if what in self.sizes:
+ return self.sizes[what]
+ return 0
+
class file:
"""Parse a config file."""
diff --git a/source-builder/sb/path.py b/source-builder/sb/path.py
index e2fd3d4..984f3d7 100644
--- a/source-builder/sb/path.py
+++ b/source-builder/sb/path.py
@@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2010-2016 Chris Johns (chrisj at rtems.org)
+# Copyright 2010-2018 Chris Johns (chrisj at rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@@ -53,11 +53,6 @@ def host(path):
path = u'\\'.join([u'\\\\?', path])
return path
-def is_abspath(path):
- if path is not None and len(path) > 0:
- return '/' == path[0]
- return False
-
def shell(path):
if path is not None:
if windows:
@@ -79,6 +74,11 @@ def dirname(path):
path = shell(path)
return shell(os.path.dirname(path))
+def is_abspath(path):
+ if path is not None and len(path) > 0:
+ return '/' == path[0]
+ return False
+
def join(path, *args):
path = shell(path)
for arg in args:
@@ -304,6 +304,58 @@ def copy_tree(src, dst):
else:
raise error.general('copying tree (4): %s -> %s: %s' % (hsrc, hdst, str(why)))
+def get_size(path, depth = -1):
+ #
+ # Get the size the directory tree manually to the required depth.
+ # This makes sure on Windows the files are correctly encoded to avoid
+ # the file name size limit. On Windows the os.walk fails once we
+ # get to the max path length on Windows.
+ #
+ def _isdir(path):
+ hpath = host(path)
+ return os.path.isdir(hpath) and not os.path.islink(hpath)
+
+ def _node_size(path):
+ hpath = host(path)
+ size = 0
+ if not os.path.islink(hpath):
+ size = os.path.getsize(hpath)
+ return size
+
+ def _get_size(path, depth, level = 0):
+ level += 1
+ dirs = []
+ size = 0
+ for name in listdir(path):
+ path_ = join(path, shell(name))
+ hname = host(path_)
+ if _isdir(path_):
+ dirs += [shell(name)]
+ else:
+ size += _node_size(path_)
+ if depth < 0 or level < depth:
+ for name in dirs:
+ dir = join(path, name)
+ size += _get_size(dir, depth, level)
+ return size
+
+ path = shell(path)
+ hpath = host(path)
+ size = 0
+
+ if os.path.exists(hpath):
+ size = _get_size(path, depth)
+
+ return size
+
+def get_humanize_size(path, depth = -1):
+ size = get_size(path, depth)
+ for unit in ['','K','M','G','T','P','E','Z']:
+ if abs(size) < 1024.0:
+ return "%5.3f%sB" % (size, unit)
+ size /= 1024.0
+ return "%.3f%sB" % (size, 'Y')
+
if __name__ == '__main__':
print(host('/a/b/c/d-e-f'))
print(host('//a/b//c/d-e-f'))
@@ -311,6 +363,10 @@ if __name__ == '__main__':
print(basename('/as/sd/df/fg/me.txt'))
print(dirname('/as/sd/df/fg/me.txt'))
print(join('/d', 'g', '/tyty/fgfg'))
+ print('size of . depth all: ', get_size('.'))
+ print('size of . depth 1: ', get_size('.', 1))
+ print('size of . depth 2: ', get_size('.', 2))
+ print('size of . as human : ', get_humanize_size('.'))
windows = True
print(host('/a/b/c/d-e-f'))
print(host('//a/b//c/d-e-f'))
diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py
index b7cd8f2..2e6d643 100644
--- a/source-builder/sb/setbuilder.py
+++ b/source-builder/sb/setbuilder.py
@@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2010-2016 Chris Johns (chrisj at rtems.org)
+# Copyright 2010-2018 Chris Johns (chrisj at rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@@ -447,13 +447,48 @@ class buildset:
self.install(b.name(),
b.config.expand('%{buildroot}'),
b.config.expand('%{_prefix}'))
-
+ #
+ # Sizes ...
+ #
+ if len(builds) > 1:
+ size_build = 0
+ size_installed = 0
+ size_build_max = 0
+ for b in builds:
+ s = b.get_build_size()
+ size_build += s
+ if s > size_build_max:
+ size_build_max = s
+ size_installed += b.get_installed_size()
+ size_sources = 0
+ for p in builds[0].config.expand('%{_sourcedir}').split(':'):
+ size_sources += path.get_size(p)
+ size_patches = 0
+ for p in builds[0].config.expand('%{_patchdir}').split(':'):
+ size_patches += path.get_size(p)
+ size_total = size_sources + size_patches + size_installed
+ build_size = 'usage: %s' % (build.humanize_number(size_build_max + size_installed, 'B'))
+ build_size += ' total: %s' % (build.humanize_number(size_total, 'B'))
+ build_size += ' (sources: %s' % (build.humanize_number(size_sources, 'B'))
+ build_size += ', patches: %s' % (build.humanize_number(size_patches, 'B'))
+ build_size += ', installed %s)' % (build.humanize_number(size_installed, 'B'))
+ #
+ # Cleaning ...
+ #
if deps is None and \
(not self.opts.no_clean() or self.opts.always_clean()):
for b in builds:
if not b.disabled():
log.notice('cleaning: %s' % (b.name()))
b.cleanup()
+ #
+ # Log the build size message
+ #
+ if len(builds) > 1:
+ log.notice('Build Sizes: %s' % (build_size))
+ #
+ # Clear out the builds ...
+ #
for b in builds:
del b
except error.general as gerr:
@@ -484,6 +519,9 @@ class buildset:
self.write_mail_header('')
log.notice('Mailing report: %s' % (mail['to']))
body = self.get_mail_header()
+ body += 'Sizes' + os.linesep
+ body += '=====' + os.linesep + os.linesep
+
body += 'Output' + os.linesep
body += '======' + os.linesep + os.linesep
body += os.linesep.join(mail['output'].get())
More information about the vc
mailing list