[PATCH master 4/4] waf: Implement module dependency checking in the build system
chrisj at rtems.org
chrisj at rtems.org
Tue Sep 15 02:32:35 UTC 2020
From: Chris Johns <chrisj at rtems.org>
- Do not build a test if a dependency is not enabled.
- Perform a dependency check and generate an error if an enabled module
depends on a disabled module.
Closes #4077
---
builder.py | 114 +++++++++++++++++++++++++++++++++++++-------
freebsd-to-rtems.py | 1 +
libbsd.py | 29 ++++++-----
waf_libbsd.py | 22 ++++++---
wscript | 81 ++++++-------------------------
5 files changed, 143 insertions(+), 104 deletions(-)
diff --git a/builder.py b/builder.py
index fe2158a9..7e79d5f9 100755
--- a/builder.py
+++ b/builder.py
@@ -42,6 +42,11 @@ import os
import re
import sys
+try:
+ import configparser
+except ImportError:
+ import ConfigParser as configparser
+
#
# Global controls.
#
@@ -63,6 +68,9 @@ verboseDetail = 2
verboseMoreDetail = 3
verboseDebug = 4
+BUILDSET_DIR = "buildset"
+BUILDSET_DEFAULT = "buildset/default.ini"
+
def verbose(level=verboseInfo):
return verboseLevel >= level
@@ -761,8 +769,10 @@ class File(object):
state = state and (self.pathComposer == self.pathComposer)
state = state and (self.originPath == self.originPath)
state = state and (self.forwardConverter == self.forwardConverter)
- state = state and (self.self.reverseConverter == self.self.reverseConverter)
- state = state and (self.buildSystemComposer == self.buildSystemComposer)
+ state = state and (self.self.reverseConverter
+ == self.self.reverseConverter)
+ state = state and (self.buildSystemComposer
+ == self.buildSystemComposer)
return state
def processSource(self, forward):
@@ -794,13 +804,12 @@ class Module(object):
def __init__(self, manager, name, enabled=True):
self.manager = manager
self.name = name
- self.conditionalOn = "none"
self.files = []
self.cpuDependentSourceFiles = {}
self.dependencies = []
def __str__(self):
- out = [self.name + ': conditional-on=' + self.conditionalOn]
+ out = [self.name + ':']
if len(self.dependencies) > 0:
out += [' Deps: ' + str(len(self.dependencies))]
out += [' ' + type(d).__name__ for d in self.dependencies]
@@ -981,13 +990,16 @@ class Module(object):
NoConverter(), assertSourceFile,
sourceFileBuildComposer)
- def addTest(self, testFragementComposer):
+ def addTest(self, testFragementComposer, dependencies=[]):
self.files += [
File('user', testFragementComposer.testName, PathComposer(),
NoConverter(), NoConverter(), testFragementComposer)
]
+ self.dependencies += dependencies
def addDependency(self, dep):
+ if not isinstance(dep, str):
+ raise TypeError('dependencies are a string: %s' % (self.name))
self.dependencies += [dep]
@@ -1010,18 +1022,75 @@ class ModuleManager(object):
out += [str(self.modules[m]), '']
return os.linesep.join(out)
+ def _loadIni(self, ini_file):
+ if not os.path.exists(ini_file):
+ raise FileNotFoundError('file not found: %s' % (ini_file))
+ ini = configparser.ConfigParser()
+ ini.read(ini_file)
+ if not ini.has_section('general'):
+ raise Exception(
+ "'{}' is missing a general section.".format(ini_file))
+ if not ini.has_option('general', 'name'):
+ raise Exception("'{}' is missing a general/name.".format(ini_file))
+ if ini.has_option('general', 'extends'):
+ extends = ini.get('general', 'extends')
+ extendfile = None
+ basepath = os.path.dirname(ini_file)
+ if os.path.isfile(os.path.join(basepath, extends)):
+ extendfile = os.path.join(basepath, extends)
+ elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
+ extendfile = os.path.join(BUILDSET_DIR, extends)
+ else:
+ raise Exception(
+ "'{}': Invalid file given for general/extends:'{}'".format(
+ ini_file, extends))
+ base = self._loadIni(extendfile)
+ for s in ini.sections():
+ if not base.has_section(s):
+ base.add_section(s)
+ for o in ini.options(s):
+ val = ini.get(s, o)
+ base.set(s, o, val)
+ ini = base
+ return ini
+
+ def _checkDependencies(self):
+ enabled_modules = self.getEnabledModules()
+ enabled_modules.remove('tests')
+ for mod in enabled_modules:
+ if mod not in self.modules:
+ raise KeyError('enabled module not found: %s' % (mod))
+ for dep in self.modules[mod].dependencies:
+ if dep not in self.modules:
+ print(type(dep))
+ raise KeyError('dependent module not found: %s' % (dep))
+ if dep not in enabled_modules:
+ raise Exception('module "%s" dependency "%s" not enabled' %
+ (mod, dep))
+
def getAllModules(self):
if 'modules' in self.configuration:
- return self.configuration['modules']
+ return sorted(self.configuration['modules'])
return []
def getEnabledModules(self):
if 'modules-enabled' in self.configuration:
- return self.configuration['modules-enabled']
+ return sorted(self.configuration['modules-enabled'])
return []
def addModule(self, module):
- self.modules[module.name] = module
+ name = module.name
+ if name in self.modules:
+ raise KeyError('module already added: %' % (name))
+ self.modules[name] = module
+ if 'modules' not in self.configuration:
+ self.configuration['modules'] = []
+ if 'modules-enabled' not in self.configuration:
+ self.configuration['modules-enabled'] = []
+ self.configuration['modules'] += [name]
+ self.configuration['modules-enabled'] += [name]
+ self.configuration['modules'].sort()
+ self.configuration['modules-enabled'].sort
def processSource(self, direction):
if verbose(verboseDetail):
@@ -1035,15 +1104,6 @@ class ModuleManager(object):
def getConfiguration(self):
return copy.deepcopy(self.configuration)
- def updateConfiguration(self, config):
- self.configuration.update(config)
-
- def setModuleConfigiuration(self):
- mods = sorted(self.modules.keys())
- self.configuration['modules'] = mods
- # Enabled modules are overwritten by config file. Default to all.
- self.configuration['modules-enabled'] = mods
-
def generateBuild(self, only_enabled=True):
modules_to_process = self.getEnabledModules()
# Used for copy between FreeBSD and RTEMS
@@ -1053,6 +1113,7 @@ class ModuleManager(object):
if m not in self.modules:
raise KeyError('enabled module not registered: %s' % (m))
self.modules[m].generate()
+ self._checkDependencies()
def duplicateCheck(self):
dups = []
@@ -1068,6 +1129,25 @@ class ModuleManager(object):
dups += [(m, mod, fm.getPath(), fm.getSpace())]
return dups
+ def loadConfig(self, config=BUILDSET_DEFAULT):
+ if 'name' in self.configuration:
+ raise KeyError('configuration already loaded: %s (%s)' % \
+ (self.configuration['name'], config))
+ ini = self._loadIni(config)
+ self.configuration['name'] = ini.get('general', 'name')
+ self.configuration['modules-enabled'] = []
+ mods = []
+ if ini.has_section('modules'):
+ mods = ini.options('modules')
+ for mod in mods:
+ if ini.getboolean('modules', mod):
+ self.configuration['modules-enabled'].append(mod)
+
+ def getName(self):
+ if 'name' not in self.configuration:
+ raise KeyError('configuration not loaded')
+ return self.configuration['name']
+
def setGenerators(self):
self.generator['convert'] = Converter
self.generator['no-convert'] = NoConverter
diff --git a/freebsd-to-rtems.py b/freebsd-to-rtems.py
index 8f66e589..cda3ff3a 100755
--- a/freebsd-to-rtems.py
+++ b/freebsd-to-rtems.py
@@ -150,6 +150,7 @@ try:
build = builder.ModuleManager()
libbsd.load(build)
+ build.loadConfig()
build.generateBuild(only_enabled=False)
dups = build.duplicateCheck()
diff --git a/libbsd.py b/libbsd.py
index d86cdd39..58438100 100644
--- a/libbsd.py
+++ b/libbsd.py
@@ -960,7 +960,7 @@ class dev_usb_controller(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/usb/controller/ohci.h',
@@ -999,7 +999,7 @@ class dev_usb_input(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/usb/input/usb_rdesc.h',
@@ -1027,7 +1027,7 @@ class dev_usb_net(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/usb/net/if_auereg.h',
@@ -1077,7 +1077,7 @@ class dev_usb_quirk(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/usb/quirk/usb_quirk.h',
@@ -1100,7 +1100,7 @@ class dev_usb_serial(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/usb/serial/uftdi_reg.h',
@@ -1145,7 +1145,7 @@ class dev_usb_storage(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceSourceFiles(
[
'sys/dev/usb/storage/umass.c',
@@ -1163,7 +1163,7 @@ class dev_usb_controller_bbb(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/arm/ti/ti_cpuid.h',
@@ -1200,7 +1200,7 @@ class dev_usb_wlan(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/usb/wlan/if_rsureg.h',
@@ -1250,7 +1250,7 @@ class dev_wlan_rtwn(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['dev_usb'])
+ self.addDependency('dev_usb')
self.addKernelSpaceHeaderFiles(
[
'sys/dev/rtwn/if_rtwn_beacon.h',
@@ -2875,7 +2875,7 @@ class nfsv2(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['rpc_user'])
+ self.addDependency('rpc_user')
self.addRTEMSUserSourceFiles(
[
'nfsclient/mount_prot_xdr.c',
@@ -3276,7 +3276,7 @@ class crypto_openssl(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['user_space'])
+ self.addDependency('user_space')
self.addUserSpaceHeaderFiles(
[
'crypto/openssl/crypto/aes/aes_locl.h',
@@ -4258,7 +4258,7 @@ class usr_bin_openssl(builder.Module):
def generate(self):
mm = self.manager
- self.addDependency(mm['crypto_openssl'])
+ self.addDependency('crypto_openssl')
self.addUserSpaceHeaderFiles(
[
'crypto/openssl/apps/apps.h',
@@ -5328,7 +5328,8 @@ class tests(builder.Module):
def generate(self):
mm = self.manager
self.addTest(mm.generator['test']('epoch01', ['test_main'], extraLibs = ['rtemstest']))
- self.addTest(mm.generator['test']('nfs01', ['test_main'], netTest = True))
+ self.addTest(mm.generator['test']('nfs01', ['test_main'], netTest = True),
+ ['nfsv2'])
self.addTest(mm.generator['test']('foobarclient', ['test_main'],
runTest = False, netTest = True))
self.addTest(mm.generator['test']('foobarserver', ['test_main'],
@@ -5498,7 +5499,5 @@ def load(mm):
mm.addModule(tests(mm))
- mm.setModuleConfigiuration()
-
# XXX TODO Check that no file is also listed in empty
# XXX TODO Check that no file in in two modules
diff --git a/waf_libbsd.py b/waf_libbsd.py
index c457cbbd..558690b1 100644
--- a/waf_libbsd.py
+++ b/waf_libbsd.py
@@ -46,6 +46,10 @@ import builder
import rtems_waf.rtems as rtems
+
+BUILDSET_DIR = builder.BUILDSET_DIR
+BUILDSET_DEFAULT = builder.BUILDSET_DEFAULT
+
windows = os.name == 'nt'
if windows:
@@ -138,16 +142,22 @@ class Builder(builder.ModuleManager):
self.data = {}
- for mn in self.getEnabledModules():
+ enabled_modules = self.getEnabledModules()
+ for mn in enabled_modules:
m = self[mn]
- if m.conditionalOn == "none":
+ enabled = True
+ for dep in m.dependencies:
+ if dep not in enabled_modules:
+ enabled = False
+ break
+ if enabled:
for f in m.files:
_dataInsert(self.data, 'all', f.getSpace(),
f.getFragment())
- for cpu, files in sorted(m.cpuDependentSourceFiles.items()):
- for f in files:
- _dataInsert(self.data, cpu, f.getSpace(),
- f.getFragment())
+ for cpu, files in sorted(m.cpuDependentSourceFiles.items()):
+ for f in files:
+ _dataInsert(self.data, cpu, f.getSpace(),
+ f.getFragment())
# Start here if you need to understand self.data. Add 'True or'
if self.trace:
diff --git a/wscript b/wscript
index 99a60978..745ee6f8 100644
--- a/wscript
+++ b/wscript
@@ -45,76 +45,24 @@ except:
import sys
sys.exit(1)
-import libbsd
-import waf_libbsd
import os.path
import runpy
import sys
-try:
- import configparser
-except ImportError:
- import ConfigParser as configparser
-import waflib.Options
-builders = {}
-
-BUILDSET_DIR = "buildset"
-BUILDSET_DEFAULT = "buildset/default.ini"
-
-
-def load_ini(conf, f):
- ini = configparser.ConfigParser()
- ini.read(f)
- if not ini.has_section('general'):
- conf.fatal("'{}' is missing a general section.".format(f))
- if not ini.has_option('general', 'name'):
- conf.fatal("'{}' is missing a general/name.".format(f))
- if ini.has_option('general', 'extends'):
- extends = ini.get('general', 'extends')
- extendfile = None
- basepath = os.path.dirname(f)
- if os.path.isfile(os.path.join(basepath, extends)):
- extendfile = os.path.join(basepath, extends)
- elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
- extendfile = os.path.join(BUILDSET_DIR, extends)
- else:
- conf.fatal(
- "'{}': Invalid file given for general/extends:'{}'".format(
- f, extends))
- base = load_ini(conf, extendfile)
- for s in ini.sections():
- if not base.has_section(s):
- base.add_section(s)
- for o in ini.options(s):
- val = ini.get(s, o)
- base.set(s, o, val)
- ini = base
- return ini
-
-
-def load_config(conf, f):
- ini = load_ini(conf, f)
- config = {}
+import waflib.Options
- config['name'] = ini.get('general', 'name')
+import libbsd
+import waf_libbsd
- config['modules-enabled'] = []
- mods = []
- if ini.has_section('modules'):
- mods = ini.options('modules')
- for mod in mods:
- if ini.getboolean('modules', mod):
- config['modules-enabled'].append(mod)
- return config
+builders = {}
def update_builders(ctx, buildset_opt):
global builders
builders = {}
-
buildsets = []
if buildset_opt == []:
- buildset_opt.append(BUILDSET_DEFAULT)
+ buildset_opt.append(waf_libbsd.BUILDSET_DEFAULT)
for bs in buildset_opt:
if os.path.isdir(bs):
for f in os.listdir(bs):
@@ -123,15 +71,16 @@ def update_builders(ctx, buildset_opt):
else:
for f in bs.split(','):
buildsets += [f]
-
for bs in buildsets:
- builder = waf_libbsd.Builder()
- libbsd.load(builder)
- bsconfig = load_config(ctx, bs)
- bsname = bsconfig['name']
- builder.updateConfiguration(bsconfig)
- builder.generate(rtems_version)
- builders[bsname] = builder
+ try:
+ builder = waf_libbsd.Builder()
+ libbsd.load(builder)
+ builder.loadConfig(bs)
+ builder.generate(rtems_version)
+ except Exception as exc:
+ raise
+ ctx.fatal(str(exc))
+ builders[builder.getName()] = builder
def bsp_init(ctx, env, contexts):
@@ -250,7 +199,7 @@ def configure(conf):
conf.env.OPTIMIZATION = conf.options.optimization
conf.env.BUILDSET = conf.options.buildset
if len(conf.env.BUILDSET) == 0:
- conf.env.BUILDSET += [BUILDSET_DEFAULT]
+ conf.env.BUILDSET += [waf_libbsd.BUILDSET_DEFAULT]
update_builders(conf, conf.env.BUILDSET)
rtems.configure(conf, bsp_configure)
--
2.24.1
More information about the devel
mailing list