[PATCH 6/6] waf: Add configurations with different modules.
Chris Johns
chrisj at rtems.org
Mon Apr 9 01:50:16 UTC 2018
On 06/04/2018 22:26, Christian Mauderer wrote:
> This allows multiple build configurations (buildsets) with different
> modules enabled.
>
> Update #3351
> ---
> CONTRIBUTING.md | 7 ++-
> README.waf | 3 +-
> builder.py | 13 ++---
> buildset/default.ini | 58 ++++++++++++++++++++++
> buildset/sample.ini | 10 ++++
> libbsd.py | 2 +-
> rtems_waf | 2 +-
> waf_libbsd.py | 9 ----
> wscript | 136 +++++++++++++++++++++++++++++++++++++++++++++------
> 9 files changed, 203 insertions(+), 37 deletions(-)
> create mode 100644 buildset/default.ini
> create mode 100644 buildset/sample.ini
>
> diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
> index eddefe98..9e6ee83c 100644
> --- a/CONTRIBUTING.md
> +++ b/CONTRIBUTING.md
> @@ -67,8 +67,7 @@ freebsd-to-rtems.py [args]
> ```
>
> In its default mode of operation, freebsd-to-rtems.py is used to copy code
> -from FreeBSD to the rtems-libbsd tree and perform transformations. In forward
> -mode, the script may be requested to just generate the Waf script.
> +from FreeBSD to the rtems-libbsd tree and perform transformations.
>
> In *reverse mode*, this script undoes those transformations and copies
> the source code back to the *master* FreeBSD tree. This allows us to do
> @@ -126,9 +125,9 @@ How to import code from FreeBSD
> * Run `./freebsd-to-rtems.py -R`
> * Run `./freebsd-to-rtems.py`
> * Run `git status` and make sure your working directory is clean. If you see modified files, then the `freebsd-to-rtems.py` script needs to be fixed first.
> -* Add the files to import to `libbsd.py`.
> +* Add the files to import to `libbsd.py` and your intended build set (for example `buildset/default.ini`.
> * Run `./freebsd-to-rtems.py`
> -* Immediately check in the imported files without the changes to `libbsd_waf.py`. Do not touch the imported files yourself at this point.
> +* Immediately check in the imported files without the changes to `libbsd.py` and the buildsets. Do not touch the imported files yourself at this point.
> * Port the imported files to RTEMS. See 'Rules for Modifying FreeBSD Source'.
> * Add a test to the testsuite if possible.
> * Run `./create-kernel-namespace.sh` if you imported kernel space headers. Add only your new defines via `git add -p rtemsbsd/include/machine/rtems-bsd-kernel-namespace.h`.
> diff --git a/README.waf b/README.waf
> index dddc04a4..485a1899 100644
> --- a/README.waf
> +++ b/README.waf
> @@ -73,7 +73,8 @@ cd rtems-libbsd
> git submodule init
> git submodule update rtems_waf
> waf configure --prefix="$sandbox/rtems-4.12" \
> - --rtems-bsps=arm/xilinx_zynq_a9_qemu
> + --rtems-bsps=arm/xilinx_zynq_a9_qemu \
> + --buildset=buildset/default.ini
Is this the default and so if you do not add a --buildset option you get this?
It might pay to highlight this if it is the case.
> waf
> waf install
> qemu-system-arm -no-reboot -serial null -serial mon:stdio -net none \
> diff --git a/builder.py b/builder.py
> index bb5791a8..c968e05c 100755
> --- a/builder.py
> +++ b/builder.py
> @@ -44,6 +44,7 @@ import getopt
> import filecmp
> import difflib
> import codecs
> +import copy
>
> #
> # Global controls.
> @@ -487,15 +488,11 @@ class Module(object):
> def __init__(self, manager, name, enabled = True):
> self.manager = manager
> self.name = name
> - self.enabled = enabled
> self.conditionalOn = "none"
> self.files = []
> self.cpuDependentSourceFiles = {}
> self.dependencies = []
>
> - def isEnabled(self):
> - return self.enabled
> -
> def initCPUDependencies(self, cpu):
> if cpu not in self.cpuDependentSourceFiles:
> self.cpuDependentSourceFiles[cpu] = []
> @@ -664,15 +661,19 @@ class ModuleManager(object):
> self.modules[m].processSource(direction)
>
> def setConfiguration(self, config):
> - self.configuration = config
> + self.configuration = copy.deepcopy(config)
>
> def getConfiguration(self):
> return self.configuration
>
> + def updateConfiguration(self, config):
> + self.configuration.update(config)
> +
> def setModuleConfigiuration(self):
> mods = sorted(self.modules.keys())
> self.configuration['modules'] = mods
> - self.configuration['modules-enabled'] = [m for m in mods if self.modules[m].isEnabled()]
> + # Enabled modules are overwritten by config file. Default to all.
> + self.configuration['modules-enabled'] = mods
>
> def generateBuild(self, only_enabled=True):
> modules_to_process = self.getEnabledModules()
> diff --git a/buildset/default.ini b/buildset/default.ini
> new file mode 100644
> index 00000000..15b57df4
> --- /dev/null
> +++ b/buildset/default.ini
> @@ -0,0 +1,58 @@
> +#
> +# Default configuration.
> +#
> +
> +[general]
> +name = default
> +
> +[modules-enabled]
Is there a 'modules-disabled' section? I see you have an integer bool in the
list below so that means we can disable a module in the enable section which is
confusing. Maybe this should be called 'modules'?
> +altq = 1
Can this please be ''enable/disable'?
> +base = 1
> +cam = 1
> +contrib_expat = 1
> +contrib_libpcap = 1
> +crypto = 1
> +crypto_openssl = 1
> +dev_input = 1
> +dev_net = 1
> +dev_nic = 1
> +dev_nic_broadcomm = 1
> +dev_nic_dc = 1
> +dev_nic_e1000 = 1
> +dev_nic_fxp = 1
> +dev_nic_re = 1
> +dev_nic_smc = 1
> +dev_usb = 1
> +dev_usb_controller = 1
> +dev_usb_controller_bbb = 1
> +dev_usb_input = 1
> +dev_usb_net = 1
> +dev_usb_quirk = 1
> +dev_usb_serial = 1
> +dev_usb_storage = 1
> +dev_usb_wlan = 1
> +dev_wlan_rtwn = 1
> +dhcpcd = 1
> +dpaa = 1
> +evdev = 1
> +fdt = 1
> +in_cksum = 1
> +ipfw = 1
> +mdnsresponder = 1
> +mghttpd = 1
> +mmc = 1
> +mmc_ti = 1
> +net = 1
> +net80211 = 1
> +netinet = 1
> +netinet6 = 1
> +opencrypto = 1
> +pci = 1
> +pf = 1
> +rtems = 1
> +tests = 1
> +tty = 1
> +user_space = 1
> +user_space_wlanstats = 1
> +usr_sbin_tcpdump = 1
> +usr_sbin_wpa_supplicant = 1
> diff --git a/buildset/sample.ini b/buildset/sample.ini
> new file mode 100644
> index 00000000..f30c1491
> --- /dev/null
> +++ b/buildset/sample.ini
> @@ -0,0 +1,10 @@
> +#
> +# Currently this is mostly a sample configuration.
> +#
> +
> +[general]
> +name = sample
> +extends = default.ini
> +
> +[modules-enabled]
> +dev_nic_broadcomm = 0
> diff --git a/libbsd.py b/libbsd.py
> index e3c0a3ac..a256594e 100644
> --- a/libbsd.py
> +++ b/libbsd.py
> @@ -1996,7 +1996,7 @@ class netinet6(builder.Module):
> class netipsec(builder.Module):
>
> def __init__(self, manager):
> - super(netipsec, self).__init__(manager, type(self).__name__, enabled = False)
> + super(netipsec, self).__init__(manager, type(self).__name__)
>
> def generate(self):
> mm = self.manager
> diff --git a/rtems_waf b/rtems_waf
> index d793097d..4ef2f41e 160000
> --- a/rtems_waf
> +++ b/rtems_waf
> @@ -1 +1 @@
> -Subproject commit d793097d2032ca0ab2d5cf93fa84730865db6c03
> +Subproject commit 4ef2f41eb7a6b0fd84791eafff9062da1a9d94c4
> diff --git a/waf_libbsd.py b/waf_libbsd.py
> index ae8a1ea0..fffdf9fe 100644
> --- a/waf_libbsd.py
> +++ b/waf_libbsd.py
> @@ -295,12 +295,6 @@ class Builder(builder.ModuleManager):
> import pprint
> pprint.pprint(self.data)
>
> - def init(self, ctx):
> - pass
> -
> - def options(self, opt):
> - pass
> -
Why remove?
> def bsp_configure(self, conf, arch_bsp):
> if 'configure' in self.data:
> for cfg in self.data['configure']:
> @@ -310,9 +304,6 @@ class Builder(builder.ModuleManager):
> includes = conf.env.IFLAGS,
> mandatory = False)
>
> - def configure(self, conf):
> - pass
> -
Also why?
> def build(self, bld):
> # This is only necessary for build. But it depends on the waflib which
> # is only available in a waf context. But we need the module manager for
> diff --git a/wscript b/wscript
> index 6c063965..1dacb3b2 100644
> --- a/wscript
> +++ b/wscript
> @@ -44,23 +44,122 @@ except:
> #import libbsd_waf
> import libbsd
> import waf_libbsd
> +import os.path
> +import runpy
> +import sys
> +try:
> + import configparser
> +except ImportError:
> + import ConfigParser as configparser
> +import waflib.Options
> +
> +builders = {}
> +
> +BUILDSET_DIR = "buildset"
> +BUILDSET_DEFAULT = "buildset/default.ini"
> +
> +def load_ini(conf, f):
> + ini = configparser.ConfigParser()
> + ini.read(f)
> + if not ini.has_section('general'):
> + conf.fatal("'{}' is missing a general section.".format(f))
> + if not ini.has_option('general', 'name'):
> + conf.fatal("'{}' is missing a general/name.".format(f))
> + if ini.has_option('general', 'extends'):
> + extends = ini.get('general', 'extends')
> + extendfile = None
> + basepath = os.path.dirname(f)
> + if os.path.isfile(os.path.join(basepath, extends)):
> + extendfile = extends
> + elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
> + extendfile = os.path.join(BUILDSET_DIR, extends)
> + else:
> + conf.fatal("'{}': Invalid file given for general/extends:'{}'"
> + .format(f, extends))
> + base = load_ini(conf, extendfile)
> + for s in ini.sections():
> + if not base.has_section(s):
> + base.add_section(s)
> + for o in ini.options(s):
> + val = ini.get(s, o)
> + base.set(s, o, val)
> + ini = base
> + return ini
> +
> +def load_config(conf, f):
> + ini = load_ini(conf, f)
> + config = {}
> +
> + config['name'] = ini.get('general', 'name')
>
> -builder = None
> + config['modules-enabled'] = []
> + mods = []
> + if ini.has_section('modules-enabled'):
> + mods = ini.options('modules-enabled')
> + for mod in mods:
> + if ini.getboolean('modules-enabled', mod):
> + config['modules-enabled'].append(mod)
> + return config
>
> -def create_builder():
> - global builder
> - if builder is None:
> +def check_buildsets(ctx, buildset_opt):
> + buildsets = []
> + if buildset_opt == []:
> + buildset_opt.append(BUILDSET_DEFAULT)
> + for bs in buildset_opt:
> + if os.path.isdir(bs):
> + for f in os.listdir(bs):
> + if f[-4:] == ".ini":
> + buildsets += [os.path.join(bs,f)]
> + else:
> + for f in bs.split(','):
> + buildsets += [f]
> + return buildsets
> +
Ah ok. Maybe this should be in a separate module with a class so the source
management and waf build can both share?
We are starting to collect a few python modules so I wonder if they should be
moved into something like 'buildsys', what do you think?
> +def bsp_init(ctx, env, contexts):
> + buildsets = []
> + if 'buildset' in env.options:
> + buildsets = check_buildsets(ctx, env.options['buildset'])
> + else:
> + buildsets = check_buildsets(ctx, [BUILDSET_DEFAULT])
> + bsnames = []
> +
> + global builders
> +
> + for bs in buildsets:
> + # create builder objects
> builder = waf_libbsd.Builder()
buildsets = buildsets.BuildSets()
?
> libbsd.load(builder)
libbsd.load(builder, buildsets)
?
Would something like that work?
Chris
> + bsconfig = load_config(ctx, bs)
> + bsname = bsconfig['name']
> + bsnames += [bsname]
> + builder.updateConfiguration(bsconfig)
> builder.generate(rtems_version)
> + builders[bsname]=builder
> +
> + # Update the contextes for build variants
> + for y in contexts:
> + newcmd = y.cmd + '-' + bsname
> + newvariant = y.variant + '-' + bsname
> + class context(y):
> + cmd = newcmd
> + variant = newvariant
> + libbsd_buildset_name = bsname
> +
> + # Transform the commands to per build variant commands
> + commands = []
> + for cmd in waflib.Options.commands:
> + if cmd.startswith(('build', 'clean', 'install')):
> + for bsname in bsnames:
> + commands += [cmd + '-' + bsname]
> + else:
> + commands += [cmd]
> + waflib.Options.commands = commands
>
> def init(ctx):
> - create_builder();
> - rtems.init(ctx, version = rtems_version, long_commands = True)
> - builder.init(ctx)
> + rtems.init(ctx, version = rtems_version, long_commands = True,
> + bsp_init = bsp_init)
>
> def options(opt):
> - create_builder();
> rtems.options(opt)
> opt.add_option("--enable-auto-regen",
> action = "store_true",
> @@ -86,20 +185,29 @@ def options(opt):
> default = "2",
> dest = "optimization",
> help = "Set optimization level to OPTIMIZATION (-On compiler flag). Default is 2 (-O2).")
> - builder.options(opt)
> + opt.add_option("--buildset",
> + action = "append",
> + default = [],
> + dest = "buildset",
> + help = "Select build sets to build. If set to a directory, all .ini file in this directory will be used.")
>
> def bsp_configure(conf, arch_bsp):
> - create_builder();
> conf.check(header_name = "dlfcn.h", features = "c")
> conf.check(header_name = "rtems/pci.h", features = "c", mandatory = False)
> if not rtems.check_posix(conf):
> conf.fatal("RTEMS kernel POSIX support is disabled; configure RTEMS with --enable-posix")
> if rtems.check_networking(conf):
> conf.fatal("RTEMS kernel contains the old network support; configure RTEMS with --disable-networking")
> - builder.bsp_configure(conf, arch_bsp)
> + env = conf.env.derive()
> + for builder in builders:
> + ab = conf.env.RTEMS_ARCH_BSP
> + variant = ab + "-" + builder
> + conf.msg('Configure variant: ', variant)
> + conf.setenv(variant, env)
> + builders[builder].bsp_configure(conf, arch_bsp)
> + conf.setenv(ab)
>
> def configure(conf):
> - create_builder();
> if conf.options.auto_regen:
> conf.find_program("lex", mandatory = True)
> conf.find_program("rpcgen", mandatory = True)
> @@ -110,9 +218,7 @@ def configure(conf):
> conf.env.FREEBSD_OPTIONS =conf.options.freebsd_options
> conf.env.OPTIMIZATION = conf.options.optimization
> rtems.configure(conf, bsp_configure)
> - builder.configure(conf)
>
> def build(bld):
> - create_builder();
> rtems.build(bld)
> - builder.build(bld)
> + builders[bld.libbsd_buildset_name].build(bld)
>
More information about the devel
mailing list