[RSB PATCH] sb: Add sb-rtems-pkg to update the RTEMS package hashes and checksums
Kinsey Moore
kinsey.moore at oarcorp.com
Wed Apr 3 16:51:53 UTC 2024
Looks fine overall. Minor nits:
"host" is set to "freebsd" and is never used.
Numeric indexes for repo config details are nice for brevity, but not for
readability.
Kinsey
On Wed, Apr 3, 2024 at 11:29 AM Joel Sherrill <joel.sherrill at gmail.com>
wrote:
> This looks handy. I don't have an issue with it.
>
> On Tue, Apr 2, 2024 at 11:25 PM <chrisj at rtems.org> wrote:
>
>> From: Chris Johns <chrisj at rtems.org>
>>
>> ---
>> source-builder/sb-rtems-pkg | 29 ++++
>> source-builder/sb/download.py | 5 +-
>> source-builder/sb/git.py | 12 ++
>> source-builder/sb/rtemspkg.py | 279 ++++++++++++++++++++++++++++++++++
>> 4 files changed, 324 insertions(+), 1 deletion(-)
>> create mode 100755 source-builder/sb-rtems-pkg
>> create mode 100644 source-builder/sb/rtemspkg.py
>>
>> diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg
>> new file mode 100755
>> index 0000000..99ed26c
>> --- /dev/null
>> +++ b/source-builder/sb-rtems-pkg
>> @@ -0,0 +1,29 @@
>> +#! /usr/bin/env python
>> +#
>> +# RTEMS Tools Project (http://www.rtems.org/)
>> +# Copyright 2024 Chris Johns (chrisj at rtems.org)
>> +# All rights reserved.
>> +#
>> +# This file is part of the RTEMS Tools package in 'rtems-tools'.
>> +#
>> +# Permission to use, copy, modify, and/or distribute this software for
>> any
>> +# purpose with or without fee is hereby granted, provided that the above
>> +# copyright notice and this permission notice appear in all copies.
>> +#
>> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
>> WARRANTIES
>> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
>> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
>> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
>> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
>> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
>> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
>> +
>> +from __future__ import print_function
>> +
>> +try:
>> + import sb.rtemspkg
>> + sb.rtemspkg.run()
>> +except ImportError:
>> + import sys
>> + print("Incorrect Source Builder installation", file = sys.stderr)
>> + sys.exit(1)
>> diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py
>> index 6e6f9f2..0201675 100644
>> --- a/source-builder/sb/download.py
>> +++ b/source-builder/sb/download.py
>> @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True):
>> hash = hash.split()
>> if len(hash) != 2:
>> raise error.internal('invalid hash format: %s' % (file_))
>> + if hash[0] == 'NO-HASH':
>> + return not failed
>> try:
>> hashlib_algorithms = hashlib.algorithms
>> except:
>> @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts):
>> else:
>> repo.clean(['-f', '-d'])
>> repo.reset('--hard')
>> - repo.checkout('master')
>> + default_branch = repo.default_branch()
>> + repo.checkout(default_branch)
>> for a in us[1:]:
>> _as = a.split('=')
>> if _as[0] == 'branch' or _as[0] == 'checkout':
>> diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py
>> index 237e690..0aa4da9 100644
>> --- a/source-builder/sb/git.py
>> +++ b/source-builder/sb/git.py
>> @@ -226,6 +226,18 @@ class repo:
>> hash = l1[len('commit '):]
>> return hash
>>
>> + def default_branch(self):
>> + ec, output = self._run(['remote', 'show'])
>> + if ec == 0:
>> + origin = output.split('\n')[0]
>> + ec, output = self._run(['remote', 'show', origin])
>> + if ec == 0:
>> + for l in output.split('\n'):
>> + l = l.strip()
>> + if l.startswith('HEAD branch: '):
>> + return l[len('HEAD branch: '):]
>> + return None
>> +
>> if __name__ == '__main__':
>> import os.path
>> import sys
>> diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py
>> new file mode 100644
>> index 0000000..0b43d23
>> --- /dev/null
>> +++ b/source-builder/sb/rtemspkg.py
>> @@ -0,0 +1,279 @@
>> +#
>> +# RTEMS Tools Project (http://www.rtems.org/)
>> +# Copyright 2024 Chris Johns (chrisj at rtems.org)
>> +# All rights reserved.
>> +#
>> +# This file is part of the RTEMS Tools package in 'rtems-tools'.
>> +#
>> +# Permission to use, copy, modify, and/or distribute this software for
>> any
>> +# purpose with or without fee is hereby granted, provided that the above
>> +# copyright notice and this permission notice appear in all copies.
>> +#
>> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
>> WARRANTIES
>> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
>> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
>> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
>> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
>> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
>> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
>> +
>> +#
>> +# This code builds a package compiler tool suite given a tool set. A tool
>> +# set lists the various tools. These are specific tool configurations.
>> +#
>> +
>> +from __future__ import print_function
>> +
>> +import argparse
>> +import base64
>> +import copy
>> +import datetime
>> +import hashlib
>> +import os
>> +import sys
>> +
>> +try:
>> + from . import build
>> + from . import download
>> + from . import error
>> + from . import git
>> + from . import log
>> + from . import path
>> + from . import simhost
>> + from . import version
>> +except KeyboardInterrupt:
>> + print('abort: user terminated', file=sys.stderr)
>> + sys.exit(1)
>> +except:
>> + raise
>> +
>> +#
>> +# RTEMS Packages we maintian a git hash of in the RSB
>> +#
>> +rtems_pkg_cfgs = [
>> + [
>> + 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg',
>> + 'rtems_tools_version', 'git://git.rtems.org/rtems-tools',
>> + 'rtems-tools.git', 'master',
>> + '
>> https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2
>> ',
>> + 'rtems-tools-%{rtems_tools_version}.tar.bz2'
>> + ],
>> + [
>> + 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg',
>> + 'rtems_kernel_version', 'git://git.rtems.org/rtems',
>> 'rtems.git',
>> + 'master',
>> + '
>> https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2
>> ',
>> + 'rtems-kernel-%{rtems_kernel_version}.tar.bz2'
>> + ],
>> + [
>> + 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg',
>> + 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd',
>> + 'rtems-libbsd.git', '6-freebsd-12',
>> + '
>> https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}
>> ',
>> + 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}'
>> + ],
>> + [
>> + 'RTEMS Net Legacy',
>> 'tools/rtems-net-legacy-%{rtems_version}.cfg',
>> + 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy',
>> + 'rtems-net-legacy.git', 'main',
>> + '
>> https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}
>> ',
>> + 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}'
>> + ],
>> + [
>> + 'RTEMS Net Services', 'net/net-services-1.cfg',
>> + 'rtems_net_services_version', 'git://
>> git.rtems.org/rtems-net-services',
>> + 'rtems-net-services.git', 'master',
>> + '
>> https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}
>> ',
>> +
>> 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}'
>> + ],
>> +]
>> +
>> +
>> +def clean_line(line):
>> + line = line[0:-1]
>> + b = line.find('#')
>> + if b >= 0:
>> + line = line[1:b] + ('\\' if line[-1] == '\\' else '')
>> + return line.strip()
>> +
>> +
>> +def clean_and_pack(line, last_line):
>> + leading_ws = ' ' if len(line) > 0 and line[0].isspace() else ''
>> + line = clean_line(line)
>> + if len(last_line) > 0:
>> + line = last_line + leading_ws + line
>> + return line
>> +
>> +
>> +def config_patch(configdir, config, version_label, config_hash,
>> repo_hash,
>> + checksum):
>> + for cd in configdir.split(':'):
>> + cf = path.join(cd, config)
>> + if path.exists(cf):
>> + try:
>> + with open(cf) as f:
>> + lines = f.readlines()
>> + except IOError as err:
>> + raise error.general('config: %s: read error: %s' %
>> + (config, str(err)))
>> + new_config = []
>> + new_lines = []
>> + last_line = ''
>> + for line in lines:
>> + new_lines += [line]
>> + line = clean_and_pack(line, last_line)
>> + if len(line) > 0:
>> + if line[-1] == '\\':
>> + last_line = line[:-1]
>> + continue
>> + last_line = ''
>> + if version_label in line and not 'rsb_version' in
>> line:
>> + if line.startswith('%define ' + version_label):
>> + new_lines = [
>> + '%define ' + version_label + ' ' +
>> repo_hash +
>> + os.linesep
>> + ]
>> + elif line.startswith('%hash '):
>> + ls = line.split()
>> + if len(ls) != 4:
>> + raise error.general('invalid %hash: ' +
>> line)
>> + new_lines = [
>> + ' '.join(ls[0:3]) + ' \\' + os.linesep,
>> + ' ' + checksum + os.linesep
>> + ]
>> + new_config += new_lines
>> + new_lines = []
>> + try:
>> + with open(cf, 'w') as f:
>> + f.writelines(new_config)
>> + except IOError as err:
>> + raise error.general('config: %s: write error: %s' %
>> + (config, str(err)))
>> + return
>> + raise error.general('could not find: ' + config)
>> +
>> +
>> +def checksum_shar512_base64(tarball):
>> + hasher = hashlib.new('sha512')
>> + try:
>> + with open(path.host(tarball), 'rb') as f:
>> + hasher.update(f.read())
>> + except IOError as err:
>> + log.notice('hash: %s: read error: %s' % (file_, str(err)))
>> + except:
>> + raise
>> + raise error.general('cannot hash the tar file')
>> + hash_hex = hasher.hexdigest()
>> + hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8')
>> + return hash_base64
>> +
>> +
>> +def run(args=sys.argv):
>> + ec = 0
>> + output = []
>> + try:
>> + #
>> + # The RSB options support cannot be used because it loads the
>> defaults
>> + # for the host which we cannot do here.
>> + #
>> + description = 'RTEMS Track Dependencies a build set has for all
>> hosts.'
>> +
>> + argsp = argparse.ArgumentParser(prog='sb-rtems-pkg',
>> + description=description)
>> + argsp.add_argument('--rtems-version',
>> + help='Set the RTEMS version.',
>> + type=str,
>> + default=version.version())
>> + argsp.add_argument('--output',
>> + help='Output file.',
>> + type=str,
>> + default=None)
>> + argsp.add_argument('--log',
>> + help='Log file.',
>> + type=str,
>> + default=simhost.log_default('rtems-pkg'))
>> + argsp.add_argument('--trace',
>> + help='Enable trace logging for debugging.',
>> + action='store_true')
>> + argsp.add_argument('--dry-run',
>> + help='Dry run, do not update the
>> configurations',
>> + action='store_true')
>> + argsp.add_argument('bsets', nargs='*', help='Build sets.')
>> +
>> + argopts = argsp.parse_args(args[1:])
>> +
>> + simhost.load_log(argopts.log)
>> + log.notice('RTEMS Source Builder - RTEMS Package Update, %s' %
>> + (version.string()))
>> + log.tracing = argopts.trace
>> +
>> + opts = simhost.load_options(args, argopts,
>> extras=['--with-download'])
>> + opts.defaults['_rsb_getting_source'] = '1'
>> + opts.defaults[
>> + 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}'
>> + host = 'freebsd'
>> +
>> + for cfg in rtems_pkg_cfgs:
>> + b = None
>> + try:
>> + bopts = copy.copy(opts)
>> + bmacros = copy.copy(opts.defaults)
>> + b = build.build(cfg[1], False, bopts, bmacros)
>> + git_hash_key = b.macros.find(cfg[2])
>> + if len(git_hash_key) == 0:
>> + raise error.general(cfg[0] + ': cannot find version
>> macro')
>> + source_dir = b.macros.expand('%{_sourcedir}')
>> + config_hash = b.macros.expand('%{' + cfg[2] + '}')
>> + repo_path = path.join(source_dir, cfg[4])
>> + download.get_file(cfg[3] + '?fetch?checkout=' + cfg[5],
>> + repo_path, bopts, b)
>> + repo = git.repo(repo_path)
>> + repo_hash = repo.head()
>> + if config_hash != repo_hash:
>> + update = True
>> + update_str = 'UPDATE'
>> + else:
>> + update = False
>> + update_str = 'matching'
>> + print(cfg[0] + ': ' + update_str + ' config:' +
>> config_hash +
>> + ' repo:' + repo_hash)
>> + b.macros[cfg[2]] = repo_hash
>> + tarball = b.macros.expand(cfg[7])
>> + b.macros.set_write_map('hashes')
>> + b.macros[tarball] = 'NO-HASH NO-HASH'
>> + b.macros.unset_write_map()
>> + tarball_path = path.join(source_dir,
>> b.macros.expand(cfg[7]))
>> + download.get_file(b.macros.expand(cfg[6]), tarball_path,
>> bopts,
>> + b)
>> + tarball_hash = checksum_shar512_base64(tarball_path)
>> + if update:
>> + config_patch(b.macros.expand('%{_configdir}'),
>> + b.macros.expand(cfg[1]), cfg[2],
>> config_hash,
>> + repo_hash, tarball_hash)
>> + del b
>> + except error.general as gerr:
>> + log.stderr(str(gerr))
>> + log.stderr('Build FAILED')
>> + b = None
>> + except error.general as gerr:
>> + log.stderr(str(gerr))
>> + log.stderr('Build FAILED')
>> + ec = 1
>> + except error.internal as ierr:
>> + log.stderr(str(ierr))
>> + log.stderr('Internal Build FAILED')
>> + ec = 1
>> + except error.exit as eerr:
>> + pass
>> + except KeyboardInterrupt:
>> + log.notice('abort: user terminated')
>> + ec = 1
>> + except:
>> + raise
>> + log.notice('abort: unknown error')
>> + ec = 1
>> + sys.exit(ec)
>> +
>> +
>> +if __name__ == "__main__":
>> + run()
>> --
>> 2.42.0
>>
>> _______________________________________________
>> devel mailing list
>> devel at rtems.org
>> http://lists.rtems.org/mailman/listinfo/devel
>>
> _______________________________________________
> devel mailing list
> devel at rtems.org
> http://lists.rtems.org/mailman/listinfo/devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rtems.org/pipermail/devel/attachments/20240403/d4b2cc45/attachment-0001.htm>
More information about the devel
mailing list