[RSB PATCH] sb: Add sb-rtems-pkg to update the RTEMS package hashes and checksums
Joel Sherrill
joel at rtems.org
Wed Apr 3 16:59:24 UTC 2024
Chris.. if you post a V2 addressing those, I will commit it.
On Wed, Apr 3, 2024, 11:52 AM Kinsey Moore <kinsey.moore at oarcorp.com> wrote:
> Looks fine overall. Minor nits:
> "host" is set to "freebsd" and is never used.
> Numeric indexes for repo config details are nice for brevity, but not for
> readability.
>
> Kinsey
>
> On Wed, Apr 3, 2024 at 11:29 AM Joel Sherrill <joel.sherrill at gmail.com>
> wrote:
>
>> This looks handy. I don't have an issue with it.
>>
>> On Tue, Apr 2, 2024 at 11:25 PM <chrisj at rtems.org> wrote:
>>
>>> From: Chris Johns <chrisj at rtems.org>
>>>
>>> ---
>>> source-builder/sb-rtems-pkg | 29 ++++
>>> source-builder/sb/download.py | 5 +-
>>> source-builder/sb/git.py | 12 ++
>>> source-builder/sb/rtemspkg.py | 279 ++++++++++++++++++++++++++++++++++
>>> 4 files changed, 324 insertions(+), 1 deletion(-)
>>> create mode 100755 source-builder/sb-rtems-pkg
>>> create mode 100644 source-builder/sb/rtemspkg.py
>>>
>>> diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg
>>> new file mode 100755
>>> index 0000000..99ed26c
>>> --- /dev/null
>>> +++ b/source-builder/sb-rtems-pkg
>>> @@ -0,0 +1,29 @@
>>> +#! /usr/bin/env python
>>> +#
>>> +# RTEMS Tools Project (http://www.rtems.org/)
>>> +# Copyright 2024 Chris Johns (chrisj at rtems.org)
>>> +# All rights reserved.
>>> +#
>>> +# This file is part of the RTEMS Tools package in 'rtems-tools'.
>>> +#
>>> +# Permission to use, copy, modify, and/or distribute this software for
>>> any
>>> +# purpose with or without fee is hereby granted, provided that the above
>>> +# copyright notice and this permission notice appear in all copies.
>>> +#
>>> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
>>> WARRANTIES
>>> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
>>> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE
>>> FOR
>>> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
>>> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
>>> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
>>> OF
>>> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
>>> +
>>> +from __future__ import print_function
>>> +
>>> +try:
>>> + import sb.rtemspkg
>>> + sb.rtemspkg.run()
>>> +except ImportError:
>>> + import sys
>>> + print("Incorrect Source Builder installation", file = sys.stderr)
>>> + sys.exit(1)
>>> diff --git a/source-builder/sb/download.py
>>> b/source-builder/sb/download.py
>>> index 6e6f9f2..0201675 100644
>>> --- a/source-builder/sb/download.py
>>> +++ b/source-builder/sb/download.py
>>> @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True):
>>> hash = hash.split()
>>> if len(hash) != 2:
>>> raise error.internal('invalid hash format: %s' % (file_))
>>> + if hash[0] == 'NO-HASH':
>>> + return not failed
>>> try:
>>> hashlib_algorithms = hashlib.algorithms
>>> except:
>>> @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts):
>>> else:
>>> repo.clean(['-f', '-d'])
>>> repo.reset('--hard')
>>> - repo.checkout('master')
>>> + default_branch = repo.default_branch()
>>> + repo.checkout(default_branch)
>>> for a in us[1:]:
>>> _as = a.split('=')
>>> if _as[0] == 'branch' or _as[0] == 'checkout':
>>> diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py
>>> index 237e690..0aa4da9 100644
>>> --- a/source-builder/sb/git.py
>>> +++ b/source-builder/sb/git.py
>>> @@ -226,6 +226,18 @@ class repo:
>>> hash = l1[len('commit '):]
>>> return hash
>>>
>>> + def default_branch(self):
>>> + ec, output = self._run(['remote', 'show'])
>>> + if ec == 0:
>>> + origin = output.split('\n')[0]
>>> + ec, output = self._run(['remote', 'show', origin])
>>> + if ec == 0:
>>> + for l in output.split('\n'):
>>> + l = l.strip()
>>> + if l.startswith('HEAD branch: '):
>>> + return l[len('HEAD branch: '):]
>>> + return None
>>> +
>>> if __name__ == '__main__':
>>> import os.path
>>> import sys
>>> diff --git a/source-builder/sb/rtemspkg.py
>>> b/source-builder/sb/rtemspkg.py
>>> new file mode 100644
>>> index 0000000..0b43d23
>>> --- /dev/null
>>> +++ b/source-builder/sb/rtemspkg.py
>>> @@ -0,0 +1,279 @@
>>> +#
>>> +# RTEMS Tools Project (http://www.rtems.org/)
>>> +# Copyright 2024 Chris Johns (chrisj at rtems.org)
>>> +# All rights reserved.
>>> +#
>>> +# This file is part of the RTEMS Tools package in 'rtems-tools'.
>>> +#
>>> +# Permission to use, copy, modify, and/or distribute this software for
>>> any
>>> +# purpose with or without fee is hereby granted, provided that the above
>>> +# copyright notice and this permission notice appear in all copies.
>>> +#
>>> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
>>> WARRANTIES
>>> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
>>> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE
>>> FOR
>>> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
>>> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
>>> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
>>> OF
>>> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
>>> +
>>> +#
>>> +# This code builds a package compiler tool suite given a tool set. A
>>> tool
>>> +# set lists the various tools. These are specific tool configurations.
>>> +#
>>> +
>>> +from __future__ import print_function
>>> +
>>> +import argparse
>>> +import base64
>>> +import copy
>>> +import datetime
>>> +import hashlib
>>> +import os
>>> +import sys
>>> +
>>> +try:
>>> + from . import build
>>> + from . import download
>>> + from . import error
>>> + from . import git
>>> + from . import log
>>> + from . import path
>>> + from . import simhost
>>> + from . import version
>>> +except KeyboardInterrupt:
>>> + print('abort: user terminated', file=sys.stderr)
>>> + sys.exit(1)
>>> +except:
>>> + raise
>>> +
>>> +#
>>> +# RTEMS Packages we maintian a git hash of in the RSB
>>> +#
>>> +rtems_pkg_cfgs = [
>>> + [
>>> + 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg',
>>> + 'rtems_tools_version', 'git://git.rtems.org/rtems-tools',
>>> + 'rtems-tools.git', 'master',
>>> + '
>>> https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2
>>> ',
>>> + 'rtems-tools-%{rtems_tools_version}.tar.bz2'
>>> + ],
>>> + [
>>> + 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg',
>>> + 'rtems_kernel_version', 'git://git.rtems.org/rtems',
>>> 'rtems.git',
>>> + 'master',
>>> + '
>>> https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2
>>> ',
>>> + 'rtems-kernel-%{rtems_kernel_version}.tar.bz2'
>>> + ],
>>> + [
>>> + 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg',
>>> + 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd',
>>> + 'rtems-libbsd.git', '6-freebsd-12',
>>> + '
>>> https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}
>>> ',
>>> + 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}'
>>> + ],
>>> + [
>>> + 'RTEMS Net Legacy',
>>> 'tools/rtems-net-legacy-%{rtems_version}.cfg',
>>> + 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy',
>>> + 'rtems-net-legacy.git', 'main',
>>> + '
>>> https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}
>>> ',
>>> + 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}'
>>> + ],
>>> + [
>>> + 'RTEMS Net Services', 'net/net-services-1.cfg',
>>> + 'rtems_net_services_version', 'git://
>>> git.rtems.org/rtems-net-services',
>>> + 'rtems-net-services.git', 'master',
>>> + '
>>> https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}
>>> ',
>>> +
>>> 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}'
>>> + ],
>>> +]
>>> +
>>> +
>>> +def clean_line(line):
>>> + line = line[0:-1]
>>> + b = line.find('#')
>>> + if b >= 0:
>>> + line = line[1:b] + ('\\' if line[-1] == '\\' else '')
>>> + return line.strip()
>>> +
>>> +
>>> +def clean_and_pack(line, last_line):
>>> + leading_ws = ' ' if len(line) > 0 and line[0].isspace() else ''
>>> + line = clean_line(line)
>>> + if len(last_line) > 0:
>>> + line = last_line + leading_ws + line
>>> + return line
>>> +
>>> +
>>> +def config_patch(configdir, config, version_label, config_hash,
>>> repo_hash,
>>> + checksum):
>>> + for cd in configdir.split(':'):
>>> + cf = path.join(cd, config)
>>> + if path.exists(cf):
>>> + try:
>>> + with open(cf) as f:
>>> + lines = f.readlines()
>>> + except IOError as err:
>>> + raise error.general('config: %s: read error: %s' %
>>> + (config, str(err)))
>>> + new_config = []
>>> + new_lines = []
>>> + last_line = ''
>>> + for line in lines:
>>> + new_lines += [line]
>>> + line = clean_and_pack(line, last_line)
>>> + if len(line) > 0:
>>> + if line[-1] == '\\':
>>> + last_line = line[:-1]
>>> + continue
>>> + last_line = ''
>>> + if version_label in line and not 'rsb_version' in
>>> line:
>>> + if line.startswith('%define ' + version_label):
>>> + new_lines = [
>>> + '%define ' + version_label + ' ' +
>>> repo_hash +
>>> + os.linesep
>>> + ]
>>> + elif line.startswith('%hash '):
>>> + ls = line.split()
>>> + if len(ls) != 4:
>>> + raise error.general('invalid %hash: ' +
>>> line)
>>> + new_lines = [
>>> + ' '.join(ls[0:3]) + ' \\' + os.linesep,
>>> + ' ' + checksum + os.linesep
>>> + ]
>>> + new_config += new_lines
>>> + new_lines = []
>>> + try:
>>> + with open(cf, 'w') as f:
>>> + f.writelines(new_config)
>>> + except IOError as err:
>>> + raise error.general('config: %s: write error: %s' %
>>> + (config, str(err)))
>>> + return
>>> + raise error.general('could not find: ' + config)
>>> +
>>> +
>>> +def checksum_shar512_base64(tarball):
>>> + hasher = hashlib.new('sha512')
>>> + try:
>>> + with open(path.host(tarball), 'rb') as f:
>>> + hasher.update(f.read())
>>> + except IOError as err:
>>> + log.notice('hash: %s: read error: %s' % (file_, str(err)))
>>> + except:
>>> + raise
>>> + raise error.general('cannot hash the tar file')
>>> + hash_hex = hasher.hexdigest()
>>> + hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8')
>>> + return hash_base64
>>> +
>>> +
>>> +def run(args=sys.argv):
>>> + ec = 0
>>> + output = []
>>> + try:
>>> + #
>>> + # The RSB options support cannot be used because it loads the
>>> defaults
>>> + # for the host which we cannot do here.
>>> + #
>>> + description = 'RTEMS Track Dependencies a build set has for all
>>> hosts.'
>>> +
>>> + argsp = argparse.ArgumentParser(prog='sb-rtems-pkg',
>>> + description=description)
>>> + argsp.add_argument('--rtems-version',
>>> + help='Set the RTEMS version.',
>>> + type=str,
>>> + default=version.version())
>>> + argsp.add_argument('--output',
>>> + help='Output file.',
>>> + type=str,
>>> + default=None)
>>> + argsp.add_argument('--log',
>>> + help='Log file.',
>>> + type=str,
>>> + default=simhost.log_default('rtems-pkg'))
>>> + argsp.add_argument('--trace',
>>> + help='Enable trace logging for debugging.',
>>> + action='store_true')
>>> + argsp.add_argument('--dry-run',
>>> + help='Dry run, do not update the
>>> configurations',
>>> + action='store_true')
>>> + argsp.add_argument('bsets', nargs='*', help='Build sets.')
>>> +
>>> + argopts = argsp.parse_args(args[1:])
>>> +
>>> + simhost.load_log(argopts.log)
>>> + log.notice('RTEMS Source Builder - RTEMS Package Update, %s' %
>>> + (version.string()))
>>> + log.tracing = argopts.trace
>>> +
>>> + opts = simhost.load_options(args, argopts,
>>> extras=['--with-download'])
>>> + opts.defaults['_rsb_getting_source'] = '1'
>>> + opts.defaults[
>>> + 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}'
>>> + host = 'freebsd'
>>> +
>>> + for cfg in rtems_pkg_cfgs:
>>> + b = None
>>> + try:
>>> + bopts = copy.copy(opts)
>>> + bmacros = copy.copy(opts.defaults)
>>> + b = build.build(cfg[1], False, bopts, bmacros)
>>> + git_hash_key = b.macros.find(cfg[2])
>>> + if len(git_hash_key) == 0:
>>> + raise error.general(cfg[0] + ': cannot find version
>>> macro')
>>> + source_dir = b.macros.expand('%{_sourcedir}')
>>> + config_hash = b.macros.expand('%{' + cfg[2] + '}')
>>> + repo_path = path.join(source_dir, cfg[4])
>>> + download.get_file(cfg[3] + '?fetch?checkout=' + cfg[5],
>>> + repo_path, bopts, b)
>>> + repo = git.repo(repo_path)
>>> + repo_hash = repo.head()
>>> + if config_hash != repo_hash:
>>> + update = True
>>> + update_str = 'UPDATE'
>>> + else:
>>> + update = False
>>> + update_str = 'matching'
>>> + print(cfg[0] + ': ' + update_str + ' config:' +
>>> config_hash +
>>> + ' repo:' + repo_hash)
>>> + b.macros[cfg[2]] = repo_hash
>>> + tarball = b.macros.expand(cfg[7])
>>> + b.macros.set_write_map('hashes')
>>> + b.macros[tarball] = 'NO-HASH NO-HASH'
>>> + b.macros.unset_write_map()
>>> + tarball_path = path.join(source_dir,
>>> b.macros.expand(cfg[7]))
>>> + download.get_file(b.macros.expand(cfg[6]),
>>> tarball_path, bopts,
>>> + b)
>>> + tarball_hash = checksum_shar512_base64(tarball_path)
>>> + if update:
>>> + config_patch(b.macros.expand('%{_configdir}'),
>>> + b.macros.expand(cfg[1]), cfg[2],
>>> config_hash,
>>> + repo_hash, tarball_hash)
>>> + del b
>>> + except error.general as gerr:
>>> + log.stderr(str(gerr))
>>> + log.stderr('Build FAILED')
>>> + b = None
>>> + except error.general as gerr:
>>> + log.stderr(str(gerr))
>>> + log.stderr('Build FAILED')
>>> + ec = 1
>>> + except error.internal as ierr:
>>> + log.stderr(str(ierr))
>>> + log.stderr('Internal Build FAILED')
>>> + ec = 1
>>> + except error.exit as eerr:
>>> + pass
>>> + except KeyboardInterrupt:
>>> + log.notice('abort: user terminated')
>>> + ec = 1
>>> + except:
>>> + raise
>>> + log.notice('abort: unknown error')
>>> + ec = 1
>>> + sys.exit(ec)
>>> +
>>> +
>>> +if __name__ == "__main__":
>>> + run()
>>> --
>>> 2.42.0
>>>
>>> _______________________________________________
>>> devel mailing list
>>> devel at rtems.org
>>> http://lists.rtems.org/mailman/listinfo/devel
>>>
>> _______________________________________________
>> devel mailing list
>> devel at rtems.org
>> http://lists.rtems.org/mailman/listinfo/devel
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rtems.org/pipermail/devel/attachments/20240403/694540ce/attachment-0001.htm>
More information about the devel
mailing list