[RSB PATCH v3] sb: Add sb-rtems-pkg to update the RTEMS package hashes and checksums

Kinsey Moore kinsey.moore at oarcorp.com
Thu Apr 4 03:04:03 UTC 2024


Looks good to me.

Kinsey

On Wed, Apr 3, 2024 at 6:28 PM <chrisj at rtems.org> wrote:

> From: Chris Johns <chrisj at rtems.org>
>
> ---
>  source-builder/sb-rtems-pkg   |  29 ++++
>  source-builder/sb/download.py |   5 +-
>  source-builder/sb/git.py      |  12 ++
>  source-builder/sb/rtemspkg.py | 287 ++++++++++++++++++++++++++++++++++
>  4 files changed, 332 insertions(+), 1 deletion(-)
>  create mode 100755 source-builder/sb-rtems-pkg
>  create mode 100644 source-builder/sb/rtemspkg.py
>
> diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg
> new file mode 100755
> index 0000000..99ed26c
> --- /dev/null
> +++ b/source-builder/sb-rtems-pkg
> @@ -0,0 +1,29 @@
> +#! /usr/bin/env python
> +#
> +# RTEMS Tools Project (http://www.rtems.org/)
> +# Copyright 2024 Chris Johns (chrisj at rtems.org)
> +# All rights reserved.
> +#
> +# This file is part of the RTEMS Tools package in 'rtems-tools'.
> +#
> +# Permission to use, copy, modify, and/or distribute this software for any
> +# purpose with or without fee is hereby granted, provided that the above
> +# copyright notice and this permission notice appear in all copies.
> +#
> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> +
> +from __future__ import print_function
> +
> +try:
> +    import sb.rtemspkg
> +    sb.rtemspkg.run()
> +except ImportError:
> +    import sys
> +    print("Incorrect Source Builder installation", file = sys.stderr)
> +    sys.exit(1)
> diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py
> index 6e6f9f2..0201675 100644
> --- a/source-builder/sb/download.py
> +++ b/source-builder/sb/download.py
> @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True):
>          hash = hash.split()
>          if len(hash) != 2:
>              raise error.internal('invalid hash format: %s' % (file_))
> +        if hash[0] == 'NO-HASH':
> +            return not failed
>          try:
>              hashlib_algorithms = hashlib.algorithms
>          except:
> @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts):
>      else:
>          repo.clean(['-f', '-d'])
>          repo.reset('--hard')
> -        repo.checkout('master')
> +        default_branch = repo.default_branch()
> +        repo.checkout(default_branch)
>      for a in us[1:]:
>          _as = a.split('=')
>          if _as[0] == 'branch' or _as[0] == 'checkout':
> diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py
> index 237e690..0aa4da9 100644
> --- a/source-builder/sb/git.py
> +++ b/source-builder/sb/git.py
> @@ -226,6 +226,18 @@ class repo:
>                  hash = l1[len('commit '):]
>          return hash
>
> +    def default_branch(self):
> +        ec, output = self._run(['remote', 'show'])
> +        if ec == 0:
> +            origin = output.split('\n')[0]
> +            ec, output = self._run(['remote', 'show', origin])
> +            if ec == 0:
> +                for l in output.split('\n'):
> +                    l = l.strip()
> +                    if l.startswith('HEAD branch: '):
> +                        return l[len('HEAD branch: '):]
> +        return None
> +
>  if __name__ == '__main__':
>      import os.path
>      import sys
> diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py
> new file mode 100644
> index 0000000..492eb59
> --- /dev/null
> +++ b/source-builder/sb/rtemspkg.py
> @@ -0,0 +1,287 @@
> +#
> +# RTEMS Tools Project (http://www.rtems.org/)
> +# Copyright 2024 Chris Johns (chrisj at rtems.org)
> +# All rights reserved.
> +#
> +# This file is part of the RTEMS Tools package in 'rtems-tools'.
> +#
> +# Permission to use, copy, modify, and/or distribute this software for any
> +# purpose with or without fee is hereby granted, provided that the above
> +# copyright notice and this permission notice appear in all copies.
> +#
> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> +
> +#
> +# This code builds a package compiler tool suite given a tool set. A tool
> +# set lists the various tools. These are specific tool configurations.
> +#
> +
> +from __future__ import print_function
> +
> +import argparse
> +import base64
> +import copy
> +import datetime
> +import hashlib
> +import os
> +import sys
> +
> +try:
> +    from . import build
> +    from . import download
> +    from . import error
> +    from . import git
> +    from . import log
> +    from . import path
> +    from . import simhost
> +    from . import version
> +except KeyboardInterrupt:
> +    print('abort: user terminated', file=sys.stderr)
> +    sys.exit(1)
> +except:
> +    raise
> +
> +#
> +# RTEMS Packages we maintian a git hash of in the RSB
> +#
> +rpc_label = 0
> +rpc_config = 1
> +rpc_version = 2
> +rpc_repo = 3
> +rpc_repo_name = 4
> +rpc_branch = 5
> +rpc_snapshot = 6
> +rpc_package = 7
> +rtems_pkg_cfgs = [
> +    [
> +        'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg',
> +        'rtems_tools_version', 'git://git.rtems.org/rtems-tools',
> +        'rtems-tools.git', 'master',
> +        '
> https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2
> ',
> +        'rtems-tools-%{rtems_tools_version}.tar.bz2'
> +    ],
> +    [
> +        'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg',
> +        'rtems_kernel_version', 'git://git.rtems.org/rtems', 'rtems.git',
> +        'master',
> +        '
> https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2
> ',
> +        'rtems-kernel-%{rtems_kernel_version}.tar.bz2'
> +    ],
> +    [
> +        'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg',
> +        'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd',
> +        'rtems-libbsd.git', '6-freebsd-12',
> +        '
> https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}
> ',
> +        'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}'
> +    ],
> +    [
> +        'RTEMS Net Legacy', 'tools/rtems-net-legacy-%{rtems_version}.cfg',
> +        'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy',
> +        'rtems-net-legacy.git', 'main',
> +        '
> https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}
> ',
> +        'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}'
> +    ],
> +    [
> +        'RTEMS Net Services', 'net/net-services-1.cfg',
> +        'rtems_net_services_version', 'git://
> git.rtems.org/rtems-net-services',
> +        'rtems-net-services.git', 'master',
> +        '
> https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}
> ',
> +
> 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}'
> +    ],
> +]
> +
> +
> +def clean_line(line):
> +    line = line[0:-1]
> +    b = line.find('#')
> +    if b >= 0:
> +        line = line[1:b] + ('\\' if line[-1] == '\\' else '')
> +    return line.strip()
> +
> +
> +def clean_and_pack(line, last_line):
> +    leading_ws = ' ' if len(line) > 0 and line[0].isspace() else ''
> +    line = clean_line(line)
> +    if len(last_line) > 0:
> +        line = last_line + leading_ws + line
> +    return line
> +
> +
> +def config_patch(configdir, config, version_label, config_hash, repo_hash,
> +                 checksum):
> +    for cd in configdir.split(':'):
> +        cf = path.join(cd, config)
> +        if path.exists(cf):
> +            try:
> +                with open(cf) as f:
> +                    lines = f.readlines()
> +            except IOError as err:
> +                raise error.general('config: %s: read error: %s' %
> +                                    (config, str(err)))
> +            new_config = []
> +            new_lines = []
> +            last_line = ''
> +            for line in lines:
> +                new_lines += [line]
> +                line = clean_and_pack(line, last_line)
> +                if len(line) > 0:
> +                    if line[-1] == '\\':
> +                        last_line = line[:-1]
> +                        continue
> +                    last_line = ''
> +                    if version_label in line and not 'rsb_version' in
> line:
> +                        if line.startswith('%define ' + version_label):
> +                            new_lines = [
> +                                '%define ' + version_label + ' ' +
> repo_hash +
> +                                os.linesep
> +                            ]
> +                        elif line.startswith('%hash '):
> +                            ls = line.split()
> +                            if len(ls) != 4:
> +                                raise error.general('invalid %hash: ' +
> line)
> +                            new_lines = [
> +                                ' '.join(ls[0:3]) + ' \\' + os.linesep,
> +                                '              ' + checksum + os.linesep
> +                            ]
> +                new_config += new_lines
> +                new_lines = []
> +            try:
> +                with open(cf, 'w') as f:
> +                    f.writelines(new_config)
> +            except IOError as err:
> +                raise error.general('config: %s: write error: %s' %
> +                                    (config, str(err)))
> +            return
> +    raise error.general('could not find: ' + config)
> +
> +
> +def checksum_sha512_base64(tarball):
> +    hasher = hashlib.new('sha512')
> +    try:
> +        with open(path.host(tarball), 'rb') as f:
> +            hasher.update(f.read())
> +    except IOError as err:
> +        log.notice('hash: %s: read error: %s' % (file_, str(err)))
> +    except:
> +        raise
> +        raise error.general('cannot hash the tar file')
> +    hash_hex = hasher.hexdigest()
> +    hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8')
> +    return hash_base64
> +
> +
> +def run(args=sys.argv):
> +    ec = 0
> +    output = []
> +    try:
> +        #
> +        # The RSB options support cannot be used because it loads the
> defaults
> +        # for the host which we cannot do here.
> +        #
> +        description = 'RTEMS Track Dependencies a build set has for all
> hosts.'
> +
> +        argsp = argparse.ArgumentParser(prog='sb-rtems-pkg',
> +                                        description=description)
> +        argsp.add_argument('--rtems-version',
> +                           help='Set the RTEMS version.',
> +                           type=str,
> +                           default=version.version())
> +        argsp.add_argument('--log',
> +                           help='Log file.',
> +                           type=str,
> +                           default=simhost.log_default('rtems-pkg'))
> +        argsp.add_argument('--trace',
> +                           help='Enable trace logging for debugging.',
> +                           action='store_true')
> +        argsp.add_argument('--dry-run',
> +                           help='Dry run, do not update the
> configurations',
> +                           action='store_true')
> +        argsp.add_argument('bsets', nargs='*', help='Build sets.')
> +
> +        argopts = argsp.parse_args(args[1:])
> +
> +        simhost.load_log(argopts.log)
> +        log.notice('RTEMS Source Builder - RTEMS Package Update, %s' %
> +                   (version.string()))
> +        log.tracing = argopts.trace
> +
> +        opts = simhost.load_options(args, argopts,
> extras=['--with-download'])
> +        opts.defaults['_rsb_getting_source'] = '1'
> +        opts.defaults[
> +            'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}'
> +        opts.defaults['rtems_version'] = argopts.rtems_version
> +
> +        for cfg in rtems_pkg_cfgs:
> +            b = None
> +            try:
> +                bopts = copy.copy(opts)
> +                bmacros = copy.copy(opts.defaults)
> +                b = build.build(cfg[rpc_config], False, bopts, bmacros)
> +                git_hash_key = b.macros.find(cfg[rpc_version])
> +                if len(git_hash_key) == 0:
> +                    raise error.general(cfg[rpc_label] +
> +                                        ': cannot find version macro')
> +                source_dir = b.macros.expand('%{_sourcedir}')
> +                config_hash = b.macros.expand('%{' + cfg[rpc_version] +
> '}')
> +                repo_path = path.join(source_dir, cfg[rpc_repo_name])
> +                download.get_file(
> +                    cfg[rpc_repo] + '?fetch?checkout=' + cfg[rpc_branch],
> +                    repo_path, bopts, b)
> +                repo = git.repo(repo_path)
> +                repo_hash = repo.head()
> +                if config_hash != repo_hash:
> +                    update = True
> +                    update_str = 'UPDATE'
> +                else:
> +                    update = False
> +                    update_str = 'matching'
> +                print(cfg[rpc_label] + ': ' + update_str + ' config:' +
> +                      config_hash + ' repo:' + repo_hash)
> +                b.macros[cfg[rpc_version]] = repo_hash
> +                tarball = b.macros.expand(cfg[rpc_package])
> +                b.macros.set_write_map('hashes')
> +                b.macros[tarball] = 'NO-HASH NO-HASH'
> +                b.macros.unset_write_map()
> +                tarball_path = path.join(source_dir,
> +
>  b.macros.expand(cfg[rpc_package]))
> +                download.get_file(b.macros.expand(cfg[rpc_snapshot]),
> +                                  tarball_path, bopts, b)
> +                tarball_hash = checksum_sha512_base64(tarball_path)
> +                if update and not argopts.dry_run:
> +                    config_patch(b.macros.expand('%{_configdir}'),
> +                                 b.macros.expand(cfg[rpc_config]),
> +                                 cfg[rpc_version], config_hash, repo_hash,
> +                                 tarball_hash)
> +                del b
> +            except error.general as gerr:
> +                log.stderr(str(gerr))
> +                log.stderr('Configuration load FAILED')
> +                b = None
> +    except error.general as gerr:
> +        log.stderr(str(gerr))
> +        log.stderr('Build FAILED')
> +        ec = 1
> +    except error.internal as ierr:
> +        log.stderr(str(ierr))
> +        log.stderr('Internal Build FAILED')
> +        ec = 1
> +    except error.exit as eerr:
> +        pass
> +    except KeyboardInterrupt:
> +        log.notice('abort: user terminated')
> +        ec = 1
> +    except:
> +        raise
> +        log.notice('abort: unknown error')
> +        ec = 1
> +    sys.exit(ec)
> +
> +
> +if __name__ == "__main__":
> +    run()
> --
> 2.42.0
>
> _______________________________________________
> devel mailing list
> devel at rtems.org
> http://lists.rtems.org/mailman/listinfo/devel
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rtems.org/pipermail/devel/attachments/20240403/a2277525/attachment-0001.htm>


More information about the devel mailing list