[RSB PATCH v3] sb: Add sb-rtems-pkg to update the RTEMS package hashes and checksums
chrisj at rtems.org
chrisj at rtems.org
Wed Apr 3 23:27:21 UTC 2024
From: Chris Johns <chrisj at rtems.org>
---
source-builder/sb-rtems-pkg | 29 ++++
source-builder/sb/download.py | 5 +-
source-builder/sb/git.py | 12 ++
source-builder/sb/rtemspkg.py | 287 ++++++++++++++++++++++++++++++++++
4 files changed, 332 insertions(+), 1 deletion(-)
create mode 100755 source-builder/sb-rtems-pkg
create mode 100644 source-builder/sb/rtemspkg.py
diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg
new file mode 100755
index 0000000..99ed26c
--- /dev/null
+++ b/source-builder/sb-rtems-pkg
@@ -0,0 +1,29 @@
+#! /usr/bin/env python
+#
+# RTEMS Tools Project (http://www.rtems.org/)
+# Copyright 2024 Chris Johns (chrisj at rtems.org)
+# All rights reserved.
+#
+# This file is part of the RTEMS Tools package in 'rtems-tools'.
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import print_function
+
+try:
+ import sb.rtemspkg
+ sb.rtemspkg.run()
+except ImportError:
+ import sys
+ print("Incorrect Source Builder installation", file = sys.stderr)
+ sys.exit(1)
diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py
index 6e6f9f2..0201675 100644
--- a/source-builder/sb/download.py
+++ b/source-builder/sb/download.py
@@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True):
hash = hash.split()
if len(hash) != 2:
raise error.internal('invalid hash format: %s' % (file_))
+ if hash[0] == 'NO-HASH':
+ return not failed
try:
hashlib_algorithms = hashlib.algorithms
except:
@@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts):
else:
repo.clean(['-f', '-d'])
repo.reset('--hard')
- repo.checkout('master')
+ default_branch = repo.default_branch()
+ repo.checkout(default_branch)
for a in us[1:]:
_as = a.split('=')
if _as[0] == 'branch' or _as[0] == 'checkout':
diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py
index 237e690..0aa4da9 100644
--- a/source-builder/sb/git.py
+++ b/source-builder/sb/git.py
@@ -226,6 +226,18 @@ class repo:
hash = l1[len('commit '):]
return hash
+ def default_branch(self):
+ ec, output = self._run(['remote', 'show'])
+ if ec == 0:
+ origin = output.split('\n')[0]
+ ec, output = self._run(['remote', 'show', origin])
+ if ec == 0:
+ for l in output.split('\n'):
+ l = l.strip()
+ if l.startswith('HEAD branch: '):
+ return l[len('HEAD branch: '):]
+ return None
+
if __name__ == '__main__':
import os.path
import sys
diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py
new file mode 100644
index 0000000..492eb59
--- /dev/null
+++ b/source-builder/sb/rtemspkg.py
@@ -0,0 +1,287 @@
+#
+# RTEMS Tools Project (http://www.rtems.org/)
+# Copyright 2024 Chris Johns (chrisj at rtems.org)
+# All rights reserved.
+#
+# This file is part of the RTEMS Tools package in 'rtems-tools'.
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+#
+# This code builds a package compiler tool suite given a tool set. A tool
+# set lists the various tools. These are specific tool configurations.
+#
+
+from __future__ import print_function
+
+import argparse
+import base64
+import copy
+import datetime
+import hashlib
+import os
+import sys
+
+try:
+ from . import build
+ from . import download
+ from . import error
+ from . import git
+ from . import log
+ from . import path
+ from . import simhost
+ from . import version
+except KeyboardInterrupt:
+ print('abort: user terminated', file=sys.stderr)
+ sys.exit(1)
+except:
+ raise
+
+#
+# RTEMS Packages we maintian a git hash of in the RSB
+#
+rpc_label = 0
+rpc_config = 1
+rpc_version = 2
+rpc_repo = 3
+rpc_repo_name = 4
+rpc_branch = 5
+rpc_snapshot = 6
+rpc_package = 7
+rtems_pkg_cfgs = [
+ [
+ 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg',
+ 'rtems_tools_version', 'git://git.rtems.org/rtems-tools',
+ 'rtems-tools.git', 'master',
+ 'https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2',
+ 'rtems-tools-%{rtems_tools_version}.tar.bz2'
+ ],
+ [
+ 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg',
+ 'rtems_kernel_version', 'git://git.rtems.org/rtems', 'rtems.git',
+ 'master',
+ 'https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2',
+ 'rtems-kernel-%{rtems_kernel_version}.tar.bz2'
+ ],
+ [
+ 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg',
+ 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd',
+ 'rtems-libbsd.git', '6-freebsd-12',
+ 'https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}',
+ 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}'
+ ],
+ [
+ 'RTEMS Net Legacy', 'tools/rtems-net-legacy-%{rtems_version}.cfg',
+ 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy',
+ 'rtems-net-legacy.git', 'main',
+ 'https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}',
+ 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}'
+ ],
+ [
+ 'RTEMS Net Services', 'net/net-services-1.cfg',
+ 'rtems_net_services_version', 'git://git.rtems.org/rtems-net-services',
+ 'rtems-net-services.git', 'master',
+ 'https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}',
+ 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}'
+ ],
+]
+
+
+def clean_line(line):
+ line = line[0:-1]
+ b = line.find('#')
+ if b >= 0:
+ line = line[1:b] + ('\\' if line[-1] == '\\' else '')
+ return line.strip()
+
+
+def clean_and_pack(line, last_line):
+ leading_ws = ' ' if len(line) > 0 and line[0].isspace() else ''
+ line = clean_line(line)
+ if len(last_line) > 0:
+ line = last_line + leading_ws + line
+ return line
+
+
+def config_patch(configdir, config, version_label, config_hash, repo_hash,
+ checksum):
+ for cd in configdir.split(':'):
+ cf = path.join(cd, config)
+ if path.exists(cf):
+ try:
+ with open(cf) as f:
+ lines = f.readlines()
+ except IOError as err:
+ raise error.general('config: %s: read error: %s' %
+ (config, str(err)))
+ new_config = []
+ new_lines = []
+ last_line = ''
+ for line in lines:
+ new_lines += [line]
+ line = clean_and_pack(line, last_line)
+ if len(line) > 0:
+ if line[-1] == '\\':
+ last_line = line[:-1]
+ continue
+ last_line = ''
+ if version_label in line and not 'rsb_version' in line:
+ if line.startswith('%define ' + version_label):
+ new_lines = [
+ '%define ' + version_label + ' ' + repo_hash +
+ os.linesep
+ ]
+ elif line.startswith('%hash '):
+ ls = line.split()
+ if len(ls) != 4:
+ raise error.general('invalid %hash: ' + line)
+ new_lines = [
+ ' '.join(ls[0:3]) + ' \\' + os.linesep,
+ ' ' + checksum + os.linesep
+ ]
+ new_config += new_lines
+ new_lines = []
+ try:
+ with open(cf, 'w') as f:
+ f.writelines(new_config)
+ except IOError as err:
+ raise error.general('config: %s: write error: %s' %
+ (config, str(err)))
+ return
+ raise error.general('could not find: ' + config)
+
+
+def checksum_sha512_base64(tarball):
+ hasher = hashlib.new('sha512')
+ try:
+ with open(path.host(tarball), 'rb') as f:
+ hasher.update(f.read())
+ except IOError as err:
+ log.notice('hash: %s: read error: %s' % (file_, str(err)))
+ except:
+ raise
+ raise error.general('cannot hash the tar file')
+ hash_hex = hasher.hexdigest()
+ hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8')
+ return hash_base64
+
+
+def run(args=sys.argv):
+ ec = 0
+ output = []
+ try:
+ #
+ # The RSB options support cannot be used because it loads the defaults
+ # for the host which we cannot do here.
+ #
+ description = 'RTEMS Track Dependencies a build set has for all hosts.'
+
+ argsp = argparse.ArgumentParser(prog='sb-rtems-pkg',
+ description=description)
+ argsp.add_argument('--rtems-version',
+ help='Set the RTEMS version.',
+ type=str,
+ default=version.version())
+ argsp.add_argument('--log',
+ help='Log file.',
+ type=str,
+ default=simhost.log_default('rtems-pkg'))
+ argsp.add_argument('--trace',
+ help='Enable trace logging for debugging.',
+ action='store_true')
+ argsp.add_argument('--dry-run',
+ help='Dry run, do not update the configurations',
+ action='store_true')
+ argsp.add_argument('bsets', nargs='*', help='Build sets.')
+
+ argopts = argsp.parse_args(args[1:])
+
+ simhost.load_log(argopts.log)
+ log.notice('RTEMS Source Builder - RTEMS Package Update, %s' %
+ (version.string()))
+ log.tracing = argopts.trace
+
+ opts = simhost.load_options(args, argopts, extras=['--with-download'])
+ opts.defaults['_rsb_getting_source'] = '1'
+ opts.defaults[
+ 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}'
+ opts.defaults['rtems_version'] = argopts.rtems_version
+
+ for cfg in rtems_pkg_cfgs:
+ b = None
+ try:
+ bopts = copy.copy(opts)
+ bmacros = copy.copy(opts.defaults)
+ b = build.build(cfg[rpc_config], False, bopts, bmacros)
+ git_hash_key = b.macros.find(cfg[rpc_version])
+ if len(git_hash_key) == 0:
+ raise error.general(cfg[rpc_label] +
+ ': cannot find version macro')
+ source_dir = b.macros.expand('%{_sourcedir}')
+ config_hash = b.macros.expand('%{' + cfg[rpc_version] + '}')
+ repo_path = path.join(source_dir, cfg[rpc_repo_name])
+ download.get_file(
+ cfg[rpc_repo] + '?fetch?checkout=' + cfg[rpc_branch],
+ repo_path, bopts, b)
+ repo = git.repo(repo_path)
+ repo_hash = repo.head()
+ if config_hash != repo_hash:
+ update = True
+ update_str = 'UPDATE'
+ else:
+ update = False
+ update_str = 'matching'
+ print(cfg[rpc_label] + ': ' + update_str + ' config:' +
+ config_hash + ' repo:' + repo_hash)
+ b.macros[cfg[rpc_version]] = repo_hash
+ tarball = b.macros.expand(cfg[rpc_package])
+ b.macros.set_write_map('hashes')
+ b.macros[tarball] = 'NO-HASH NO-HASH'
+ b.macros.unset_write_map()
+ tarball_path = path.join(source_dir,
+ b.macros.expand(cfg[rpc_package]))
+ download.get_file(b.macros.expand(cfg[rpc_snapshot]),
+ tarball_path, bopts, b)
+ tarball_hash = checksum_sha512_base64(tarball_path)
+ if update and not argopts.dry_run:
+ config_patch(b.macros.expand('%{_configdir}'),
+ b.macros.expand(cfg[rpc_config]),
+ cfg[rpc_version], config_hash, repo_hash,
+ tarball_hash)
+ del b
+ except error.general as gerr:
+ log.stderr(str(gerr))
+ log.stderr('Configuration load FAILED')
+ b = None
+ except error.general as gerr:
+ log.stderr(str(gerr))
+ log.stderr('Build FAILED')
+ ec = 1
+ except error.internal as ierr:
+ log.stderr(str(ierr))
+ log.stderr('Internal Build FAILED')
+ ec = 1
+ except error.exit as eerr:
+ pass
+ except KeyboardInterrupt:
+ log.notice('abort: user terminated')
+ ec = 1
+ except:
+ raise
+ log.notice('abort: unknown error')
+ ec = 1
+ sys.exit(ec)
+
+
+if __name__ == "__main__":
+ run()
--
2.42.0
More information about the devel
mailing list