<div dir="auto">Chris.. if you post a V2 addressing those, I will commit it. </div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, Apr 3, 2024, 11:52 AM Kinsey Moore <<a href="mailto:kinsey.moore@oarcorp.com">kinsey.moore@oarcorp.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"><div dir="ltr"><div>Looks fine overall. Minor nits:</div><div>"host" is set to "freebsd" and is never used.</div><div>Numeric indexes for repo config details are nice for brevity, but not for readability.</div><div><br></div><div>Kinsey<br> </div></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, Apr 3, 2024 at 11:29 AM Joel Sherrill <<a href="mailto:joel.sherrill@gmail.com" target="_blank" rel="noreferrer">joel.sherrill@gmail.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div dir="ltr"><div dir="ltr">This looks handy. I don't have an issue with it. </div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Tue, Apr 2, 2024 at 11:25 PM <<a href="mailto:chrisj@rtems.org" target="_blank" rel="noreferrer">chrisj@rtems.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">From: Chris Johns <<a href="mailto:chrisj@rtems.org" target="_blank" rel="noreferrer">chrisj@rtems.org</a>><br>
<br>
---<br>
source-builder/sb-rtems-pkg | 29 ++++<br>
source-builder/sb/download.py | 5 +-<br>
source-builder/sb/git.py | 12 ++<br>
source-builder/sb/rtemspkg.py | 279 ++++++++++++++++++++++++++++++++++<br>
4 files changed, 324 insertions(+), 1 deletion(-)<br>
create mode 100755 source-builder/sb-rtems-pkg<br>
create mode 100644 source-builder/sb/rtemspkg.py<br>
<br>
diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg<br>
new file mode 100755<br>
index 0000000..99ed26c<br>
--- /dev/null<br>
+++ b/source-builder/sb-rtems-pkg<br>
@@ -0,0 +1,29 @@<br>
+#! /usr/bin/env python<br>
+#<br>
+# RTEMS Tools Project (<a href="http://www.rtems.org/" rel="noreferrer noreferrer" target="_blank">http://www.rtems.org/</a>)<br>
+# Copyright 2024 Chris Johns (<a href="mailto:chrisj@rtems.org" target="_blank" rel="noreferrer">chrisj@rtems.org</a>)<br>
+# All rights reserved.<br>
+#<br>
+# This file is part of the RTEMS Tools package in 'rtems-tools'.<br>
+#<br>
+# Permission to use, copy, modify, and/or distribute this software for any<br>
+# purpose with or without fee is hereby granted, provided that the above<br>
+# copyright notice and this permission notice appear in all copies.<br>
+#<br>
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES<br>
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF<br>
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR<br>
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES<br>
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN<br>
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF<br>
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.<br>
+<br>
+from __future__ import print_function<br>
+<br>
+try:<br>
+ import sb.rtemspkg<br>
+ sb.rtemspkg.run()<br>
+except ImportError:<br>
+ import sys<br>
+ print("Incorrect Source Builder installation", file = sys.stderr)<br>
+ sys.exit(1)<br>
diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py<br>
index 6e6f9f2..0201675 100644<br>
--- a/source-builder/sb/download.py<br>
+++ b/source-builder/sb/download.py<br>
@@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True):<br>
hash = hash.split()<br>
if len(hash) != 2:<br>
raise error.internal('invalid hash format: %s' % (file_))<br>
+ if hash[0] == 'NO-HASH':<br>
+ return not failed<br>
try:<br>
hashlib_algorithms = hashlib.algorithms<br>
except:<br>
@@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts):<br>
else:<br>
repo.clean(['-f', '-d'])<br>
repo.reset('--hard')<br>
- repo.checkout('master')<br>
+ default_branch = repo.default_branch()<br>
+ repo.checkout(default_branch)<br>
for a in us[1:]:<br>
_as = a.split('=')<br>
if _as[0] == 'branch' or _as[0] == 'checkout':<br>
diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py<br>
index 237e690..0aa4da9 100644<br>
--- a/source-builder/sb/git.py<br>
+++ b/source-builder/sb/git.py<br>
@@ -226,6 +226,18 @@ class repo:<br>
hash = l1[len('commit '):]<br>
return hash<br>
<br>
+ def default_branch(self):<br>
+ ec, output = self._run(['remote', 'show'])<br>
+ if ec == 0:<br>
+ origin = output.split('\n')[0]<br>
+ ec, output = self._run(['remote', 'show', origin])<br>
+ if ec == 0:<br>
+ for l in output.split('\n'):<br>
+ l = l.strip()<br>
+ if l.startswith('HEAD branch: '):<br>
+ return l[len('HEAD branch: '):]<br>
+ return None<br>
+<br>
if __name__ == '__main__':<br>
import os.path<br>
import sys<br>
diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py<br>
new file mode 100644<br>
index 0000000..0b43d23<br>
--- /dev/null<br>
+++ b/source-builder/sb/rtemspkg.py<br>
@@ -0,0 +1,279 @@<br>
+#<br>
+# RTEMS Tools Project (<a href="http://www.rtems.org/" rel="noreferrer noreferrer" target="_blank">http://www.rtems.org/</a>)<br>
+# Copyright 2024 Chris Johns (<a href="mailto:chrisj@rtems.org" target="_blank" rel="noreferrer">chrisj@rtems.org</a>)<br>
+# All rights reserved.<br>
+#<br>
+# This file is part of the RTEMS Tools package in 'rtems-tools'.<br>
+#<br>
+# Permission to use, copy, modify, and/or distribute this software for any<br>
+# purpose with or without fee is hereby granted, provided that the above<br>
+# copyright notice and this permission notice appear in all copies.<br>
+#<br>
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES<br>
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF<br>
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR<br>
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES<br>
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN<br>
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF<br>
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.<br>
+<br>
+#<br>
+# This code builds a package compiler tool suite given a tool set. A tool<br>
+# set lists the various tools. These are specific tool configurations.<br>
+#<br>
+<br>
+from __future__ import print_function<br>
+<br>
+import argparse<br>
+import base64<br>
+import copy<br>
+import datetime<br>
+import hashlib<br>
+import os<br>
+import sys<br>
+<br>
+try:<br>
+ from . import build<br>
+ from . import download<br>
+ from . import error<br>
+ from . import git<br>
+ from . import log<br>
+ from . import path<br>
+ from . import simhost<br>
+ from . import version<br>
+except KeyboardInterrupt:<br>
+ print('abort: user terminated', file=sys.stderr)<br>
+ sys.exit(1)<br>
+except:<br>
+ raise<br>
+<br>
+#<br>
+# RTEMS Packages we maintian a git hash of in the RSB<br>
+#<br>
+rtems_pkg_cfgs = [<br>
+ [<br>
+ 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg',<br>
+ 'rtems_tools_version', 'git://<a href="http://git.rtems.org/rtems-tools" rel="noreferrer noreferrer" target="_blank">git.rtems.org/rtems-tools</a>',<br>
+ 'rtems-tools.git', 'master',<br>
+ '<a href="https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%%7Brtems_tools_version%7D.tar.bz2" rel="noreferrer noreferrer" target="_blank">https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2</a>',<br>
+ 'rtems-tools-%{rtems_tools_version}.tar.bz2'<br>
+ ],<br>
+ [<br>
+ 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg',<br>
+ 'rtems_kernel_version', 'git://<a href="http://git.rtems.org/rtems" rel="noreferrer noreferrer" target="_blank">git.rtems.org/rtems</a>', 'rtems.git',<br>
+ 'master',<br>
+ '<a href="https://git.rtems.org/rtems/snapshot/rtems-%%7Brtems_kernel_version%7D.tar.bz2" rel="noreferrer noreferrer" target="_blank">https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2</a>',<br>
+ 'rtems-kernel-%{rtems_kernel_version}.tar.bz2'<br>
+ ],<br>
+ [<br>
+ 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg',<br>
+ 'rtems_libbsd_version', 'git://<a href="http://git.rtems.org/rtems-libbsd" rel="noreferrer noreferrer" target="_blank">git.rtems.org/rtems-libbsd</a>',<br>
+ 'rtems-libbsd.git', '6-freebsd-12',<br>
+ '<a href="https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%%7Brtems_libbsd_version%7D.tar.%%7Brtems_libbsd_ext%7D" rel="noreferrer noreferrer" target="_blank">https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}</a>',<br>
+ 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}'<br>
+ ],<br>
+ [<br>
+ 'RTEMS Net Legacy', 'tools/rtems-net-legacy-%{rtems_version}.cfg',<br>
+ 'rtems_net_version', 'git://<a href="http://git.rtems.org/rtems-net-legacy" rel="noreferrer noreferrer" target="_blank">git.rtems.org/rtems-net-legacy</a>',<br>
+ 'rtems-net-legacy.git', 'main',<br>
+ '<a href="https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%%7Brtems_net_version%7D.tar.%%7Brtems_net_ext%7D" rel="noreferrer noreferrer" target="_blank">https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}</a>',<br>
+ 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}'<br>
+ ],<br>
+ [<br>
+ 'RTEMS Net Services', 'net/net-services-1.cfg',<br>
+ 'rtems_net_services_version', 'git://<a href="http://git.rtems.org/rtems-net-services" rel="noreferrer noreferrer" target="_blank">git.rtems.org/rtems-net-services</a>',<br>
+ 'rtems-net-services.git', 'master',<br>
+ '<a href="https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%%7Brtems_net_services_version%7D.tar.%%7Brtems_net_services_ext%7D" rel="noreferrer noreferrer" target="_blank">https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}</a>',<br>
+ 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}'<br>
+ ],<br>
+]<br>
+<br>
+<br>
+def clean_line(line):<br>
+ line = line[0:-1]<br>
+ b = line.find('#')<br>
+ if b >= 0:<br>
+ line = line[1:b] + ('\\' if line[-1] == '\\' else '')<br>
+ return line.strip()<br>
+<br>
+<br>
+def clean_and_pack(line, last_line):<br>
+ leading_ws = ' ' if len(line) > 0 and line[0].isspace() else ''<br>
+ line = clean_line(line)<br>
+ if len(last_line) > 0:<br>
+ line = last_line + leading_ws + line<br>
+ return line<br>
+<br>
+<br>
+def config_patch(configdir, config, version_label, config_hash, repo_hash,<br>
+ checksum):<br>
+ for cd in configdir.split(':'):<br>
+ cf = path.join(cd, config)<br>
+ if path.exists(cf):<br>
+ try:<br>
+ with open(cf) as f:<br>
+ lines = f.readlines()<br>
+ except IOError as err:<br>
+ raise error.general('config: %s: read error: %s' %<br>
+ (config, str(err)))<br>
+ new_config = []<br>
+ new_lines = []<br>
+ last_line = ''<br>
+ for line in lines:<br>
+ new_lines += [line]<br>
+ line = clean_and_pack(line, last_line)<br>
+ if len(line) > 0:<br>
+ if line[-1] == '\\':<br>
+ last_line = line[:-1]<br>
+ continue<br>
+ last_line = ''<br>
+ if version_label in line and not 'rsb_version' in line:<br>
+ if line.startswith('%define ' + version_label):<br>
+ new_lines = [<br>
+ '%define ' + version_label + ' ' + repo_hash +<br>
+ os.linesep<br>
+ ]<br>
+ elif line.startswith('%hash '):<br>
+ ls = line.split()<br>
+ if len(ls) != 4:<br>
+ raise error.general('invalid %hash: ' + line)<br>
+ new_lines = [<br>
+ ' '.join(ls[0:3]) + ' \\' + os.linesep,<br>
+ ' ' + checksum + os.linesep<br>
+ ]<br>
+ new_config += new_lines<br>
+ new_lines = []<br>
+ try:<br>
+ with open(cf, 'w') as f:<br>
+ f.writelines(new_config)<br>
+ except IOError as err:<br>
+ raise error.general('config: %s: write error: %s' %<br>
+ (config, str(err)))<br>
+ return<br>
+ raise error.general('could not find: ' + config)<br>
+<br>
+<br>
+def checksum_shar512_base64(tarball):<br>
+ hasher = hashlib.new('sha512')<br>
+ try:<br>
+ with open(path.host(tarball), 'rb') as f:<br>
+ hasher.update(f.read())<br>
+ except IOError as err:<br>
+ log.notice('hash: %s: read error: %s' % (file_, str(err)))<br>
+ except:<br>
+ raise<br>
+ raise error.general('cannot hash the tar file')<br>
+ hash_hex = hasher.hexdigest()<br>
+ hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8')<br>
+ return hash_base64<br>
+<br>
+<br>
+def run(args=sys.argv):<br>
+ ec = 0<br>
+ output = []<br>
+ try:<br>
+ #<br>
+ # The RSB options support cannot be used because it loads the defaults<br>
+ # for the host which we cannot do here.<br>
+ #<br>
+ description = 'RTEMS Track Dependencies a build set has for all hosts.'<br>
+<br>
+ argsp = argparse.ArgumentParser(prog='sb-rtems-pkg',<br>
+ description=description)<br>
+ argsp.add_argument('--rtems-version',<br>
+ help='Set the RTEMS version.',<br>
+ type=str,<br>
+ default=version.version())<br>
+ argsp.add_argument('--output',<br>
+ help='Output file.',<br>
+ type=str,<br>
+ default=None)<br>
+ argsp.add_argument('--log',<br>
+ help='Log file.',<br>
+ type=str,<br>
+ default=simhost.log_default('rtems-pkg'))<br>
+ argsp.add_argument('--trace',<br>
+ help='Enable trace logging for debugging.',<br>
+ action='store_true')<br>
+ argsp.add_argument('--dry-run',<br>
+ help='Dry run, do not update the configurations',<br>
+ action='store_true')<br>
+ argsp.add_argument('bsets', nargs='*', help='Build sets.')<br>
+<br>
+ argopts = argsp.parse_args(args[1:])<br>
+<br>
+ simhost.load_log(argopts.log)<br>
+ log.notice('RTEMS Source Builder - RTEMS Package Update, %s' %<br>
+ (version.string()))<br>
+ log.tracing = argopts.trace<br>
+<br>
+ opts = simhost.load_options(args, argopts, extras=['--with-download'])<br>
+ opts.defaults['_rsb_getting_source'] = '1'<br>
+ opts.defaults[<br>
+ 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}'<br>
+ host = 'freebsd'<br>
+<br>
+ for cfg in rtems_pkg_cfgs:<br>
+ b = None<br>
+ try:<br>
+ bopts = copy.copy(opts)<br>
+ bmacros = copy.copy(opts.defaults)<br>
+ b = build.build(cfg[1], False, bopts, bmacros)<br>
+ git_hash_key = b.macros.find(cfg[2])<br>
+ if len(git_hash_key) == 0:<br>
+ raise error.general(cfg[0] + ': cannot find version macro')<br>
+ source_dir = b.macros.expand('%{_sourcedir}')<br>
+ config_hash = b.macros.expand('%{' + cfg[2] + '}')<br>
+ repo_path = path.join(source_dir, cfg[4])<br>
+ download.get_file(cfg[3] + '?fetch?checkout=' + cfg[5],<br>
+ repo_path, bopts, b)<br>
+ repo = git.repo(repo_path)<br>
+ repo_hash = repo.head()<br>
+ if config_hash != repo_hash:<br>
+ update = True<br>
+ update_str = 'UPDATE'<br>
+ else:<br>
+ update = False<br>
+ update_str = 'matching'<br>
+ print(cfg[0] + ': ' + update_str + ' config:' + config_hash +<br>
+ ' repo:' + repo_hash)<br>
+ b.macros[cfg[2]] = repo_hash<br>
+ tarball = b.macros.expand(cfg[7])<br>
+ b.macros.set_write_map('hashes')<br>
+ b.macros[tarball] = 'NO-HASH NO-HASH'<br>
+ b.macros.unset_write_map()<br>
+ tarball_path = path.join(source_dir, b.macros.expand(cfg[7]))<br>
+ download.get_file(b.macros.expand(cfg[6]), tarball_path, bopts,<br>
+ b)<br>
+ tarball_hash = checksum_shar512_base64(tarball_path)<br>
+ if update:<br>
+ config_patch(b.macros.expand('%{_configdir}'),<br>
+ b.macros.expand(cfg[1]), cfg[2], config_hash,<br>
+ repo_hash, tarball_hash)<br>
+ del b<br>
+ except error.general as gerr:<br>
+ log.stderr(str(gerr))<br>
+ log.stderr('Build FAILED')<br>
+ b = None<br>
+ except error.general as gerr:<br>
+ log.stderr(str(gerr))<br>
+ log.stderr('Build FAILED')<br>
+ ec = 1<br>
+ except error.internal as ierr:<br>
+ log.stderr(str(ierr))<br>
+ log.stderr('Internal Build FAILED')<br>
+ ec = 1<br>
+ except error.exit as eerr:<br>
+ pass<br>
+ except KeyboardInterrupt:<br>
+ log.notice('abort: user terminated')<br>
+ ec = 1<br>
+ except:<br>
+ raise<br>
+ log.notice('abort: unknown error')<br>
+ ec = 1<br>
+ sys.exit(ec)<br>
+<br>
+<br>
+if __name__ == "__main__":<br>
+ run()<br>
-- <br>
2.42.0<br>
<br>
_______________________________________________<br>
devel mailing list<br>
<a href="mailto:devel@rtems.org" target="_blank" rel="noreferrer">devel@rtems.org</a><br>
<a href="http://lists.rtems.org/mailman/listinfo/devel" rel="noreferrer noreferrer" target="_blank">http://lists.rtems.org/mailman/listinfo/devel</a><br>
</blockquote></div></div>
_______________________________________________<br>
devel mailing list<br>
<a href="mailto:devel@rtems.org" target="_blank" rel="noreferrer">devel@rtems.org</a><br>
<a href="http://lists.rtems.org/mailman/listinfo/devel" rel="noreferrer noreferrer" target="_blank">http://lists.rtems.org/mailman/listinfo/devel</a></blockquote></div>
</blockquote></div>