[rtems-central commit] qdp: Add scripts to build a QDP
Sebastian Huber
sebh at rtems.org
Tue Nov 21 13:35:41 UTC 2023
Module: rtems-central
Branch: master
Commit: 6d96924f5b3e5cbfa1d416543d33e45e7eb72176
Changeset: http://git.rtems.org/rtems-central/commit/?id=6d96924f5b3e5cbfa1d416543d33e45e7eb72176
Author: Sebastian Huber <sebastian.huber at embedded-brains.de>
Date: Tue Nov 21 11:13:16 2023 +0100
qdp: Add scripts to build a QDP
---
Makefile | 56 +++
config/base.yml | 146 ++++++++
config/variant-sparc-gr712rc-smp.yml | 45 +++
config/variant-sparc-gr712rc-uni.yml | 44 +++
config/variant-sparc-gr740-smp.yml | 45 +++
config/variant-sparc-gr740-uni.yml | 44 +++
qdp_workspace.py | 526 +++++++++++++++++++++++++++++
rtemsspec/tests/test_util.py | 18 +-
rtemsspec/util.py | 20 ++
spec-qdp/qdp/deployment/archive.yml | 13 +
spec-qdp/qdp/deployment/verify-package.yml | 13 +
spec-qdp/qdp/package-build.yml | 9 +
spec-qdp/qdp/steps/archive.yml | 19 ++
spec-qdp/spec/qdp-dummy.yml | 22 ++
spec-qdp/spec/qdp-uuid.yml | 26 ++
workspace/.gitignore | 3 +
16 files changed, 1045 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index 6627d317..22f9e3c0 100644
--- a/Makefile
+++ b/Makefile
@@ -32,3 +32,59 @@ env:
. env/bin/activate && pip install --upgrade pip && pip install -r requirements.txt
echo -e "#!/bin/sh\n$$(which python3-config) "'$$@' > env/bin/python3-config
chmod +x env/bin/python3-config
+
+PREFIX = /opt/rtems
+
+PACKAGE_VERSION = 0
+
+RTEMS_API = 6
+
+LOG_LEVEL = DEBUG
+
+GR712RC_SMP_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr712rc-smp-$(PACKAGE_VERSION)
+
+GR712RC_SMP_LOG = $(GR712RC_SMP_PKG)-log.txt
+
+gr712rc-smp-clean:
+ rm -rf $(GR712RC_SMP_PKG) $(GR712RC_SMP_LOG)
+
+gr712rc-smp-update:
+ ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR712RC_SMP_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr712rc-smp.yml
+
+gr712rc-smp-new: gr712rc-smp-clean gr712rc-smp-update
+
+GR712RC_UNI_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr712rc-uni-$(PACKAGE_VERSION)
+
+GR712RC_UNI_LOG = $(GR712RC_UNI_PKG)-log.txt
+
+gr712rc-uni-clean:
+ rm -rf $(GR712RC_UNI_PKG) $(GR712RC_UNI_LOG)
+
+gr712rc-uni-update:
+ ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR712RC_UNI_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr712rc-uni.yml
+
+gr712rc-uni-new: gr712rc-uni-clean gr712rc-uni-update
+
+GR740_SMP_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr740-smp-$(PACKAGE_VERSION)
+
+GR740_SMP_LOG = $(GR740_SMP_PKG)-log.txt
+
+gr740-smp-clean:
+ rm -rf $(GR740_SMP_PKG) $(GR740_SMP_LOG)
+
+gr740-smp-update:
+ ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR740_SMP_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr740-smp.yml
+
+gr740-smp-new: gr740-smp-clean gr740-smp-update
+
+GR740_UNI_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr740-uni-$(PACKAGE_VERSION)
+
+GR740_UNI_LOG = $(GR740_UNI_PKG)-log.txt
+
+gr740-uni-clean:
+ rm -rf $(GR740_UNI_PKG) $(GR740_UNI_LOG)
+
+gr740-uni-update:
+ ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR740_UNI_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr740-uni.yml
+
+gr740-uni-new: gr740-uni-clean gr740-uni-update
diff --git a/config/base.yml b/config/base.yml
new file mode 100644
index 00000000..b01672ba
--- /dev/null
+++ b/config/base.yml
@@ -0,0 +1,146 @@
+workspace-actions:
+- action-name: base-load-items
+ action-type: load-items
+ action-when: 1000
+ enabled-by: true
+ paths:
+ - ${.:/toolchain-directory}/spec-spec
+ - ${.:/toolchain-directory}/spec-glossary
+ - ${.:/toolchain-directory}/spec-qdp
+ - ${.:/toolchain-directory}/spec
+ set-types:
+ - type: qdp/variant
+ uid: /qdp/variant
+- action-name: base-deployment-directory
+ action-type: make-deployment-directory
+ action-when: 3000
+ enabled-by: true
+- action-name: base-workspace-items-load
+ action-type: load-workspace-items
+ action-when: 3000
+ enabled-by: true
+ path: ${/qdp/variant:/build-directory}/spec
+ set-types:
+ - type: qdp/variant
+ uid: /qdp/variant
+- action-name: base-make-uuid
+ action-type: make-uuid-item
+ action-when: 3000
+ enabled-by: true
+ uid: /qdp/uuid
+- action-name: base-gitignore
+ action-type: copy-directory
+ action-when: 4000
+ copyrights-by-license: {}
+ destination-directory: ${../variant:/deployment-directory}
+ enabled-by: true
+ files:
+ - file: .gitignore
+ hash: null
+ links: []
+ patterns: []
+ source-directory: ${.:/toolchain-directory}/workspace
+ uid: /qdp/source/gitignore
+- action-name: qt-modules
+ action-type: copy-directory
+ action-when: 4000
+ copyrights-by-license: {}
+ destination-directory: ${../variant:/build-directory}
+ enabled-by: true
+ files: []
+ links: []
+ patterns:
+ - exclude:
+ - '*/.*'
+ include: rtemsspec/*.py
+ - exclude: []
+ include: qdp_build.py
+ source-directory: ${.:/toolchain-directory}
+ uid: /qdp/source/qt-modules
+- action-name: base-rtems
+ action-type: git-clone
+ action-when: 4000
+ branch: qdp
+ commit: 42c9cdf35f6aa27f41d20b9b170d6e4e83a76913
+ copyrights-by-license:
+ description: |
+ RTEMS and all third-party software distributed with RTEMS which may be
+ linked to the application is licensed under permissive open source
+ licenses. This means that the licenses do not propagate to the
+ application software. Most of the original RTEMS code is now under the
+ BSD-2-Clause license. Some code of RTEMS is under a legacy license, the
+ modified GPL-2.0 or later license with an exception for static linking.
+ It exposes no license requirements on application code. RTMES is a
+ collection of software from several sources. Each file may have its own
+ copyright/license that is embedded in the source file.
+ files:
+ - LICENSE
+ - LICENSE.Apache-2.0
+ - LICENSE.BSD-2-Clause
+ - LICENSE.BSD-3-Clause
+ - LICENSE.CC-BY-SA-4.0
+ - LICENSE.Freescale
+ - LICENSE.GPL-2.0
+ - LICENSE.JFFS2
+ - LICENSE.LLVM
+ description: |
+ This repository contains the RTEMS sources. It is used to provide the BSPs
+ shipped with the QDP.
+ destination-directory: ${../variant:/deployment-directory}/src/rtems
+ directory-state-invalidates: []
+ enabled-by: true
+ links:
+ - role: repository
+ uid: ../variant
+ - hash: null
+ name: member
+ role: input-to
+ uid: ../steps/archive
+ origin-branch: master
+ origin-commit: 71c024eaca2b16c32447a0d9d712310717d17af8
+ origin-commit-url: https://git.rtems.org/rtems/commit/?id=${.:/origin-commit}
+ origin-fetch: []
+ origin-url: git://git.rtems.org/rtems.git
+ post-clone-commands: []
+ source-directory: ${.:/toolchain-directory}/modules/rtems
+ uid: /qdp/source/rtems
+- action-name: base-rtems-load-spec
+ action-type: load-items
+ action-when: 4000
+ enabled-by: true
+ paths:
+ - ${/qdp/variant:/deployment-directory}/src/rtems/spec
+ set-types: []
+- action-name: base-rtems-docs
+ action-type: git-clone
+ action-when: 4000
+ branch: qdp
+ commit: 2c88912893ebbcc3b9fa14d4fcc100c42252d0df
+ copyrights-by-license: {}
+ description: |
+ This repository contains the RTEMS Documentation sources. It is used to
+ provide the RTEMS Documentation shipped with the QDP.
+ destination-directory: ${../variant:/deployment-directory}/src/rtems-docs
+ directory-state-invalidates: []
+ enabled-by: true
+ links:
+ - role: repository
+ uid: ../variant
+ - hash: null
+ name: member
+ role: input-to
+ uid: ../steps/archive
+ origin-branch: master
+ origin-commit: 2c88912893ebbcc3b9fa14d4fcc100c42252d0df
+ origin-commit-url: https://git.rtems.org/rtems-docs/commit/?id=${.:/origin-commit}
+ origin-fetch: []
+ origin-url: git://git.rtems.org/rtems-docs.git
+ post-clone-commands: []
+ source-directory: ${.:/toolchain-directory}/modules/rtems-docs
+ uid: /qdp/source/rtems-docs
+- action-name: base-workspace-items-finalize
+ action-type: finalize-workspace-items
+ action-when: 6000
+ enabled-by: true
+ spec-type-root-uid: /spec/root
+ verify: true
diff --git a/config/variant-sparc-gr712rc-smp.yml b/config/variant-sparc-gr712rc-smp.yml
new file mode 100644
index 00000000..384e0666
--- /dev/null
+++ b/config/variant-sparc-gr712rc-smp.yml
@@ -0,0 +1,45 @@
+workspace-actions:
+- action-name: sparc-gr712rc-smp
+ action-type: make-item
+ action-when: 500
+ data:
+ SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+ arch: sparc
+ bsp: gr712rc
+ bsp-family: leon3
+ build-directory: ${.:/deployment-directory}/build
+ config: smp
+ copyrights:
+ - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG
+ deployment-directory: ${.:/prefix-directory}/${.:/package-directory}
+ enabled:
+ - ${.:/arch}
+ - bsps/${.:/arch}/${.:/bsp-family}
+ - ${.:/arch}/${.:/bsp}
+ - RTEMS_QUAL
+ - RTEMS_SMP
+ - __GNUC__
+ - target/evaluation-board
+ - target-hash/cpI09Ju6orF2eoJcmJi4igeIarypsRNwUxTrZSs9LMg=
+ - target/simulator
+ - target-hash/qYOFDHUGg5--JyB28V7llk_t6WYeA3VAogeqwGLZeCM=
+ enabled-by: true
+ ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version}
+ links:
+ - role: package-build
+ uid: package-build
+ name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version}
+ package-directory: rtems-${.:/rtems-version}-${.:/name}
+ package-version: '0'
+ params:
+ makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq
+ ${.:sis-extirq} -dumbio -r $$<
+ sis-cpus: '2'
+ sis-extirq: '12'
+ sis-target: leon3
+ prefix-directory: /opt/rtems
+ qdp-type: variant
+ rtems-version: '6'
+ type: qdp
+ enabled-by: true
+ uid: /qdp/variant
diff --git a/config/variant-sparc-gr712rc-uni.yml b/config/variant-sparc-gr712rc-uni.yml
new file mode 100644
index 00000000..b1ee37f3
--- /dev/null
+++ b/config/variant-sparc-gr712rc-uni.yml
@@ -0,0 +1,44 @@
+workspace-actions:
+- action-name: sparc-gr712rc-uni
+ action-type: make-item
+ action-when: 500
+ data:
+ SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+ arch: sparc
+ bsp: gr712rc
+ bsp-family: leon3
+ build-directory: ${.:/deployment-directory}/build
+ config: uni
+ copyrights:
+ - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG
+ deployment-directory: ${.:/prefix-directory}/${.:/package-directory}
+ enabled:
+ - ${.:/arch}
+ - bsps/${.:/arch}/${.:/bsp-family}
+ - ${.:/arch}/${.:/bsp}
+ - RTEMS_QUAL
+ - __GNUC__
+ - target/evaluation-board
+ - target-hash/cpI09Ju6orF2eoJcmJi4igeIarypsRNwUxTrZSs9LMg=
+ - target/simulator
+ - target-hash/qYOFDHUGg5--JyB28V7llk_t6WYeA3VAogeqwGLZeCM=
+ enabled-by: true
+ ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version}
+ links:
+ - role: package-build
+ uid: package-build
+ name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version}
+ package-directory: rtems-${.:/rtems-version}-${.:/name}
+ package-version: '0'
+ params:
+ makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq
+ ${.:sis-extirq} -dumbio -r $$<
+ sis-cpus: '1'
+ sis-extirq: '12'
+ sis-target: leon3
+ prefix-directory: /opt/rtems
+ qdp-type: variant
+ rtems-version: '6'
+ type: qdp
+ enabled-by: true
+ uid: /qdp/variant
diff --git a/config/variant-sparc-gr740-smp.yml b/config/variant-sparc-gr740-smp.yml
new file mode 100644
index 00000000..bd2ce838
--- /dev/null
+++ b/config/variant-sparc-gr740-smp.yml
@@ -0,0 +1,45 @@
+workspace-actions:
+- action-name: sparc-gr740-smp
+ action-type: make-item
+ action-when: 500
+ data:
+ SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+ arch: sparc
+ bsp: gr740
+ bsp-family: leon3
+ build-directory: ${.:/deployment-directory}/build
+ config: smp
+ copyrights:
+ - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG
+ deployment-directory: ${.:/prefix-directory}/${.:/package-directory}
+ enabled:
+ - ${.:/arch}
+ - bsps/${.:/arch}/${.:/bsp-family}
+ - ${.:/arch}/${.:/bsp}
+ - RTEMS_QUAL
+ - RTEMS_SMP
+ - __GNUC__
+ - target/evaluation-board
+ - target-hash/c1ZkBOsUIJ-siPI7pK7knk0z6uni1pxOFlZ2eLDflYc=
+ - target/simulator
+ - target-hash/_xQeTNJwSla2bVbhWPVcI0emLk2bE_GVQfvzt9CN84k=
+ enabled-by: true
+ ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version}
+ links:
+ - role: package-build
+ uid: package-build
+ name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version}
+ package-directory: rtems-${.:/rtems-version}-${.:/name}
+ package-version: '0'
+ params:
+ makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq
+ ${.:sis-extirq} -dumbio -r $$<
+ sis-cpus: '4'
+ sis-extirq: '10'
+ sis-target: gr740
+ prefix-directory: /opt/rtems
+ qdp-type: variant
+ rtems-version: '6'
+ type: qdp
+ enabled-by: true
+ uid: /qdp/variant
diff --git a/config/variant-sparc-gr740-uni.yml b/config/variant-sparc-gr740-uni.yml
new file mode 100644
index 00000000..9433818e
--- /dev/null
+++ b/config/variant-sparc-gr740-uni.yml
@@ -0,0 +1,44 @@
+workspace-actions:
+- action-name: sparc-gr740-uni
+ action-type: make-item
+ action-when: 500
+ data:
+ SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+ arch: sparc
+ bsp: gr740
+ bsp-family: leon3
+ build-directory: ${.:/deployment-directory}/build
+ config: uni
+ copyrights:
+ - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG
+ deployment-directory: ${.:/prefix-directory}/${.:/package-directory}
+ enabled:
+ - ${.:/arch}
+ - bsps/${.:/arch}/${.:/bsp-family}
+ - ${.:/arch}/${.:/bsp}
+ - RTEMS_QUAL
+ - __GNUC__
+ - target/evaluation-board
+ - target-hash/c1ZkBOsUIJ-siPI7pK7knk0z6uni1pxOFlZ2eLDflYc=
+ - target/simulator
+ - target-hash/_xQeTNJwSla2bVbhWPVcI0emLk2bE_GVQfvzt9CN84k=
+ enabled-by: true
+ ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version}
+ links:
+ - role: package-build
+ uid: package-build
+ name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version}
+ package-directory: rtems-${.:/rtems-version}-${.:/name}
+ package-version: '0'
+ params:
+ makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq
+ ${.:sis-extirq} -dumbio -r $$<
+ sis-cpus: '1'
+ sis-extirq: '10'
+ sis-target: gr740
+ prefix-directory: /opt/rtems
+ qdp-type: variant
+ rtems-version: '6'
+ type: qdp
+ enabled-by: true
+ uid: /qdp/variant
diff --git a/qdp_workspace.py b/qdp_workspace.py
new file mode 100755
index 00000000..0f38429a
--- /dev/null
+++ b/qdp_workspace.py
@@ -0,0 +1,526 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: BSD-2-Clause
+""" Creates a QDP workspace directory according to the configuration file. """
+
+# Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import copy
+import logging
+import os
+import subprocess
+import sys
+from typing import Any, Callable, Dict, List, Optional, Set
+import uuid
+
+from rtemsspec.items import Item, ItemCache, JSONItemCache, is_enabled
+from rtemsspec.directorystate import DirectoryState
+from rtemsspec.packagebuild import BuildItem, BuildItemFactory, \
+ PackageBuildDirector
+from rtemsspec.packagebuildfactory import create_build_item_factory
+from rtemsspec.specverify import verify
+from rtemsspec.util import create_build_argument_parser, hash_file, \
+ init_logging, load_config, run_command
+
+
+class _ConfigItem(BuildItem):
+
+ def __init__(self, director: PackageBuildDirector, item: Item,
+ factory: BuildItemFactory):
+ super().__init__(director, item)
+ workspace_cache_config: Dict[str, Any] = {
+ "enabled": [],
+ "initialize-links": False,
+ "paths": [],
+ "resolve-proxies": False,
+ "spec-type-root-uid": None
+ }
+ self.workspace_cache = JSONItemCache(workspace_cache_config)
+ self.workspace_director = PackageBuildDirector(self.workspace_cache,
+ factory)
+
+ def is_enabled(self, enabled_by: Any) -> bool:
+ """
+ Returns true, if the enabled by expression evaluates to true for the
+ enabled set of the configuration item, otherwise returns false.
+ """
+ try:
+ enabled_set = self.enabled_set
+ except KeyError:
+ enabled_set = []
+ return is_enabled(self.substitute(enabled_set), enabled_by)
+
+ def set_file(self, item: Item) -> None:
+ """ Sets the file of the item. """
+ file = os.path.join(self["spec-directory"], f"{item.uid[1:]}.json")
+ os.makedirs(os.path.dirname(file), exist_ok=True)
+ item.file = file
+
+ def try_save(self, build_item: BuildItem, _reason: str) -> None:
+ """ Tries to set the file of the item and save it. """
+ if "spec-directory" in self:
+ self.set_file(build_item.item)
+ build_item.item.save()
+ self.item.cache[build_item.uid].data.clear()
+ self.item.cache[build_item.uid].data.update(build_item.item.data)
+ else:
+ logging.info("%s: cannot save item", build_item.uid)
+
+
+def _create_item(
+ config: _ConfigItem,
+ action: Dict[str, Any],
+ data: Dict[str, Any],
+ type_name: str,
+ prepare: Optional[Callable[[_ConfigItem, Dict[str, Any]], None]] = None
+) -> Optional[BuildItem]:
+ uid = action["uid"]
+ item = config.item.cache.create_item(uid, data)
+ item["_type"] = type_name
+ if prepare is not None:
+ prepare(config, action)
+ workspace_item = config.workspace_cache.create_item(
+ uid, copy.deepcopy(data))
+ workspace_item["_type"] = type_name
+ build_item = config.workspace_director[uid]
+ return build_item
+
+
+def _make_root_data(config: _ConfigItem, type_name: str) -> Any:
+ return {
+ "SPDX-License-Identifier": config.variant["SPDX-License-Identifier"],
+ "copyrights": config.variant["copyrights"],
+ "enabled-by": True,
+ "links": [],
+ "qdp-type": type_name,
+ "type": "qdp"
+ }
+
+
+def _make_directory_state_data(config: _ConfigItem, directory: str,
+ directory_state_type: str) -> Any:
+ data = _make_root_data(config, "directory-state")
+ data["copyrights-by-license"] = {}
+ data["directory"] = directory
+ data["directory-state-type"] = directory_state_type
+ data["files"] = []
+ data["hash"] = None
+ data["patterns"] = []
+ return data
+
+
+def _load_directory(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ directory_state = config.director[action["uid"]]
+ assert isinstance(directory_state, DirectoryState)
+ directory_state.load()
+ directory_state["patterns"] = []
+
+
+def _action_copy_directory(config: _ConfigItem, action: Dict[str,
+ Any]) -> None:
+ source_directory = config.substitute(action["source-directory"])
+ data = _make_directory_state_data(config, source_directory, "generic")
+ data["copyrights-by-license"] = action["copyrights-by-license"]
+ data["patterns"] = action["patterns"]
+ data["files"] = action["files"]
+ data["links"] = action["links"]
+ workspace_directory_state = _create_item(config, action, data,
+ "qdp/directory-state/generic",
+ _load_directory)
+ assert isinstance(workspace_directory_state, DirectoryState)
+ directory_state = config.director[action["uid"]]
+ assert isinstance(directory_state, DirectoryState)
+ workspace_directory_state["directory"] = action["destination-directory"]
+ workspace_directory_state.clear()
+ workspace_directory_state.lazy_clone(directory_state)
+ config.try_save(workspace_directory_state, "Update directory state")
+
+
+def _copy_file(config: _ConfigItem, action: Dict[str, Any],
+ the_type: str) -> None:
+ source_file = config.substitute(action["source-file"])
+ data = _make_directory_state_data(config, os.path.dirname(source_file),
+ the_type)
+ data["files"] = [{"file": os.path.basename(source_file), "hash": None}]
+ data["links"] = action["links"]
+ workspace_directory_state = _create_item(
+ config, action, data, f"qdp/directory-state/{the_type}",
+ _load_directory)
+ assert isinstance(workspace_directory_state, DirectoryState)
+ directory_state = config.director[action["uid"]]
+ assert isinstance(directory_state, DirectoryState)
+ workspace_directory_state["directory"] = action["destination-directory"]
+ workspace_directory_state.clear()
+ workspace_directory_state.copy_file(source_file,
+ action["destination-file"])
+ workspace_directory_state.load()
+ config.try_save(workspace_directory_state, "Update directory state")
+
+
+def _action_copy_file(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ _copy_file(config, action, "generic")
+
+
+def _action_copy_test_log(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ _copy_file(config, action, "test-log")
+
+
+def _git_clone(config: _ConfigItem, action: Dict[str, Any],
+ repository: DirectoryState) -> None:
+ source_directory = config.substitute(action["source-directory"])
+ status = run_command(["git", "fetch", "origin"], source_directory)
+ assert status == 0
+ branch = action["branch"]
+ commit = action["commit"]
+ status = run_command(["git", "branch", "-f", branch, commit],
+ source_directory)
+ assert status == 0
+ destination_directory = repository.directory
+ status = run_command([
+ "git", "clone", "--branch", branch, "--single-branch",
+ f"file://{source_directory}", destination_directory
+ ], source_directory)
+ assert status == 0
+ origin_url = action["origin-url"]
+ if origin_url:
+ status = run_command(
+ ["git", "remote", "add", "tmp",
+ config.substitute(origin_url)], destination_directory)
+ assert status == 0
+ status = run_command(["git", "remote", "remove", "origin"],
+ destination_directory)
+ assert status == 0
+ status = run_command(["git", "remote", "rename", "tmp", "origin"],
+ destination_directory)
+ assert status == 0
+ for fetch in action["origin-fetch"]:
+ status = run_command(
+ ["git", "fetch", "origin",
+ config.substitute(fetch)], destination_directory)
+ assert status == 0
+ origin_branch = action["origin-branch"]
+ origin_commit = action["origin-commit"]
+ if origin_branch and origin_commit:
+ status = run_command(
+ ["git", "checkout", "-b", origin_branch, origin_commit],
+ destination_directory)
+ assert status == 0
+ status = run_command(["git", "checkout", branch],
+ destination_directory)
+ assert status == 0
+ status = run_command([
+ "git", "symbolic-ref", "refs/remotes/origin/HEAD",
+ f"refs/remotes/origin/{origin_branch}"
+ ], destination_directory)
+ assert status == 0
+ for command in action["post-clone-commands"]:
+ status = run_command(config.substitute(command), destination_directory)
+ assert status == 0
+ repository.load()
+ config.try_save(repository, "Clone Git repository")
+
+
+def _action_git_clone(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ data = _make_directory_state_data(config, action["destination-directory"],
+ "repository")
+ data["patterns"] = [{"include": "**/*", "exclude": []}]
+ for key in [
+ "branch", "commit", "copyrights-by-license", "description",
+ "links", "origin-branch", "origin-commit", "origin-commit-url",
+ "origin-url"
+ ]:
+ data[key] = action[key]
+ repository = _create_item(config, action, data,
+ "qdp/directory-state/repository")
+ assert isinstance(repository, DirectoryState)
+ destination_directory = repository.directory
+ assert not os.path.exists(destination_directory)
+ _git_clone(config, action, repository)
+
+
+def _add_item(config: _ConfigItem, action: Dict[str, Any],
+ data: Dict[str, Any]) -> None:
+ config.item.cache.create_item(action["uid"], data)
+
+
+def _action_add_item(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ source_file = config.substitute(action["source"])
+ data = config.item.cache.load_data(source_file, action["uid"])
+ _add_item(config, action, data)
+
+
+def _action_make_item(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ _add_item(config, action, action["data"])
+
+
+def _action_make_uuid_item(config: _ConfigItem, action: Dict[str,
+ Any]) -> None:
+ data = _make_root_data(config, "uuid")
+ data["uuid"] = str(uuid.uuid4())
+ _add_item(config, action, data)
+
+
+def _set_types(item_cache: ItemCache, action: Dict[str, Any]) -> None:
+ for set_type in action["set-types"]:
+ item_cache[set_type["uid"]]["_type"] = set_type["type"]
+
+
+def _action_load_items(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ action_name = action["action-name"]
+ item_cache = config.item.cache
+ for path in action["paths"]:
+ path = config.substitute(path)
+ logging.info("%s: load items from: %s", action_name, path)
+ item_cache.load_items(path)
+ _set_types(item_cache, action)
+ config.director.clear()
+ config.workspace_director.clear()
+
+
+def _new_workspace_items(config: _ConfigItem, action_name: str,
+ new: Set[str]) -> None:
+ for uid in sorted(new):
+ logging.debug("%s: new item: %s", action_name, uid)
+ data = config.item.cache[uid].data
+ item = config.workspace_cache.create_item(uid, copy.deepcopy(data))
+ item["_type"] = data["_type"]
+ config.set_file(item)
+ item.save()
+
+
+def _action_load_workspace_items(config: _ConfigItem,
+ action: Dict[str, Any]) -> None:
+ action_name = action["action-name"]
+ path = config.substitute(action["path"])
+ config["spec-directory"] = path
+ config.variant.item["enabled"] = config.variant["enabled"]
+ logging.info("%s: add workspace items to: %s", action_name, path)
+ new = set(config.item.cache.keys())
+ _new_workspace_items(config, action_name, new)
+ _set_types(config.workspace_cache, action)
+
+
+def _action_finalize_workspace_items(config: _ConfigItem,
+ action: Dict[str, Any]) -> None:
+ action_name = action["action-name"]
+ new = set(config.item.cache.keys())
+ _new_workspace_items(config, action_name, new)
+ logging.info("%s: initialize workspace item links", action_name)
+ config.workspace_cache.initialize_links()
+ enabled_set = config.variant["enabled"]
+ logging.info("%s: set workspace enabled: %s", action_name, enabled_set)
+ config.workspace_cache.set_enabled(enabled_set)
+ logging.info("%s: set workspace item types", action_name)
+ config.workspace_cache.set_types(action["spec-type-root-uid"])
+ config.workspace_director.clear()
+ if action["verify"] and config["spec-verify"]:
+ logging.info("%s: verify workspace items", action_name)
+ logger = logging.getLogger()
+ level = logger.getEffectiveLevel()
+ logger.setLevel(logging.ERROR)
+ verify_config = {"root-type": action["spec-type-root-uid"]}
+ status = verify(verify_config, config.workspace_cache)
+ assert status.critical == 0
+ assert status.error == 0
+ logger.setLevel(level)
+ logging.debug("%s: finished verifying workspace items", action_name)
+
+
+def _action_make_deployment_directory(config: _ConfigItem,
+ _action: Dict[str, Any]) -> None:
+ deployment_directory = config.variant["deployment-directory"]
+ assert not os.path.exists(deployment_directory)
+ logging.info("workspace: create deployment directory: %s",
+ deployment_directory)
+ os.makedirs(deployment_directory)
+
+
+def _create_symbolic_links(action: Dict[str, Any],
+ unpacked_archive: DirectoryState) -> None:
+ for symbolic_link in action["archive-symbolic-links"]:
+ link = unpacked_archive.substitute(symbolic_link["link"])
+ link_path = os.path.join(unpacked_archive.directory, link)
+ target = unpacked_archive.substitute(symbolic_link["target"])
+ target = os.path.relpath(target, os.path.dirname(link_path))
+ logging.info("%s: create symbolic link from '%s' to '%s'",
+ action["action-name"], link_path, target)
+ os.symlink(target, link_path)
+ unpacked_archive.add_files([link])
+
+
+def _apply_patches(config: _ConfigItem, action: Dict[str, Any],
+ unpacked_archive: DirectoryState) -> None:
+ for patch in action["archive-patches"]:
+ if patch["type"] == "inline":
+ source = "-"
+ stdin = config.substitute(patch["patch"]).encode("utf-8")
+ logging.info("%s: apply inline patch: %s", action["action-name"],
+ stdin)
+ else:
+ assert patch["type"] == "file"
+ source = patch["file"]
+ stdin = None
+ logging.info("%s: apply patch: %s", action["action-name"], source)
+ command = [
+ "git", "apply", "--apply", "--numstat", "--directory",
+ unpacked_archive.directory, "-p", "1", "--unsafe-paths", source
+ ]
+ output = subprocess.check_output(command,
+ stdin=stdin,
+ cwd=config["toolchain-directory"])
+ unpacked_archive.add_files([
+ line.decode("utf-8").split("\t")[2]
+ for line in output.splitlines()
+ ])
+
+
+def _action_unpack_archive(config: _ConfigItem, action: Dict[str,
+ Any]) -> None:
+ data = _make_directory_state_data(config, action["destination-directory"],
+ "unpacked-archive")
+ archive_file = config.substitute(action["archive-file"])
+ data["archive-file"] = os.path.basename(archive_file)
+ for key in [
+ "archive-hash", "archive-patches", "archive-symbolic-links",
+ "archive-url", "copyrights-by-license", "description",
+ "enabled-by", "links"
+ ]:
+ data[key] = action[key]
+ unpacked_archive = _create_item(config, action, data,
+ "qdp/directory-state/unpacked-archive")
+ assert isinstance(unpacked_archive, DirectoryState)
+ unpacked_archive.add_tarfile_members(archive_file,
+ unpacked_archive.directory, True)
+ assert hash_file(archive_file) == unpacked_archive["archive-hash"]
+ unpacked_archive.compact()
+ _create_symbolic_links(action, unpacked_archive)
+ _apply_patches(config, action, unpacked_archive)
+ unpacked_archive.load()
+ config.try_save(unpacked_archive, "Unpack archive")
+
+
+def _create_dummy(config: _ConfigItem, action: Dict[str, Any]) -> None:
+ data = _make_root_data(config, "dummy")
+ data["enabled-by"] = action["enabled-by"]
+ dummy = _create_item(config, action, data, "qdp/dummy")
+ if dummy is None:
+ return
+ config.try_save(dummy, "Add dummy item")
+
+
+_ACTIONS = {
+ "add-item": _action_add_item,
+ "copy-directory": _action_copy_directory,
+ "copy-file": _action_copy_file,
+ "copy-test-log": _action_copy_test_log,
+ "finalize-workspace-items": _action_finalize_workspace_items,
+ "git-clone": _action_git_clone,
+ "load-items": _action_load_items,
+ "load-workspace-items": _action_load_workspace_items,
+ "make-deployment-directory": _action_make_deployment_directory,
+ "make-item": _action_make_item,
+ "make-uuid-item": _action_make_uuid_item,
+ "unpack-archive": _action_unpack_archive
+}
+
+_NEEDS_DUMMY = [
+ "add-item", "copy-directory", "copy-file", "copy-test-log", "git-clone",
+ "make-item", "unpack-archive"
+]
+
+
+def _run_actions(config: _ConfigItem, when: int,
+ actions: List[Dict[str, Any]]) -> None:
+ logging.info("workspace: run actions at: %i", when)
+ for action in actions:
+ action_type = action["action-type"]
+ if config.is_enabled(action["enabled-by"]):
+ logging.info("%s: run action", action["action-name"])
+ _ACTIONS[action_type](config, action)
+ else:
+ logging.info("%s: action is disabled", action["action-name"])
+ if action_type in _NEEDS_DUMMY:
+ _create_dummy(config, action)
+
+
+def _toolchain_commit(toolchain_directory: str) -> str:
+ stdout: List[str] = []
+ status = run_command(["git", "rev-parse", "HEAD"], toolchain_directory,
+ stdout)
+ assert status == 0
+ return stdout[0].strip()
+
+
+def main(argv: List[str]) -> None:
+ """
+ Creates a QDP workspace directory according to the configuration files.
+ """
+ parser = create_build_argument_parser()
+ parser.add_argument("--prefix",
+ help="the prefix directory",
+ default="/tmp/qdp")
+ parser.add_argument("--cache-directory",
+ help="the config cache directory",
+ default="config-cache")
+ parser.add_argument('config_files', nargs='+')
+ args = parser.parse_args(argv)
+ init_logging(args)
+ config_cache_config: Dict[str, Any] = {
+ "cache-directory": os.path.abspath(args.cache_directory),
+ "enabled": [],
+ "resolve-proxies": False,
+ "initialize-links": False,
+ "paths": [],
+ "spec-type-root-uid": None,
+ }
+ config_cache = ItemCache(config_cache_config)
+ factory = create_build_item_factory()
+ director = PackageBuildDirector(config_cache, factory)
+ config_item = Item(
+ config_cache, "/qdp/config", {
+ "SPDX-License-Identifier":
+ "CC-BY-SA-4.0 OR BSD-2-Clause",
+ "copyrights":
+ ["Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG"]
+ })
+ config_item["_type"] = "config"
+ config = _ConfigItem(director, config_item, factory)
+ config["prefix-directory"] = os.path.abspath(args.prefix)
+ config["spec-verify"] = not args.no_spec_verify
+ toolchain_directory = os.path.abspath(os.path.dirname(__file__))
+ logging.info("workspace: toolchain directory: %s", toolchain_directory)
+ config["toolchain-directory"] = toolchain_directory
+ config["toolchain-commit"] = _toolchain_commit(toolchain_directory)
+ config["enabled"] = []
+ actions_at: Dict[int, List[Dict[str, Any]]] = {}
+ for file in args.config_files:
+ logging.info("workspace: load configuration from: %s", file)
+ for action in load_config(file)["workspace-actions"]:
+ actions_at.setdefault(action["action-when"], []).append(action)
+ for when, actions in sorted(actions_at.items()):
+ _run_actions(config, when, actions)
+ config.workspace_director.build_package(args.only, args.force)
+
+
+if __name__ == "__main__": # pragma: no cover
+ main(sys.argv[1:])
diff --git a/rtemsspec/tests/test_util.py b/rtemsspec/tests/test_util.py
index 3eab1641..0d4db2b3 100644
--- a/rtemsspec/tests/test_util.py
+++ b/rtemsspec/tests/test_util.py
@@ -28,7 +28,8 @@ import os
import logging
from rtemsspec.util import copy_file, copy_files, create_argument_parser, \
- base64_to_hex, init_logging, load_config, run_command
+ create_build_argument_parser, base64_to_hex, init_logging, load_config, \
+ run_command
from rtemsspec.tests.util import get_and_clear_log
@@ -79,13 +80,22 @@ DEBUG A"""
def test_args():
- parser = create_argument_parser()
+ parser = create_build_argument_parser()
args = parser.parse_args([])
- init_logging(args)
assert args.log_level == "INFO"
assert args.log_file is None
+ assert args.only is None
+ assert args.force is None
+ assert not args.no_spec_verify
+ init_logging(args)
log_file = "log.txt"
- args = parser.parse_args(["--log-level=DEBUG", f"--log-file={log_file}"])
+ args = parser.parse_args([
+ "--log-level=DEBUG", f"--log-file={log_file}", "--only", "abc",
+ "--force", "def", "--no-spec-verify"
+ ])
assert args.log_level == "DEBUG"
assert args.log_file == log_file
+ assert args.only == ["abc"]
+ assert args.force == ["def"]
+ assert args.no_spec_verify
init_logging(args)
diff --git a/rtemsspec/util.py b/rtemsspec/util.py
index a1f46372..296c6137 100644
--- a/rtemsspec/util.py
+++ b/rtemsspec/util.py
@@ -188,6 +188,26 @@ def create_argument_parser(
return parser
+def create_build_argument_parser(
+ default_log_level: str = "INFO") -> argparse.ArgumentParser:
+ """ Creates an argument parser with default build options. """
+ parser = create_argument_parser(default_log_level)
+ parser.add_argument('--only',
+ type=str,
+ nargs='*',
+ default=None,
+ help="build only these steps")
+ parser.add_argument('--force',
+ type=str,
+ nargs='*',
+ default=None,
+ help="force to build these steps")
+ parser.add_argument('--no-spec-verify',
+ action="store_true",
+ help="do not verify the specification")
+ return parser
+
+
def init_logging(args: argparse.Namespace) -> None:
""" Initializes the logging module. """
handlers: List[Any] = [logging.StreamHandler()]
diff --git a/spec-qdp/qdp/deployment/archive.yml b/spec-qdp/qdp/deployment/archive.yml
new file mode 100644
index 00000000..823071e8
--- /dev/null
+++ b/spec-qdp/qdp/deployment/archive.yml
@@ -0,0 +1,13 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2020 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/build-directory}
+directory-state-type: generic
+enabled-by: true
+files: []
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/spec-qdp/qdp/deployment/verify-package.yml b/spec-qdp/qdp/deployment/verify-package.yml
new file mode 100644
index 00000000..34ca7d60
--- /dev/null
+++ b/spec-qdp/qdp/deployment/verify-package.yml
@@ -0,0 +1,13 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2021 embedded brains GmbH & Co. KG
+copyrights-by-license: {}
+directory: ${../variant:/deployment-directory}
+directory-state-type: generic
+enabled-by: true
+files: []
+hash: null
+links: []
+patterns: []
+qdp-type: directory-state
+type: qdp
diff --git a/spec-qdp/qdp/package-build.yml b/spec-qdp/qdp/package-build.yml
new file mode 100644
index 00000000..1e999798
--- /dev/null
+++ b/spec-qdp/qdp/package-build.yml
@@ -0,0 +1,9 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2020 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: build-step
+ uid: steps/archive
+qdp-type: package-build
+type: qdp
diff --git a/spec-qdp/qdp/steps/archive.yml b/spec-qdp/qdp/steps/archive.yml
new file mode 100644
index 00000000..019542f7
--- /dev/null
+++ b/spec-qdp/qdp/steps/archive.yml
@@ -0,0 +1,19 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+archive-file: rtems-${../variant:/rtems-version}-${../variant:/name}.tar.xz
+archive-strip-prefix: ${../variant:/prefix-directory}/
+build-step-type: archive
+copyrights:
+- Copyright (C) 2020, 2021 embedded brains GmbH & Co. KG
+description: |
+ Packs all deployed components into the archive file handed over to end users.
+enabled-by: true
+links:
+- name: archive
+ role: output
+ uid: ../deployment/archive
+- name: verify-package
+ role: output
+ uid: ../deployment/verify-package
+qdp-type: build-step
+type: qdp
+verification-script: verify_package.py
diff --git a/spec-qdp/spec/qdp-dummy.yml b/spec-qdp/spec/qdp-dummy.yml
new file mode 100644
index 00000000..c1051559
--- /dev/null
+++ b/spec-qdp/spec/qdp-dummy.yml
@@ -0,0 +1,22 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: qdp-type
+ spec-value: dummy
+ uid: qdp-root
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes: {}
+ description: |
+ This set of attributes specifies a dummy.
+ mandatory-attributes: all
+spec-name: Dummy Item Type
+spec-type: qdp-dummy
+type: spec
diff --git a/spec-qdp/spec/qdp-uuid.yml b/spec-qdp/spec/qdp-uuid.yml
new file mode 100644
index 00000000..6cad9270
--- /dev/null
+++ b/spec-qdp/spec/qdp-uuid.yml
@@ -0,0 +1,26 @@
+SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause
+copyrights:
+- Copyright (C) 2023 embedded brains GmbH & Co. KG
+enabled-by: true
+links:
+- role: spec-member
+ uid: root
+- role: spec-refinement
+ spec-key: qdp-type
+ spec-value: uuid
+ uid: qdp-root
+spec-description: null
+spec-example: null
+spec-info:
+ dict:
+ attributes:
+ uuid:
+ description: |
+ It shall be the UUID.
+ spec-type: str
+ description: |
+ This set of attributes provides an UUID.
+ mandatory-attributes: all
+spec-name: UUID Item Type
+spec-type: qdp-uuid
+type: spec
diff --git a/workspace/.gitignore b/workspace/.gitignore
new file mode 100644
index 00000000..ca53c44c
--- /dev/null
+++ b/workspace/.gitignore
@@ -0,0 +1,3 @@
+b-*
+*.pyc
+__pycache__
More information about the vc
mailing list