[PATCH 15/27] score: Add SMP scheduler make/clean sticky
Sebastian Huber
sebastian.huber at embedded-brains.de
Mon Nov 15 17:12:47 UTC 2021
This patch fixes the following broken behaviour:
While a thread is scheduled on a helping scheduler, while it does not
own a MrsP semaphore, if it obtains a MrsP semaphore, then no
scheduler node using an idle thread and the ceiling priority of the
semaphore is unblocked for the home scheduler.
This could lead to priority inversion issues and is not in line
with the MrsP protocol.
Introduce two new scheduler operations which are only enabled if
RTEMS_SMP is defined. The operations are used to make the scheduler
node of the home scheduler sticky and to clean the sticky property.
This helps to keep the sticky handing out of the frequently used
priority update operation.
Close #4532.
---
cpukit/include/rtems/score/mrspimpl.h | 19 ++-
cpukit/include/rtems/score/scheduler.h | 54 +++++++
cpukit/include/rtems/score/scheduleredfsmp.h | 32 +++++
cpukit/include/rtems/score/schedulerimpl.h | 59 --------
.../score/schedulerpriorityaffinitysmp.h | 32 +++++
.../rtems/score/schedulerprioritysmp.h | 32 +++++
.../include/rtems/score/schedulersimplesmp.h | 32 +++++
cpukit/include/rtems/score/schedulersmpimpl.h | 84 +++++++++++
.../include/rtems/score/schedulerstrongapa.h | 62 ++++++++
cpukit/include/rtems/score/threadimpl.h | 28 ++--
.../src/schedulerdefaultmakecleansticky.c | 52 +++++++
cpukit/score/src/scheduleredfsmp.c | 36 ++++-
.../score/src/schedulerpriorityaffinitysmp.c | 39 +++++-
cpukit/score/src/schedulerprioritysmp.c | 37 ++++-
cpukit/score/src/schedulersimplesmp.c | 37 ++++-
cpukit/score/src/schedulerstrongapa.c | 37 ++++-
cpukit/score/src/threadchangepriority.c | 132 +++++++++++++++++-
cpukit/score/src/threadqenqueue.c | 6 +-
spec/build/cpukit/objsmp.yml | 1 +
19 files changed, 720 insertions(+), 91 deletions(-)
create mode 100644 cpukit/score/src/schedulerdefaultmakecleansticky.c
diff --git a/cpukit/include/rtems/score/mrspimpl.h b/cpukit/include/rtems/score/mrspimpl.h
index 3e64ad94e6..444586b4ab 100644
--- a/cpukit/include/rtems/score/mrspimpl.h
+++ b/cpukit/include/rtems/score/mrspimpl.h
@@ -268,7 +268,7 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Claim_ownership(
_MRSP_Set_owner( mrsp, executing );
cpu_self = _Thread_queue_Dispatch_disable( queue_context );
_MRSP_Release( mrsp, queue_context );
- _Thread_Priority_and_sticky_update( executing, 1 );
+ _Thread_Priority_update_and_make_sticky( executing );
_Thread_Dispatch_enable( cpu_self );
return STATUS_SUCCESSFUL;
}
@@ -384,13 +384,6 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Wait_for_ownership(
_MRSP_Replace_priority( mrsp, executing, &ceiling_priority );
} else {
Per_CPU_Control *cpu_self;
- int sticky_level_change;
-
- if ( status != STATUS_DEADLOCK ) {
- sticky_level_change = -1;
- } else {
- sticky_level_change = 0;
- }
_ISR_lock_ISR_disable( &queue_context->Lock_context.Lock_context );
_MRSP_Remove_priority( executing, &ceiling_priority, queue_context );
@@ -398,7 +391,13 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Wait_for_ownership(
&queue_context->Lock_context.Lock_context
);
_ISR_lock_ISR_enable( &queue_context->Lock_context.Lock_context );
- _Thread_Priority_and_sticky_update( executing, sticky_level_change );
+
+ if ( status != STATUS_DEADLOCK ) {
+ _Thread_Priority_update_and_clean_sticky( executing );
+ } else {
+ _Thread_Priority_update_only( executing );
+ }
+
_Thread_Dispatch_enable( cpu_self );
}
@@ -493,7 +492,7 @@ RTEMS_INLINE_ROUTINE Status_Control _MRSP_Surrender(
&queue_context->Lock_context.Lock_context
);
_MRSP_Release( mrsp, queue_context );
- _Thread_Priority_and_sticky_update( executing, -1 );
+ _Thread_Priority_update_and_clean_sticky( executing );
_Thread_Dispatch_enable( cpu_self );
return STATUS_SUCCESSFUL;
}
diff --git a/cpukit/include/rtems/score/scheduler.h b/cpukit/include/rtems/score/scheduler.h
index ad9d630023..2fd1fc567a 100644
--- a/cpukit/include/rtems/score/scheduler.h
+++ b/cpukit/include/rtems/score/scheduler.h
@@ -134,6 +134,40 @@ typedef struct {
Thread_Scheduler_state next_state
);
+ /**
+ * @brief Makes the node sticky.
+ *
+ * This operation is used by _Thread_Priority_update_and_make_sticky().
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+ void ( *make_sticky )(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+ );
+
+ /**
+ * @brief Cleans the sticky property from the node.
+ *
+ * This operation is used by _Thread_Priority_update_and_clean_sticky().
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+ void ( *clean_sticky )(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+ );
+
/**
* @brief Pin thread operation.
*
@@ -397,6 +431,24 @@ Priority_Control _Scheduler_default_Unmap_priority(
);
#if defined(RTEMS_SMP)
+ /**
+ * @brief Does nothing.
+ *
+ * This default implementation for the make and clean sticky operations
+ * should be used by uniprocessor schedulers if SMP support is enabled.
+ *
+ * @param scheduler is an unused parameter.
+ *
+ * @param the_thread is an unused parameter.
+ *
+ * @param node is an unused parameter.
+ */
+ void _Scheduler_default_Sticky_do_nothing(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+ );
+
/**
* @brief Does nothing.
*
@@ -459,6 +511,8 @@ Priority_Control _Scheduler_default_Unmap_priority(
NULL, \
NULL, \
NULL, \
+ _Scheduler_default_Sticky_do_nothing, \
+ _Scheduler_default_Sticky_do_nothing, \
_Scheduler_default_Pin_or_unpin_do_nothing, \
_Scheduler_default_Pin_or_unpin_do_nothing, \
NULL, \
diff --git a/cpukit/include/rtems/score/scheduleredfsmp.h b/cpukit/include/rtems/score/scheduleredfsmp.h
index e749b3d419..2c73c5535a 100644
--- a/cpukit/include/rtems/score/scheduleredfsmp.h
+++ b/cpukit/include/rtems/score/scheduleredfsmp.h
@@ -129,6 +129,8 @@ typedef struct {
_Scheduler_EDF_SMP_Ask_for_help, \
_Scheduler_EDF_SMP_Reconsider_help_request, \
_Scheduler_EDF_SMP_Withdraw_node, \
+ _Scheduler_EDF_SMP_Make_sticky, \
+ _Scheduler_EDF_SMP_Clean_sticky, \
_Scheduler_EDF_SMP_Pin, \
_Scheduler_EDF_SMP_Unpin, \
_Scheduler_EDF_SMP_Add_processor, \
@@ -248,6 +250,36 @@ void _Scheduler_EDF_SMP_Withdraw_node(
Thread_Scheduler_state next_state
);
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+void _Scheduler_EDF_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+void _Scheduler_EDF_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
/**
* @brief Pin thread operation.
*
diff --git a/cpukit/include/rtems/score/schedulerimpl.h b/cpukit/include/rtems/score/schedulerimpl.h
index 7319cc4e4c..eb279876c7 100644
--- a/cpukit/include/rtems/score/schedulerimpl.h
+++ b/cpukit/include/rtems/score/schedulerimpl.h
@@ -405,65 +405,6 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Update_priority( Thread_Control *the_thread
#endif
}
-#if defined(RTEMS_SMP)
-/**
- * @brief Changes the sticky level of the home scheduler node and propagates a
- * priority change of a thread to the scheduler.
- *
- * @param the_thread The thread changing its priority or sticky level.
- *
- * @see _Scheduler_Update_priority().
- */
-RTEMS_INLINE_ROUTINE void _Scheduler_Priority_and_sticky_update(
- Thread_Control *the_thread,
- int sticky_level_change
-)
-{
- Chain_Node *node;
- const Chain_Node *tail;
- Scheduler_Node *scheduler_node;
- const Scheduler_Control *scheduler;
- ISR_lock_Context lock_context;
-
- _Thread_Scheduler_process_requests( the_thread );
-
- node = _Chain_First( &the_thread->Scheduler.Scheduler_nodes );
- scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node );
- scheduler = _Scheduler_Node_get_scheduler( scheduler_node );
-
- _Scheduler_Acquire_critical( scheduler, &lock_context );
-
- scheduler_node->sticky_level += sticky_level_change;
- _Assert( scheduler_node->sticky_level >= 0 );
-
- ( *scheduler->Operations.update_priority )(
- scheduler,
- the_thread,
- scheduler_node
- );
-
- _Scheduler_Release_critical( scheduler, &lock_context );
-
- tail = _Chain_Immutable_tail( &the_thread->Scheduler.Scheduler_nodes );
- node = _Chain_Next( node );
-
- while ( node != tail ) {
- scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node );
- scheduler = _Scheduler_Node_get_scheduler( scheduler_node );
-
- _Scheduler_Acquire_critical( scheduler, &lock_context );
- ( *scheduler->Operations.update_priority )(
- scheduler,
- the_thread,
- scheduler_node
- );
- _Scheduler_Release_critical( scheduler, &lock_context );
-
- node = _Chain_Next( node );
- }
-}
-#endif
-
/**
* @brief Maps a thread priority from the user domain to the scheduler domain.
*
diff --git a/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h b/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h
index e997e81128..d77629b39d 100644
--- a/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h
+++ b/cpukit/include/rtems/score/schedulerpriorityaffinitysmp.h
@@ -65,6 +65,8 @@ extern "C" {
_Scheduler_priority_affinity_SMP_Ask_for_help, \
_Scheduler_priority_affinity_SMP_Reconsider_help_request, \
_Scheduler_priority_affinity_SMP_Withdraw_node, \
+ _Scheduler_priority_affinity_SMP_Make_sticky, \
+ _Scheduler_priority_affinity_SMP_Clean_sticky, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_priority_affinity_SMP_Add_processor, \
@@ -180,6 +182,36 @@ void _Scheduler_priority_affinity_SMP_Withdraw_node(
Thread_Scheduler_state next_state
);
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+void _Scheduler_priority_affinity_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+void _Scheduler_priority_affinity_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
/**
* @brief Adds @a idle to @a scheduler.
*
diff --git a/cpukit/include/rtems/score/schedulerprioritysmp.h b/cpukit/include/rtems/score/schedulerprioritysmp.h
index fe314fb05b..9ece9ae143 100644
--- a/cpukit/include/rtems/score/schedulerprioritysmp.h
+++ b/cpukit/include/rtems/score/schedulerprioritysmp.h
@@ -93,6 +93,8 @@ typedef struct {
_Scheduler_priority_SMP_Ask_for_help, \
_Scheduler_priority_SMP_Reconsider_help_request, \
_Scheduler_priority_SMP_Withdraw_node, \
+ _Scheduler_priority_SMP_Make_sticky, \
+ _Scheduler_priority_SMP_Clean_sticky, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_priority_SMP_Add_processor, \
@@ -214,6 +216,36 @@ void _Scheduler_priority_SMP_Withdraw_node(
Thread_Scheduler_state next_state
);
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+void _Scheduler_priority_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+void _Scheduler_priority_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
/**
* @brief Adds @a idle to @a scheduler.
*
diff --git a/cpukit/include/rtems/score/schedulersimplesmp.h b/cpukit/include/rtems/score/schedulersimplesmp.h
index c8394781c9..3b6f43869e 100644
--- a/cpukit/include/rtems/score/schedulersimplesmp.h
+++ b/cpukit/include/rtems/score/schedulersimplesmp.h
@@ -75,6 +75,8 @@ typedef struct {
_Scheduler_simple_SMP_Ask_for_help, \
_Scheduler_simple_SMP_Reconsider_help_request, \
_Scheduler_simple_SMP_Withdraw_node, \
+ _Scheduler_simple_SMP_Make_sticky, \
+ _Scheduler_simple_SMP_Clean_sticky, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_simple_SMP_Add_processor, \
@@ -194,6 +196,36 @@ void _Scheduler_simple_SMP_Withdraw_node(
Thread_Scheduler_state next_state
);
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+void _Scheduler_simple_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+void _Scheduler_simple_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
/**
* @brief Adds @a idle to @a scheduler.
*
diff --git a/cpukit/include/rtems/score/schedulersmpimpl.h b/cpukit/include/rtems/score/schedulersmpimpl.h
index 54ed976e85..68ae3718a0 100644
--- a/cpukit/include/rtems/score/schedulersmpimpl.h
+++ b/cpukit/include/rtems/score/schedulersmpimpl.h
@@ -1672,6 +1672,90 @@ static inline void _Scheduler_SMP_Withdraw_node(
}
}
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+static inline void _Scheduler_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node,
+ Scheduler_SMP_Update update,
+ Scheduler_SMP_Enqueue enqueue
+)
+{
+ Scheduler_SMP_Node_state node_state;
+
+ node_state = _Scheduler_SMP_Node_state( node );
+
+ if ( node_state == SCHEDULER_SMP_NODE_BLOCKED ) {
+ Scheduler_Context *context;
+ Priority_Control insert_priority;
+ Priority_Control priority;
+
+ context = _Scheduler_Get_context( scheduler );
+ priority = _Scheduler_Node_get_priority( node );
+ priority = SCHEDULER_PRIORITY_PURIFY( priority );
+
+ if ( priority != _Scheduler_SMP_Node_priority( node ) ) {
+ ( *update )( context, node, priority );
+ }
+
+ _Scheduler_SMP_Node_change_state( node, SCHEDULER_SMP_NODE_READY );
+ insert_priority = SCHEDULER_PRIORITY_APPEND( priority );
+ (void) ( *enqueue )( context, node, insert_priority );
+ }
+}
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+static inline void _Scheduler_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node,
+ Scheduler_SMP_Extract extract_from_scheduled,
+ Scheduler_SMP_Extract extract_from_ready,
+ Scheduler_SMP_Get_highest_ready get_highest_ready,
+ Scheduler_SMP_Move move_from_ready_to_scheduled,
+ Scheduler_SMP_Allocate_processor allocate_processor
+)
+{
+ Scheduler_SMP_Node_state node_state;
+
+ node_state = _Scheduler_SMP_Node_state( node );
+
+ if ( node_state == SCHEDULER_SMP_NODE_SCHEDULED && node->idle != NULL ) {
+ Scheduler_Context *context;
+
+ context = _Scheduler_Get_context( scheduler );
+ _Scheduler_SMP_Node_change_state( node, SCHEDULER_SMP_NODE_BLOCKED );
+
+ ( *extract_from_scheduled )( context, node );
+
+ _Scheduler_SMP_Schedule_highest_ready(
+ context,
+ node,
+ _Thread_Get_CPU( node->idle ),
+ extract_from_ready,
+ get_highest_ready,
+ move_from_ready_to_scheduled,
+ allocate_processor
+ );
+ }
+}
+
/**
* @brief Starts the idle thread on the given processor.
*
diff --git a/cpukit/include/rtems/score/schedulerstrongapa.h b/cpukit/include/rtems/score/schedulerstrongapa.h
index bbded1b493..8db3ae8634 100644
--- a/cpukit/include/rtems/score/schedulerstrongapa.h
+++ b/cpukit/include/rtems/score/schedulerstrongapa.h
@@ -161,6 +161,8 @@ typedef struct {
_Scheduler_strong_APA_Ask_for_help, \
_Scheduler_strong_APA_Reconsider_help_request, \
_Scheduler_strong_APA_Withdraw_node, \
+ _Scheduler_strong_APA_Make_sticky, \
+ _Scheduler_strong_APA_Clean_sticky, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_default_Pin_or_unpin_not_supported, \
_Scheduler_strong_APA_Add_processor, \
@@ -278,6 +280,66 @@ void _Scheduler_strong_APA_Withdraw_node(
Thread_Scheduler_state next_state
);
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+void _Scheduler_strong_APA_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+void _Scheduler_strong_APA_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Makes the node sticky.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to make sticky.
+ */
+void _Scheduler_strong_APA_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
+/**
+ * @brief Cleans the sticky property from the node.
+ *
+ * @param scheduler is the scheduler of the node.
+ *
+ * @param[in, out] the_thread is the thread owning the node.
+ *
+ * @param[in, out] node is the scheduler node to clean the sticky property.
+ */
+void _Scheduler_strong_APA_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+);
+
/**
* @brief Adds the idle thread to a processor.
*
diff --git a/cpukit/include/rtems/score/threadimpl.h b/cpukit/include/rtems/score/threadimpl.h
index 934b56468a..e35a13abd6 100644
--- a/cpukit/include/rtems/score/threadimpl.h
+++ b/cpukit/include/rtems/score/threadimpl.h
@@ -790,17 +790,29 @@ void _Thread_Priority_replace(
*/
void _Thread_Priority_update( Thread_queue_Context *queue_context );
+#if defined(RTEMS_SMP)
/**
- * @brief Updates the priority of the thread and changes it sticky level.
+ * @brief Updates the priority of the thread and makes its home scheduler node
+ * sticky.
*
- * @param the_thread The thread.
- * @param sticky_level_change The new value for the sticky level.
+ * @param the_thread is the thread to work on.
*/
-#if defined(RTEMS_SMP)
-void _Thread_Priority_and_sticky_update(
- Thread_Control *the_thread,
- int sticky_level_change
-);
+void _Thread_Priority_update_and_make_sticky( Thread_Control *the_thread );
+
+/**
+ * @brief Updates the priority of the thread and cleans the sticky property of
+ * its home scheduler node.
+ *
+ * @param the_thread is the thread to work on.
+ */
+void _Thread_Priority_update_and_clean_sticky( Thread_Control *the_thread );
+
+/**
+ * @brief Updates the priority of the thread.
+ *
+ * @param the_thread is the thread to update the priority.
+ */
+void _Thread_Priority_update_only( Thread_Control *the_thread );
#endif
/**
diff --git a/cpukit/score/src/schedulerdefaultmakecleansticky.c b/cpukit/score/src/schedulerdefaultmakecleansticky.c
new file mode 100644
index 0000000000..e2b2d659f1
--- /dev/null
+++ b/cpukit/score/src/schedulerdefaultmakecleansticky.c
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+
+/**
+ * @file
+ *
+ * @ingroup RTEMSScoreScheduler
+ *
+ * @brief This source file contains the implementation of
+ * _Scheduler_default_Sticky_do_nothing().
+ */
+
+/*
+ * Copyright (C) 2021 embedded brains GmbH (http://www.embedded-brains.de)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <rtems/score/scheduler.h>
+
+void _Scheduler_default_Sticky_do_nothing(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ (void) scheduler;
+ (void) the_thread;
+ (void) node;
+}
diff --git a/cpukit/score/src/scheduleredfsmp.c b/cpukit/score/src/scheduleredfsmp.c
index 27be08ac40..0b0ee6ed21 100644
--- a/cpukit/score/src/scheduleredfsmp.c
+++ b/cpukit/score/src/scheduleredfsmp.c
@@ -11,7 +11,8 @@
* _Scheduler_EDF_SMP_Remove_processor(), _Scheduler_EDF_SMP_Set_affinity(),
* _Scheduler_EDF_SMP_Start_idle(), _Scheduler_EDF_SMP_Unblock(),
* _Scheduler_EDF_SMP_Unpin(), _Scheduler_EDF_SMP_Update_priority(),
- * _Scheduler_EDF_SMP_Withdraw_node(), and _Scheduler_EDF_SMP_Yield().
+ * _Scheduler_EDF_SMP_Withdraw_node(), _Scheduler_EDF_SMP_Make_sticky(),
+ * _Scheduler_EDF_SMP_Clean_sticky(), and _Scheduler_EDF_SMP_Yield().
*/
/*
@@ -619,6 +620,39 @@ void _Scheduler_EDF_SMP_Withdraw_node(
);
}
+void _Scheduler_EDF_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Make_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_EDF_SMP_Do_update,
+ _Scheduler_EDF_SMP_Enqueue
+ );
+}
+
+void _Scheduler_EDF_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Clean_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_EDF_SMP_Extract_from_scheduled,
+ _Scheduler_EDF_SMP_Extract_from_ready,
+ _Scheduler_EDF_SMP_Get_highest_ready,
+ _Scheduler_EDF_SMP_Move_from_ready_to_scheduled,
+ _Scheduler_EDF_SMP_Allocate_processor
+ );
+}
+
static inline void _Scheduler_EDF_SMP_Register_idle(
Scheduler_Context *context,
Scheduler_Node *idle_base,
diff --git a/cpukit/score/src/schedulerpriorityaffinitysmp.c b/cpukit/score/src/schedulerpriorityaffinitysmp.c
index 4bbf2f6e17..648c243589 100644
--- a/cpukit/score/src/schedulerpriorityaffinitysmp.c
+++ b/cpukit/score/src/schedulerpriorityaffinitysmp.c
@@ -12,8 +12,10 @@
* _Scheduler_priority_affinity_SMP_Remove_processor(),
* _Scheduler_priority_affinity_SMP_Set_affinity(),
* _Scheduler_priority_affinity_SMP_Unblock(),
- * _Scheduler_priority_affinity_SMP_Update_priority(), and
- * _Scheduler_priority_affinity_SMP_Withdraw_node().
+ * _Scheduler_priority_affinity_SMP_Update_priority(),
+ * _Scheduler_priority_affinity_SMP_Withdraw_node(),
+ * _Scheduler_priority_affinity_SMP_Make_sticky(), and
+ * _Scheduler_priority_affinity_SMP_Clean_sticky().
*/
/*
@@ -512,6 +514,39 @@ void _Scheduler_priority_affinity_SMP_Withdraw_node(
);
}
+void _Scheduler_priority_affinity_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Make_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_priority_SMP_Do_update,
+ _Scheduler_priority_affinity_SMP_Enqueue
+ );
+}
+
+void _Scheduler_priority_affinity_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Clean_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_SMP_Extract_from_scheduled,
+ _Scheduler_priority_SMP_Extract_from_ready,
+ _Scheduler_priority_affinity_SMP_Get_highest_ready,
+ _Scheduler_priority_SMP_Move_from_ready_to_scheduled,
+ _Scheduler_SMP_Allocate_processor_exact
+ );
+}
+
void _Scheduler_priority_affinity_SMP_Add_processor(
const Scheduler_Control *scheduler,
Thread_Control *idle
diff --git a/cpukit/score/src/schedulerprioritysmp.c b/cpukit/score/src/schedulerprioritysmp.c
index b0b920c960..7262f48e8e 100644
--- a/cpukit/score/src/schedulerprioritysmp.c
+++ b/cpukit/score/src/schedulerprioritysmp.c
@@ -12,7 +12,9 @@
* _Scheduler_priority_SMP_Remove_processor(),
* _Scheduler_priority_SMP_Unblock(),
* _Scheduler_priority_SMP_Update_priority(),
- * _Scheduler_priority_SMP_Withdraw_node(), and
+ * _Scheduler_priority_SMP_Withdraw_node(),
+ * _Scheduler_priority_SMP_Make_sticky(),
+ * _Scheduler_priority_SMP_Clean_sticky(), and
* _Scheduler_priority_SMP_Yield().
*/
@@ -265,6 +267,39 @@ void _Scheduler_priority_SMP_Withdraw_node(
);
}
+void _Scheduler_priority_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Make_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_priority_SMP_Do_update,
+ _Scheduler_priority_SMP_Enqueue
+ );
+}
+
+void _Scheduler_priority_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Clean_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_SMP_Extract_from_scheduled,
+ _Scheduler_priority_SMP_Extract_from_ready,
+ _Scheduler_priority_SMP_Get_highest_ready,
+ _Scheduler_priority_SMP_Move_from_ready_to_scheduled,
+ _Scheduler_SMP_Allocate_processor_lazy
+ );
+}
+
void _Scheduler_priority_SMP_Add_processor(
const Scheduler_Control *scheduler,
Thread_Control *idle
diff --git a/cpukit/score/src/schedulersimplesmp.c b/cpukit/score/src/schedulersimplesmp.c
index 02e4579aa1..b7814b0fcb 100644
--- a/cpukit/score/src/schedulersimplesmp.c
+++ b/cpukit/score/src/schedulersimplesmp.c
@@ -11,7 +11,9 @@
* _Scheduler_simple_SMP_Reconsider_help_request(),
* _Scheduler_simple_SMP_Remove_processor(), _Scheduler_simple_SMP_Unblock(),
* _Scheduler_simple_SMP_Update_priority(),
- * _Scheduler_simple_SMP_Withdraw_node(), and _Scheduler_simple_SMP_Yield().
+ * _Scheduler_simple_SMP_Withdraw_node(),
+ * _Scheduler_simple_SMP_Make_sticky(), _Scheduler_simple_SMP_Clean_sticky(),
+ * and _Scheduler_simple_SMP_Yield().
*/
/*
@@ -335,6 +337,39 @@ void _Scheduler_simple_SMP_Withdraw_node(
);
}
+void _Scheduler_simple_SMP_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Make_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_simple_SMP_Do_update,
+ _Scheduler_simple_SMP_Enqueue
+ );
+}
+
+void _Scheduler_simple_SMP_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Clean_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_SMP_Extract_from_scheduled,
+ _Scheduler_simple_SMP_Extract_from_ready,
+ _Scheduler_simple_SMP_Get_highest_ready,
+ _Scheduler_simple_SMP_Move_from_ready_to_scheduled,
+ _Scheduler_SMP_Allocate_processor_lazy
+ );
+}
+
void _Scheduler_simple_SMP_Add_processor(
const Scheduler_Control *scheduler,
Thread_Control *idle
diff --git a/cpukit/score/src/schedulerstrongapa.c b/cpukit/score/src/schedulerstrongapa.c
index afd9fcc709..6c7ab942fa 100644
--- a/cpukit/score/src/schedulerstrongapa.c
+++ b/cpukit/score/src/schedulerstrongapa.c
@@ -31,7 +31,9 @@
* _Scheduler_strong_APA_Set_affinity(),
* _Scheduler_strong_APA_Set_scheduled(), _Scheduler_strong_APA_Start_idle(),
* _Scheduler_strong_APA_Unblock(), _Scheduler_strong_APA_Update_priority(),
- * _Scheduler_strong_APA_Withdraw_node(), and _Scheduler_strong_APA_Yield().
+ * _Scheduler_strong_APA_Withdraw_node(),
+ * _Scheduler_strong_APA_Make_sticky(), _Scheduler_strong_APA_Clean_sticky(),
+ * and _Scheduler_strong_APA_Yield().
*/
/*
@@ -935,6 +937,39 @@ void _Scheduler_strong_APA_Withdraw_node(
);
}
+void _Scheduler_strong_APA_Make_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Make_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_strong_APA_Do_update,
+ _Scheduler_strong_APA_Enqueue
+ );
+}
+
+void _Scheduler_strong_APA_Clean_sticky(
+ const Scheduler_Control *scheduler,
+ Thread_Control *the_thread,
+ Scheduler_Node *node
+)
+{
+ _Scheduler_SMP_Clean_sticky(
+ scheduler,
+ the_thread,
+ node,
+ _Scheduler_SMP_Extract_from_scheduled,
+ _Scheduler_strong_APA_Extract_from_ready,
+ _Scheduler_strong_APA_Get_highest_ready,
+ _Scheduler_strong_APA_Move_from_ready_to_scheduled,
+ _Scheduler_strong_APA_Allocate_processor
+ );
+}
+
static inline void _Scheduler_strong_APA_Register_idle(
Scheduler_Context *context,
Scheduler_Node *idle_base,
diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c
index bd4fef279b..135e3f60ab 100644
--- a/cpukit/score/src/threadchangepriority.c
+++ b/cpukit/score/src/threadchangepriority.c
@@ -372,18 +372,140 @@ void _Thread_Priority_update( Thread_queue_Context *queue_context )
}
#if defined(RTEMS_SMP)
-void _Thread_Priority_and_sticky_update(
+static void _Thread_Priority_update_helping(
Thread_Control *the_thread,
- int sticky_level_change
+ Chain_Node *first_node
)
{
- ISR_lock_Context lock_context;
+ const Chain_Node *tail;
+ Chain_Node *node;
+
+ tail = _Chain_Immutable_tail( &the_thread->Scheduler.Scheduler_nodes );
+ node = _Chain_Next( first_node );
+
+ while ( node != tail ) {
+ Scheduler_Node *scheduler_node;
+ const Scheduler_Control *scheduler;
+ ISR_lock_Context lock_context;
+
+ scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node );
+ scheduler = _Scheduler_Node_get_scheduler( scheduler_node );
+
+ _Scheduler_Acquire_critical( scheduler, &lock_context );
+ ( *scheduler->Operations.update_priority )(
+ scheduler,
+ the_thread,
+ scheduler_node
+ );
+ _Scheduler_Release_critical( scheduler, &lock_context );
+
+ node = _Chain_Next( node );
+ }
+}
+
+void _Thread_Priority_update_and_make_sticky( Thread_Control *the_thread )
+{
+ ISR_lock_Context lock_context;
+ ISR_lock_Context lock_context_2;
+ Chain_Node *node;
+ Scheduler_Node *scheduler_node;
+ const Scheduler_Control *scheduler;
+ int new_sticky_level;
+ int make_sticky_level;
_Thread_State_acquire( the_thread, &lock_context );
- _Scheduler_Priority_and_sticky_update(
+ _Thread_Scheduler_process_requests( the_thread );
+
+ node = _Chain_First( &the_thread->Scheduler.Scheduler_nodes );
+ scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node );
+ scheduler = _Scheduler_Node_get_scheduler( scheduler_node );
+
+ _Scheduler_Acquire_critical( scheduler, &lock_context_2 );
+
+ new_sticky_level = scheduler_node->sticky_level + 1;
+ scheduler_node->sticky_level = new_sticky_level;
+ _Assert( new_sticky_level >= 1 );
+
+ /*
+ * The sticky level is incremented by the scheduler block operation, so for a
+ * ready thread, the change to sticky happens at a level of two.
+ */
+ make_sticky_level = 1 + (int) _Thread_Is_ready( the_thread );
+
+ if ( new_sticky_level == make_sticky_level ) {
+ ( *scheduler->Operations.make_sticky )(
+ scheduler,
+ the_thread,
+ scheduler_node
+ );
+ }
+
+ ( *scheduler->Operations.update_priority )(
+ scheduler,
the_thread,
- sticky_level_change
+ scheduler_node
);
+
+ _Scheduler_Release_critical( scheduler, &lock_context_2 );
+ _Thread_Priority_update_helping( the_thread, node );
+ _Thread_State_release( the_thread, &lock_context );
+}
+
+void _Thread_Priority_update_and_clean_sticky( Thread_Control *the_thread )
+{
+ ISR_lock_Context lock_context;
+ ISR_lock_Context lock_context_2;
+ Chain_Node *node;
+ Scheduler_Node *scheduler_node;
+ const Scheduler_Control *scheduler;
+ int new_sticky_level;
+ int clean_sticky_level;
+
+ _Thread_State_acquire( the_thread, &lock_context );
+ _Thread_Scheduler_process_requests( the_thread );
+
+ node = _Chain_First( &the_thread->Scheduler.Scheduler_nodes );
+ scheduler_node = SCHEDULER_NODE_OF_THREAD_SCHEDULER_NODE( node );
+ scheduler = _Scheduler_Node_get_scheduler( scheduler_node );
+
+ _Scheduler_Acquire_critical( scheduler, &lock_context_2 );
+
+ new_sticky_level = scheduler_node->sticky_level - 1;
+ scheduler_node->sticky_level = new_sticky_level;
+ _Assert( new_sticky_level >= 0 );
+
+ /*
+ * The sticky level is incremented by the scheduler block operation, so for a
+ * ready thread, the change to sticky happens at a level of one.
+ */
+ clean_sticky_level = (int) _Thread_Is_ready( the_thread );
+
+ if ( new_sticky_level == clean_sticky_level ) {
+ ( *scheduler->Operations.clean_sticky )(
+ scheduler,
+ the_thread,
+ scheduler_node
+ );
+ }
+
+ ( *scheduler->Operations.update_priority )(
+ scheduler,
+ the_thread,
+ scheduler_node
+ );
+
+ _Scheduler_Release_critical( scheduler, &lock_context_2 );
+ _Thread_Priority_update_helping( the_thread, node );
+ _Thread_State_release( the_thread, &lock_context );
+}
+
+void _Thread_Priority_update_only( Thread_Control *the_thread )
+{
+ ISR_lock_Context lock_context;
+
+ _Thread_State_acquire( the_thread, &lock_context );
+ _Thread_Scheduler_process_requests( the_thread );
+ _Scheduler_Update_priority( the_thread );
_Thread_State_release( the_thread, &lock_context );
}
#endif
diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c
index 5fc357ec82..2e31095510 100644
--- a/cpukit/score/src/threadqenqueue.c
+++ b/cpukit/score/src/threadqenqueue.c
@@ -518,7 +518,7 @@ Status_Control _Thread_queue_Enqueue_sticky(
);
_Thread_Priority_update( queue_context );
- _Thread_Priority_and_sticky_update( the_thread, 1 );
+ _Thread_Priority_update_and_make_sticky( the_thread );
_Thread_Dispatch_enable( cpu_self );
while (
@@ -899,8 +899,8 @@ void _Thread_queue_Surrender_sticky(
queue,
&queue_context->Lock_context.Lock_context
);
- _Thread_Priority_and_sticky_update( previous_owner, -1 );
- _Thread_Priority_and_sticky_update( new_owner, 0 );
+ _Thread_Priority_update_and_clean_sticky( previous_owner );
+ _Thread_Priority_update_only( new_owner );
_Thread_Dispatch_enable( cpu_self );
}
#endif
diff --git a/spec/build/cpukit/objsmp.yml b/spec/build/cpukit/objsmp.yml
index 1a55708638..ac57bb27b1 100644
--- a/spec/build/cpukit/objsmp.yml
+++ b/spec/build/cpukit/objsmp.yml
@@ -13,6 +13,7 @@ source:
- cpukit/score/src/percpujobs.c
- cpukit/score/src/percpustatewait.c
- cpukit/score/src/profilingsmplock.c
+- cpukit/score/src/schedulerdefaultmakecleansticky.c
- cpukit/score/src/schedulerdefaultpinunpin.c
- cpukit/score/src/schedulerdefaultpinunpindonothing.c
- cpukit/score/src/schedulerdefaultsetaffinity.c
--
2.26.2
More information about the devel
mailing list