[PATCH v2 4/5] score: Turn thread lock into thread wait lock
Sebastian Huber
sebastian.huber at embedded-brains.de
Tue Jul 26 09:20:51 UTC 2016
The _Thread_Lock_acquire() function had a potentially infinite run-time
due to the lack of fairness at atomic operations level.
Update #2412.
Update #2556.
Update #2765.
---
cpukit/libmisc/monitor/mon-task.c | 7 +-
cpukit/posix/src/pthreadgetschedparam.c | 8 +-
cpukit/rtems/include/rtems/rtems/ratemonimpl.h | 4 +-
cpukit/rtems/src/eventreceive.c | 4 +-
cpukit/rtems/src/eventseize.c | 6 +-
cpukit/rtems/src/eventsurrender.c | 6 +-
cpukit/rtems/src/systemeventreceive.c | 4 +-
cpukit/rtems/src/tasksetscheduler.c | 18 +-
cpukit/score/include/rtems/score/schedulerimpl.h | 14 +
cpukit/score/include/rtems/score/thread.h | 125 ++----
cpukit/score/include/rtems/score/threadimpl.h | 515 +++++++++++++----------
cpukit/score/include/rtems/score/threadq.h | 97 ++++-
cpukit/score/include/rtems/score/threadqimpl.h | 148 ++++++-
cpukit/score/src/threadchangepriority.c | 30 +-
cpukit/score/src/threadinitialize.c | 10 +-
cpukit/score/src/threadqenqueue.c | 91 +++-
cpukit/score/src/threadqops.c | 63 ++-
cpukit/score/src/threadrestart.c | 3 +-
cpukit/score/src/threadtimeout.c | 30 +-
testsuites/sptests/spthreadq01/init.c | 7 +-
20 files changed, 766 insertions(+), 424 deletions(-)
diff --git a/cpukit/libmisc/monitor/mon-task.c b/cpukit/libmisc/monitor/mon-task.c
index 96891e2..eedba3e 100644
--- a/cpukit/libmisc/monitor/mon-task.c
+++ b/cpukit/libmisc/monitor/mon-task.c
@@ -20,17 +20,16 @@ rtems_monitor_task_wait_info(
Thread_Control *rtems_thread
)
{
- ISR_lock_Context lock_context;
- void *lock;
+ Thread_queue_Context queue_context;
- lock = _Thread_Lock_acquire( rtems_thread, &lock_context );
+ _Thread_Wait_acquire( rtems_thread, &queue_context );
canonical_task->state = rtems_thread->current_state;
canonical_task->wait_id = _Thread_Wait_get_id( rtems_thread );
canonical_task->wait_queue = rtems_thread->Wait.queue;
canonical_task->wait_operations = rtems_thread->Wait.operations;
- _Thread_Lock_release( lock, &lock_context );
+ _Thread_Wait_release( rtems_thread, &queue_context );
}
void
diff --git a/cpukit/posix/src/pthreadgetschedparam.c b/cpukit/posix/src/pthreadgetschedparam.c
index a0a4c6e..1b7c731 100644
--- a/cpukit/posix/src/pthreadgetschedparam.c
+++ b/cpukit/posix/src/pthreadgetschedparam.c
@@ -36,7 +36,7 @@ int pthread_getschedparam(
)
{
Thread_Control *the_thread;
- ISR_lock_Context lock_context;
+ Thread_queue_Context queue_context;
POSIX_API_Control *api;
const Scheduler_Control *scheduler;
Priority_Control priority;
@@ -45,7 +45,7 @@ int pthread_getschedparam(
return EINVAL;
}
- the_thread = _Thread_Get( thread, &lock_context );
+ the_thread = _Thread_Get( thread, &queue_context.Lock_context );
if ( the_thread == NULL ) {
return ESRCH;
@@ -53,7 +53,7 @@ int pthread_getschedparam(
api = the_thread->API_Extensions[ THREAD_API_POSIX ];
- _Thread_Lock_acquire_default_critical( the_thread, &lock_context );
+ _Thread_Wait_acquire_critical( the_thread, &queue_context );
*policy = api->Attributes.schedpolicy;
*param = api->Attributes.schedparam;
@@ -61,7 +61,7 @@ int pthread_getschedparam(
scheduler = _Scheduler_Get_own( the_thread );
priority = the_thread->real_priority;
- _Thread_Lock_release_default( the_thread, &lock_context );
+ _Thread_Wait_release( the_thread, &queue_context );
param->sched_priority = _POSIX_Priority_From_core( scheduler, priority );
return 0;
diff --git a/cpukit/rtems/include/rtems/rtems/ratemonimpl.h b/cpukit/rtems/include/rtems/rtems/ratemonimpl.h
index 61ebb5a..9963cab 100644
--- a/cpukit/rtems/include/rtems/rtems/ratemonimpl.h
+++ b/cpukit/rtems/include/rtems/rtems/ratemonimpl.h
@@ -73,7 +73,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Acquire_critical(
ISR_lock_Context *lock_context
)
{
- _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+ _Thread_Wait_acquire_default_critical( the_thread, lock_context );
}
RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release(
@@ -81,7 +81,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release(
ISR_lock_Context *lock_context
)
{
- _Thread_Lock_release_default( the_thread, lock_context );
+ _Thread_Wait_release_default( the_thread, lock_context );
}
RTEMS_INLINE_ROUTINE Rate_monotonic_Control *_Rate_monotonic_Get(
diff --git a/cpukit/rtems/src/eventreceive.c b/cpukit/rtems/src/eventreceive.c
index e03ff27..193960b 100644
--- a/cpukit/rtems/src/eventreceive.c
+++ b/cpukit/rtems/src/eventreceive.c
@@ -38,7 +38,7 @@ rtems_status_code rtems_event_receive(
RTEMS_API_Control *api;
Event_Control *event;
- executing = _Thread_Lock_acquire_default_for_executing( &lock_context );
+ executing = _Thread_Wait_acquire_default_for_executing( &lock_context );
api = executing->API_Extensions[ THREAD_API_RTEMS ];
event = &api->Event;
@@ -56,7 +56,7 @@ rtems_status_code rtems_event_receive(
);
} else {
*event_out = event->pending_events;
- _Thread_Lock_release_default( executing, &lock_context );
+ _Thread_Wait_release_default( executing, &lock_context );
sc = RTEMS_SUCCESSFUL;
}
} else {
diff --git a/cpukit/rtems/src/eventseize.c b/cpukit/rtems/src/eventseize.c
index 7f5903d..696481c 100644
--- a/cpukit/rtems/src/eventseize.c
+++ b/cpukit/rtems/src/eventseize.c
@@ -50,13 +50,13 @@ rtems_status_code _Event_Seize(
(seized_events == event_in || _Options_Is_any( option_set )) ) {
event->pending_events =
_Event_sets_Clear( pending_events, seized_events );
- _Thread_Lock_release_default( executing, lock_context );
+ _Thread_Wait_release_default( executing, lock_context );
*event_out = seized_events;
return RTEMS_SUCCESSFUL;
}
if ( _Options_Is_no_wait( option_set ) ) {
- _Thread_Lock_release_default( executing, lock_context );
+ _Thread_Wait_release_default( executing, lock_context );
*event_out = seized_events;
return RTEMS_UNSATISFIED;
}
@@ -78,7 +78,7 @@ rtems_status_code _Event_Seize(
_Thread_Wait_flags_set( executing, intend_to_block );
cpu_self = _Thread_Dispatch_disable_critical( lock_context );
- _Thread_Lock_release_default( executing, lock_context );
+ _Thread_Wait_release_default( executing, lock_context );
if ( ticks ) {
_Thread_Timer_insert_relative(
diff --git a/cpukit/rtems/src/eventsurrender.c b/cpukit/rtems/src/eventsurrender.c
index a9bef59..c805b0e 100644
--- a/cpukit/rtems/src/eventsurrender.c
+++ b/cpukit/rtems/src/eventsurrender.c
@@ -77,7 +77,7 @@ rtems_status_code _Event_Surrender(
rtems_event_set seized_events;
bool unblock;
- _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+ _Thread_Wait_acquire_default_critical( the_thread, lock_context );
_Event_sets_Post( event_in, &event->pending_events );
pending_events = event->pending_events;
@@ -116,14 +116,14 @@ rtems_status_code _Event_Surrender(
Per_CPU_Control *cpu_self;
cpu_self = _Thread_Dispatch_disable_critical( lock_context );
- _Thread_Lock_release_default( the_thread, lock_context );
+ _Thread_Wait_release_default( the_thread, lock_context );
_Thread_Timer_remove( the_thread );
_Thread_Unblock( the_thread );
_Thread_Dispatch_enable( cpu_self );
} else {
- _Thread_Lock_release_default( the_thread, lock_context );
+ _Thread_Wait_release_default( the_thread, lock_context );
}
return RTEMS_SUCCESSFUL;
diff --git a/cpukit/rtems/src/systemeventreceive.c b/cpukit/rtems/src/systemeventreceive.c
index a2215fa..93a5694 100644
--- a/cpukit/rtems/src/systemeventreceive.c
+++ b/cpukit/rtems/src/systemeventreceive.c
@@ -44,7 +44,7 @@ rtems_status_code rtems_event_system_receive(
RTEMS_API_Control *api;
Event_Control *event;
- executing = _Thread_Lock_acquire_default_for_executing( &lock_context );
+ executing = _Thread_Wait_acquire_default_for_executing( &lock_context );
api = executing->API_Extensions[ THREAD_API_RTEMS ];
event = &api->System_event;
@@ -62,7 +62,7 @@ rtems_status_code rtems_event_system_receive(
);
} else {
*event_out = event->pending_events;
- _Thread_Lock_release_default( executing, &lock_context );
+ _Thread_Wait_release_default( executing, &lock_context );
sc = RTEMS_SUCCESSFUL;
}
} else {
diff --git a/cpukit/rtems/src/tasksetscheduler.c b/cpukit/rtems/src/tasksetscheduler.c
index 175f235..0ff74d9 100644
--- a/cpukit/rtems/src/tasksetscheduler.c
+++ b/cpukit/rtems/src/tasksetscheduler.c
@@ -28,10 +28,9 @@ rtems_status_code rtems_task_set_scheduler(
{
const Scheduler_Control *scheduler;
Thread_Control *the_thread;
- ISR_lock_Context lock_context;
- ISR_lock_Context state_lock_context;
+ Thread_queue_Context queue_context;
+ ISR_lock_Context state_context;
Per_CPU_Control *cpu_self;
- void *lock;
bool valid;
Priority_Control core_priority;
Status_Control status;
@@ -45,7 +44,7 @@ rtems_status_code rtems_task_set_scheduler(
return RTEMS_INVALID_PRIORITY;
}
- the_thread = _Thread_Get( task_id, &lock_context );
+ the_thread = _Thread_Get( task_id, &queue_context.Lock_context );
if ( the_thread == NULL ) {
#if defined(RTEMS_MULTIPROCESSING)
@@ -57,16 +56,15 @@ rtems_status_code rtems_task_set_scheduler(
return RTEMS_INVALID_ID;
}
- cpu_self = _Thread_Dispatch_disable_critical( &lock_context );
- _ISR_lock_ISR_enable( &lock_context );
+ cpu_self = _Thread_Dispatch_disable_critical( &queue_context.Lock_context );
- lock = _Thread_Lock_acquire( the_thread, &lock_context );
- _Thread_State_acquire_critical( the_thread, &state_lock_context );
+ _Thread_Wait_acquire_critical( the_thread, &queue_context );
+ _Thread_State_acquire_critical( the_thread, &state_context );
status = _Scheduler_Set( scheduler, the_thread, core_priority );
- _Thread_State_release_critical( the_thread, &state_lock_context );
- _Thread_Lock_release( lock, &lock_context );
+ _Thread_State_release_critical( the_thread, &state_context );
+ _Thread_Wait_release( the_thread, &queue_context );
_Thread_Dispatch_enable( cpu_self );
return _Status_Get( status );
}
diff --git a/cpukit/score/include/rtems/score/schedulerimpl.h b/cpukit/score/include/rtems/score/schedulerimpl.h
index db4be99..f8c29e2 100644
--- a/cpukit/score/include/rtems/score/schedulerimpl.h
+++ b/cpukit/score/include/rtems/score/schedulerimpl.h
@@ -825,6 +825,20 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Node_set_priority(
#endif
}
+RTEMS_INLINE_ROUTINE void _Scheduler_Thread_set_priority(
+ Thread_Control *the_thread,
+ Priority_Control new_priority,
+ bool prepend_it
+)
+{
+ Scheduler_Node *own_node;
+
+ own_node = _Scheduler_Thread_get_own_node( the_thread );
+ _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
+
+ the_thread->current_priority = new_priority;
+}
+
#if defined(RTEMS_SMP)
/**
* @brief Gets an idle thread from the scheduler instance.
diff --git a/cpukit/score/include/rtems/score/thread.h b/cpukit/score/include/rtems/score/thread.h
index 46c222f..85d759c 100644
--- a/cpukit/score/include/rtems/score/thread.h
+++ b/cpukit/score/include/rtems/score/thread.h
@@ -11,7 +11,7 @@
* COPYRIGHT (c) 1989-2014.
* On-Line Applications Research Corporation (OAR).
*
- * Copyright (c) 2014 embedded brains GmbH.
+ * Copyright (c) 2014, 2016 embedded brains GmbH.
*
* The license and distribution terms for this file may be
* found in the file LICENSE in this distribution or at
@@ -288,16 +288,6 @@ typedef struct {
uint32_t return_code;
/**
- * @brief The current thread queue.
- *
- * In case this field is @c NULL, then the thread is not blocked on a thread
- * queue. This field is protected by the thread lock.
- *
- * @see _Thread_Lock_set() and _Thread_Wait_set_queue().
- */
- Thread_queue_Queue *queue;
-
- /**
* @brief This field contains several flags used to control the wait class
* and state of a thread in case fine-grained locking is used.
*/
@@ -307,12 +297,54 @@ typedef struct {
Thread_Wait_flags flags;
#endif
+#if defined(RTEMS_SMP)
+ /**
+ * @brief Thread wait lock control block.
+ *
+ * Parts of the thread wait information is protected by the thread wait
+ * default lock and additionally a thread queue lock in case the thread
+ * currently blocks on a thread queue.
+ *
+ * The thread wait lock mechanism protects the following thread variables
+ * - POSIX_API_Control::Attributes,
+ * - Thread_Control::current_priority,
+ * - Thread_Control::Wait::Lock::Pending_requests,
+ * - Thread_Control::Wait::queue, and
+ * - Thread_Control::Wait::operations.
+ *
+ * @see _Thread_Wait_acquire(), _Thread_Wait_release(), _Thread_Wait_claim()
+ * and _Thread_Wait_restore_default().
+ */
+ struct {
+ /**
+ * @brief Thread wait default lock.
+ */
+ ISR_lock_Control Default;
+
+ /**
+ * @brief The pending thread wait lock acquire or tranquilize requests in
+ * case the thread is blocked on a thread queue.
+ */
+ Chain_Control Pending_requests;
+ } Lock;
+#endif
+
+ /**
+ * @brief The current thread queue.
+ *
+ * In case this field is @c NULL, then the thread is not blocked on a thread
+ * queue. This field is protected by the thread wait default lock.
+ *
+ * @see _Thread_Wait_claim().
+ */
+ Thread_queue_Queue *queue;
+
/**
* @brief The current thread queue operations.
*
- * This field is protected by the thread lock.
+ * This field is protected by the thread lock wait default lock.
*
- * @see _Thread_Lock_set() and _Thread_Wait_set_operations().
+ * @see _Thread_Wait_claim().
*/
const Thread_queue_Operations *operations;
@@ -639,66 +671,6 @@ typedef struct {
void * control;
}Thread_Capture_control;
-#if defined(RTEMS_SMP)
-/**
- * @brief Thread lock control.
- *
- * The thread lock is either the default lock or the lock of the resource on
- * which the thread is currently blocked. The generation number takes care
- * that the up to date lock is used. Only resources using fine grained locking
- * provide their own lock.
- *
- * The thread lock protects the following thread variables
- * - POSIX_API_Control::Attributes,
- * - Thread_Control::current_priority,
- * - Thread_Control::Wait::queue, and
- * - Thread_Control::Wait::operations.
- *
- * @see _Thread_Lock_acquire(), _Thread_Lock_release(), _Thread_Lock_set() and
- * _Thread_Lock_restore_default().
- */
-typedef struct {
- /**
- * @brief The current thread lock.
- *
- * This is a plain ticket lock without SMP lock statistics support. This
- * enables external libraries to use thread locks since they are independent
- * of the actual RTEMS build configuration, e.g. profiling enabled or
- * disabled.
- */
- union {
- /**
- * @brief The current thread lock as an atomic unsigned integer pointer value.
- */
- Atomic_Uintptr atomic;
-
- /**
- * @brief The current thread lock as a normal pointer.
- *
- * Only provided for debugging purposes.
- */
- SMP_ticket_lock_Control *normal;
- } current;
-
- /**
- * @brief The default thread lock in case the thread is not blocked on a
- * resource.
- */
- SMP_ticket_lock_Control Default;
-
-#if defined(RTEMS_PROFILING)
- /**
- * @brief The thread lock statistics.
- *
- * These statistics are used by the executing thread in case it acquires a
- * thread lock. Thus the statistics are an aggregation of acquire and
- * release operations of different locks.
- */
- SMP_lock_Stats Stats;
-#endif
-} Thread_Lock_control;
-#endif
-
/**
* This structure defines the Thread Control Block (TCB).
*
@@ -770,13 +742,6 @@ struct _Thread_Control {
#endif
/*================= end of common block =================*/
-#if defined(RTEMS_SMP)
- /**
- * @brief Thread lock control.
- */
- Thread_Lock_control Lock;
-#endif
-
#if defined(RTEMS_SMP) && defined(RTEMS_PROFILING)
/**
* @brief Potpourri lock statistics.
diff --git a/cpukit/score/include/rtems/score/threadimpl.h b/cpukit/score/include/rtems/score/threadimpl.h
index 92968a2..c42292f 100644
--- a/cpukit/score/include/rtems/score/threadimpl.h
+++ b/cpukit/score/include/rtems/score/threadimpl.h
@@ -11,7 +11,7 @@
* COPYRIGHT (c) 1989-2008.
* On-Line Applications Research Corporation (OAR).
*
- * Copyright (c) 2014-2015 embedded brains GmbH.
+ * Copyright (c) 2014, 2016 embedded brains GmbH.
*
* The license and distribution terms for this file may be
* found in the file LICENSE in this distribution or at
@@ -933,44 +933,35 @@ RTEMS_INLINE_ROUTINE bool _Thread_Owns_resources(
}
/**
- * @brief Acquires the default thread lock inside a critical section
+ * @brief Acquires the thread wait default lock inside a critical section
* (interrupts disabled).
*
* @param[in] the_thread The thread.
* @param[in] lock_context The lock context used for the corresponding lock
- * release.
+ * release.
*
- * @see _Thread_Lock_release_default().
+ * @see _Thread_Wait_release_default_critical().
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default_critical(
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default_critical(
Thread_Control *the_thread,
ISR_lock_Context *lock_context
)
{
- _Assert( _ISR_Get_level() != 0 );
-#if defined(RTEMS_SMP)
- _SMP_ticket_lock_Acquire(
- &the_thread->Lock.Default,
- &_Thread_Executing->Lock.Stats,
- &lock_context->Lock_context.Stats_context
- );
-#else
- (void) the_thread;
- (void) lock_context;
-#endif
+ _ISR_lock_Acquire( &the_thread->Wait.Lock.Default, lock_context );
}
/**
- * @brief Acquires the default thread lock and returns the executing thread.
+ * @brief Acquires the thread wait default lock and returns the executing
+ * thread.
*
* @param[in] lock_context The lock context used for the corresponding lock
- * release.
+ * release.
*
* @return The executing thread.
*
- * @see _Thread_Lock_release_default().
+ * @see _Thread_Wait_release_default().
*/
-RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing(
+RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Wait_acquire_default_for_executing(
ISR_lock_Context *lock_context
)
{
@@ -978,247 +969,398 @@ RTEMS_INLINE_ROUTINE Thread_Control *_Thread_Lock_acquire_default_for_executing(
_ISR_lock_ISR_disable( lock_context );
executing = _Thread_Executing;
- _Thread_Lock_acquire_default_critical( executing, lock_context );
+ _Thread_Wait_acquire_default_critical( executing, lock_context );
return executing;
}
/**
- * @brief Acquires the default thread lock.
+ * @brief Acquires the thread wait default lock and disables interrupts.
*
* @param[in] the_thread The thread.
* @param[in] lock_context The lock context used for the corresponding lock
- * release.
+ * release.
*
- * @see _Thread_Lock_release_default().
+ * @see _Thread_Wait_release_default().
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_acquire_default(
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_default(
Thread_Control *the_thread,
ISR_lock_Context *lock_context
)
{
_ISR_lock_ISR_disable( lock_context );
- _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+ _Thread_Wait_acquire_default_critical( the_thread, lock_context );
}
/**
- * @brief Releases the thread lock inside a critical section (interrupts
- * disabled).
+ * @brief Releases the thread wait default lock inside a critical section
+ * (interrupts disabled).
*
* The previous interrupt status is not restored.
*
- * @param[in] lock The lock.
+ * @param[in] the_thread The thread.
* @param[in] lock_context The lock context used for the corresponding lock
- * acquire.
+ * acquire.
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release_critical(
- void *lock,
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default_critical(
+ Thread_Control *the_thread,
ISR_lock_Context *lock_context
)
{
-#if defined(RTEMS_SMP)
- _SMP_ticket_lock_Release(
- (SMP_ticket_lock_Control *) lock,
- &lock_context->Lock_context.Stats_context
- );
-#else
- (void) lock;
- (void) lock_context;
-#endif
+ _ISR_lock_Release( &the_thread->Wait.Lock.Default, lock_context );
}
/**
- * @brief Releases the thread lock.
+ * @brief Releases the thread wait default lock and restores the previous
+ * interrupt status.
*
- * @param[in] lock The lock returned by _Thread_Lock_acquire().
- * @param[in] lock_context The lock context used for _Thread_Lock_acquire().
+ * @param[in] the_thread The thread.
+ * @param[in] lock_context The lock context used for the corresponding lock
+ * acquire.
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release(
- void *lock,
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_default(
+ Thread_Control *the_thread,
ISR_lock_Context *lock_context
)
{
- _Thread_Lock_release_critical( lock, lock_context );
+ _Thread_Wait_release_default_critical( the_thread, lock_context );
_ISR_lock_ISR_enable( lock_context );
}
+#if defined(RTEMS_SMP)
+#define THREAD_QUEUE_CONTEXT_OF_REQUEST( node ) \
+ RTEMS_CONTAINER_OF( node, Thread_queue_Context, Wait.Gate.Node )
+
+RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request_locked(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
+)
+{
+ _Chain_Extract_unprotected( &queue_context->Wait.Gate.Node );
+
+ if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) {
+ Thread_queue_Context *first;
+
+ first = THREAD_QUEUE_CONTEXT_OF_REQUEST(
+ _Chain_First( &the_thread->Wait.Lock.Pending_requests )
+ );
+
+ _Thread_queue_Gate_open( &first->Wait.Gate );
+ }
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_queue_critical(
+ SMP_ticket_lock_Control *queue_lock,
+ Thread_queue_Context *queue_context
+)
+{
+ _SMP_ticket_lock_Acquire(
+ queue_lock,
+ &_Thread_Executing->Potpourri_stats,
+ &queue_context->Lock_context.Lock_context.Stats_context
+ );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_queue_critical(
+ SMP_ticket_lock_Control *queue_lock,
+ Thread_queue_Context *queue_context
+)
+{
+ _SMP_ticket_lock_Release(
+ queue_lock,
+ &queue_context->Lock_context.Lock_context.Stats_context
+ );
+}
+#endif
+
/**
- * @brief Releases the default thread lock inside a critical section
- * (interrupts disabled).
- *
- * The previous interrupt status is not restored.
+ * @brief Acquires the thread wait lock inside a critical section (interrupts
+ * disabled).
*
* @param[in] the_thread The thread.
- * @param[in] lock_context The lock context used for the corresponding lock
- * acquire.
+ * @param[in] queue_context The thread queue context for the corresponding
+ * _Thread_Wait_release_critical().
+ *
+ * @see _Thread_queue_Context_initialize().
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default_critical(
- Thread_Control *the_thread,
- ISR_lock_Context *lock_context
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire_critical(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
)
{
- _Thread_Lock_release_critical(
#if defined(RTEMS_SMP)
- &the_thread->Lock.Default,
+ Thread_queue_Queue *queue;
+
+ _Thread_Wait_acquire_default_critical(
+ the_thread,
+ &queue_context->Lock_context
+ );
+
+ queue = the_thread->Wait.queue;
+ queue_context->Wait.queue = queue;
+ queue_context->Wait.operations = the_thread->Wait.operations;
+
+ if ( queue != NULL ) {
+ queue_context->Wait.queue_lock = &queue->Lock;
+ _Chain_Initialize_node( &queue_context->Wait.Gate.Node );
+ _Chain_Append_unprotected(
+ &the_thread->Wait.Lock.Pending_requests,
+ &queue_context->Wait.Gate.Node
+ );
+ _Thread_Wait_release_default_critical(
+ the_thread,
+ &queue_context->Lock_context
+ );
+ _Thread_Wait_acquire_queue_critical( &queue->Lock, queue_context );
+ } else {
+ queue_context->Wait.queue_lock = NULL;
+ }
#else
- NULL,
+ (void) the_thread;
+ (void) queue_context;
#endif
- lock_context
- );
}
/**
- * @brief Releases the default thread lock.
+ * @brief Acquires the thread wait default lock and disables interrupts.
*
* @param[in] the_thread The thread.
- * @param[in] lock_context The lock context used for the corresponding lock
- * acquire.
+ * @param[in] queue_context The thread queue context for the corresponding
+ * _Thread_Wait_release().
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_release_default(
- Thread_Control *the_thread,
- ISR_lock_Context *lock_context
+RTEMS_INLINE_ROUTINE void _Thread_Wait_acquire(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
)
{
- _Thread_Lock_release_default_critical( the_thread, lock_context );
- _ISR_lock_ISR_enable( lock_context );
+ _Thread_queue_Context_initialize( queue_context );
+ _ISR_lock_ISR_disable( &queue_context->Lock_context );
+ _Thread_Wait_acquire_critical( the_thread, queue_context );
}
/**
- * @brief Acquires the thread lock.
+ * @brief Releases the thread wait lock inside a critical section (interrupts
+ * disabled).
*
- * @param[in] the_thread The thread.
- * @param[in] lock_context The lock context for _Thread_Lock_release().
+ * The previous interrupt status is not restored.
*
- * @return The lock required by _Thread_Lock_release().
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ * _Thread_Wait_acquire_critical().
*/
-RTEMS_INLINE_ROUTINE void *_Thread_Lock_acquire(
- Thread_Control *the_thread,
- ISR_lock_Context *lock_context
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release_critical(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
)
{
#if defined(RTEMS_SMP)
- SMP_ticket_lock_Control *lock_0;
-
- while ( true ) {
- SMP_ticket_lock_Control *lock_1;
-
- _ISR_lock_ISR_disable( lock_context );
-
- /*
- * We assume that a normal load of pointer is identical to a relaxed atomic
- * load. Here, we may read an out-of-date lock. However, only the owner
- * of this out-of-date lock is allowed to set a new one. Thus, we read at
- * least this new lock ...
- */
- lock_0 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr(
- &the_thread->Lock.current.atomic,
- ATOMIC_ORDER_RELAXED
- );
+ SMP_ticket_lock_Control *queue_lock;
- _SMP_ticket_lock_Acquire(
- lock_0,
- &_Thread_Executing->Lock.Stats,
- &lock_context->Lock_context.Stats_context
- );
+ queue_lock = queue_context->Wait.queue_lock;
- /*
- * We must use a load acquire here paired with the store release in
- * _Thread_Lock_set_unprotected() to observe corresponding thread wait
- * queue and thread wait operations. It is important to do this after the
- * lock acquire, since we may have the following scenario.
- *
- * - We read the default lock and try to acquire it.
- * - The thread lock changes to a thread queue lock.
- * - The thread lock is restored to the default lock.
- * - We acquire the default lock and read it here again.
- * - Now, we must read the restored default thread wait queue and thread
- * wait operations and this is not synchronized via the default thread
- * lock.
- */
- lock_1 = (SMP_ticket_lock_Control *) _Atomic_Load_uintptr(
- &the_thread->Lock.current.atomic,
- ATOMIC_ORDER_ACQUIRE
+ if ( queue_lock != NULL ) {
+ _Thread_Wait_release_queue_critical( queue_lock, queue_context );
+ _Thread_Wait_acquire_default_critical(
+ the_thread,
+ &queue_context->Lock_context
);
- /*
- * ... here, and so on.
- */
- if ( lock_0 == lock_1 ) {
- return lock_0;
- }
-
- _Thread_Lock_release( lock_0, lock_context );
+ _Thread_Wait_remove_request_locked( the_thread, queue_context );
}
-#else
- _ISR_Local_disable( lock_context->isr_level );
- return NULL;
+ _Thread_Wait_release_default_critical(
+ the_thread,
+ &queue_context->Lock_context
+ );
+#else
+ (void) the_thread;
+ (void) queue_context;
#endif
}
-#if defined(RTEMS_SMP)
-/*
- * Internal function, use _Thread_Lock_set() or _Thread_Lock_restore_default()
- * instead.
+/**
+ * @brief Releases the thread wait lock and restores the previous interrupt
+ * status.
+ *
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ * _Thread_Wait_acquire().
*/
-RTEMS_INLINE_ROUTINE void _Thread_Lock_set_unprotected(
- Thread_Control *the_thread,
- SMP_ticket_lock_Control *new_lock
+RTEMS_INLINE_ROUTINE void _Thread_Wait_release(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
)
{
- _Atomic_Store_uintptr(
- &the_thread->Lock.current.atomic,
- (uintptr_t) new_lock,
- ATOMIC_ORDER_RELEASE
- );
+ _Thread_Wait_release_critical( the_thread, queue_context );
+ _ISR_lock_ISR_enable( &queue_context->Lock_context );
}
-#endif
/**
- * @brief Sets a new thread lock.
+ * @brief Claims the thread wait queue and operations.
*
- * The caller must not be the owner of the default thread lock. The caller
- * must be the owner of the new lock.
+ * The caller must not be the owner of the default thread wait lock. The
+ * caller must be the owner of the corresponding thread queue lock.
*
* @param[in] the_thread The thread.
- * @param[in] new_lock The new thread lock.
+ * @param[in] queue The new thread queue.
+ * @param[in] operations The new thread operations.
+ *
+ * @see _Thread_Wait_restore_default().
*/
-#if defined(RTEMS_SMP)
-RTEMS_INLINE_ROUTINE void _Thread_Lock_set(
- Thread_Control *the_thread,
- SMP_ticket_lock_Control *new_lock
+RTEMS_INLINE_ROUTINE void _Thread_Wait_claim(
+ Thread_Control *the_thread,
+ Thread_queue_Queue *queue,
+ const Thread_queue_Operations *operations
)
{
ISR_lock_Context lock_context;
- _Thread_Lock_acquire_default_critical( the_thread, &lock_context );
- _Assert( the_thread->Lock.current.normal == &the_thread->Lock.Default );
- _Thread_Lock_set_unprotected( the_thread, new_lock );
- _Thread_Lock_release_default_critical( the_thread, &lock_context );
+ _Thread_Wait_acquire_default_critical( the_thread, &lock_context );
+
+ _Assert( the_thread->Wait.queue == NULL );
+ the_thread->Wait.queue = queue;
+ the_thread->Wait.operations = operations;
+
+ _Thread_Wait_release_default_critical( the_thread, &lock_context );
}
+
+/**
+ * @brief Removes a thread wait lock request.
+ *
+ * On SMP configurations, removes a thread wait lock request.
+ *
+ * On other configurations, this function does nothing.
+ *
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ * _Thread_Wait_acquire().
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Wait_remove_request(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
+)
+{
+#if defined(RTEMS_SMP)
+ ISR_lock_Context lock_context;
+
+ _Thread_Wait_acquire_default( the_thread, &lock_context );
+ _Thread_Wait_remove_request_locked( the_thread, queue_context );
+ _Thread_Wait_release_default( the_thread, &lock_context );
#else
-#define _Thread_Lock_set( the_thread, new_lock ) \
- do { } while ( 0 )
+ (void) the_thread;
+ (void) queue_context;
#endif
+}
/**
- * @brief Restores the default thread lock.
+ * @brief Restores the default thread wait queue and operations.
*
- * The caller must be the owner of the current thread lock.
+ * The caller must be the owner of the current thread wait queue lock.
+ *
+ * On SMP configurations, the pending requests are updated to use the stale
+ * thread queue operations.
*
* @param[in] the_thread The thread.
+ *
+ * @see _Thread_Wait_claim().
*/
-#if defined(RTEMS_SMP)
-RTEMS_INLINE_ROUTINE void _Thread_Lock_restore_default(
+RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default(
Thread_Control *the_thread
)
{
- _Thread_Lock_set_unprotected( the_thread, &the_thread->Lock.Default );
+#if defined(RTEMS_SMP)
+ ISR_lock_Context lock_context;
+ Chain_Node *node;
+ const Chain_Node *tail;
+
+ _Thread_Wait_acquire_default_critical(
+ the_thread,
+ &lock_context
+ );
+
+ node = _Chain_First( &the_thread->Wait.Lock.Pending_requests );
+ tail = _Chain_Immutable_tail( &the_thread->Wait.Lock.Pending_requests );
+
+ while ( node != tail ) {
+ Thread_queue_Context *queue_context;
+
+ queue_context = THREAD_QUEUE_CONTEXT_OF_REQUEST( node );
+ queue_context->Wait.queue = NULL;
+ queue_context->Wait.operations = &_Thread_queue_Operations_stale_queue;
+
+ node = _Chain_Next( node );
+ }
+#endif
+
+ the_thread->Wait.queue = NULL;
+ the_thread->Wait.operations = &_Thread_queue_Operations_default;
+
+#if defined(RTEMS_SMP)
+ _Thread_Wait_release_default_critical(
+ the_thread,
+ &lock_context
+ );
+#endif
}
+
+/**
+ * @brief Tranquilizes a thread wait on a thread queue procedure.
+ *
+ * On SMP configurations, ensures that all pending thread wait lock requests
+ * completed before the thread is able to begin a new thread wait procedure.
+ *
+ * On other configurations, this function does nothing.
+ *
+ * @param[in] the_thread The thread.
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Wait_tranquilize(
+ Thread_Control *the_thread
+)
+{
+#if defined(RTEMS_SMP)
+ Thread_queue_Context queue_context;
+
+ _Thread_queue_Context_initialize( &queue_context );
+ _Thread_Wait_acquire_default( the_thread, &queue_context.Lock_context );
+
+ if ( !_Chain_Is_empty( &the_thread->Wait.Lock.Pending_requests ) ) {
+ _Thread_queue_Gate_add(
+ &the_thread->Wait.Lock.Pending_requests,
+ &queue_context.Wait.Gate
+ );
+ _Thread_Wait_release_default( the_thread, &queue_context.Lock_context );
+ _Thread_queue_Gate_wait( &queue_context.Wait.Gate );
+ _Thread_Wait_remove_request( the_thread, &queue_context );
+ } else {
+ _Thread_Wait_release_default( the_thread, &queue_context.Lock_context );
+ }
#else
-#define _Thread_Lock_restore_default( the_thread ) \
- do { } while ( 0 )
+ (void) the_thread;
#endif
+}
+
+/**
+ * @brief Cancels a thread wait on a thread queue.
+ *
+ * @param[in] the_thread The thread.
+ * @param[in] queue_context The thread queue context used for corresponding
+ * _Thread_Wait_acquire().
+ */
+RTEMS_INLINE_ROUTINE void _Thread_Wait_cancel(
+ Thread_Control *the_thread,
+ Thread_queue_Context *queue_context
+)
+{
+ _Thread_queue_Context_extract( queue_context, the_thread );
+
+#if defined(RTEMS_SMP)
+ if ( queue_context->Wait.queue != NULL ) {
+#endif
+ _Thread_Wait_restore_default( the_thread );
+#if defined(RTEMS_SMP)
+ }
+#endif
+}
/**
* @brief The initial thread wait flags value set by _Thread_Initialize().
@@ -1388,58 +1530,6 @@ RTEMS_INLINE_ROUTINE bool _Thread_Wait_flags_try_change_acquire(
}
/**
- * @brief Sets the thread queue.
- *
- * The caller must be the owner of the thread lock.
- *
- * @param[in] the_thread The thread.
- * @param[in] new_queue The new queue.
- *
- * @see _Thread_Lock_set().
- */
-RTEMS_INLINE_ROUTINE void _Thread_Wait_set_queue(
- Thread_Control *the_thread,
- Thread_queue_Queue *new_queue
-)
-{
- the_thread->Wait.queue = new_queue;
-}
-
-/**
- * @brief Sets the thread queue operations.
- *
- * The caller must be the owner of the thread lock.
- *
- * @param[in] the_thread The thread.
- * @param[in] new_operations The new queue operations.
- *
- * @see _Thread_Lock_set() and _Thread_Wait_restore_default_operations().
- */
-RTEMS_INLINE_ROUTINE void _Thread_Wait_set_operations(
- Thread_Control *the_thread,
- const Thread_queue_Operations *new_operations
-)
-{
- the_thread->Wait.operations = new_operations;
-}
-
-/**
- * @brief Restores the default thread queue operations.
- *
- * The caller must be the owner of the thread lock.
- *
- * @param[in] the_thread The thread.
- *
- * @see _Thread_Wait_set_operations().
- */
-RTEMS_INLINE_ROUTINE void _Thread_Wait_restore_default_operations(
- Thread_Control *the_thread
-)
-{
- the_thread->Wait.operations = &_Thread_queue_Operations_default;
-}
-
-/**
* @brief Returns the object identifier of the object containing the current
* thread wait queue.
*
@@ -1541,6 +1631,7 @@ RTEMS_INLINE_ROUTINE void _Thread_Remove_timer_and_unblock(
Thread_queue_Queue *queue
)
{
+ _Thread_Wait_tranquilize( the_thread );
_Thread_Timer_remove( the_thread );
#if defined(RTEMS_MULTIPROCESSING)
diff --git a/cpukit/score/include/rtems/score/threadq.h b/cpukit/score/include/rtems/score/threadq.h
index 75aab20..9a17804 100644
--- a/cpukit/score/include/rtems/score/threadq.h
+++ b/cpukit/score/include/rtems/score/threadq.h
@@ -43,6 +43,10 @@ extern "C" {
typedef struct _Thread_Control Thread_Control;
+typedef struct Thread_queue_Queue Thread_queue_Queue;
+
+typedef struct Thread_queue_Operations Thread_queue_Operations;
+
typedef struct Thread_queue_Path Thread_queue_Path;
#if defined(RTEMS_MULTIPROCESSING)
@@ -53,6 +57,8 @@ typedef struct Thread_queue_Path Thread_queue_Path;
* control is actually a thread proxy if and only if
* _Objects_Is_local_id( the_proxy->Object.id ) is false.
* @param mp_id Object identifier of the object containing the thread queue.
+ *
+ * @see _Thread_queue_Context_set_MP_callout().
*/
typedef void ( *Thread_queue_MP_callout )(
Thread_Control *the_proxy,
@@ -60,6 +66,24 @@ typedef void ( *Thread_queue_MP_callout )(
);
#endif
+#if defined(RTEMS_SMP)
+/**
+ * @brief The thread queue gate is an SMP synchronization means.
+ *
+ * The gates are added to a list of requests. A busy wait is performed to make
+ * sure that preceding requests are carried out. Each predecessor notifies its
+ * successor about on request completion.
+ *
+ * @see _Thread_queue_Gate_add(), _Thread_queue_Gate_wait(), and
+ * _Thread_queue_Gate_open().
+ */
+typedef struct {
+ Chain_Node Node;
+
+ Atomic_Uint go_ahead;
+} Thread_queue_Gate;
+#endif
+
/**
* @brief Thread queue context for the thread queue methods.
*
@@ -105,8 +129,64 @@ typedef struct {
#if defined(RTEMS_MULTIPROCESSING)
Thread_queue_MP_callout mp_callout;
#endif
+
+#if defined(RTEMS_SMP)
+ /**
+ * @brief Data to support thread queue enqueue operations.
+ */
+ struct {
+ /**
+ * @brief Gate to synchronize thread wait lock requests.
+ *
+ * @see _Thread_Wait_acquire_critical() and _Thread_Wait_tranquilize().
+ */
+ Thread_queue_Gate Gate;
+
+ /**
+ * @brief The thread queue lock in case the thread is blocked on a thread
+ * queue at thread wait lock acquire time.
+ */
+ SMP_ticket_lock_Control *queue_lock;
+
+ /**
+ * @brief The thread queue after thread wait lock acquire.
+ *
+ * In case the thread queue is NULL and the thread queue lock is non-NULL
+ * in this context, then we have a stale thread queue. This happens in
+ * case the thread wait default is restored while we wait on the thread
+ * queue lock, e.g. during a mutex ownership transfer.
+ *
+ * @see _Thread_Wait_restore_default().
+ */
+ Thread_queue_Queue *queue;
+
+ /**
+ * @brief The thread queue operations after thread wait lock acquire.
+ */
+ const Thread_queue_Operations *operations;
+ } Wait;
+#endif
} Thread_queue_Context;
+#if defined(RTEMS_SMP)
+/**
+ * @brief A thread queue link from one thread to another specified by the
+ * thread queue owner and thread wait queue relationships.
+ */
+typedef struct {
+ /**
+ * @brief The owner of this thread queue link.
+ */
+ Thread_Control *owner;
+
+ /**
+ * @brief The queue context used to acquire the thread wait lock of the
+ * owner.
+ */
+ Thread_queue_Context Queue_context;
+} Thread_queue_Link;
+#endif
+
/**
* @brief Thread priority queue.
*/
@@ -191,7 +271,7 @@ typedef struct _Thread_queue_Heads {
sizeof( Thread_queue_Heads )
#endif
-typedef struct {
+struct Thread_queue_Queue {
/**
* @brief Lock to protect this thread queue.
*
@@ -221,13 +301,15 @@ typedef struct {
* @brief The thread queue owner.
*/
Thread_Control *owner;
-} Thread_queue_Queue;
+};
/**
* @brief Thread queue priority change operation.
*
* @param[in] the_thread The thread.
* @param[in] new_priority The new priority value.
+ * @param[in] prepend_it In case this is true, then the thread is prepended to
+ * its priority group in its scheduler instance, otherwise it is appended.
* @param[in] queue The actual thread queue.
*
* @see Thread_queue_Operations.
@@ -235,6 +317,7 @@ typedef struct {
typedef void ( *Thread_queue_Priority_change_operation )(
Thread_Control *the_thread,
Priority_Control new_priority,
+ bool prepend_it,
Thread_queue_Queue *queue
);
@@ -247,8 +330,6 @@ typedef void ( *Thread_queue_Priority_change_operation )(
*
* @param[in] queue The actual thread queue.
* @param[in] the_thread The thread to enqueue on the queue.
- *
- * @see _Thread_Wait_set_operations().
*/
typedef void ( *Thread_queue_Enqueue_operation )(
Thread_queue_Queue *queue,
@@ -261,8 +342,6 @@ typedef void ( *Thread_queue_Enqueue_operation )(
*
* @param[in] queue The actual thread queue.
* @param[in] the_thread The thread to extract from the thread queue.
- *
- * @see _Thread_Wait_set_operations().
*/
typedef void ( *Thread_queue_Extract_operation )(
Thread_queue_Queue *queue,
@@ -277,8 +356,6 @@ typedef void ( *Thread_queue_Extract_operation )(
* @retval NULL No thread is present on the thread queue.
* @retval first The first thread of the thread queue according to the insert
* order. This thread remains on the thread queue.
- *
- * @see _Thread_Wait_set_operations().
*/
typedef Thread_Control *( *Thread_queue_First_operation )(
Thread_queue_Heads *heads
@@ -289,7 +366,7 @@ typedef Thread_Control *( *Thread_queue_First_operation )(
*
* @see _Thread_wait_Set_operations().
*/
-typedef struct {
+struct Thread_queue_Operations {
/**
* @brief Thread queue priority change operation.
*
@@ -320,7 +397,7 @@ typedef struct {
* @brief Thread queue first operation.
*/
Thread_queue_First_operation first;
-} Thread_queue_Operations;
+};
/**
* This is the structure used to manage sets of tasks which are blocked
diff --git a/cpukit/score/include/rtems/score/threadqimpl.h b/cpukit/score/include/rtems/score/threadqimpl.h
index 1502b09..11dc6c5 100644
--- a/cpukit/score/include/rtems/score/threadqimpl.h
+++ b/cpukit/score/include/rtems/score/threadqimpl.h
@@ -26,6 +26,10 @@
#include <rtems/score/smp.h>
#include <rtems/score/thread.h>
+#if defined(RTEMS_DEBUG)
+#include <string.h>
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -46,6 +50,13 @@ extern "C" {
* relationships.
*/
struct Thread_queue_Path {
+#if defined(RTEMS_SMP)
+ /**
+ * @brief The start of a thread queue path.
+ */
+ Thread_queue_Link Start;
+#endif
+
/**
* @brief A potential thread to update the priority via
* _Thread_Update_priority().
@@ -84,10 +95,8 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_initialize(
)
{
#if defined(RTEMS_DEBUG)
+ memset( queue_context, 0, sizeof( *queue_context ) );
queue_context->expected_thread_dispatch_disable_level = 0xdeadbeef;
-#if defined(RTEMS_MULTIPROCESSING)
- queue_context->mp_callout = NULL;
-#endif
#else
(void) queue_context;
#endif
@@ -187,6 +196,135 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_set_MP_callout(
} while ( 0 )
#endif
+/**
+ * @brief Gets the thread wait queue of the thread queue context.
+ *
+ * On SMP configurations, the value is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \
+ ( queue_context )->Wait.queue
+#else
+#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \
+ ( the_thread )->Wait.queue
+#endif
+
+/**
+ * @brief Gets the thread wait operations of the thread queue context.
+ *
+ * On SMP configurations, the value is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \
+ ( queue_context )->Wait.operations
+#else
+#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \
+ ( the_thread )->Wait.operations
+#endif
+
+/**
+ * @brief Thread priority change by means of the thread queue context.
+ *
+ * On SMP configurations, the used data is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ * @param new_priority The new thread priority.
+ * @param prepend_it Prepend it to its priority group or not.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_priority_change( \
+ queue_context, \
+ the_thread, \
+ new_priority, \
+ prepend_it \
+ ) \
+ ( *( queue_context )->Wait.operations->priority_change )( \
+ the_thread, \
+ new_priority, \
+ prepend_it, \
+ ( queue_context )->Wait.queue \
+ )
+#else
+#define _Thread_queue_Context_priority_change( \
+ queue_context, \
+ the_thread, \
+ new_priority, \
+ prepend_it \
+ ) \
+ ( *( the_thread )->Wait.operations->priority_change )( \
+ the_thread, \
+ new_priority, \
+ prepend_it, \
+ ( the_thread )->Wait.queue \
+ )
+#endif
+
+/**
+ * @brief Thread queue extract by means of the thread queue context.
+ *
+ * On SMP configurations, the used data is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_extract( \
+ queue_context, \
+ the_thread \
+ ) \
+ ( *( queue_context )->Wait.operations->extract )( \
+ ( queue_context )->Wait.queue, \
+ the_thread \
+ )
+#else
+#define _Thread_queue_Context_extract( \
+ queue_context, \
+ the_thread \
+ ) \
+ ( *( the_thread )->Wait.operations->extract )( \
+ ( the_thread )->Wait.queue, \
+ the_thread \
+ )
+#endif
+
+#if defined(RTEMS_SMP)
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_add(
+ Chain_Control *chain,
+ Thread_queue_Gate *gate
+)
+{
+ _Atomic_Store_uint( &gate->go_ahead, 0, ATOMIC_ORDER_RELAXED );
+ _Chain_Append_unprotected( chain, &gate->Node );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_open(
+ Thread_queue_Gate *gate
+)
+{
+ _Atomic_Store_uint( &gate->go_ahead, 1, ATOMIC_ORDER_RELAXED );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_wait(
+ Thread_queue_Gate *gate
+)
+{
+ while ( _Atomic_Load_uint( &gate->go_ahead, ATOMIC_ORDER_RELAXED ) == 0 ) {
+ /* Wait */
+ }
+}
+#endif
+
RTEMS_INLINE_ROUTINE void _Thread_queue_Heads_initialize(
Thread_queue_Heads *heads
)
@@ -911,6 +1049,10 @@ extern const Thread_queue_Operations _Thread_queue_Operations_priority;
extern const Thread_queue_Operations _Thread_queue_Operations_priority_inherit;
+#if defined(RTEMS_SMP)
+extern const Thread_queue_Operations _Thread_queue_Operations_stale_queue;
+#endif
+
/**@}*/
#ifdef __cplusplus
diff --git a/cpukit/score/src/threadchangepriority.c b/cpukit/score/src/threadchangepriority.c
index e3cf4a1..c569530 100644
--- a/cpukit/score/src/threadchangepriority.c
+++ b/cpukit/score/src/threadchangepriority.c
@@ -27,7 +27,8 @@ static Thread_Control *_Thread_Apply_priority_locked(
Priority_Control new_priority,
void *arg,
Thread_Change_priority_filter filter,
- bool prepend_it
+ bool prepend_it,
+ Thread_queue_Context *queue_context
)
{
/*
@@ -45,17 +46,11 @@ static Thread_Control *_Thread_Apply_priority_locked(
* we are not REALLY changing priority.
*/
if ( ( *filter )( the_thread, &new_priority, arg ) ) {
- Scheduler_Node *own_node;
-
- own_node = _Scheduler_Thread_get_own_node( the_thread );
- _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
-
- the_thread->current_priority = new_priority;
-
- ( *the_thread->Wait.operations->priority_change )(
+ _Thread_queue_Context_priority_change(
+ queue_context,
the_thread,
new_priority,
- the_thread->Wait.queue
+ prepend_it
);
} else {
the_thread = NULL;
@@ -72,19 +67,20 @@ Thread_Control *_Thread_Apply_priority(
bool prepend_it
)
{
- ISR_lock_Context lock_context;
- ISR_lock_Control *lock;
+ Thread_queue_Context queue_context;
+ Thread_Control *the_thread_to_update;
- lock = _Thread_Lock_acquire( the_thread, &lock_context );
- the_thread = _Thread_Apply_priority_locked(
+ _Thread_Wait_acquire( the_thread, &queue_context );
+ the_thread_to_update = _Thread_Apply_priority_locked(
the_thread,
new_priority,
arg,
filter,
- prepend_it
+ prepend_it,
+ &queue_context
);
- _Thread_Lock_release( lock, &lock_context );
- return the_thread;
+ _Thread_Wait_release( the_thread, &queue_context );
+ return the_thread_to_update;
}
void _Thread_Update_priority( Thread_Control *the_thread )
diff --git a/cpukit/score/src/threadinitialize.c b/cpukit/score/src/threadinitialize.c
index 940537f..209be9e 100644
--- a/cpukit/score/src/threadinitialize.c
+++ b/cpukit/score/src/threadinitialize.c
@@ -182,13 +182,11 @@ bool _Thread_Initialize(
the_thread->Scheduler.control = scheduler;
the_thread->Scheduler.own_node = scheduler_node;
_Resource_Node_initialize( &the_thread->Resource_node );
- _Atomic_Store_uintptr(
- &the_thread->Lock.current.atomic,
- (uintptr_t) &the_thread->Lock.Default,
- ATOMIC_ORDER_RELAXED
+ _ISR_lock_Initialize(
+ &the_thread->Wait.Lock.Default,
+ "Thread Wait Default Lock"
);
- _SMP_ticket_lock_Initialize( &the_thread->Lock.Default );
- _SMP_lock_Stats_initialize( &the_thread->Lock.Stats, "Thread Lock" );
+ _Chain_Initialize_empty( &the_thread->Wait.Lock.Pending_requests );
_SMP_lock_Stats_initialize( &the_thread->Potpourri_stats, "Thread Potpourri" );
#endif
diff --git a/cpukit/score/src/threadqenqueue.c b/cpukit/score/src/threadqenqueue.c
index 818073c..19c345b 100644
--- a/cpukit/score/src/threadqenqueue.c
+++ b/cpukit/score/src/threadqenqueue.c
@@ -34,6 +34,51 @@
#define THREAD_QUEUE_READY_AGAIN \
(THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_READY_AGAIN)
+static void _Thread_queue_Path_release( Thread_queue_Path *path )
+{
+#if defined(RTEMS_SMP)
+ Thread_queue_Link *link;
+
+ link = &path->Start;
+
+ if ( link->owner != NULL ) {
+ _Thread_Wait_release_critical( link->owner, &link->Queue_context );
+ }
+#else
+ (void) path;
+#endif
+}
+
+static void _Thread_queue_Path_acquire(
+ Thread_Control *the_thread,
+ Thread_queue_Queue *queue,
+ Thread_queue_Path *path
+)
+{
+#if defined(RTEMS_SMP)
+ Thread_Control *owner;
+ Thread_queue_Link *link;
+
+ owner = queue->owner;
+
+ if ( owner == NULL ) {
+ return;
+ }
+
+ link = &path->Start;
+ link->owner = owner;
+
+ _Thread_Wait_acquire_default_critical(
+ owner,
+ &link->Queue_context.Lock_context
+ );
+#else
+ (void) the_thread;
+ (void) queue;
+ (void) path;
+#endif
+}
+
void _Thread_queue_Enqueue_critical(
Thread_queue_Queue *queue,
const Thread_queue_Operations *operations,
@@ -52,14 +97,13 @@ void _Thread_queue_Enqueue_critical(
}
#endif
- _Thread_Lock_set( the_thread, &queue->Lock );
-
- the_thread->Wait.return_code = STATUS_SUCCESSFUL;
- _Thread_Wait_set_queue( the_thread, queue );
- _Thread_Wait_set_operations( the_thread, operations );
+ _Thread_Wait_claim( the_thread, queue, operations );
+ _Thread_queue_Path_acquire( the_thread, queue, &path );
( *operations->enqueue )( queue, the_thread, &path );
+ _Thread_queue_Path_release( &path );
+ the_thread->Wait.return_code = STATUS_SUCCESSFUL;
_Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK );
cpu_self = _Thread_Dispatch_disable_critical( &queue_context->Lock_context );
_Thread_queue_Queue_release( queue, &queue_context->Lock_context );
@@ -173,9 +217,7 @@ bool _Thread_queue_Do_extract_locked(
unblock = true;
}
- _Thread_Wait_set_queue( the_thread, NULL );
- _Thread_Wait_restore_default_operations( the_thread );
- _Thread_Lock_restore_default( the_thread );
+ _Thread_Wait_restore_default( the_thread );
return unblock;
}
@@ -227,30 +269,35 @@ void _Thread_queue_Extract_critical(
void _Thread_queue_Extract( Thread_Control *the_thread )
{
- Thread_queue_Context queue_context;
- void *lock;
- Thread_queue_Queue *queue;
+ Thread_queue_Context queue_context;
_Thread_queue_Context_initialize( &queue_context );
- lock = _Thread_Lock_acquire( the_thread, &queue_context.Lock_context );
+ _Thread_Wait_acquire( the_thread, &queue_context );
- queue = the_thread->Wait.queue;
-
- if ( queue != NULL ) {
- _SMP_Assert( lock == &queue->Lock );
+ if (
+ _Thread_queue_Context_get_queue( &queue_context, the_thread ) != NULL
+ ) {
+ bool unblock;
+ _Thread_Wait_remove_request( the_thread, &queue_context );
_Thread_queue_Context_set_MP_callout(
&queue_context,
_Thread_queue_MP_callout_do_nothing
);
- _Thread_queue_Extract_critical(
- queue,
- the_thread->Wait.operations,
+ unblock = _Thread_queue_Extract_locked(
+ _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+ _Thread_queue_Context_get_operations( &queue_context, the_thread ),
the_thread,
- &queue_context
+ &queue_context.Lock_context
+ );
+ _Thread_queue_Unblock_critical(
+ unblock,
+ _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+ the_thread,
+ &queue_context.Lock_context
);
} else {
- _Thread_Lock_release( lock, &queue_context.Lock_context );
+ _Thread_Wait_release( the_thread, &queue_context );
}
}
@@ -273,8 +320,6 @@ Thread_Control *_Thread_queue_Do_dequeue(
the_thread = _Thread_queue_First_locked( the_thread_queue, operations );
if ( the_thread != NULL ) {
- _SMP_Assert( the_thread->Lock.current.normal == &the_thread_queue->Queue.Lock );
-
_Thread_queue_Extract_critical(
&the_thread_queue->Queue,
operations,
diff --git a/cpukit/score/src/threadqops.c b/cpukit/score/src/threadqops.c
index c288cee..8059446 100644
--- a/cpukit/score/src/threadqops.c
+++ b/cpukit/score/src/threadqops.c
@@ -22,13 +22,16 @@
#include <rtems/score/rbtreeimpl.h>
#include <rtems/score/schedulerimpl.h>
-static void _Thread_queue_Do_nothing_priority_change(
+static void _Thread_queue_Default_priority_change(
Thread_Control *the_thread,
Priority_Control new_priority,
+ bool prepend_it,
Thread_queue_Queue *queue
)
{
- /* Do nothing */
+ (void) queue;
+
+ _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
}
static void _Thread_queue_Do_nothing_extract(
@@ -39,6 +42,31 @@ static void _Thread_queue_Do_nothing_extract(
/* Do nothing */
}
+#if defined(RTEMS_SMP)
+static void _Thread_queue_Stale_queue_priority_change(
+ Thread_Control *the_thread,
+ Priority_Control new_priority,
+ bool prepend_it,
+ Thread_queue_Queue *queue
+)
+{
+ ISR_lock_Context lock_context;
+
+ (void) queue;
+
+ /*
+ * This operation is used to change the priority in case we have a thread
+ * queue context with a stale thread queue. We own the thread queue lock of
+ * the former thread queue. In addition, we need the thread wait default
+ * lock, see _Thread_Wait_restore_default().
+ */
+
+ _Thread_Wait_acquire_default_critical( the_thread, &lock_context );
+ _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+ _Thread_Wait_release_default_critical( the_thread, &lock_context );
+}
+#endif
+
static Thread_queue_Heads *_Thread_queue_Queue_enqueue(
Thread_queue_Queue *queue,
Thread_Control *the_thread,
@@ -190,6 +218,7 @@ static bool _Thread_queue_Priority_less(
static void _Thread_queue_Priority_priority_change(
Thread_Control *the_thread,
Priority_Control new_priority,
+ bool prepend_it,
Thread_queue_Queue *queue
)
{
@@ -198,6 +227,8 @@ static void _Thread_queue_Priority_priority_change(
_Assert( heads != NULL );
+ _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+
priority_queue = _Thread_queue_Priority_queue( heads, the_thread );
_RBTree_Extract(
@@ -353,22 +384,16 @@ static void _Thread_queue_Priority_inherit_enqueue(
#endif
if ( priority < owner->current_priority ) {
- Scheduler_Node *own_node;
-
path->update_priority = owner;
owner->priority_restore_hint = true;
_Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
- own_node = _Scheduler_Thread_get_own_node( owner );
- _Scheduler_Node_set_priority( own_node, priority, false );
-
- owner->current_priority = priority;
-
- ( *owner->Wait.operations->priority_change )(
+ _Thread_queue_Context_priority_change(
+ &path->Start.Queue_context,
owner,
priority,
- owner->Wait.queue
+ false
);
} else {
path->update_priority = NULL;
@@ -385,24 +410,21 @@ void _Thread_queue_Boost_priority(
if ( !_Chain_Has_only_one_node( &heads->Heads.Fifo ) ) {
const Scheduler_Control *scheduler;
- Scheduler_Node *own_node;
Priority_Control boost_priority;
the_thread->priority_restore_hint = true;
_Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
scheduler = _Scheduler_Get_own( the_thread );
- own_node = _Scheduler_Thread_get_own_node( the_thread );
boost_priority = _Scheduler_Map_priority( scheduler, PRIORITY_PSEUDO_ISR );
- _Scheduler_Node_set_priority( own_node, boost_priority, false );
- the_thread->current_priority = boost_priority;
+ _Scheduler_Thread_set_priority( the_thread, boost_priority, false );
}
}
#endif
const Thread_queue_Operations _Thread_queue_Operations_default = {
- .priority_change = _Thread_queue_Do_nothing_priority_change,
+ .priority_change = _Thread_queue_Default_priority_change,
.extract = _Thread_queue_Do_nothing_extract
/*
* The default operations are only used in _Thread_Change_priority() and
@@ -412,7 +434,7 @@ const Thread_queue_Operations _Thread_queue_Operations_default = {
};
const Thread_queue_Operations _Thread_queue_Operations_FIFO = {
- .priority_change = _Thread_queue_Do_nothing_priority_change,
+ .priority_change = _Thread_queue_Default_priority_change,
.enqueue = _Thread_queue_FIFO_enqueue,
.extract = _Thread_queue_FIFO_extract,
.first = _Thread_queue_FIFO_first
@@ -431,3 +453,10 @@ const Thread_queue_Operations _Thread_queue_Operations_priority_inherit = {
.extract = _Thread_queue_Priority_extract,
.first = _Thread_queue_Priority_first
};
+
+#if defined(RTEMS_SMP)
+const Thread_queue_Operations _Thread_queue_Operations_stale_queue = {
+ .priority_change = _Thread_queue_Stale_queue_priority_change,
+ .extract = _Thread_queue_Do_nothing_extract
+};
+#endif
diff --git a/cpukit/score/src/threadrestart.c b/cpukit/score/src/threadrestart.c
index 757f552..81ba73d 100644
--- a/cpukit/score/src/threadrestart.c
+++ b/cpukit/score/src/threadrestart.c
@@ -215,8 +215,7 @@ static void _Thread_Free( Thread_Control *the_thread )
_Workspace_Free( the_thread->Start.tls_area );
#if defined(RTEMS_SMP)
- _SMP_ticket_lock_Destroy( &the_thread->Lock.Default );
- _SMP_lock_Stats_destroy( &the_thread->Lock.Stats );
+ _ISR_lock_Destroy( &the_thread->Wait.Lock.Default );
_SMP_lock_Stats_destroy( &the_thread->Potpourri_stats );
#endif
diff --git a/cpukit/score/src/threadtimeout.c b/cpukit/score/src/threadtimeout.c
index a2ba61f..b6b6cc4 100644
--- a/cpukit/score/src/threadtimeout.c
+++ b/cpukit/score/src/threadtimeout.c
@@ -22,28 +22,15 @@
#include <rtems/score/threadimpl.h>
#include <rtems/score/status.h>
-static void _Thread_Do_timeout( Thread_Control *the_thread )
-{
- the_thread->Wait.return_code = STATUS_TIMEOUT;
- ( *the_thread->Wait.operations->extract )(
- the_thread->Wait.queue,
- the_thread
- );
- _Thread_Wait_set_queue( the_thread, NULL );
- _Thread_Wait_restore_default_operations( the_thread );
- _Thread_Lock_restore_default( the_thread );
-}
-
void _Thread_Timeout( Watchdog_Control *watchdog )
{
- Thread_Control *the_thread;
- void *thread_lock;
- ISR_lock_Context lock_context;
- Thread_Wait_flags wait_flags;
- bool unblock;
+ Thread_Control *the_thread;
+ Thread_queue_Context queue_context;
+ Thread_Wait_flags wait_flags;
+ bool unblock;
the_thread = RTEMS_CONTAINER_OF( watchdog, Thread_Control, Timer.Watchdog );
- thread_lock = _Thread_Lock_acquire( the_thread, &lock_context );
+ _Thread_Wait_acquire( the_thread, &queue_context );
wait_flags = _Thread_Wait_flags_get( the_thread );
@@ -52,7 +39,9 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
Thread_Wait_flags ready_again;
bool success;
- _Thread_Do_timeout( the_thread );
+ _Thread_Wait_cancel( the_thread, &queue_context );
+
+ the_thread->Wait.return_code = STATUS_TIMEOUT;
wait_class = wait_flags & THREAD_WAIT_CLASS_MASK;
ready_again = wait_class | THREAD_WAIT_STATE_READY_AGAIN;
@@ -76,9 +65,10 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
unblock = false;
}
- _Thread_Lock_release( thread_lock, &lock_context );
+ _Thread_Wait_release( the_thread, &queue_context );
if ( unblock ) {
+ _Thread_Wait_tranquilize( the_thread );
_Thread_Unblock( the_thread );
#if defined(RTEMS_MULTIPROCESSING)
diff --git a/testsuites/sptests/spthreadq01/init.c b/testsuites/sptests/spthreadq01/init.c
index 85df6bd..afaa652 100644
--- a/testsuites/sptests/spthreadq01/init.c
+++ b/testsuites/sptests/spthreadq01/init.c
@@ -68,13 +68,12 @@ static void wake_up_master(test_context *ctx)
static rtems_id get_wait_id(test_context *ctx)
{
- ISR_lock_Context lock_context;
- void *lock;
+ Thread_queue_Context queue_context;
rtems_id id;
- lock = _Thread_Lock_acquire(ctx->master, &lock_context);
+ _Thread_Wait_acquire(ctx->master, &queue_context);
id = _Thread_Wait_get_id(ctx->master);
- _Thread_Lock_release(lock, &lock_context);
+ _Thread_Wait_release(ctx->master, &queue_context);
return id;
}
--
1.8.4.5
More information about the devel
mailing list