RTOS/rtems
1
0
Fork 0
forked from Imagelibrary/rtems
Code Pull Requests Activity

score: Rework SMP multicast action

Browse Source 
Use a FIFO list of jobs per processor to carry out the SMP multicast
action.  Use a done indicator per job to reduce the bus traffic a bit.
This commit is contained in:
Sebastian Huber
2019-04-11 15:16:40 +02:00
parent df8d7bd76f
commit e90486ab41
6 changed files with 224 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
cpukit/include/rtems/score/percpu.h
View File

@@ -74,6 +74,8 @@ struct _Thread_Control;
struct Scheduler_Context;
struct Per_CPU_Job;
/**
* @defgroup PerCPU RTEMS Per CPU Information
*
@@ -494,6 +496,37 @@ typedef struct Per_CPU_Control {
*/
Atomic_Uintptr before_multitasking_action;
/**
* @brief FIFO list of jobs to be performed by this processor.
*
* @see _SMP_Multicast_action().
*/
struct {
/**
* @brief Lock to protect the FIFO list of jobs to be performed by this
* processor.
*/
ISR_lock_Control Lock;
/**
* @brief Head of the FIFO list of jobs to be performed by this
* processor.
*
* This member is protected by the Per_CPU_Control::Jobs::Lock lock.
*/
struct Per_CPU_Job *head;
/**
* @brief Tail of the FIFO list of jobs to be performed by this
* processor.
*
* This member is only valid if the head is not @c NULL.
*
* This member is protected by the Per_CPU_Control::Jobs::Lock lock.
*/
struct Per_CPU_Job **tail;
} Jobs;
/**
* @brief Indicates if the processor has been successfully started via
* _CPU_SMP_Start_processor().
@@ -710,6 +743,13 @@ bool _Per_CPU_State_wait_for_non_initial_state(
uint32_t timeout_in_ns
);
/**
* @brief Performs the jobs of the specified processor.
*
* @param[in, out] cpu The jobs of this processor will be performed.
*/
void _Per_CPU_Perform_jobs( Per_CPU_Control *cpu );
#endif /* defined( RTEMS_SMP ) */
/*

13
cpukit/include/rtems/score/smpimpl.h
View File

@@ -51,11 +51,11 @@ extern "C" {
#define SMP_MESSAGE_TEST 0x2UL
/**
* @brief SMP message to request a multicast action.
* @brief SMP message to perform per-processor jobs.
*
* @see _SMP_Send_message().
*/
#define SMP_MESSAGE_MULTICAST_ACTION 0x4UL
#define SMP_MESSAGE_PERFORM_JOBS 0x4UL
/**
* @brief SMP message to request a clock tick.
@@ -157,11 +157,6 @@ static inline void _SMP_Set_test_message_handler(
_SMP_Test_message_handler = handler;
}
/**
* @brief Processes all pending multicast actions.
*/
void _SMP_Multicast_actions_process( void );
/**
* @brief Interrupt handler for inter-processor interrupts.
*
@@ -195,8 +190,8 @@ static inline long unsigned _SMP_Inter_processor_interrupt_handler(
( *_SMP_Test_message_handler )( cpu_self );
}
if ( ( message & SMP_MESSAGE_MULTICAST_ACTION ) != 0 ) {
_SMP_Multicast_actions_process();
if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) {
_Per_CPU_Perform_jobs( cpu_self );
}
}

1
cpukit/score/src/smp.c
View File

@@ -118,6 +118,7 @@ void _SMP_Handler_initialize( void )
cpu = _Per_CPU_Get_by_index( cpu_index );
_ISR_lock_Set_name( &cpu->Lock, "Per-CPU" );
_ISR_lock_Set_name( &cpu->Jobs.Lock, "Per-CPU Jobs" );
_ISR_lock_Set_name( &cpu->Watchdog.Lock, "Per-CPU Watchdog" );
_Chain_Initialize_empty( &cpu->Threads_in_need_for_help );
}

247
cpukit/score/src/smpmulticastaction.c
View File

@@ -1,91 +1,194 @@
/*
* Copyright (c) 2014 Aeroflex Gaisler AB. All rights reserved.
* SPDX-License-Identifier: BSD-2-Clause
*
* The license and distribution terms for this file may be
* found in the file LICENSE in this distribution or at
* http://www.rtems.org/license/LICENSE.
* Copyright (C) 2019 embedded brains GmbH
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#include "config.h"
#endif
#include <rtems/score/smpimpl.h>
#include <rtems/score/isrlock.h>
#include <rtems/score/chainimpl.h>
#include <rtems/score/assert.h>
#include <rtems/score/threaddispatch.h>
#include <rtems/score/sysstate.h>
typedef struct {
Chain_Node Node;
SMP_Action_handler handler;
void *arg;
Processor_mask targets;
Atomic_Ulong done;
} SMP_Multicast_action;
typedef struct Per_CPU_Job Per_CPU_Job;
typedef struct {
ISR_lock_Control Lock;
Chain_Control Actions;
} SMP_Multicast_context;
typedef struct Per_CPU_Jobs Per_CPU_Jobs;
static SMP_Multicast_context _SMP_Multicast = {
.Lock = ISR_LOCK_INITIALIZER( "SMP Multicast Action" ),
.Actions = CHAIN_INITIALIZER_EMPTY( _SMP_Multicast.Actions )
/**
* @brief A per-processor job.
*/
struct Per_CPU_Job {
union {
/**
* @brief The next job in the corresponding per-processor job list.
*/
Per_CPU_Job *next;
/**
* @brief Indication if the job is done.
*
* A job is done if this member has the value one. This assumes that one
* is not a valid pointer value.
*/
Atomic_Ulong done;
};
/**
* @brief Back pointer to the jobs to get the handler and argument.
*/
Per_CPU_Jobs *jobs;
};
void _SMP_Multicast_actions_process( void )
/**
* @brief A collection of jobs, one for each processor.
*/
struct Per_CPU_Jobs {
/**
* @brief The job handler.
*/
SMP_Action_handler handler;
/**
* @brief The job handler argument.
*/
void *arg;
/**
* @brief One job for each potential processor.
*/
Per_CPU_Job Jobs[ CPU_MAXIMUM_PROCESSORS ];
};
void _Per_CPU_Perform_jobs( Per_CPU_Control *cpu )
{
ISR_lock_Context lock_context;
uint32_t cpu_self_index;
SMP_Multicast_action *node;
SMP_Multicast_action *next;
ISR_lock_Context lock_context;
Per_CPU_Job *job;
_ISR_lock_ISR_disable_and_acquire( &_SMP_Multicast.Lock, &lock_context );
cpu_self_index = _SMP_Get_current_processor();
node = (SMP_Multicast_action *) _Chain_First( &_SMP_Multicast.Actions );
_ISR_lock_ISR_disable( &lock_context );
_Per_CPU_Acquire( cpu, &lock_context );
while ( !_Chain_Is_tail( &_SMP_Multicast.Actions, &node->Node ) ) {
next = (SMP_Multicast_action *) _Chain_Next( &node->Node );
while ( ( job = cpu->Jobs.head ) != NULL ) {
Per_CPU_Jobs *jobs;
if ( _Processor_mask_Is_set( &node->targets, cpu_self_index ) ) {
_Processor_mask_Clear( &node->targets, cpu_self_index );
cpu->Jobs.head = job->next;
_Per_CPU_Release( cpu, &lock_context );
_ISR_lock_ISR_enable( &lock_context );
( *node->handler )( node->arg );
jobs = job->jobs;
( *jobs->handler )( jobs->arg );
_Atomic_Store_ulong( &job->done, 1, ATOMIC_ORDER_RELEASE );
if ( _Processor_mask_Is_zero( &node->targets ) ) {
_Chain_Extract_unprotected( &node->Node );
_Atomic_Store_ulong( &node->done, 1, ATOMIC_ORDER_RELEASE );
}
}
node = next;
_ISR_lock_ISR_disable( &lock_context );
_Per_CPU_Acquire( cpu, &lock_context );
}
_ISR_lock_Release_and_ISR_enable( &_SMP_Multicast.Lock, &lock_context );
_Per_CPU_Release( cpu, &lock_context );
_ISR_lock_ISR_enable( &lock_context );
}
static void
_SMP_Multicasts_try_process( void )
static void _Per_CPU_Try_perform_jobs( Per_CPU_Control *cpu_self )
{
unsigned long message;
Per_CPU_Control *cpu_self;
ISR_Level isr_level;
_ISR_Local_disable( isr_level );
cpu_self = _Per_CPU_Get();
message = _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED );
if ( message & SMP_MESSAGE_MULTICAST_ACTION ) {
if ( _Atomic_Compare_exchange_ulong( &cpu_self->message, &message,
message & ~SMP_MESSAGE_MULTICAST_ACTION, ATOMIC_ORDER_RELAXED,
ATOMIC_ORDER_RELAXED ) ) {
_SMP_Multicast_actions_process();
if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) {
bool success;
success = _Atomic_Compare_exchange_ulong(
&cpu_self->message, &message,
message & ~SMP_MESSAGE_PERFORM_JOBS, ATOMIC_ORDER_RELAXED,
ATOMIC_ORDER_RELAXED
);
if ( success ) {
_Per_CPU_Perform_jobs( cpu_self );
}
}
}
_ISR_Local_enable( isr_level );
static void _SMP_Issue_action_jobs(
const Processor_mask *targets,
Per_CPU_Jobs *jobs,
uint32_t cpu_max
)
{
uint32_t cpu_index;
for ( cpu_index = 0; cpu_index < cpu_max; ++cpu_index ) {
if ( _Processor_mask_Is_set( targets, cpu_index ) ) {
ISR_lock_Context lock_context;
Per_CPU_Job *job;
Per_CPU_Control *cpu;
job = &jobs->Jobs[ cpu_index ];
_Atomic_Store_ulong( &job->done, 0, ATOMIC_ORDER_RELAXED );
_Assert( job->next == NULL );
job->jobs = jobs;
cpu = _Per_CPU_Get_by_index( cpu_index );
_ISR_lock_ISR_disable( &lock_context );
_Per_CPU_Acquire( cpu, &lock_context );
if ( cpu->Jobs.head == NULL ) {
cpu->Jobs.head = job;
} else {
*cpu->Jobs.tail = job;
}
cpu->Jobs.tail = &job->next;
_Per_CPU_Release( cpu, &lock_context );
_ISR_lock_ISR_enable( &lock_context );
_SMP_Send_message( cpu_index, SMP_MESSAGE_PERFORM_JOBS );
}
}
}
static void _SMP_Wait_for_action_jobs(
const Processor_mask *targets,
const Per_CPU_Jobs *jobs,
uint32_t cpu_max,
Per_CPU_Control *cpu_self
)
{
uint32_t cpu_index;
for ( cpu_index = 0; cpu_index < cpu_max; ++cpu_index ) {
if ( _Processor_mask_Is_set( targets, cpu_index ) ) {
const Per_CPU_Job *job;
job = &jobs->Jobs[ cpu_index ];
while ( _Atomic_Load_ulong( &job->done, ATOMIC_ORDER_ACQUIRE ) == 0 ) {
_Per_CPU_Try_perform_jobs( cpu_self );
}
}
}
}
void _SMP_Multicast_action(
@@ -94,33 +197,27 @@ void _SMP_Multicast_action(
void *arg
)
{
SMP_Multicast_action node;
ISR_lock_Context lock_context;
uint32_t i;
Per_CPU_Jobs jobs;
uint32_t cpu_max;
Per_CPU_Control *cpu_self;
cpu_max = _SMP_Get_processor_maximum();
_Assert( cpu_max <= CPU_MAXIMUM_PROCESSORS );
if ( ! _System_state_Is_up( _System_state_Get() ) ) {
( *handler )( arg );
return;
}
if( targets == NULL ) {
if ( targets == NULL ) {
targets = _SMP_Get_online_processors();
}
_Chain_Initialize_node( &node.Node );
node.handler = handler;
node.arg = arg;
_Processor_mask_Assign( &node.targets, targets );
_Atomic_Store_ulong( &node.done, 0, ATOMIC_ORDER_RELAXED );
jobs.handler = handler;
jobs.arg = arg;
_ISR_lock_ISR_disable_and_acquire( &_SMP_Multicast.Lock, &lock_context );
_Chain_Prepend_unprotected( &_SMP_Multicast.Actions, &node.Node );
_ISR_lock_Release_and_ISR_enable( &_SMP_Multicast.Lock, &lock_context );
_SMP_Send_message_multicast( targets, SMP_MESSAGE_MULTICAST_ACTION );
_SMP_Multicasts_try_process();
while ( _Atomic_Load_ulong( &node.done, ATOMIC_ORDER_ACQUIRE ) == 0 ) {
/* Wait */
};
cpu_self = _Thread_Dispatch_disable();
_SMP_Issue_action_jobs( targets, &jobs, cpu_max );
_SMP_Wait_for_action_jobs( targets, &jobs, cpu_max, cpu_self );
_Thread_Dispatch_enable( cpu_self );
}

5
testsuites/smptests/smpcache01/init.c
View File

@@ -126,12 +126,15 @@ static void call_tests_isr_disabled( SMP_barrier_State *bs )
broadcast_test_init();
for (i = 0; i < RTEMS_ARRAY_SIZE( test_cases ); ++i) {
ISR_Level isr_level;
Per_CPU_Control *cpu_self;
ISR_Level isr_level;
cpu_self = _Thread_Dispatch_disable();
_ISR_Local_disable( isr_level );
barrier( bs );
( *test_cases[ i ] )();
_ISR_Local_enable( isr_level );
_Thread_Dispatch_enable( cpu_self );
barrier( bs );
}

3
testsuites/smptests/smpmulticast01/init.c
View File

@@ -59,11 +59,14 @@ static void multicast_action_irq_disabled(
void *arg
)
{
Per_CPU_Control *cpu_self;
rtems_interrupt_level level;
cpu_self = _Thread_Dispatch_disable();
rtems_interrupt_local_disable(level);
_SMP_Multicast_action(targets, handler, arg);
rtems_interrupt_local_enable(level);
_Thread_Dispatch_enable(cpu_self);
}
static void multicast_action_dispatch_disabled(