41 #include "kmp_error.h"
43 #define MAX_MESSAGE 512
64 if (__kmp_ignore_mppbeg() == FALSE) {
65 __kmp_internal_begin();
67 KC_TRACE( 10, (
"__kmpc_begin: called\n" ) );
85 if (__kmp_ignore_mppend() == FALSE) {
86 KC_TRACE( 10, (
"__kmpc_end: called\n" ) );
87 KA_TRACE( 30, (
"__kmpc_end\n" ));
89 __kmp_internal_end_thread( -1 );
115 kmp_int32 gtid = __kmp_entry_gtid();
117 KC_TRACE( 10, (
"__kmpc_global_thread_num: T#%d\n", gtid ) );
138 KC_TRACE( 10, (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
140 return TCR_4(__kmp_nth);
152 KC_TRACE( 10, (
"__kmpc_bound_thread_num: called\n" ) );
153 return __kmp_tid_from_gtid( __kmp_entry_gtid() );
164 KC_TRACE( 10, (
"__kmpc_bound_num_threads: called\n" ) );
166 return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
188 if (__kmp_par_range == 0) {
195 semi2 = strchr(semi2,
';');
199 semi2 = strchr(semi2 + 1,
';');
203 if (__kmp_par_range_filename[0]) {
204 const char *name = semi2 - 1;
205 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
208 if ((*name ==
'/') || (*name ==
';')) {
211 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
212 return __kmp_par_range < 0;
215 semi3 = strchr(semi2 + 1,
';');
216 if (__kmp_par_range_routine[0]) {
217 if ((semi3 != NULL) && (semi3 > semi2)
218 && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
219 return __kmp_par_range < 0;
222 if (sscanf(semi3 + 1,
"%d", &line_no) == 1) {
223 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
224 return __kmp_par_range > 0;
226 return __kmp_par_range < 0;
242 return __kmp_entry_thread() -> th.th_root -> r.r_active;
257 KA_TRACE( 20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
258 global_tid, num_threads ) );
260 __kmp_push_num_threads( loc, global_tid, num_threads );
264 __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid )
266 KA_TRACE( 20, (
"__kmpc_pop_num_threads: enter\n" ) );
275 __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
277 KA_TRACE( 20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
278 global_tid, proc_bind ) );
280 __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
298 int gtid = __kmp_entry_gtid();
302 va_start( ap, microtask );
304 __kmp_fork_call( loc, gtid, TRUE,
306 VOLATILE_CAST(microtask_t) microtask,
307 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
309 #
if KMP_ARCH_X86_64 && KMP_OS_LINUX
315 __kmp_join_call( loc, gtid );
335 KA_TRACE( 20, (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
336 global_tid, num_teams, num_threads ) );
338 __kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
353 int gtid = __kmp_entry_gtid();
354 kmp_info_t *this_thr = __kmp_threads[ gtid ];
356 va_start( ap, microtask );
359 this_thr->th.th_team_microtask = microtask;
360 this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level;
363 if ( this_thr->th.th_set_nth_teams == 0 ) {
364 __kmp_push_num_teams( loc, gtid, 0, 0 );
366 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
367 KMP_DEBUG_ASSERT(this_thr->th.th_set_nth_teams >= 1);
369 __kmp_fork_call( loc, gtid, TRUE,
371 VOLATILE_CAST(microtask_t) __kmp_teams_master,
372 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
373 #
if KMP_ARCH_X86_64 && KMP_OS_LINUX
379 __kmp_join_call( loc, gtid );
380 this_thr->th.th_team_microtask = NULL;
381 this_thr->th.th_teams_level = 0;
395 __kmpc_invoke_task_func(
int gtid )
397 return __kmp_invoke_task_func( gtid );
415 kmp_info_t *this_thr;
416 kmp_team_t *serial_team;
418 KC_TRACE( 10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid ) );
425 if( ! TCR_4( __kmp_init_parallel ) )
426 __kmp_parallel_initialize();
428 this_thr = __kmp_threads[ global_tid ];
429 serial_team = this_thr -> th.th_serial_team;
432 KMP_DEBUG_ASSERT( serial_team );
436 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
437 KMP_DEBUG_ASSERT( this_thr -> th.th_task_team == this_thr -> th.th_team -> t.t_task_team );
438 KMP_DEBUG_ASSERT( serial_team -> t.t_task_team == NULL );
439 KA_TRACE( 20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / team %p, new task_team = NULL\n",
440 global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
441 this_thr -> th.th_task_team = NULL;
443 #endif // OMP_30_ENABLED
446 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
447 if ( this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false ) {
448 proc_bind = proc_bind_false;
450 else if ( proc_bind == proc_bind_default ) {
455 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
460 this_thr->th.th_set_proc_bind = proc_bind_default;
463 if( this_thr -> th.th_team != serial_team ) {
466 int level = this_thr->th.th_team->t.t_level;
468 if( serial_team -> t.t_serialized ) {
471 kmp_team_t *new_team;
472 int tid = this_thr->th.th_info.ds.ds_tid;
474 __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
476 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
481 & this_thr->th.th_current_task->td_icvs,
483 this_thr->th.th_team->t.t_set_nproc[tid],
484 this_thr->th.th_team->t.t_set_dynamic[tid],
485 this_thr->th.th_team->t.t_set_nested[tid],
486 this_thr->th.th_team->t.t_set_blocktime[tid],
487 this_thr->th.th_team->t.t_set_bt_intervals[tid],
488 this_thr->th.th_team->t.t_set_bt_set[tid],
491 __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
492 KMP_ASSERT( new_team );
495 new_team -> t.t_threads[0] = this_thr;
496 new_team -> t.t_parent = this_thr -> th.th_team;
497 serial_team = new_team;
498 this_thr -> th.th_serial_team = serial_team;
500 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
501 global_tid, serial_team ) );
508 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
509 global_tid, serial_team ) );
513 KMP_DEBUG_ASSERT( serial_team->t.t_threads );
514 KMP_DEBUG_ASSERT( serial_team->t.t_threads[0] == this_thr );
515 KMP_DEBUG_ASSERT( this_thr->th.th_team != serial_team );
516 serial_team -> t.t_ident = loc;
517 serial_team -> t.t_serialized = 1;
518 serial_team -> t.t_nproc = 1;
519 serial_team -> t.t_parent = this_thr->th.th_team;
521 serial_team -> t.t_sched = this_thr->th.th_team->t.t_sched;
522 #endif // OMP_30_ENABLED
523 this_thr -> th.th_team = serial_team;
524 serial_team -> t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
527 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#d curtask=%p\n",
528 global_tid, this_thr->th.th_current_task ) );
529 KMP_ASSERT( this_thr->th.th_current_task->td_flags.executing == 1 );
530 this_thr->th.th_current_task->td_flags.executing = 0;
532 __kmp_push_current_task_to_thread( this_thr, serial_team, 0 );
537 & this_thr->th.th_current_task->td_icvs,
538 & this_thr->th.th_current_task->td_parent->td_icvs );
541 if ( __kmp_nested_nth.used && ( level + 1 < __kmp_nested_nth.used ) ) {
542 this_thr->th.th_current_task->td_icvs.nproc = __kmp_nested_nth.nth[ level + 1 ];
546 if ( __kmp_nested_proc_bind.used && ( level + 1 < __kmp_nested_proc_bind.used ) ) {
547 this_thr->th.th_current_task->td_icvs.proc_bind
548 = __kmp_nested_proc_bind.bind_types[ level + 1 ];
553 serial_team -> t.t_set_nproc[0] = serial_team->t.t_parent->
554 t.t_set_nproc[serial_team->
556 serial_team -> t.t_set_dynamic[0] = serial_team->t.t_parent->
557 t.t_set_dynamic[serial_team->
559 serial_team -> t.t_set_nested[0] = serial_team->t.t_parent->
560 t.t_set_nested[serial_team->
562 serial_team -> t.t_set_blocktime[0] = serial_team->t.t_parent->
563 t.t_set_blocktime[serial_team->
565 serial_team -> t.t_set_bt_intervals[0] = serial_team->t.t_parent->
566 t.t_set_bt_intervals[serial_team->
568 serial_team -> t.t_set_bt_set[0] = serial_team->t.t_parent->
569 t.t_set_bt_set[serial_team->
571 #endif // OMP_30_ENABLED
572 this_thr -> th.th_info.ds.ds_tid = 0;
575 this_thr -> th.th_team_nproc = 1;
576 this_thr -> th.th_team_master = this_thr;
577 this_thr -> th.th_team_serialized = 1;
580 serial_team -> t.t_level = serial_team -> t.t_parent -> t.t_level + 1;
581 serial_team -> t.t_active_level = serial_team -> t.t_parent -> t.t_active_level;
582 #endif // OMP_30_ENABLED
584 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
585 if ( __kmp_inherit_fp_control ) {
586 __kmp_store_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
587 __kmp_store_mxcsr( &serial_team->t.t_mxcsr );
588 serial_team->t.t_mxcsr &= KMP_X86_MXCSR_MASK;
589 serial_team->t.t_fp_control_saved = TRUE;
591 serial_team->t.t_fp_control_saved = FALSE;
595 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
596 if ( !serial_team->t.t_dispatch->th_disp_buffer ) {
597 serial_team->t.t_dispatch->th_disp_buffer = (dispatch_private_info_t *)
598 __kmp_allocate(
sizeof( dispatch_private_info_t ) );
600 this_thr -> th.th_dispatch = serial_team->t.t_dispatch;
607 KMP_DEBUG_ASSERT( this_thr->th.th_team == serial_team );
608 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
609 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
610 ++ serial_team -> t.t_serialized;
611 this_thr -> th.th_team_serialized = serial_team -> t.t_serialized;
615 int level = this_thr->th.th_team->t.t_level;
617 if ( __kmp_nested_nth.used && ( level + 1 < __kmp_nested_nth.used ) ) {
618 this_thr->th.th_current_task->td_icvs.nproc = __kmp_nested_nth.nth[ level + 1 ];
620 serial_team -> t.t_level++;
621 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level of serial team %p to %d\n",
622 global_tid, serial_team, serial_team -> t.t_level ) );
624 KF_TRACE( 10, (
"__kmpc_serialized_parallel: T#%d reusing team %p for nested serialized parallel region\n",
625 global_tid, serial_team ) );
626 #endif // OMP_30_ENABLED
629 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
631 dispatch_private_info_t * disp_buffer = (dispatch_private_info_t *)
632 __kmp_allocate(
sizeof( dispatch_private_info_t ) );
633 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
634 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
636 this_thr -> th.th_dispatch = serial_team->t.t_dispatch;
641 if ( __kmp_env_consistency_check )
642 __kmp_push_parallel( global_tid, NULL );
646 if ( ( __itt_frame_begin_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG )
648 __kmp_itt_region_forking( global_tid, 1 );
651 if( __kmp_forkjoin_frames_mode == 1 && __kmp_itt_csv_file )
653 if( this_thr->th.th_team->t.t_level == 1 ) {
655 #if defined( __GNUC__ )
656 # if !defined( __INTEL_COMPILER )
657 fr_begin = __kmp_hardware_timestamp();
659 fr_begin = __rdtsc();
662 fr_begin = __rdtsc();
664 this_thr->th.th_frame_time_serialized = fr_begin;
681 kmp_internal_control_t *top;
682 kmp_info_t *this_thr;
683 kmp_team_t *serial_team;
685 KC_TRACE( 10, (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
693 if( ! TCR_4( __kmp_init_parallel ) )
694 __kmp_parallel_initialize();
696 this_thr = __kmp_threads[ global_tid ];
697 serial_team = this_thr->th.th_serial_team;
700 KMP_DEBUG_ASSERT( serial_team );
701 KMP_ASSERT( serial_team -> t.t_serialized );
702 KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
703 KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
704 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
705 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
708 top = serial_team -> t.t_control_stack_top;
709 if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
712 &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs,
715 serial_team -> t.t_set_nproc[0] = top -> nproc;
716 serial_team -> t.t_set_dynamic[0] = top -> dynamic;
717 serial_team -> t.t_set_nested[0] = top -> nested;
718 serial_team -> t.t_set_blocktime[0] = top -> blocktime;
719 serial_team -> t.t_set_bt_intervals[0] = top -> bt_intervals;
720 serial_team -> t.t_set_bt_set[0] = top -> bt_set;
721 #endif // OMP_30_ENABLED
722 serial_team -> t.t_control_stack_top = top -> next;
728 serial_team -> t.t_level--;
729 #endif // OMP_30_ENABLED
732 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
734 dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
735 serial_team->t.t_dispatch->th_disp_buffer =
736 serial_team->t.t_dispatch->th_disp_buffer->next;
737 __kmp_free( disp_buffer );
740 -- serial_team -> t.t_serialized;
741 if ( serial_team -> t.t_serialized == 0 ) {
745 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
746 if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
747 __kmp_clear_x87_fpu_status_word();
748 __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
749 __kmp_load_mxcsr( &serial_team->t.t_mxcsr );
753 this_thr -> th.th_team = serial_team -> t.t_parent;
754 this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
757 this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc;
758 this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0];
759 this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
762 this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
763 t.t_dispatch[ serial_team -> t.t_master_tid ];
766 __kmp_pop_current_task_from_thread( this_thr );
768 KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
769 this_thr -> th.th_current_task -> td_flags.executing = 1;
771 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
776 if ( ( this_thr -> th.th_task_team = this_thr -> th.th_team -> t.t_task_team ) != NULL ) {
777 this_thr -> th.th_task_state = this_thr -> th.th_task_team -> tt.tt_state;
779 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
780 global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
782 #endif // OMP_30_ENABLED
788 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
789 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
790 global_tid, serial_team, serial_team -> t.t_serialized ) );
792 #endif // OMP_30_ENABLED
798 if ( ( __itt_frame_end_v3_ptr && __kmp_forkjoin_frames && ! __kmp_forkjoin_frames_mode ) || KMP_ITT_DEBUG )
800 __kmp_itt_region_joined( global_tid, 1 );
803 if( __kmp_forkjoin_frames_mode == 1 && __kmp_itt_csv_file )
805 if( this_thr->th.th_team->t.t_level == 0 ) {
806 ident_t * loc = this_thr->th.th_ident;
810 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 1 );
813 #if defined( __GNUC__ )
814 # if !defined( __INTEL_COMPILER )
815 fr_end = __kmp_hardware_timestamp();
822 K_DIAG( 3, (
"__kmpc_end_serialized_parallel: T#%d frame_begin = %llu, frame_end = %llu\n",
823 global_tid, this_thr->th.th_frame_time, fr_end ) );
825 __kmp_str_buf_print( &__kmp_itt_frame_buffer,
"%s$omp$frame@%s:%d:%d,%llu,%llu,,\n",
826 str_loc.func, str_loc.file, str_loc.line, str_loc.col, this_thr->th.th_frame_time_serialized, fr_end );
827 __kmp_str_loc_free( &str_loc );
833 if ( __kmp_env_consistency_check )
834 __kmp_pop_parallel( global_tid, NULL );
852 KC_TRACE( 10, (
"__kmpc_flush: called\n" ) );
861 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
874 if ( ! __kmp_cpuinfo.initialized ) {
875 __kmp_query_cpuid( & __kmp_cpuinfo );
877 if ( ! __kmp_cpuinfo.sse2 ) {
880 #if defined( __GNUC__ ) && !defined( __INTEL_COMPILER )
881 __sync_synchronize();
888 #error Unknown or unsupported architecture
890 #endif // OMP_30_ENABLED
908 int explicit_barrier_flag;
909 KC_TRACE( 10, (
"__kmpc_barrier: called T#%d\n", global_tid ) );
911 if (! TCR_4(__kmp_init_parallel))
912 __kmp_parallel_initialize();
914 if ( __kmp_env_consistency_check ) {
916 KMP_WARNING( ConstructIdentInvalid );
919 __kmp_check_barrier( global_tid, ct_barrier, loc );
922 __kmp_threads[ global_tid ]->th.th_ident = loc;
930 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
945 KC_TRACE( 10, (
"__kmpc_master: called T#%d\n", global_tid ) );
947 if( ! TCR_4( __kmp_init_parallel ) )
948 __kmp_parallel_initialize();
950 if( KMP_MASTER_GTID( global_tid ))
953 if ( __kmp_env_consistency_check ) {
955 __kmp_push_sync( global_tid, ct_master, loc, NULL );
957 __kmp_check_sync( global_tid, ct_master, loc, NULL );
974 KC_TRACE( 10, (
"__kmpc_end_master: called T#%d\n", global_tid ) );
976 KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
978 if ( __kmp_env_consistency_check ) {
980 KMP_WARNING( ThreadIdentInvalid );
982 if( KMP_MASTER_GTID( global_tid ))
983 __kmp_pop_sync( global_tid, ct_master, loc );
999 KMP_DEBUG_ASSERT( __kmp_init_serial );
1001 KC_TRACE( 10, (
"__kmpc_ordered: called T#%d\n", gtid ));
1003 if (! TCR_4(__kmp_init_parallel))
1004 __kmp_parallel_initialize();
1007 __kmp_itt_ordered_prep( gtid );
1011 th = __kmp_threads[ gtid ];
1013 if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
1014 (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
1016 __kmp_parallel_deo( & gtid, & cid, loc );
1019 __kmp_itt_ordered_start( gtid );
1036 KC_TRACE( 10, (
"__kmpc_end_ordered: called T#%d\n", gtid ) );
1039 __kmp_itt_ordered_end( gtid );
1043 th = __kmp_threads[ gtid ];
1045 if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
1046 (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
1048 __kmp_parallel_dxo( & gtid, & cid, loc );
1052 __kmp_static_yield(
int arg ) {
1056 static kmp_user_lock_p
1057 __kmp_get_critical_section_ptr( kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid )
1059 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1065 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1067 if ( lck == NULL ) {
1072 lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
1073 __kmp_init_user_lock_with_checks( lck );
1074 __kmp_set_user_lock_location( lck, loc );
1076 __kmp_itt_critical_creating( lck );
1089 int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
1091 if ( status == 0 ) {
1094 __kmp_itt_critical_destroyed( lck );
1098 __kmp_destroy_user_lock_with_checks( lck );
1099 __kmp_user_lock_free( &idx, gtid, lck );
1100 lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1101 KMP_DEBUG_ASSERT( lck != NULL );
1120 kmp_user_lock_p lck;
1122 KC_TRACE( 10, (
"__kmpc_critical: called T#%d\n", global_tid ) );
1126 KMP_CHECK_USER_LOCK_INIT();
1128 if ( ( __kmp_user_lock_kind == lk_tas )
1129 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1130 lck = (kmp_user_lock_p)crit;
1132 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1133 else if ( ( __kmp_user_lock_kind == lk_futex )
1134 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1135 lck = (kmp_user_lock_p)crit;
1139 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
1142 if ( __kmp_env_consistency_check )
1143 __kmp_push_sync( global_tid, ct_critical, loc, lck );
1152 __kmp_itt_critical_acquiring( lck );
1156 __kmp_acquire_user_lock_with_checks( lck, global_tid );
1159 __kmp_itt_critical_acquired( lck );
1162 KA_TRACE( 15, (
"__kmpc_critical: done T#%d\n", global_tid ));
1177 kmp_user_lock_p lck;
1179 KC_TRACE( 10, (
"__kmpc_end_critical: called T#%d\n", global_tid ));
1181 if ( ( __kmp_user_lock_kind == lk_tas )
1182 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1183 lck = (kmp_user_lock_p)crit;
1185 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1186 else if ( ( __kmp_user_lock_kind == lk_futex )
1187 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1188 lck = (kmp_user_lock_p)crit;
1192 lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
1195 KMP_ASSERT(lck != NULL);
1197 if ( __kmp_env_consistency_check )
1198 __kmp_pop_sync( global_tid, ct_critical, loc );
1201 __kmp_itt_critical_releasing( lck );
1205 __kmp_release_user_lock_with_checks( lck, global_tid );
1207 KA_TRACE( 15, (
"__kmpc_end_critical: done T#%d\n", global_tid ));
1223 KC_TRACE( 10, (
"__kmpc_barrier_master: called T#%d\n", global_tid ) );
1225 if (! TCR_4(__kmp_init_parallel))
1226 __kmp_parallel_initialize();
1228 if ( __kmp_env_consistency_check )
1229 __kmp_check_barrier( global_tid, ct_barrier, loc );
1231 status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
1233 return (status != 0) ? 0 : 1;
1248 KC_TRACE( 10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid ));
1250 __kmp_end_split_barrier ( bs_plain_barrier, global_tid );
1268 KC_TRACE( 10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
1270 if (! TCR_4(__kmp_init_parallel))
1271 __kmp_parallel_initialize();
1273 if ( __kmp_env_consistency_check ) {
1275 KMP_WARNING( ConstructIdentInvalid );
1277 __kmp_check_barrier( global_tid, ct_barrier, loc );
1280 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
1284 if ( __kmp_env_consistency_check ) {
1288 if ( global_tid < 0 ) {
1289 KMP_WARNING( ThreadIdentInvalid );
1295 __kmp_pop_sync( global_tid, ct_master, loc );
1317 kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
1333 __kmp_exit_single( global_tid );
1346 KE_TRACE( 10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1348 if ( __kmp_env_consistency_check )
1349 __kmp_pop_workshare( global_tid, ct_pdo, loc );
1358 ompc_set_num_threads(
int arg )
1361 __kmp_set_num_threads( arg, __kmp_entry_gtid() );
1365 ompc_set_dynamic(
int flag )
1370 thread = __kmp_entry_thread();
1372 __kmp_save_internal_controls( thread );
1374 set__dynamic( thread, flag ? TRUE : FALSE );
1378 ompc_set_nested(
int flag )
1383 thread = __kmp_entry_thread();
1385 __kmp_save_internal_controls( thread );
1387 set__nested( thread, flag ? TRUE : FALSE );
1393 ompc_set_max_active_levels(
int max_active_levels )
1399 __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
1403 ompc_set_schedule( omp_sched_t kind,
int modifier )
1406 __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
1410 ompc_get_ancestor_thread_num(
int level )
1412 return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
1416 ompc_get_team_size(
int level )
1418 return __kmp_get_team_size( __kmp_entry_gtid(), level );
1421 #endif // OMP_30_ENABLED
1424 kmpc_set_stacksize(
int arg )
1427 __kmp_aux_set_stacksize( arg );
1431 kmpc_set_stacksize_s(
size_t arg )
1434 __kmp_aux_set_stacksize( arg );
1438 kmpc_set_blocktime(
int arg )
1443 gtid = __kmp_entry_gtid();
1444 tid = __kmp_tid_from_gtid(gtid);
1445 thread = __kmp_thread_from_gtid(gtid);
1447 __kmp_aux_set_blocktime( arg, thread, tid );
1451 kmpc_set_library(
int arg )
1454 __kmp_user_set_library( (
enum library_type)arg );
1458 kmpc_set_defaults(
char const * str )
1461 __kmp_aux_set_defaults( str, strlen( str ) );
1464 #ifdef OMP_30_ENABLED
1467 kmpc_set_affinity_mask_proc(
int proc,
void **mask )
1469 #if defined(KMP_STUB) || !(KMP_OS_WINDOWS || KMP_OS_LINUX)
1472 if ( ! TCR_4(__kmp_init_middle) ) {
1473 __kmp_middle_initialize();
1475 return __kmp_aux_set_affinity_mask_proc( proc, mask );
1480 kmpc_unset_affinity_mask_proc(
int proc,
void **mask )
1482 #if defined(KMP_STUB) || !(KMP_OS_WINDOWS || KMP_OS_LINUX)
1485 if ( ! TCR_4(__kmp_init_middle) ) {
1486 __kmp_middle_initialize();
1488 return __kmp_aux_unset_affinity_mask_proc( proc, mask );
1493 kmpc_get_affinity_mask_proc(
int proc,
void **mask )
1495 #if defined(KMP_STUB) || !(KMP_OS_WINDOWS || KMP_OS_LINUX)
1498 if ( ! TCR_4(__kmp_init_middle) ) {
1499 __kmp_middle_initialize();
1501 return __kmp_aux_get_affinity_mask_proc( proc, mask );
1553 KC_TRACE( 10, (
"__kmpc_copyprivate: called T#%d\n", gtid ));
1557 data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
1559 if ( __kmp_env_consistency_check ) {
1561 KMP_WARNING( ConstructIdentInvalid );
1567 if (didit) *data_ptr = cpy_data;
1570 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1572 if (! didit) (*cpy_func)( cpy_data, *data_ptr );
1577 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1582 #define INIT_LOCK __kmp_init_user_lock_with_checks
1583 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
1584 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
1585 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
1586 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
1587 #define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed
1588 #define RELEASE_LOCK __kmp_release_user_lock_with_checks
1589 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
1590 #define TEST_LOCK __kmp_test_user_lock_with_checks
1591 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
1592 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
1593 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
1603 __kmpc_init_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1604 static char const *
const func =
"omp_init_lock";
1605 kmp_user_lock_p lck;
1606 KMP_DEBUG_ASSERT( __kmp_init_serial );
1608 if ( __kmp_env_consistency_check ) {
1609 if ( user_lock == NULL ) {
1610 KMP_FATAL( LockIsUninitialized, func );
1614 KMP_CHECK_USER_LOCK_INIT();
1616 if ( ( __kmp_user_lock_kind == lk_tas )
1617 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1618 lck = (kmp_user_lock_p)user_lock;
1620 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1621 else if ( ( __kmp_user_lock_kind == lk_futex )
1622 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1623 lck = (kmp_user_lock_p)user_lock;
1627 lck = __kmp_user_lock_allocate( user_lock, gtid );
1630 __kmp_set_user_lock_location( lck, loc );
1633 __kmp_itt_lock_creating( lck );
1639 __kmpc_init_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1640 static char const *
const func =
"omp_init_nest_lock";
1641 kmp_user_lock_p lck;
1642 KMP_DEBUG_ASSERT( __kmp_init_serial );
1644 if ( __kmp_env_consistency_check ) {
1645 if ( user_lock == NULL ) {
1646 KMP_FATAL( LockIsUninitialized, func );
1650 KMP_CHECK_USER_LOCK_INIT();
1652 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1653 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1654 lck = (kmp_user_lock_p)user_lock;
1656 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1657 else if ( ( __kmp_user_lock_kind == lk_futex )
1658 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1659 <= OMP_NEST_LOCK_T_SIZE ) ) {
1660 lck = (kmp_user_lock_p)user_lock;
1664 lck = __kmp_user_lock_allocate( user_lock, gtid );
1667 INIT_NESTED_LOCK( lck );
1668 __kmp_set_user_lock_location( lck, loc );
1671 __kmp_itt_lock_creating( lck );
1676 __kmpc_destroy_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1678 kmp_user_lock_p lck;
1680 if ( ( __kmp_user_lock_kind == lk_tas )
1681 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1682 lck = (kmp_user_lock_p)user_lock;
1684 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1685 else if ( ( __kmp_user_lock_kind == lk_futex )
1686 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1687 lck = (kmp_user_lock_p)user_lock;
1691 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_lock" );
1695 __kmp_itt_lock_destroyed( lck );
1697 DESTROY_LOCK( lck );
1699 if ( ( __kmp_user_lock_kind == lk_tas )
1700 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1703 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1704 else if ( ( __kmp_user_lock_kind == lk_futex )
1705 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1710 __kmp_user_lock_free( user_lock, gtid, lck );
1716 __kmpc_destroy_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1718 kmp_user_lock_p lck;
1720 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1721 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1722 lck = (kmp_user_lock_p)user_lock;
1724 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1725 else if ( ( __kmp_user_lock_kind == lk_futex )
1726 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1727 <= OMP_NEST_LOCK_T_SIZE ) ) {
1728 lck = (kmp_user_lock_p)user_lock;
1732 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_nest_lock" );
1736 __kmp_itt_lock_destroyed( lck );
1739 DESTROY_NESTED_LOCK( lck );
1741 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1742 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1745 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1746 else if ( ( __kmp_user_lock_kind == lk_futex )
1747 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1748 <= OMP_NEST_LOCK_T_SIZE ) ) {
1753 __kmp_user_lock_free( user_lock, gtid, lck );
1758 __kmpc_set_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1759 kmp_user_lock_p lck;
1761 if ( ( __kmp_user_lock_kind == lk_tas )
1762 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1763 lck = (kmp_user_lock_p)user_lock;
1765 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1766 else if ( ( __kmp_user_lock_kind == lk_futex )
1767 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1768 lck = (kmp_user_lock_p)user_lock;
1772 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_lock" );
1776 __kmp_itt_lock_acquiring( lck );
1779 ACQUIRE_LOCK( lck, gtid );
1782 __kmp_itt_lock_acquired( lck );
1788 __kmpc_set_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1789 kmp_user_lock_p lck;
1791 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1792 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1793 lck = (kmp_user_lock_p)user_lock;
1795 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1796 else if ( ( __kmp_user_lock_kind == lk_futex )
1797 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1798 <= OMP_NEST_LOCK_T_SIZE ) ) {
1799 lck = (kmp_user_lock_p)user_lock;
1803 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_nest_lock" );
1807 __kmp_itt_lock_acquiring( lck );
1810 ACQUIRE_NESTED_LOCK( lck, gtid );
1813 __kmp_itt_lock_acquired( lck );
1818 __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1820 kmp_user_lock_p lck;
1825 if ( ( __kmp_user_lock_kind == lk_tas )
1826 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1827 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1830 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
1832 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
1836 lck = (kmp_user_lock_p)user_lock;
1839 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1840 else if ( ( __kmp_user_lock_kind == lk_futex )
1841 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1842 lck = (kmp_user_lock_p)user_lock;
1846 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_lock" );
1850 __kmp_itt_lock_releasing( lck );
1853 RELEASE_LOCK( lck, gtid );
1858 __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1860 kmp_user_lock_p lck;
1864 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1865 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1866 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1868 kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
1870 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
1872 if ( --(tl->lk.depth_locked) == 0 ) {
1873 TCW_4(tl->lk.poll, 0);
1878 lck = (kmp_user_lock_p)user_lock;
1881 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1882 else if ( ( __kmp_user_lock_kind == lk_futex )
1883 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1884 <= OMP_NEST_LOCK_T_SIZE ) ) {
1885 lck = (kmp_user_lock_p)user_lock;
1889 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_nest_lock" );
1893 __kmp_itt_lock_releasing( lck );
1896 RELEASE_NESTED_LOCK( lck, gtid );
1901 __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1903 kmp_user_lock_p lck;
1906 if ( ( __kmp_user_lock_kind == lk_tas )
1907 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1908 lck = (kmp_user_lock_p)user_lock;
1910 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1911 else if ( ( __kmp_user_lock_kind == lk_futex )
1912 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1913 lck = (kmp_user_lock_p)user_lock;
1917 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_lock" );
1921 __kmp_itt_lock_acquiring( lck );
1924 rc = TEST_LOCK( lck, gtid );
1927 __kmp_itt_lock_acquired( lck );
1929 __kmp_itt_lock_cancelled( lck );
1932 return ( rc ? FTN_TRUE : FTN_FALSE );
1939 __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
1941 kmp_user_lock_p lck;
1944 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1945 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1946 lck = (kmp_user_lock_p)user_lock;
1948 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1949 else if ( ( __kmp_user_lock_kind == lk_futex )
1950 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1951 <= OMP_NEST_LOCK_T_SIZE ) ) {
1952 lck = (kmp_user_lock_p)user_lock;
1956 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_nest_lock" );
1960 __kmp_itt_lock_acquiring( lck );
1963 rc = TEST_NESTED_LOCK( lck, gtid );
1966 __kmp_itt_lock_acquired( lck );
1968 __kmp_itt_lock_cancelled( lck );
1986 #define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \
1987 ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) )
1989 #define __KMP_GET_REDUCTION_METHOD(gtid) \
1990 ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method )
1996 static __forceinline
void
1997 __kmp_enter_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2003 kmp_user_lock_p lck;
2008 if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
2009 lck = (kmp_user_lock_p)crit;
2012 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
2014 KMP_DEBUG_ASSERT( lck != NULL );
2016 if ( __kmp_env_consistency_check )
2017 __kmp_push_sync( global_tid, ct_critical, loc, lck );
2019 __kmp_acquire_user_lock_with_checks( lck, global_tid );
2023 static __forceinline
void
2024 __kmp_end_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2026 kmp_user_lock_p lck;
2030 if ( __kmp_base_user_lock_size > 32 ) {
2031 lck = *( (kmp_user_lock_p *) crit );
2032 KMP_ASSERT( lck != NULL );
2034 lck = (kmp_user_lock_p) crit;
2037 if ( __kmp_env_consistency_check )
2038 __kmp_pop_sync( global_tid, ct_critical, loc );
2040 __kmp_release_user_lock_with_checks( lck, global_tid );
2061 ident_t *loc, kmp_int32 global_tid,
2062 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2063 kmp_critical_name *lck ) {
2066 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2068 KA_TRACE( 10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
2075 if( ! TCR_4( __kmp_init_parallel ) )
2076 __kmp_parallel_initialize();
2079 if ( __kmp_env_consistency_check )
2080 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2093 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2094 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2096 if( packed_reduction_method == critical_reduce_block ) {
2098 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2101 }
else if( packed_reduction_method == empty_reduce_block ) {
2106 }
else if( packed_reduction_method == atomic_reduce_block ) {
2113 if ( __kmp_env_consistency_check )
2114 __kmp_pop_sync( global_tid, ct_reduce, loc );
2116 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2128 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
2129 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2133 if ( __kmp_env_consistency_check ) {
2135 __kmp_pop_sync( global_tid, ct_reduce, loc );
2146 KA_TRACE( 10, (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2162 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2164 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
2166 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2168 if( packed_reduction_method == critical_reduce_block ) {
2170 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2172 }
else if( packed_reduction_method == empty_reduce_block ) {
2176 }
else if( packed_reduction_method == atomic_reduce_block ) {
2183 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2194 if ( __kmp_env_consistency_check )
2195 __kmp_pop_sync( global_tid, ct_reduce, loc );
2197 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2219 ident_t *loc, kmp_int32 global_tid,
2220 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
2221 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2222 kmp_critical_name *lck )
2225 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2227 KA_TRACE( 10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid ) );
2234 if( ! TCR_4( __kmp_init_parallel ) )
2235 __kmp_parallel_initialize();
2238 if ( __kmp_env_consistency_check )
2239 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2243 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2244 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2246 if( packed_reduction_method == critical_reduce_block ) {
2248 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2251 }
else if( packed_reduction_method == empty_reduce_block ) {
2256 }
else if( packed_reduction_method == atomic_reduce_block ) {
2260 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2265 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
2266 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2270 if ( __kmp_env_consistency_check ) {
2272 __kmp_pop_sync( global_tid, ct_reduce, loc );
2283 KA_TRACE( 10, (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2300 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2302 KA_TRACE( 10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
2304 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2309 if( packed_reduction_method == critical_reduce_block ) {
2311 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2314 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2316 }
else if( packed_reduction_method == empty_reduce_block ) {
2321 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2323 }
else if( packed_reduction_method == atomic_reduce_block ) {
2326 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2328 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2331 __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
2340 if ( __kmp_env_consistency_check )
2341 __kmp_pop_sync( global_tid, ct_reduce, loc );
2343 KA_TRACE( 10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2348 #undef __KMP_GET_REDUCTION_METHOD
2349 #undef __KMP_SET_REDUCTION_METHOD
2354 __kmpc_get_taskid() {
2359 kmp_info_t * thread;
2361 gtid = __kmp_get_gtid();
2365 thread = __kmp_thread_from_gtid( gtid );
2366 return thread->th.th_current_task->td_task_id;
2378 __kmpc_get_parent_taskid() {
2383 kmp_info_t * thread;
2384 kmp_taskdata_t * parent_task;
2386 gtid = __kmp_get_gtid();
2390 thread = __kmp_thread_from_gtid( gtid );
2391 parent_task = thread->th.th_current_task->td_parent;
2392 return ( parent_task == NULL ? 0 : parent_task->td_task_id );
2402 void __kmpc_place_threads(
int nC,
int nT,
int nO)
2405 if ( ! __kmp_init_serial ) {
2406 __kmp_serial_initialize();
2408 __kmp_place_num_cores = nC;
2409 __kmp_place_num_threads_per_core = nT;
2410 __kmp_place_core_offset = nO;