45 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
47 # include <sys/syscall.h>
63 # define __kmp_static_delay( arg )
67 __kmp_static_delay(
int arg )
70 # if KMP_ARCH_X86_64 && KMP_OS_LINUX
71 KMP_ASSERT( arg != 0 );
73 KMP_ASSERT( arg >= 0 );
79 __kmp_static_yield(
int arg )
88 __kmp_validate_locks(
void )
94 x = ~((kmp_uint32) 0) - 2;
97 for (i = 0; i < 8; ++i, ++x, ++y) {
98 kmp_uint32 z = (x - y);
102 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
120 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
122 return TCR_4( lck->lk.poll ) - 1;
126 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
128 return lck->lk.depth_locked != -1;
131 __forceinline
static void
132 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
136 #ifdef USE_LOCK_PROFILE
137 kmp_uint32 curr = TCR_4( lck->lk.poll );
138 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
139 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
143 if ( ( lck->lk.poll == 0 )
144 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, gtid + 1 ) ) {
145 KMP_FSYNC_ACQUIRED(lck);
150 KMP_FSYNC_PREPARE( lck );
151 KMP_INIT_YIELD( spins );
152 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
157 KMP_YIELD_SPIN( spins );
160 while ( ( lck->lk.poll != 0 ) ||
161 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, gtid + 1 ) ) ) {
165 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
170 KMP_YIELD_SPIN( spins );
173 KMP_FSYNC_ACQUIRED( lck );
177 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
179 __kmp_acquire_tas_lock_timed_template( lck, gtid );
183 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
185 if ( __kmp_env_consistency_check ) {
186 char const *
const func =
"omp_set_lock";
187 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
188 && __kmp_is_tas_lock_nestable( lck ) ) {
189 KMP_FATAL( LockNestableUsedAsSimple, func );
191 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
192 KMP_FATAL( LockIsAlreadyOwned, func );
195 __kmp_acquire_tas_lock( lck, gtid );
199 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
201 if ( ( lck->lk.poll == 0 )
202 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, gtid + 1 ) ) {
203 KMP_FSYNC_ACQUIRED( lck );
210 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
212 if ( __kmp_env_consistency_check ) {
213 char const *
const func =
"omp_test_lock";
214 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
215 && __kmp_is_tas_lock_nestable( lck ) ) {
216 KMP_FATAL( LockNestableUsedAsSimple, func );
219 return __kmp_test_tas_lock( lck, gtid );
223 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
227 KMP_FSYNC_RELEASING(lck);
228 KMP_ST_REL32( &(lck->lk.poll), 0 );
232 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
237 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
239 if ( __kmp_env_consistency_check ) {
240 char const *
const func =
"omp_unset_lock";
242 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
243 && __kmp_is_tas_lock_nestable( lck ) ) {
244 KMP_FATAL( LockNestableUsedAsSimple, func );
246 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
247 KMP_FATAL( LockUnsettingFree, func );
249 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
250 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
251 KMP_FATAL( LockUnsettingSetByAnother, func );
254 __kmp_release_tas_lock( lck, gtid );
258 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
260 TCW_4( lck->lk.poll, 0 );
264 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
266 __kmp_init_tas_lock( lck );
270 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
276 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
278 if ( __kmp_env_consistency_check ) {
279 char const *
const func =
"omp_destroy_lock";
280 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
281 && __kmp_is_tas_lock_nestable( lck ) ) {
282 KMP_FATAL( LockNestableUsedAsSimple, func );
284 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
285 KMP_FATAL( LockStillOwned, func );
288 __kmp_destroy_tas_lock( lck );
297 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
299 KMP_DEBUG_ASSERT( gtid >= 0 );
301 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
302 lck->lk.depth_locked += 1;
305 __kmp_acquire_tas_lock_timed_template( lck, gtid );
306 lck->lk.depth_locked = 1;
311 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
313 if ( __kmp_env_consistency_check ) {
314 char const *
const func =
"omp_set_nest_lock";
315 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
316 KMP_FATAL( LockSimpleUsedAsNestable, func );
319 __kmp_acquire_nested_tas_lock( lck, gtid );
323 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
327 KMP_DEBUG_ASSERT( gtid >= 0 );
329 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
330 retval = ++lck->lk.depth_locked;
332 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
337 retval = lck->lk.depth_locked = 1;
343 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
345 if ( __kmp_env_consistency_check ) {
346 char const *
const func =
"omp_test_nest_lock";
347 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
348 KMP_FATAL( LockSimpleUsedAsNestable, func );
351 return __kmp_test_nested_tas_lock( lck, gtid );
355 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
357 KMP_DEBUG_ASSERT( gtid >= 0 );
360 if ( --(lck->lk.depth_locked) == 0 ) {
361 __kmp_release_tas_lock( lck, gtid );
366 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
368 if ( __kmp_env_consistency_check ) {
369 char const *
const func =
"omp_unset_nest_lock";
371 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
372 KMP_FATAL( LockSimpleUsedAsNestable, func );
374 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
375 KMP_FATAL( LockUnsettingFree, func );
377 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
378 KMP_FATAL( LockUnsettingSetByAnother, func );
381 __kmp_release_nested_tas_lock( lck, gtid );
385 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
387 __kmp_init_tas_lock( lck );
388 lck->lk.depth_locked = 0;
392 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
394 __kmp_init_nested_tas_lock( lck );
398 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
400 __kmp_destroy_tas_lock( lck );
401 lck->lk.depth_locked = 0;
405 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
407 if ( __kmp_env_consistency_check ) {
408 char const *
const func =
"omp_destroy_nest_lock";
409 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
410 KMP_FATAL( LockSimpleUsedAsNestable, func );
412 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
413 KMP_FATAL( LockStillOwned, func );
416 __kmp_destroy_nested_tas_lock( lck );
420 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
431 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
433 return ( TCR_4( lck->lk.poll ) >> 1 ) - 1;
437 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
439 return lck->lk.depth_locked != -1;
442 __forceinline
static void
443 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
445 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
449 #ifdef USE_LOCK_PROFILE
450 kmp_uint32 curr = TCR_4( lck->lk.poll );
451 if ( ( curr != 0 ) && ( curr != gtid_code ) )
452 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
456 KMP_FSYNC_PREPARE( lck );
457 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
458 lck, lck->lk.poll, gtid ) );
461 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), 0,
462 gtid_code ) ) != 0 ) {
463 kmp_int32 cond = poll_val & 1;
464 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
465 lck, gtid, poll_val, cond ) );
480 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ),
481 poll_val, poll_val | 1 ) ) {
482 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
483 lck, lck->lk.poll, gtid ) );
488 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
489 lck, lck->lk.poll, gtid ) );
492 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
493 lck, gtid, poll_val ) );
496 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
497 poll_val, NULL, NULL, 0 ) ) != 0 ) {
498 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
499 lck, gtid, poll_val, rc, errno ) );
503 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
504 lck, gtid, poll_val ) );
514 KMP_FSYNC_ACQUIRED( lck );
515 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
516 lck, lck->lk.poll, gtid ) );
520 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
522 __kmp_acquire_futex_lock_timed_template( lck, gtid );
526 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
528 if ( __kmp_env_consistency_check ) {
529 char const *
const func =
"omp_set_lock";
530 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
531 && __kmp_is_futex_lock_nestable( lck ) ) {
532 KMP_FATAL( LockNestableUsedAsSimple, func );
534 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
535 KMP_FATAL( LockIsAlreadyOwned, func );
538 __kmp_acquire_futex_lock( lck, gtid );
542 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
544 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, ( gtid + 1 ) << 1 ) ) {
545 KMP_FSYNC_ACQUIRED( lck );
552 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
554 if ( __kmp_env_consistency_check ) {
555 char const *
const func =
"omp_test_lock";
556 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
557 && __kmp_is_futex_lock_nestable( lck ) ) {
558 KMP_FATAL( LockNestableUsedAsSimple, func );
561 return __kmp_test_futex_lock( lck, gtid );
565 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
569 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
570 lck, lck->lk.poll, gtid ) );
572 KMP_FSYNC_RELEASING(lck);
574 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), 0 );
576 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
577 lck, gtid, poll_val ) );
579 if ( poll_val & 1 ) {
580 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
582 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, 1, NULL, NULL, 0 );
587 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
588 lck, lck->lk.poll, gtid ) );
590 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
595 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
597 if ( __kmp_env_consistency_check ) {
598 char const *
const func =
"omp_unset_lock";
600 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
601 && __kmp_is_futex_lock_nestable( lck ) ) {
602 KMP_FATAL( LockNestableUsedAsSimple, func );
604 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
605 KMP_FATAL( LockUnsettingFree, func );
607 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
608 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
609 KMP_FATAL( LockUnsettingSetByAnother, func );
612 __kmp_release_futex_lock( lck, gtid );
616 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
618 TCW_4( lck->lk.poll, 0 );
622 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
624 __kmp_init_futex_lock( lck );
628 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
634 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
636 if ( __kmp_env_consistency_check ) {
637 char const *
const func =
"omp_destroy_lock";
638 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
639 && __kmp_is_futex_lock_nestable( lck ) ) {
640 KMP_FATAL( LockNestableUsedAsSimple, func );
642 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
643 KMP_FATAL( LockStillOwned, func );
646 __kmp_destroy_futex_lock( lck );
655 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
657 KMP_DEBUG_ASSERT( gtid >= 0 );
659 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
660 lck->lk.depth_locked += 1;
663 __kmp_acquire_futex_lock_timed_template( lck, gtid );
664 lck->lk.depth_locked = 1;
669 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
671 if ( __kmp_env_consistency_check ) {
672 char const *
const func =
"omp_set_nest_lock";
673 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
674 KMP_FATAL( LockSimpleUsedAsNestable, func );
677 __kmp_acquire_nested_futex_lock( lck, gtid );
681 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
685 KMP_DEBUG_ASSERT( gtid >= 0 );
687 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
688 retval = ++lck->lk.depth_locked;
690 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
695 retval = lck->lk.depth_locked = 1;
701 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
703 if ( __kmp_env_consistency_check ) {
704 char const *
const func =
"omp_test_nest_lock";
705 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
706 KMP_FATAL( LockSimpleUsedAsNestable, func );
709 return __kmp_test_nested_futex_lock( lck, gtid );
713 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
715 KMP_DEBUG_ASSERT( gtid >= 0 );
718 if ( --(lck->lk.depth_locked) == 0 ) {
719 __kmp_release_futex_lock( lck, gtid );
724 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
726 if ( __kmp_env_consistency_check ) {
727 char const *
const func =
"omp_unset_nest_lock";
729 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
730 KMP_FATAL( LockSimpleUsedAsNestable, func );
732 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
733 KMP_FATAL( LockUnsettingFree, func );
735 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
736 KMP_FATAL( LockUnsettingSetByAnother, func );
739 __kmp_release_nested_futex_lock( lck, gtid );
743 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
745 __kmp_init_futex_lock( lck );
746 lck->lk.depth_locked = 0;
750 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
752 __kmp_init_nested_futex_lock( lck );
756 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
758 __kmp_destroy_futex_lock( lck );
759 lck->lk.depth_locked = 0;
763 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
765 if ( __kmp_env_consistency_check ) {
766 char const *
const func =
"omp_destroy_nest_lock";
767 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
768 KMP_FATAL( LockSimpleUsedAsNestable, func );
770 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
771 KMP_FATAL( LockStillOwned, func );
774 __kmp_destroy_nested_futex_lock( lck );
777 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
784 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
786 return TCR_4( lck->lk.owner_id ) - 1;
790 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
792 return lck->lk.depth_locked != -1;
796 __kmp_bakery_check(kmp_uint value, kmp_uint checker)
798 register kmp_uint32 pause;
800 if (value == checker) {
803 for (pause = checker - value; pause != 0; --pause) {
804 __kmp_static_delay(TRUE);
809 __forceinline
static void
810 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
812 kmp_uint32 my_ticket;
815 my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
817 #ifdef USE_LOCK_PROFILE
818 if ( TCR_4( lck->lk.now_serving ) != my_ticket )
819 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
823 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
824 KMP_FSYNC_ACQUIRED(lck);
827 KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
828 KMP_FSYNC_ACQUIRED(lck);
832 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
834 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
838 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
840 if ( __kmp_env_consistency_check ) {
841 char const *
const func =
"omp_set_lock";
842 if ( lck->lk.initialized != lck ) {
843 KMP_FATAL( LockIsUninitialized, func );
845 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
846 KMP_FATAL( LockNestableUsedAsSimple, func );
848 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
849 KMP_FATAL( LockIsAlreadyOwned, func );
853 __kmp_acquire_ticket_lock( lck, gtid );
855 if ( __kmp_env_consistency_check ) {
856 lck->lk.owner_id = gtid + 1;
861 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
863 kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
864 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
865 kmp_uint32 next_ticket = my_ticket + 1;
866 if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
867 my_ticket, next_ticket ) ) {
868 KMP_FSYNC_ACQUIRED( lck );
876 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
878 if ( __kmp_env_consistency_check ) {
879 char const *
const func =
"omp_test_lock";
880 if ( lck->lk.initialized != lck ) {
881 KMP_FATAL( LockIsUninitialized, func );
883 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
884 KMP_FATAL( LockNestableUsedAsSimple, func );
888 int retval = __kmp_test_ticket_lock( lck, gtid );
890 if ( __kmp_env_consistency_check && retval ) {
891 lck->lk.owner_id = gtid + 1;
897 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
903 KMP_FSYNC_RELEASING(lck);
904 distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
906 KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
911 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
915 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
917 if ( __kmp_env_consistency_check ) {
918 char const *
const func =
"omp_unset_lock";
920 if ( lck->lk.initialized != lck ) {
921 KMP_FATAL( LockIsUninitialized, func );
923 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
924 KMP_FATAL( LockNestableUsedAsSimple, func );
926 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
927 KMP_FATAL( LockUnsettingFree, func );
929 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
930 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
931 KMP_FATAL( LockUnsettingSetByAnother, func );
933 lck->lk.owner_id = 0;
935 __kmp_release_ticket_lock( lck, gtid );
939 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
941 lck->lk.location = NULL;
942 TCW_4( lck->lk.next_ticket, 0 );
943 TCW_4( lck->lk.now_serving, 0 );
944 lck->lk.owner_id = 0;
945 lck->lk.depth_locked = -1;
946 lck->lk.initialized = (kmp_ticket_lock *)lck;
950 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
952 __kmp_init_ticket_lock( lck );
956 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
958 lck->lk.initialized = NULL;
959 lck->lk.location = NULL;
960 lck->lk.next_ticket = 0;
961 lck->lk.now_serving = 0;
962 lck->lk.owner_id = 0;
963 lck->lk.depth_locked = -1;
967 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
969 if ( __kmp_env_consistency_check ) {
970 char const *
const func =
"omp_destroy_lock";
971 if ( lck->lk.initialized != lck ) {
972 KMP_FATAL( LockIsUninitialized, func );
974 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
975 KMP_FATAL( LockNestableUsedAsSimple, func );
977 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
978 KMP_FATAL( LockStillOwned, func );
981 __kmp_destroy_ticket_lock( lck );
990 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
992 KMP_DEBUG_ASSERT( gtid >= 0 );
994 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
995 lck->lk.depth_locked += 1;
998 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
1000 lck->lk.depth_locked = 1;
1002 lck->lk.owner_id = gtid + 1;
1007 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1009 if ( __kmp_env_consistency_check ) {
1010 char const *
const func =
"omp_set_nest_lock";
1011 if ( lck->lk.initialized != lck ) {
1012 KMP_FATAL( LockIsUninitialized, func );
1014 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1015 KMP_FATAL( LockSimpleUsedAsNestable, func );
1018 __kmp_acquire_nested_ticket_lock( lck, gtid );
1022 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1026 KMP_DEBUG_ASSERT( gtid >= 0 );
1028 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
1029 retval = ++lck->lk.depth_locked;
1031 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
1036 retval = lck->lk.depth_locked = 1;
1038 lck->lk.owner_id = gtid + 1;
1044 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
1047 if ( __kmp_env_consistency_check ) {
1048 char const *
const func =
"omp_test_nest_lock";
1049 if ( lck->lk.initialized != lck ) {
1050 KMP_FATAL( LockIsUninitialized, func );
1052 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1053 KMP_FATAL( LockSimpleUsedAsNestable, func );
1056 return __kmp_test_nested_ticket_lock( lck, gtid );
1060 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1062 KMP_DEBUG_ASSERT( gtid >= 0 );
1065 if ( --(lck->lk.depth_locked) == 0 ) {
1067 lck->lk.owner_id = 0;
1068 __kmp_release_ticket_lock( lck, gtid );
1073 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1075 if ( __kmp_env_consistency_check ) {
1076 char const *
const func =
"omp_unset_nest_lock";
1078 if ( lck->lk.initialized != lck ) {
1079 KMP_FATAL( LockIsUninitialized, func );
1081 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1082 KMP_FATAL( LockSimpleUsedAsNestable, func );
1084 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1085 KMP_FATAL( LockUnsettingFree, func );
1087 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1088 KMP_FATAL( LockUnsettingSetByAnother, func );
1091 __kmp_release_nested_ticket_lock( lck, gtid );
1095 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1097 __kmp_init_ticket_lock( lck );
1098 lck->lk.depth_locked = 0;
1102 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1104 __kmp_init_nested_ticket_lock( lck );
1108 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1110 __kmp_destroy_ticket_lock( lck );
1111 lck->lk.depth_locked = 0;
1115 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1117 if ( __kmp_env_consistency_check ) {
1118 char const *
const func =
"omp_destroy_nest_lock";
1119 if ( lck->lk.initialized != lck ) {
1120 KMP_FATAL( LockIsUninitialized, func );
1122 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1123 KMP_FATAL( LockSimpleUsedAsNestable, func );
1125 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1126 KMP_FATAL( LockStillOwned, func );
1129 __kmp_destroy_nested_ticket_lock( lck );
1138 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1140 return lck == lck->lk.initialized;
1144 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1146 return lck->lk.location;
1150 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1152 lck->lk.location = loc;
1155 static kmp_lock_flags_t
1156 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1158 return lck->lk.flags;
1162 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1164 lck->lk.flags = flags;
1223 #ifdef DEBUG_QUEUING_LOCKS
1226 #define TRACE_BUF_ELE 1024
1227 static char traces[TRACE_BUF_ELE][128] = { 0 }
1229 #define TRACE_LOCK(X,Y) sprintf( traces[tc++ % TRACE_BUF_ELE], "t%d at %s\n", X, Y );
1230 #define TRACE_LOCK_T(X,Y,Z) sprintf( traces[tc++ % TRACE_BUF_ELE], "t%d at %s%d\n", X,Y,Z );
1231 #define TRACE_LOCK_HT(X,Y,Z,Q) sprintf( traces[tc++ % TRACE_BUF_ELE], "t%d at %s %d,%d\n", X, Y, Z, Q );
1234 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1235 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1239 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1241 i = tc % TRACE_BUF_ELE;
1242 __kmp_printf_no_lock(
"%s\n", traces[i] );
1243 i = (i+1) % TRACE_BUF_ELE;
1244 while ( i != (tc % TRACE_BUF_ELE) ) {
1245 __kmp_printf_no_lock(
"%s", traces[i] );
1246 i = (i+1) % TRACE_BUF_ELE;
1248 __kmp_printf_no_lock(
"\n" );
1250 __kmp_printf_no_lock(
1251 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1252 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1255 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1257 if ( lck->lk.head_id >= 1 ) {
1258 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1260 __kmp_printf_no_lock(
"-> %d ", t );
1261 t = __kmp_threads[t-1]->th.th_next_waiting;
1264 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1265 __kmp_printf_no_lock(
"\n\n" );
1271 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1273 return TCR_4( lck->lk.owner_id ) - 1;
1277 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1279 return lck->lk.depth_locked != -1;
1283 template <
bool takeTime>
1286 __forceinline
static void
1287 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1290 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1291 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1292 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1293 volatile kmp_uint32 *spin_here_p;
1294 kmp_int32 need_mf = 1;
1296 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1298 KMP_FSYNC_PREPARE( lck );
1299 KMP_DEBUG_ASSERT( this_thr != NULL );
1300 spin_here_p = & this_thr->th.th_spin_here;
1302 #ifdef DEBUG_QUEUING_LOCKS
1303 TRACE_LOCK( gtid+1,
"acq ent" );
1305 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1306 if ( this_thr->th.th_next_waiting != 0 )
1307 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1309 KMP_DEBUG_ASSERT( !*spin_here_p );
1310 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1320 *spin_here_p = TRUE;
1333 #ifdef DEBUG_QUEUING_LOCKS
1335 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1344 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1345 KMP_PACK_64( -1, 0 ),
1346 KMP_PACK_64( gtid+1, gtid+1 ) );
1347 #ifdef DEBUG_QUEUING_LOCKS
1348 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1356 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1358 #ifdef DEBUG_QUEUING_LOCKS
1359 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1368 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1370 #ifdef DEBUG_QUEUING_LOCKS
1371 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1379 kmp_int32 grabbed_lock;
1381 #ifdef DEBUG_QUEUING_LOCKS
1383 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1388 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1390 if ( grabbed_lock ) {
1392 *spin_here_p = FALSE;
1394 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1396 #ifdef DEBUG_QUEUING_LOCKS
1397 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1399 KMP_FSYNC_ACQUIRED( lck );
1409 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1410 KMP_ASSERT( tail_thr != NULL );
1411 tail_thr->th.th_next_waiting = gtid+1;
1414 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1421 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1423 #ifdef DEBUG_QUEUING_LOCKS
1424 TRACE_LOCK( gtid+1,
"acq spin" );
1426 if ( this_thr->th.th_next_waiting != 0 )
1427 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1429 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1430 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1433 #ifdef DEBUG_QUEUING_LOCKS
1434 TRACE_LOCK( gtid+1,
"acq exit 2" );
1443 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1445 #ifdef DEBUG_QUEUING_LOCKS
1446 TRACE_LOCK( gtid+1,
"acq retry" );
1450 KMP_ASSERT2( 0,
"should not get here" );
1454 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1456 KMP_DEBUG_ASSERT( gtid >= 0 );
1458 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1462 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1465 if ( __kmp_env_consistency_check ) {
1466 char const *
const func =
"omp_set_lock";
1467 if ( lck->lk.initialized != lck ) {
1468 KMP_FATAL( LockIsUninitialized, func );
1470 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1471 KMP_FATAL( LockNestableUsedAsSimple, func );
1473 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1474 KMP_FATAL( LockIsAlreadyOwned, func );
1478 __kmp_acquire_queuing_lock( lck, gtid );
1480 if ( __kmp_env_consistency_check ) {
1481 lck->lk.owner_id = gtid + 1;
1486 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1488 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1491 kmp_info_t *this_thr;
1494 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1495 KMP_DEBUG_ASSERT( gtid >= 0 );
1497 this_thr = __kmp_thread_from_gtid( gtid );
1498 KMP_DEBUG_ASSERT( this_thr != NULL );
1499 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1508 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1509 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1510 KMP_FSYNC_ACQUIRED(lck);
1515 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1520 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1522 if ( __kmp_env_consistency_check ) {
1523 char const *
const func =
"omp_test_lock";
1524 if ( lck->lk.initialized != lck ) {
1525 KMP_FATAL( LockIsUninitialized, func );
1527 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1528 KMP_FATAL( LockNestableUsedAsSimple, func );
1532 int retval = __kmp_test_queuing_lock( lck, gtid );
1534 if ( __kmp_env_consistency_check && retval ) {
1535 lck->lk.owner_id = gtid + 1;
1541 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1543 register kmp_info_t *this_thr;
1544 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1545 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1547 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1548 KMP_DEBUG_ASSERT( gtid >= 0 );
1549 this_thr = __kmp_thread_from_gtid( gtid );
1550 KMP_DEBUG_ASSERT( this_thr != NULL );
1551 #ifdef DEBUG_QUEUING_LOCKS
1552 TRACE_LOCK( gtid+1,
"rel ent" );
1554 if ( this_thr->th.th_spin_here )
1555 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1556 if ( this_thr->th.th_next_waiting != 0 )
1557 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1559 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1560 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1562 KMP_FSYNC_RELEASING(lck);
1571 #ifdef DEBUG_QUEUING_LOCKS
1573 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1574 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1576 KMP_DEBUG_ASSERT( head != 0 );
1581 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1582 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1584 #ifdef DEBUG_QUEUING_LOCKS
1585 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1595 if ( head == tail ) {
1597 #ifdef DEBUG_QUEUING_LOCKS
1598 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1600 KMP_DEBUG_ASSERT( head > 0 );
1603 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1604 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1605 #ifdef DEBUG_QUEUING_LOCKS
1606 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1611 volatile kmp_int32 *waiting_id_p;
1612 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1613 KMP_DEBUG_ASSERT( head_thr != NULL );
1614 waiting_id_p = & head_thr->th.th_next_waiting;
1617 #ifdef DEBUG_QUEUING_LOCKS
1618 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1620 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1626 *head_id_p = (kmp_int32) KMP_WAIT_YIELD((
volatile kmp_uint*) waiting_id_p, 0, KMP_NEQ, NULL);
1627 #ifdef DEBUG_QUEUING_LOCKS
1628 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1635 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1636 KMP_DEBUG_ASSERT( head_thr != NULL );
1639 #ifdef DEBUG_QUEUING_LOCKS
1640 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1642 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1647 head_thr->th.th_next_waiting = 0;
1648 #ifdef DEBUG_QUEUING_LOCKS
1649 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1654 head_thr->th.th_spin_here = FALSE;
1656 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1658 #ifdef DEBUG_QUEUING_LOCKS
1659 TRACE_LOCK( gtid+1,
"rel exit 2" );
1665 #ifdef DEBUG_QUEUING_LOCKS
1666 TRACE_LOCK( gtid+1,
"rel retry" );
1670 KMP_ASSERT2( 0,
"should not get here" );
1674 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1677 if ( __kmp_env_consistency_check ) {
1678 char const *
const func =
"omp_unset_lock";
1680 if ( lck->lk.initialized != lck ) {
1681 KMP_FATAL( LockIsUninitialized, func );
1683 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1684 KMP_FATAL( LockNestableUsedAsSimple, func );
1686 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1687 KMP_FATAL( LockUnsettingFree, func );
1689 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1690 KMP_FATAL( LockUnsettingSetByAnother, func );
1692 lck->lk.owner_id = 0;
1694 __kmp_release_queuing_lock( lck, gtid );
1698 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1700 lck->lk.location = NULL;
1701 lck->lk.head_id = 0;
1702 lck->lk.tail_id = 0;
1703 lck->lk.next_ticket = 0;
1704 lck->lk.now_serving = 0;
1705 lck->lk.owner_id = 0;
1706 lck->lk.depth_locked = -1;
1707 lck->lk.initialized = lck;
1709 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1713 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1715 __kmp_init_queuing_lock( lck );
1719 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1721 lck->lk.initialized = NULL;
1722 lck->lk.location = NULL;
1723 lck->lk.head_id = 0;
1724 lck->lk.tail_id = 0;
1725 lck->lk.next_ticket = 0;
1726 lck->lk.now_serving = 0;
1727 lck->lk.owner_id = 0;
1728 lck->lk.depth_locked = -1;
1732 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1734 if ( __kmp_env_consistency_check ) {
1735 char const *
const func =
"omp_destroy_lock";
1736 if ( lck->lk.initialized != lck ) {
1737 KMP_FATAL( LockIsUninitialized, func );
1739 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1740 KMP_FATAL( LockNestableUsedAsSimple, func );
1742 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1743 KMP_FATAL( LockStillOwned, func );
1746 __kmp_destroy_queuing_lock( lck );
1755 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1757 KMP_DEBUG_ASSERT( gtid >= 0 );
1759 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1760 lck->lk.depth_locked += 1;
1763 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1765 lck->lk.depth_locked = 1;
1767 lck->lk.owner_id = gtid + 1;
1772 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1774 if ( __kmp_env_consistency_check ) {
1775 char const *
const func =
"omp_set_nest_lock";
1776 if ( lck->lk.initialized != lck ) {
1777 KMP_FATAL( LockIsUninitialized, func );
1779 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1780 KMP_FATAL( LockSimpleUsedAsNestable, func );
1783 __kmp_acquire_nested_queuing_lock( lck, gtid );
1787 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1791 KMP_DEBUG_ASSERT( gtid >= 0 );
1793 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1794 retval = ++lck->lk.depth_locked;
1796 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1801 retval = lck->lk.depth_locked = 1;
1803 lck->lk.owner_id = gtid + 1;
1809 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1812 if ( __kmp_env_consistency_check ) {
1813 char const *
const func =
"omp_test_nest_lock";
1814 if ( lck->lk.initialized != lck ) {
1815 KMP_FATAL( LockIsUninitialized, func );
1817 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1818 KMP_FATAL( LockSimpleUsedAsNestable, func );
1821 return __kmp_test_nested_queuing_lock( lck, gtid );
1825 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1827 KMP_DEBUG_ASSERT( gtid >= 0 );
1830 if ( --(lck->lk.depth_locked) == 0 ) {
1832 lck->lk.owner_id = 0;
1833 __kmp_release_queuing_lock( lck, gtid );
1838 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1840 if ( __kmp_env_consistency_check ) {
1841 char const *
const func =
"omp_unset_nest_lock";
1843 if ( lck->lk.initialized != lck ) {
1844 KMP_FATAL( LockIsUninitialized, func );
1846 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1847 KMP_FATAL( LockSimpleUsedAsNestable, func );
1849 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1850 KMP_FATAL( LockUnsettingFree, func );
1852 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1853 KMP_FATAL( LockUnsettingSetByAnother, func );
1856 __kmp_release_nested_queuing_lock( lck, gtid );
1860 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1862 __kmp_init_queuing_lock( lck );
1863 lck->lk.depth_locked = 0;
1867 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1869 __kmp_init_nested_queuing_lock( lck );
1873 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1875 __kmp_destroy_queuing_lock( lck );
1876 lck->lk.depth_locked = 0;
1880 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1882 if ( __kmp_env_consistency_check ) {
1883 char const *
const func =
"omp_destroy_nest_lock";
1884 if ( lck->lk.initialized != lck ) {
1885 KMP_FATAL( LockIsUninitialized, func );
1887 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1888 KMP_FATAL( LockSimpleUsedAsNestable, func );
1890 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1891 KMP_FATAL( LockStillOwned, func );
1894 __kmp_destroy_nested_queuing_lock( lck );
1903 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1905 return lck == lck->lk.initialized;
1909 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1911 return lck->lk.location;
1915 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1917 lck->lk.location = loc;
1920 static kmp_lock_flags_t
1921 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1923 return lck->lk.flags;
1927 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1929 lck->lk.flags = flags;
1932 #if KMP_USE_ADAPTIVE_LOCKS
1942 #define _XBEGIN_STARTED (~0u)
1943 #define _XABORT_EXPLICIT (1 << 0)
1944 #define _XABORT_RETRY (1 << 1)
1945 #define _XABORT_CONFLICT (1 << 2)
1946 #define _XABORT_CAPACITY (1 << 3)
1947 #define _XABORT_DEBUG (1 << 4)
1948 #define _XABORT_NESTED (1 << 5)
1949 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1952 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1954 #define STRINGIZE_INTERNAL(arg) #arg
1955 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1963 static __inline
int _xbegin()
1992 #endif // KMP_ARCH_X86_64
2002 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n"
2005 "1: movl %%eax,%0\n"
2007 :
"+r"(res)::
"memory",
"%eax");
2008 #endif // KMP_OS_WINDOWS
2015 static __inline
void _xend()
2024 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
2034 #define _xabort(ARG) \
2039 #define _xabort(ARG) \
2040 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory");
2046 #if KMP_DEBUG_ADAPTIVE_LOCKS
2051 static kmp_adaptive_lock_statistics_t destroyedStats;
2054 static kmp_adaptive_lock_t liveLocks;
2057 static kmp_bootstrap_lock_t chain_lock;
2061 __kmp_init_speculative_stats()
2063 kmp_adaptive_lock *lck = &liveLocks;
2065 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
2066 lck->stats.next = lck;
2067 lck->stats.prev = lck;
2069 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2070 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2072 __kmp_init_bootstrap_lock( &chain_lock );
2078 __kmp_remember_lock( kmp_adaptive_lock * lck )
2080 __kmp_acquire_bootstrap_lock( &chain_lock );
2082 lck->stats.next = liveLocks.stats.next;
2083 lck->stats.prev = &liveLocks;
2085 liveLocks.stats.next = lck;
2086 lck->stats.next->stats.prev = lck;
2088 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2089 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2091 __kmp_release_bootstrap_lock( &chain_lock );
2095 __kmp_forget_lock( kmp_adaptive_lock * lck )
2097 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2098 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2100 kmp_adaptive_lock * n = lck->stats.next;
2101 kmp_adaptive_lock * p = lck->stats.prev;
2108 __kmp_zero_speculative_stats( kmp_adaptive_lock * lck )
2110 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2111 __kmp_remember_lock( lck );
2115 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_t * lck )
2117 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2119 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2120 t->successfulSpeculations += s->successfulSpeculations;
2121 t->hardFailedSpeculations += s->hardFailedSpeculations;
2122 t->softFailedSpeculations += s->softFailedSpeculations;
2123 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2124 t->lemmingYields += s->lemmingYields;
2128 __kmp_accumulate_speculative_stats( kmp_adaptive_lock * lck)
2130 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2132 __kmp_acquire_bootstrap_lock( &chain_lock );
2134 __kmp_add_stats( &destroyedStats, lck );
2135 __kmp_forget_lock( lck );
2137 __kmp_release_bootstrap_lock( &chain_lock );
2141 percent (kmp_uint32 count, kmp_uint32 total)
2143 return (total == 0) ? 0.0: (100.0 * count)/total;
2147 FILE * __kmp_open_stats_file()
2149 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2152 size_t buffLen = strlen( __kmp_speculative_statsfile ) + 20;
2153 char buffer[buffLen];
2154 snprintf (&buffer[0], buffLen, __kmp_speculative_statsfile, getpid());
2155 FILE * result = fopen(&buffer[0],
"w");
2158 return result ? result : stdout;
2162 __kmp_print_speculative_stats()
2164 if (__kmp_user_lock_kind != lk_adaptive)
2167 FILE * statsFile = __kmp_open_stats_file();
2169 kmp_adaptive_lock_statistics_t total = destroyedStats;
2170 kmp_adaptive_lock *lck;
2172 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2173 __kmp_add_stats( &total, lck );
2175 kmp_adaptive_lock_statistics_t *t = &total;
2176 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2177 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2178 t->softFailedSpeculations;
2180 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2181 fprintf ( statsFile,
" Lock parameters: \n"
2182 " max_soft_retries : %10d\n"
2183 " max_badness : %10d\n",
2184 __kmp_adaptive_backoff_params.max_soft_retries,
2185 __kmp_adaptive_backoff_params.max_badness);
2186 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2187 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2188 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2189 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2190 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2191 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2192 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2194 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2195 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2196 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2197 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2198 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2199 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2200 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2202 if (statsFile != stdout)
2203 fclose( statsFile );
2206 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ )
2208 # define KMP_INC_STAT(lck,stat)
2210 #endif // KMP_DEBUG_ADAPTIVE_LOCKS
2213 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2217 bool res = lck->lk.head_id == 0;
2221 #if defined( __GNUC__ ) && !defined( __INTEL_COMPILER )
2222 __sync_synchronize();
2231 static __inline
void
2232 __kmp_update_badness_after_success( kmp_queuing_lock_t *lck )
2235 lck->lk.adaptive.badness = 0;
2236 KMP_INC_STAT(lck,successfulSpeculations);
2240 static __inline
void
2241 __kmp_step_badness( kmp_queuing_lock_t *lck )
2243 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2244 if ( newBadness > lck->lk.adaptive.max_badness) {
2247 lck->lk.adaptive.badness = newBadness;
2253 __kmp_should_speculate( kmp_queuing_lock_t *lck, kmp_int32 gtid )
2255 kmp_uint32 badness = lck->lk.adaptive.badness;
2256 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2257 int res = (attempts & badness) == 0;
2265 __kmp_test_adaptive_lock_only( kmp_queuing_lock_t * lck, kmp_int32 gtid )
2267 int retries = lck->lk.adaptive.max_soft_retries;
2275 kmp_uint32 status = _xbegin();
2280 if (status == _XBEGIN_STARTED )
2286 if (! __kmp_is_unlocked_queuing_lock( lck ) )
2292 KMP_ASSERT2( 0, "should not get here" );
2297 if ( status & SOFT_ABORT_MASK)
2299 KMP_INC_STAT(lck,softFailedSpeculations);
2304 KMP_INC_STAT(lck,hardFailedSpeculations);
2309 }
while( retries-- );
2313 __kmp_step_badness( lck );
2321 __kmp_test_adaptive_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
2324 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2329 lck->lk.adaptive.acquire_attempts++;
2332 if ( __kmp_test_queuing_lock( lck, gtid ) )
2334 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2344 __kmp_test_adaptive_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
2346 if ( __kmp_env_consistency_check ) {
2347 char const *
const func =
"omp_test_lock";
2348 if ( lck->lk.initialized != lck ) {
2349 KMP_FATAL( LockIsUninitialized, func );
2353 int retval = __kmp_test_adaptive_lock( lck, gtid );
2355 if ( __kmp_env_consistency_check && retval ) {
2356 lck->lk.owner_id = gtid + 1;
2376 __kmp_acquire_adaptive_lock( kmp_queuing_lock_t * lck, kmp_int32 gtid )
2378 if ( __kmp_should_speculate( lck, gtid ) )
2380 if ( __kmp_is_unlocked_queuing_lock( lck ) )
2382 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2394 while ( ! __kmp_is_unlocked_queuing_lock( lck ) )
2396 KMP_INC_STAT(lck,lemmingYields);
2400 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2407 lck->lk.adaptive.acquire_attempts++;
2409 __kmp_acquire_queuing_lock_timed_template<FALSE>( lck, gtid );
2411 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2415 __kmp_acquire_adaptive_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
2417 if ( __kmp_env_consistency_check ) {
2418 char const *
const func =
"omp_set_lock";
2419 if ( lck->lk.initialized != lck ) {
2420 KMP_FATAL( LockIsUninitialized, func );
2422 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
2423 KMP_FATAL( LockIsAlreadyOwned, func );
2427 __kmp_acquire_adaptive_lock( lck, gtid );
2429 if ( __kmp_env_consistency_check ) {
2430 lck->lk.owner_id = gtid + 1;
2435 __kmp_release_adaptive_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
2437 if ( __kmp_is_unlocked_queuing_lock( lck ) )
2442 __kmp_update_badness_after_success( lck );
2447 __kmp_release_queuing_lock( lck, gtid );
2452 __kmp_release_adaptive_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
2454 if ( __kmp_env_consistency_check ) {
2455 char const *
const func =
"omp_unset_lock";
2457 if ( lck->lk.initialized != lck ) {
2458 KMP_FATAL( LockIsUninitialized, func );
2460 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
2461 KMP_FATAL( LockUnsettingFree, func );
2463 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
2464 KMP_FATAL( LockUnsettingSetByAnother, func );
2466 lck->lk.owner_id = 0;
2468 __kmp_release_adaptive_lock( lck, gtid );
2472 __kmp_init_adaptive_lock( kmp_queuing_lock_t *lck )
2474 __kmp_init_queuing_lock( lck );
2475 lck->lk.adaptive.badness = 0;
2476 lck->lk.adaptive.acquire_attempts = 0;
2477 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2478 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2479 #if KMP_DEBUG_ADAPTIVE_LOCKS
2480 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2482 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2486 __kmp_init_adaptive_lock_with_checks( kmp_queuing_lock_t * lck )
2488 __kmp_init_adaptive_lock( lck );
2492 __kmp_destroy_adaptive_lock( kmp_queuing_lock_t *lck )
2494 #if KMP_DEBUG_ADAPTIVE_LOCKS
2495 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2497 __kmp_destroy_queuing_lock (lck);
2502 __kmp_destroy_adaptive_lock_with_checks( kmp_queuing_lock_t *lck )
2504 if ( __kmp_env_consistency_check ) {
2505 char const *
const func =
"omp_destroy_lock";
2506 if ( lck->lk.initialized != lck ) {
2507 KMP_FATAL( LockIsUninitialized, func );
2509 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
2510 KMP_FATAL( LockStillOwned, func );
2513 __kmp_destroy_adaptive_lock( lck );
2517 #endif // KMP_USE_ADAPTIVE_LOCKS
2525 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2527 return TCR_4( lck->lk.owner_id ) - 1;
2531 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2533 return lck->lk.depth_locked != -1;
2536 __forceinline
static void
2537 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2539 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2540 kmp_uint64 mask = TCR_8(lck->lk.mask);
2541 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2542 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2543 TCR_PTR(lck->lk.polls);
2545 #ifdef USE_LOCK_PROFILE
2546 if (TCR_8(polls[ticket & mask].poll) != ticket)
2547 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2563 KMP_FSYNC_PREPARE(lck);
2564 KMP_INIT_YIELD(spins);
2565 while (TCR_8(polls[ticket & mask]).poll < ticket) {
2566 __kmp_static_delay(TRUE);
2573 KMP_YIELD(TCR_4(__kmp_nth)
2574 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2575 KMP_YIELD_SPIN(spins);
2586 mask = TCR_8(lck->lk.mask);
2587 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2588 TCR_PTR(lck->lk.polls);
2594 KMP_FSYNC_ACQUIRED(lck);
2595 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2597 lck->lk.now_serving = ticket;
2606 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2607 __kmp_free((
void *)lck->lk.old_polls);
2608 lck->lk.old_polls = NULL;
2609 lck->lk.cleanup_ticket = 0;
2617 if (lck->lk.old_polls == NULL) {
2618 bool reconfigure =
false;
2619 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2620 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2622 if (TCR_4(__kmp_nth)
2623 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2628 if (num_polls > 1) {
2630 num_polls = TCR_4(lck->lk.num_polls);
2633 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2634 __kmp_allocate(num_polls *
sizeof(*polls));
2635 polls[0].poll = ticket;
2644 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2645 if (num_waiting > num_polls) {
2646 kmp_uint32 old_num_polls = num_polls;
2649 mask = (mask << 1) | 1;
2651 }
while (num_polls <= num_waiting);
2659 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2660 __kmp_allocate(num_polls *
sizeof(*polls));
2662 for (i = 0; i < old_num_polls; i++) {
2663 polls[i].poll = old_polls[i].poll;
2680 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2681 ticket, lck, num_polls));
2683 lck->lk.old_polls = old_polls;
2684 lck->lk.polls = polls;
2688 lck->lk.num_polls = num_polls;
2689 lck->lk.mask = mask;
2699 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2705 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2707 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2711 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2713 if ( __kmp_env_consistency_check ) {
2714 char const *
const func =
"omp_set_lock";
2715 if ( lck->lk.initialized != lck ) {
2716 KMP_FATAL( LockIsUninitialized, func );
2718 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2719 KMP_FATAL( LockNestableUsedAsSimple, func );
2721 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2722 KMP_FATAL( LockIsAlreadyOwned, func );
2726 __kmp_acquire_drdpa_lock( lck, gtid );
2728 if ( __kmp_env_consistency_check ) {
2729 lck->lk.owner_id = gtid + 1;
2734 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2740 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2741 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2742 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2743 TCR_PTR(lck->lk.polls);
2744 kmp_uint64 mask = TCR_8(lck->lk.mask);
2745 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2746 kmp_uint64 next_ticket = ticket + 1;
2747 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2748 ticket, next_ticket)) {
2749 KMP_FSYNC_ACQUIRED(lck);
2750 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2752 lck->lk.now_serving = ticket;
2770 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2772 if ( __kmp_env_consistency_check ) {
2773 char const *
const func =
"omp_test_lock";
2774 if ( lck->lk.initialized != lck ) {
2775 KMP_FATAL( LockIsUninitialized, func );
2777 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2778 KMP_FATAL( LockNestableUsedAsSimple, func );
2782 int retval = __kmp_test_drdpa_lock( lck, gtid );
2784 if ( __kmp_env_consistency_check && retval ) {
2785 lck->lk.owner_id = gtid + 1;
2791 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2798 kmp_uint64 ticket = lck->lk.now_serving + 1;
2799 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2800 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2801 TCR_PTR(lck->lk.polls);
2802 kmp_uint64 mask = TCR_8(lck->lk.mask);
2803 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2805 KMP_FSYNC_RELEASING(lck);
2806 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2810 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2812 if ( __kmp_env_consistency_check ) {
2813 char const *
const func =
"omp_unset_lock";
2815 if ( lck->lk.initialized != lck ) {
2816 KMP_FATAL( LockIsUninitialized, func );
2818 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2819 KMP_FATAL( LockNestableUsedAsSimple, func );
2821 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2822 KMP_FATAL( LockUnsettingFree, func );
2824 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2825 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2826 KMP_FATAL( LockUnsettingSetByAnother, func );
2828 lck->lk.owner_id = 0;
2830 __kmp_release_drdpa_lock( lck, gtid );
2834 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2836 lck->lk.location = NULL;
2838 lck->lk.num_polls = 1;
2839 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2840 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2841 lck->lk.cleanup_ticket = 0;
2842 lck->lk.old_polls = NULL;
2843 lck->lk.next_ticket = 0;
2844 lck->lk.now_serving = 0;
2845 lck->lk.owner_id = 0;
2846 lck->lk.depth_locked = -1;
2847 lck->lk.initialized = lck;
2849 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2853 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2855 __kmp_init_drdpa_lock( lck );
2859 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2861 lck->lk.initialized = NULL;
2862 lck->lk.location = NULL;
2863 if (lck->lk.polls != NULL) {
2864 __kmp_free((
void *)lck->lk.polls);
2865 lck->lk.polls = NULL;
2867 if (lck->lk.old_polls != NULL) {
2868 __kmp_free((
void *)lck->lk.old_polls);
2869 lck->lk.old_polls = NULL;
2872 lck->lk.num_polls = 0;
2873 lck->lk.cleanup_ticket = 0;
2874 lck->lk.next_ticket = 0;
2875 lck->lk.now_serving = 0;
2876 lck->lk.owner_id = 0;
2877 lck->lk.depth_locked = -1;
2881 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2883 if ( __kmp_env_consistency_check ) {
2884 char const *
const func =
"omp_destroy_lock";
2885 if ( lck->lk.initialized != lck ) {
2886 KMP_FATAL( LockIsUninitialized, func );
2888 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2889 KMP_FATAL( LockNestableUsedAsSimple, func );
2891 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2892 KMP_FATAL( LockStillOwned, func );
2895 __kmp_destroy_drdpa_lock( lck );
2904 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2906 KMP_DEBUG_ASSERT( gtid >= 0 );
2908 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2909 lck->lk.depth_locked += 1;
2912 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2914 lck->lk.depth_locked = 1;
2916 lck->lk.owner_id = gtid + 1;
2921 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2923 if ( __kmp_env_consistency_check ) {
2924 char const *
const func =
"omp_set_nest_lock";
2925 if ( lck->lk.initialized != lck ) {
2926 KMP_FATAL( LockIsUninitialized, func );
2928 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2929 KMP_FATAL( LockSimpleUsedAsNestable, func );
2932 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2936 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2940 KMP_DEBUG_ASSERT( gtid >= 0 );
2942 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2943 retval = ++lck->lk.depth_locked;
2945 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2950 retval = lck->lk.depth_locked = 1;
2952 lck->lk.owner_id = gtid + 1;
2958 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2960 if ( __kmp_env_consistency_check ) {
2961 char const *
const func =
"omp_test_nest_lock";
2962 if ( lck->lk.initialized != lck ) {
2963 KMP_FATAL( LockIsUninitialized, func );
2965 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2966 KMP_FATAL( LockSimpleUsedAsNestable, func );
2969 return __kmp_test_nested_drdpa_lock( lck, gtid );
2973 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2975 KMP_DEBUG_ASSERT( gtid >= 0 );
2978 if ( --(lck->lk.depth_locked) == 0 ) {
2980 lck->lk.owner_id = 0;
2981 __kmp_release_drdpa_lock( lck, gtid );
2986 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2988 if ( __kmp_env_consistency_check ) {
2989 char const *
const func =
"omp_unset_nest_lock";
2991 if ( lck->lk.initialized != lck ) {
2992 KMP_FATAL( LockIsUninitialized, func );
2994 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2995 KMP_FATAL( LockSimpleUsedAsNestable, func );
2997 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2998 KMP_FATAL( LockUnsettingFree, func );
3000 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
3001 KMP_FATAL( LockUnsettingSetByAnother, func );
3004 __kmp_release_nested_drdpa_lock( lck, gtid );
3008 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
3010 __kmp_init_drdpa_lock( lck );
3011 lck->lk.depth_locked = 0;
3015 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
3017 __kmp_init_nested_drdpa_lock( lck );
3021 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
3023 __kmp_destroy_drdpa_lock( lck );
3024 lck->lk.depth_locked = 0;
3028 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
3030 if ( __kmp_env_consistency_check ) {
3031 char const *
const func =
"omp_destroy_nest_lock";
3032 if ( lck->lk.initialized != lck ) {
3033 KMP_FATAL( LockIsUninitialized, func );
3035 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
3036 KMP_FATAL( LockSimpleUsedAsNestable, func );
3038 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
3039 KMP_FATAL( LockStillOwned, func );
3042 __kmp_destroy_nested_drdpa_lock( lck );
3051 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
3053 return lck == lck->lk.initialized;
3057 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
3059 return lck->lk.location;
3063 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
3065 lck->lk.location = loc;
3068 static kmp_lock_flags_t
3069 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
3071 return lck->lk.flags;
3075 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
3077 lck->lk.flags = flags;
3087 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3089 size_t __kmp_base_user_lock_size = 0;
3090 size_t __kmp_user_lock_size = 0;
3092 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3093 void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3095 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3096 void ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3097 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3098 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3099 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3100 void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3102 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3103 void ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3104 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3105 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3107 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3108 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3109 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3110 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3111 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3113 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3115 switch ( user_lock_kind ) {
3121 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3122 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3124 __kmp_get_user_lock_owner_ =
3125 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3126 ( &__kmp_get_tas_lock_owner );
3128 __kmp_acquire_user_lock_with_checks_ =
3129 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3130 ( &__kmp_acquire_tas_lock_with_checks );
3132 __kmp_test_user_lock_with_checks_ =
3133 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3134 ( &__kmp_test_tas_lock_with_checks );
3136 __kmp_release_user_lock_with_checks_ =
3137 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3138 ( &__kmp_release_tas_lock_with_checks );
3140 __kmp_init_user_lock_with_checks_ =
3141 ( void ( * )( kmp_user_lock_p ) )
3142 ( &__kmp_init_tas_lock_with_checks );
3144 __kmp_destroy_user_lock_ =
3145 ( void ( * )( kmp_user_lock_p ) )
3146 ( &__kmp_destroy_tas_lock );
3148 __kmp_destroy_user_lock_with_checks_ =
3149 ( void ( * )( kmp_user_lock_p ) )
3150 ( &__kmp_destroy_tas_lock_with_checks );
3152 __kmp_acquire_nested_user_lock_with_checks_ =
3153 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3154 ( &__kmp_acquire_nested_tas_lock_with_checks );
3156 __kmp_test_nested_user_lock_with_checks_ =
3157 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3158 ( &__kmp_test_nested_tas_lock_with_checks );
3160 __kmp_release_nested_user_lock_with_checks_ =
3161 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3162 ( &__kmp_release_nested_tas_lock_with_checks );
3164 __kmp_init_nested_user_lock_with_checks_ =
3165 ( void ( * )( kmp_user_lock_p ) )
3166 ( &__kmp_init_nested_tas_lock_with_checks );
3168 __kmp_destroy_nested_user_lock_with_checks_ =
3169 ( void ( * )( kmp_user_lock_p ) )
3170 ( &__kmp_destroy_nested_tas_lock_with_checks );
3172 __kmp_is_user_lock_initialized_ =
3173 ( int ( * )( kmp_user_lock_p ) ) NULL;
3175 __kmp_get_user_lock_location_ =
3176 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3178 __kmp_set_user_lock_location_ =
3179 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3181 __kmp_get_user_lock_flags_ =
3182 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3184 __kmp_set_user_lock_flags_ =
3185 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3189 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
3192 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3193 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3195 __kmp_get_user_lock_owner_ =
3196 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3197 ( &__kmp_get_futex_lock_owner );
3199 __kmp_acquire_user_lock_with_checks_ =
3200 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3201 ( &__kmp_acquire_futex_lock_with_checks );
3203 __kmp_test_user_lock_with_checks_ =
3204 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3205 ( &__kmp_test_futex_lock_with_checks );
3207 __kmp_release_user_lock_with_checks_ =
3208 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3209 ( &__kmp_release_futex_lock_with_checks );
3211 __kmp_init_user_lock_with_checks_ =
3212 ( void ( * )( kmp_user_lock_p ) )
3213 ( &__kmp_init_futex_lock_with_checks );
3215 __kmp_destroy_user_lock_ =
3216 ( void ( * )( kmp_user_lock_p ) )
3217 ( &__kmp_destroy_futex_lock );
3219 __kmp_destroy_user_lock_with_checks_ =
3220 ( void ( * )( kmp_user_lock_p ) )
3221 ( &__kmp_destroy_futex_lock_with_checks );
3223 __kmp_acquire_nested_user_lock_with_checks_ =
3224 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3225 ( &__kmp_acquire_nested_futex_lock_with_checks );
3227 __kmp_test_nested_user_lock_with_checks_ =
3228 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3229 ( &__kmp_test_nested_futex_lock_with_checks );
3231 __kmp_release_nested_user_lock_with_checks_ =
3232 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3233 ( &__kmp_release_nested_futex_lock_with_checks );
3235 __kmp_init_nested_user_lock_with_checks_ =
3236 ( void ( * )( kmp_user_lock_p ) )
3237 ( &__kmp_init_nested_futex_lock_with_checks );
3239 __kmp_destroy_nested_user_lock_with_checks_ =
3240 ( void ( * )( kmp_user_lock_p ) )
3241 ( &__kmp_destroy_nested_futex_lock_with_checks );
3243 __kmp_is_user_lock_initialized_ =
3244 ( int ( * )( kmp_user_lock_p ) ) NULL;
3246 __kmp_get_user_lock_location_ =
3247 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3249 __kmp_set_user_lock_location_ =
3250 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3252 __kmp_get_user_lock_flags_ =
3253 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3255 __kmp_set_user_lock_flags_ =
3256 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3260 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
3263 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3264 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3266 __kmp_get_user_lock_owner_ =
3267 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3268 ( &__kmp_get_ticket_lock_owner );
3270 __kmp_acquire_user_lock_with_checks_ =
3271 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3272 ( &__kmp_acquire_ticket_lock_with_checks );
3274 __kmp_test_user_lock_with_checks_ =
3275 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3276 ( &__kmp_test_ticket_lock_with_checks );
3278 __kmp_release_user_lock_with_checks_ =
3279 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3280 ( &__kmp_release_ticket_lock_with_checks );
3282 __kmp_init_user_lock_with_checks_ =
3283 ( void ( * )( kmp_user_lock_p ) )
3284 ( &__kmp_init_ticket_lock_with_checks );
3286 __kmp_destroy_user_lock_ =
3287 ( void ( * )( kmp_user_lock_p ) )
3288 ( &__kmp_destroy_ticket_lock );
3290 __kmp_destroy_user_lock_with_checks_ =
3291 ( void ( * )( kmp_user_lock_p ) )
3292 ( &__kmp_destroy_ticket_lock_with_checks );
3294 __kmp_acquire_nested_user_lock_with_checks_ =
3295 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3296 ( &__kmp_acquire_nested_ticket_lock_with_checks );
3298 __kmp_test_nested_user_lock_with_checks_ =
3299 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3300 ( &__kmp_test_nested_ticket_lock_with_checks );
3302 __kmp_release_nested_user_lock_with_checks_ =
3303 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3304 ( &__kmp_release_nested_ticket_lock_with_checks );
3306 __kmp_init_nested_user_lock_with_checks_ =
3307 ( void ( * )( kmp_user_lock_p ) )
3308 ( &__kmp_init_nested_ticket_lock_with_checks );
3310 __kmp_destroy_nested_user_lock_with_checks_ =
3311 ( void ( * )( kmp_user_lock_p ) )
3312 ( &__kmp_destroy_nested_ticket_lock_with_checks );
3314 __kmp_is_user_lock_initialized_ =
3315 ( int ( * )( kmp_user_lock_p ) )
3316 ( &__kmp_is_ticket_lock_initialized );
3318 __kmp_get_user_lock_location_ =
3319 (
const ident_t * ( * )( kmp_user_lock_p ) )
3320 ( &__kmp_get_ticket_lock_location );
3322 __kmp_set_user_lock_location_ =
3323 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3324 ( &__kmp_set_ticket_lock_location );
3326 __kmp_get_user_lock_flags_ =
3327 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3328 ( &__kmp_get_ticket_lock_flags );
3330 __kmp_set_user_lock_flags_ =
3331 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3332 ( &__kmp_set_ticket_lock_flags );
3337 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3338 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3340 __kmp_get_user_lock_owner_ =
3341 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3342 ( &__kmp_get_queuing_lock_owner );
3344 __kmp_acquire_user_lock_with_checks_ =
3345 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3346 ( &__kmp_acquire_queuing_lock_with_checks );
3348 __kmp_test_user_lock_with_checks_ =
3349 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3350 ( &__kmp_test_queuing_lock_with_checks );
3352 __kmp_release_user_lock_with_checks_ =
3353 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3354 ( &__kmp_release_queuing_lock_with_checks );
3356 __kmp_init_user_lock_with_checks_ =
3357 ( void ( * )( kmp_user_lock_p ) )
3358 ( &__kmp_init_queuing_lock_with_checks );
3360 __kmp_destroy_user_lock_ =
3361 ( void ( * )( kmp_user_lock_p ) )
3362 ( &__kmp_destroy_queuing_lock );
3364 __kmp_destroy_user_lock_with_checks_ =
3365 ( void ( * )( kmp_user_lock_p ) )
3366 ( &__kmp_destroy_queuing_lock_with_checks );
3368 __kmp_acquire_nested_user_lock_with_checks_ =
3369 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3370 ( &__kmp_acquire_nested_queuing_lock_with_checks );
3372 __kmp_test_nested_user_lock_with_checks_ =
3373 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3374 ( &__kmp_test_nested_queuing_lock_with_checks );
3376 __kmp_release_nested_user_lock_with_checks_ =
3377 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3378 ( &__kmp_release_nested_queuing_lock_with_checks );
3380 __kmp_init_nested_user_lock_with_checks_ =
3381 ( void ( * )( kmp_user_lock_p ) )
3382 ( &__kmp_init_nested_queuing_lock_with_checks );
3384 __kmp_destroy_nested_user_lock_with_checks_ =
3385 ( void ( * )( kmp_user_lock_p ) )
3386 ( &__kmp_destroy_nested_queuing_lock_with_checks );
3388 __kmp_is_user_lock_initialized_ =
3389 ( int ( * )( kmp_user_lock_p ) )
3390 ( &__kmp_is_queuing_lock_initialized );
3392 __kmp_get_user_lock_location_ =
3393 (
const ident_t * ( * )( kmp_user_lock_p ) )
3394 ( &__kmp_get_queuing_lock_location );
3396 __kmp_set_user_lock_location_ =
3397 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3398 ( &__kmp_set_queuing_lock_location );
3400 __kmp_get_user_lock_flags_ =
3401 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3402 ( &__kmp_get_queuing_lock_flags );
3404 __kmp_set_user_lock_flags_ =
3405 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3406 ( &__kmp_set_queuing_lock_flags );
3410 #if KMP_USE_ADAPTIVE_LOCKS
3412 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3413 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3415 __kmp_get_user_lock_owner_ =
3416 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3417 ( &__kmp_get_queuing_lock_owner );
3419 __kmp_acquire_user_lock_with_checks_ =
3420 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3421 ( &__kmp_acquire_adaptive_lock_with_checks );
3423 __kmp_test_user_lock_with_checks_ =
3424 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3425 ( &__kmp_test_adaptive_lock_with_checks );
3427 __kmp_release_user_lock_with_checks_ =
3428 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3429 ( &__kmp_release_adaptive_lock_with_checks );
3431 __kmp_init_user_lock_with_checks_ =
3432 ( void ( * )( kmp_user_lock_p ) )
3433 ( &__kmp_init_adaptive_lock_with_checks );
3435 __kmp_destroy_user_lock_with_checks_ =
3436 ( void ( * )( kmp_user_lock_p ) )
3437 ( &__kmp_destroy_adaptive_lock_with_checks );
3439 __kmp_destroy_user_lock_ =
3440 ( void ( * )( kmp_user_lock_p ) )
3441 ( &__kmp_destroy_adaptive_lock );
3443 __kmp_is_user_lock_initialized_ =
3444 ( int ( * )( kmp_user_lock_p ) )
3445 ( &__kmp_is_queuing_lock_initialized );
3447 __kmp_get_user_lock_location_ =
3448 (
const ident_t * ( * )( kmp_user_lock_p ) )
3449 ( &__kmp_get_queuing_lock_location );
3451 __kmp_set_user_lock_location_ =
3452 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3453 ( &__kmp_set_queuing_lock_location );
3455 __kmp_get_user_lock_flags_ =
3456 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3457 ( &__kmp_get_queuing_lock_flags );
3459 __kmp_set_user_lock_flags_ =
3460 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3461 ( &__kmp_set_queuing_lock_flags );
3465 #endif // KMP_USE_ADAPTIVE_LOCKS
3468 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3469 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3471 __kmp_get_user_lock_owner_ =
3472 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3473 ( &__kmp_get_drdpa_lock_owner );
3475 __kmp_acquire_user_lock_with_checks_ =
3476 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3477 ( &__kmp_acquire_drdpa_lock_with_checks );
3479 __kmp_test_user_lock_with_checks_ =
3480 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3481 ( &__kmp_test_drdpa_lock_with_checks );
3483 __kmp_release_user_lock_with_checks_ =
3484 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3485 ( &__kmp_release_drdpa_lock_with_checks );
3487 __kmp_init_user_lock_with_checks_ =
3488 ( void ( * )( kmp_user_lock_p ) )
3489 ( &__kmp_init_drdpa_lock_with_checks );
3491 __kmp_destroy_user_lock_ =
3492 ( void ( * )( kmp_user_lock_p ) )
3493 ( &__kmp_destroy_drdpa_lock );
3495 __kmp_destroy_user_lock_with_checks_ =
3496 ( void ( * )( kmp_user_lock_p ) )
3497 ( &__kmp_destroy_drdpa_lock_with_checks );
3499 __kmp_acquire_nested_user_lock_with_checks_ =
3500 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3501 ( &__kmp_acquire_nested_drdpa_lock_with_checks );
3503 __kmp_test_nested_user_lock_with_checks_ =
3504 ( int ( * )( kmp_user_lock_p, kmp_int32 ) )
3505 ( &__kmp_test_nested_drdpa_lock_with_checks );
3507 __kmp_release_nested_user_lock_with_checks_ =
3508 ( void ( * )( kmp_user_lock_p, kmp_int32 ) )
3509 ( &__kmp_release_nested_drdpa_lock_with_checks );
3511 __kmp_init_nested_user_lock_with_checks_ =
3512 ( void ( * )( kmp_user_lock_p ) )
3513 ( &__kmp_init_nested_drdpa_lock_with_checks );
3515 __kmp_destroy_nested_user_lock_with_checks_ =
3516 ( void ( * )( kmp_user_lock_p ) )
3517 ( &__kmp_destroy_nested_drdpa_lock_with_checks );
3519 __kmp_is_user_lock_initialized_ =
3520 ( int ( * )( kmp_user_lock_p ) )
3521 ( &__kmp_is_drdpa_lock_initialized );
3523 __kmp_get_user_lock_location_ =
3524 (
const ident_t * ( * )( kmp_user_lock_p ) )
3525 ( &__kmp_get_drdpa_lock_location );
3527 __kmp_set_user_lock_location_ =
3528 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3529 ( &__kmp_set_drdpa_lock_location );
3531 __kmp_get_user_lock_flags_ =
3532 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3533 ( &__kmp_get_drdpa_lock_flags );
3535 __kmp_set_user_lock_flags_ =
3536 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3537 ( &__kmp_set_drdpa_lock_flags );
3547 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3548 kmp_user_lock_p __kmp_lock_pool = NULL;
3551 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3552 int __kmp_num_locks_in_block = 1;
3554 static kmp_lock_index_t
3555 __kmp_lock_table_insert( kmp_user_lock_p lck )
3558 kmp_lock_index_t index;
3559 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3560 kmp_lock_index_t size;
3561 kmp_user_lock_p *table;
3564 if ( __kmp_user_lock_table.allocated == 0 ) {
3568 size = __kmp_user_lock_table.allocated * 2;
3570 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3571 memcpy( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3572 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3577 __kmp_user_lock_table.table = table;
3578 __kmp_user_lock_table.allocated = size;
3580 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3581 index = __kmp_user_lock_table.used;
3582 __kmp_user_lock_table.table[ index ] = lck;
3583 ++ __kmp_user_lock_table.used;
3587 static kmp_user_lock_p
3588 __kmp_lock_block_allocate()
3591 static int last_index = 0;
3592 if ( ( last_index >= __kmp_num_locks_in_block )
3593 || ( __kmp_lock_blocks == NULL ) ) {
3597 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3598 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3599 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
3601 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
3602 new_block->next_block = __kmp_lock_blocks;
3603 new_block->locks = (
void *)buffer;
3606 __kmp_lock_blocks = new_block;
3608 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
3609 [ last_index * __kmp_user_lock_size ] ) );
3619 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3620 kmp_lock_flags_t flags )
3622 kmp_user_lock_p lck;
3623 kmp_lock_index_t index;
3624 KMP_DEBUG_ASSERT( user_lock );
3626 __kmp_acquire_lock( &__kmp_global_lock, gtid );
3628 if ( __kmp_lock_pool == NULL ) {
3630 if ( __kmp_num_locks_in_block <= 1 ) {
3631 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
3634 lck = __kmp_lock_block_allocate();
3639 index = __kmp_lock_table_insert( lck );
3643 lck = __kmp_lock_pool;
3644 index = __kmp_lock_pool->pool.index;
3645 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3652 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3653 * ( (kmp_lock_index_t *) user_lock ) = index;
3656 * ( (kmp_user_lock_p *) user_lock ) = lck;
3660 __kmp_set_user_lock_flags( lck, flags );
3662 __kmp_release_lock( & __kmp_global_lock, gtid );
3669 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
3671 kmp_lock_pool_t * lock_pool;
3673 KMP_DEBUG_ASSERT( user_lock != NULL );
3674 KMP_DEBUG_ASSERT( lck != NULL );
3676 __kmp_acquire_lock( & __kmp_global_lock, gtid );
3678 lck->pool.next = __kmp_lock_pool;
3679 __kmp_lock_pool = lck;
3680 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3681 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
3682 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
3683 lck->pool.index = index;
3686 __kmp_release_lock( & __kmp_global_lock, gtid );
3690 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
3692 kmp_user_lock_p lck = NULL;
3694 if ( __kmp_env_consistency_check ) {
3695 if ( user_lock == NULL ) {
3696 KMP_FATAL( LockIsUninitialized, func );
3700 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3701 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
3702 if ( __kmp_env_consistency_check ) {
3703 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
3704 KMP_FATAL( LockIsUninitialized, func );
3707 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
3708 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3709 lck = __kmp_user_lock_table.table[index];
3712 lck = *( (kmp_user_lock_p *)user_lock );
3715 if ( __kmp_env_consistency_check ) {
3716 if ( lck == NULL ) {
3717 KMP_FATAL( LockIsUninitialized, func );
3725 __kmp_cleanup_user_locks(
void )
3732 __kmp_lock_pool = NULL;
3734 #define IS_CRITICAL(lck) \
3735 ( ( __kmp_get_user_lock_flags_ != NULL ) && \
3736 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) )
3766 while ( __kmp_user_lock_table.used > 1 ) {
3773 kmp_user_lock_p lck = __kmp_user_lock_table.table[
3774 --__kmp_user_lock_table.used ];
3776 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
3777 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
3784 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
3785 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
3787 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
3788 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.func,
3789 str_loc.line, str_loc.col );
3790 __kmp_str_loc_free( &str_loc);
3794 if ( IS_CRITICAL( lck ) ) {
3795 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
3798 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
3806 __kmp_destroy_user_lock( lck );
3812 if ( __kmp_lock_blocks == NULL ) {
3822 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3823 __kmp_user_lock_table.table = NULL;
3824 __kmp_user_lock_table.allocated = 0;
3826 while ( table_ptr != NULL ) {
3831 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
3832 __kmp_free( table_ptr );
3839 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
3840 __kmp_lock_blocks = NULL;
3842 while ( block_ptr != NULL ) {
3843 kmp_block_of_locks_t *next = block_ptr->next_block;
3844 __kmp_free( block_ptr->locks );
3851 TCW_4(__kmp_init_user_locks, FALSE);