Intel® OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Groups Pages
kmp_threadprivate.c
1 /*
2  * kmp_threadprivate.c -- OpenMP threadprivate support library
3  * $Revision: 42618 $
4  * $Date: 2013-08-27 09:15:45 -0500 (Tue, 27 Aug 2013) $
5  */
6 
7 /* <copyright>
8  Copyright (c) 1997-2013 Intel Corporation. All Rights Reserved.
9 
10  Redistribution and use in source and binary forms, with or without
11  modification, are permitted provided that the following conditions
12  are met:
13 
14  * Redistributions of source code must retain the above copyright
15  notice, this list of conditions and the following disclaimer.
16  * Redistributions in binary form must reproduce the above copyright
17  notice, this list of conditions and the following disclaimer in the
18  documentation and/or other materials provided with the distribution.
19  * Neither the name of Intel Corporation nor the names of its
20  contributors may be used to endorse or promote products derived
21  from this software without specific prior written permission.
22 
23  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 
35 </copyright> */
36 
37 #include "kmp.h"
38 #include "kmp_itt.h"
39 #include "kmp_i18n.h"
40 
41 /* ------------------------------------------------------------------------ */
42 /* ------------------------------------------------------------------------ */
43 
44 #define USE_CHECKS_COMMON
45 
46 #define KMP_INLINE_SUBR 1
47 
48 
49 /* ------------------------------------------------------------------------ */
50 /* ------------------------------------------------------------------------ */
51 
52 void
53 kmp_threadprivate_insert_private_data( int gtid, void *pc_addr, void *data_addr, size_t pc_size );
54 struct private_common *
55 kmp_threadprivate_insert( int gtid, void *pc_addr, void *data_addr, size_t pc_size );
56 
57 struct shared_table __kmp_threadprivate_d_table;
58 
59 /* ------------------------------------------------------------------------ */
60 /* ------------------------------------------------------------------------ */
61 
62 static
63 #ifdef KMP_INLINE_SUBR
64 __forceinline
65 #endif
66 struct private_common *
67 __kmp_threadprivate_find_task_common( struct common_table *tbl, int gtid, void *pc_addr )
68 
69 {
70  struct private_common *tn;
71 
72 #ifdef KMP_TASK_COMMON_DEBUG
73  KC_TRACE( 10, ( "__kmp_threadprivate_find_task_common: thread#%d, called with address %p\n",
74  gtid, pc_addr ) );
75  dump_list();
76 #endif
77 
78  for (tn = tbl->data[ KMP_HASH(pc_addr) ]; tn; tn = tn->next) {
79  if (tn->gbl_addr == pc_addr) {
80 #ifdef KMP_TASK_COMMON_DEBUG
81  KC_TRACE( 10, ( "__kmp_threadprivate_find_task_common: thread#%d, found node %p on list\n",
82  gtid, pc_addr ) );
83 #endif
84  return tn;
85  }
86  }
87  return 0;
88 }
89 
90 static
91 #ifdef KMP_INLINE_SUBR
92 __forceinline
93 #endif
94 struct shared_common *
95 __kmp_find_shared_task_common( struct shared_table *tbl, int gtid, void *pc_addr )
96 {
97  struct shared_common *tn;
98 
99  for (tn = tbl->data[ KMP_HASH(pc_addr) ]; tn; tn = tn->next) {
100  if (tn->gbl_addr == pc_addr) {
101 #ifdef KMP_TASK_COMMON_DEBUG
102  KC_TRACE( 10, ( "__kmp_find_shared_task_common: thread#%d, found node %p on list\n",
103  gtid, pc_addr ) );
104 #endif
105  return tn;
106  }
107  }
108  return 0;
109 }
110 
111 
112 /*
113  * Create a template for the data initialized storage.
114  * Either the template is NULL indicating zero fill,
115  * or the template is a copy of the original data.
116  */
117 
118 static struct private_data *
119 __kmp_init_common_data( void *pc_addr, size_t pc_size )
120 {
121  struct private_data *d;
122  size_t i;
123  char *p;
124 
125  d = (struct private_data *) __kmp_allocate( sizeof( struct private_data ) );
126 /*
127  d->data = 0; // AC: commented out because __kmp_allocate zeroes the memory
128  d->next = 0;
129 */
130  d->size = pc_size;
131  d->more = 1;
132 
133  p = (char*)pc_addr;
134 
135  for (i = pc_size; i > 0; --i) {
136  if (*p++ != '\0') {
137  d->data = __kmp_allocate( pc_size );
138  memcpy( d->data, pc_addr, pc_size );
139  break;
140  }
141  }
142 
143  return d;
144 }
145 
146 /*
147  * Initialize the data area from the template.
148  */
149 
150 static void
151 __kmp_copy_common_data( void *pc_addr, struct private_data *d )
152 {
153  char *addr = (char *) pc_addr;
154  int i, offset;
155 
156  for (offset = 0; d != 0; d = d->next) {
157  for (i = d->more; i > 0; --i) {
158  if (d->data == 0)
159  memset( & addr[ offset ], '\0', d->size );
160  else
161  memcpy( & addr[ offset ], d->data, d->size );
162  offset += d->size;
163  }
164  }
165 }
166 
167 /* ------------------------------------------------------------------------ */
168 /* ------------------------------------------------------------------------ */
169 
170 /* we are called from __kmp_serial_initialize() with __kmp_initz_lock held. */
171 void
172 __kmp_common_initialize( void )
173 {
174  if( ! TCR_4(__kmp_init_common) ) {
175  int q;
176 #ifdef KMP_DEBUG
177  int gtid;
178 #endif
179 
180  __kmp_threadpriv_cache_list = NULL;
181 
182 #ifdef KMP_DEBUG
183  /* verify the uber masters were initialized */
184  for(gtid = 0 ; gtid < __kmp_threads_capacity; gtid++ )
185  if( __kmp_root[gtid] ) {
186  KMP_DEBUG_ASSERT( __kmp_root[gtid]->r.r_uber_thread );
187  for ( q = 0; q< KMP_HASH_TABLE_SIZE; ++q)
188  KMP_DEBUG_ASSERT( !__kmp_root[gtid]->r.r_uber_thread->th.th_pri_common->data[q] );
189 /* __kmp_root[ gitd ]-> r.r_uber_thread -> th.th_pri_common -> data[ q ] = 0;*/
190  }
191 #endif /* KMP_DEBUG */
192 
193  for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q)
194  __kmp_threadprivate_d_table.data[ q ] = 0;
195 
196  TCW_4(__kmp_init_common, TRUE);
197  }
198 }
199 
200 /* Call all destructors for threadprivate data belonging to all threads.
201  Currently unused! */
202 void
203 __kmp_common_destroy( void )
204 {
205  if( TCR_4(__kmp_init_common) ) {
206  int q;
207 
208  TCW_4(__kmp_init_common, FALSE);
209 
210  for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
211  int gtid;
212  struct private_common *tn;
213  struct shared_common *d_tn;
214 
215  /* C++ destructors need to be called once per thread before exiting */
216  /* don't call destructors for master thread though unless we used copy constructor */
217 
218  for (d_tn = __kmp_threadprivate_d_table.data[ q ]; d_tn; d_tn = d_tn->next) {
219  if (d_tn->is_vec) {
220  if (d_tn->dt.dtorv != 0) {
221  for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
222  if( __kmp_threads[gtid] ) {
223  if( (__kmp_foreign_tp) ? (! KMP_INITIAL_GTID (gtid)) :
224  (! KMP_UBER_GTID (gtid)) ) {
225  tn = __kmp_threadprivate_find_task_common( __kmp_threads[ gtid ]->th.th_pri_common,
226  gtid, d_tn->gbl_addr );
227  if (tn) {
228  (*d_tn->dt.dtorv) (tn->par_addr, d_tn->vec_len);
229  }
230  }
231  }
232  }
233  if (d_tn->obj_init != 0) {
234  (*d_tn->dt.dtorv) (d_tn->obj_init, d_tn->vec_len);
235  }
236  }
237  } else {
238  if (d_tn->dt.dtor != 0) {
239  for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
240  if( __kmp_threads[gtid] ) {
241  if( (__kmp_foreign_tp) ? (! KMP_INITIAL_GTID (gtid)) :
242  (! KMP_UBER_GTID (gtid)) ) {
243  tn = __kmp_threadprivate_find_task_common( __kmp_threads[ gtid ]->th.th_pri_common,
244  gtid, d_tn->gbl_addr );
245  if (tn) {
246  (*d_tn->dt.dtor) (tn->par_addr);
247  }
248  }
249  }
250  }
251  if (d_tn->obj_init != 0) {
252  (*d_tn->dt.dtor) (d_tn->obj_init);
253  }
254  }
255  }
256  }
257  __kmp_threadprivate_d_table.data[ q ] = 0;
258  }
259  }
260 }
261 
262 /* Call all destructors for threadprivate data belonging to this thread */
263 void
264 __kmp_common_destroy_gtid( int gtid )
265 {
266  struct private_common *tn;
267  struct shared_common *d_tn;
268 
269  KC_TRACE( 10, ("__kmp_common_destroy_gtid: T#%d called\n", gtid ) );
270  if( (__kmp_foreign_tp) ? (! KMP_INITIAL_GTID (gtid)) :
271  (! KMP_UBER_GTID (gtid)) ) {
272 
273  if( TCR_4(__kmp_init_common) ) {
274 
275  /* Cannot do this here since not all threads have destroyed their data */
276  /* TCW_4(__kmp_init_common, FALSE); */
277 
278  for (tn = __kmp_threads[ gtid ]->th.th_pri_head; tn; tn = tn->link) {
279 
280  d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table,
281  gtid, tn->gbl_addr );
282 
283  KMP_DEBUG_ASSERT( d_tn );
284 
285  if (d_tn->is_vec) {
286  if (d_tn->dt.dtorv != 0) {
287  (void) (*d_tn->dt.dtorv) (tn->par_addr, d_tn->vec_len);
288  }
289  if (d_tn->obj_init != 0) {
290  (void) (*d_tn->dt.dtorv) (d_tn->obj_init, d_tn->vec_len);
291  }
292  } else {
293  if (d_tn->dt.dtor != 0) {
294  (void) (*d_tn->dt.dtor) (tn->par_addr);
295  }
296  if (d_tn->obj_init != 0) {
297  (void) (*d_tn->dt.dtor) (d_tn->obj_init);
298  }
299  }
300  }
301  KC_TRACE( 30, ("__kmp_common_destroy_gtid: T#%d threadprivate destructors complete\n",
302  gtid ) );
303  }
304  }
305 }
306 
307 /* ------------------------------------------------------------------------ */
308 /* ------------------------------------------------------------------------ */
309 
310 #ifdef KMP_TASK_COMMON_DEBUG
311 static void
312 dump_list( void )
313 {
314  int p, q;
315 
316  for (p = 0; p < __kmp_all_nth; ++p) {
317  if( !__kmp_threads[p] ) continue;
318  for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
319  if (__kmp_threads[ p ]->th.th_pri_common->data[ q ]) {
320  struct private_common *tn;
321 
322  KC_TRACE( 10, ( "\tdump_list: gtid:%d addresses\n", p ) );
323 
324  for (tn = __kmp_threads[ p ]->th.th_pri_common->data[ q ]; tn; tn = tn->next) {
325  KC_TRACE( 10, ( "\tdump_list: THREADPRIVATE: Serial %p -> Parallel %p\n",
326  tn->gbl_addr, tn->par_addr ) );
327  }
328  }
329  }
330  }
331 }
332 #endif /* KMP_TASK_COMMON_DEBUG */
333 
334 
335 /*
336  * NOTE: this routine is to be called only from the serial part of the program.
337  */
338 
339 void
340 kmp_threadprivate_insert_private_data( int gtid, void *pc_addr, void *data_addr, size_t pc_size )
341 {
342  struct shared_common **lnk_tn, *d_tn;
343  KMP_DEBUG_ASSERT( __kmp_threads[ gtid ] &&
344  __kmp_threads[ gtid ] -> th.th_root -> r.r_active == 0 );
345 
346  d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table,
347  gtid, pc_addr );
348 
349  if (d_tn == 0) {
350  d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) );
351 
352  d_tn->gbl_addr = pc_addr;
353  d_tn->pod_init = __kmp_init_common_data( data_addr, pc_size );
354 /*
355  d_tn->obj_init = 0; // AC: commented out because __kmp_allocate zeroes the memory
356  d_tn->ct.ctor = 0;
357  d_tn->cct.cctor = 0;;
358  d_tn->dt.dtor = 0;
359  d_tn->is_vec = FALSE;
360  d_tn->vec_len = 0L;
361 */
362  d_tn->cmn_size = pc_size;
363 
364  __kmp_acquire_lock( &__kmp_global_lock, gtid );
365 
366  lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(pc_addr) ]);
367 
368  d_tn->next = *lnk_tn;
369  *lnk_tn = d_tn;
370 
371  __kmp_release_lock( &__kmp_global_lock, gtid );
372  }
373 }
374 
375 struct private_common *
376 kmp_threadprivate_insert( int gtid, void *pc_addr, void *data_addr, size_t pc_size )
377 {
378  struct private_common *tn, **tt;
379  struct shared_common *d_tn;
380 
381  /* +++++++++ START OF CRITICAL SECTION +++++++++ */
382 
383  __kmp_acquire_lock( & __kmp_global_lock, gtid );
384 
385  tn = (struct private_common *) __kmp_allocate( sizeof (struct private_common) );
386 
387  tn->gbl_addr = pc_addr;
388 
389  d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table,
390  gtid, pc_addr ); /* Only the MASTER data table exists. */
391 
392  if (d_tn != 0) {
393  /* This threadprivate variable has already been seen. */
394 
395  if ( d_tn->pod_init == 0 && d_tn->obj_init == 0 ) {
396  d_tn->cmn_size = pc_size;
397 
398  if (d_tn->is_vec) {
399  if (d_tn->ct.ctorv != 0) {
400  /* Construct from scratch so no prototype exists */
401  d_tn->obj_init = 0;
402  }
403  else if (d_tn->cct.cctorv != 0) {
404  /* Now data initialize the prototype since it was previously registered */
405  d_tn->obj_init = (void *) __kmp_allocate( d_tn->cmn_size );
406  (void) (*d_tn->cct.cctorv) (d_tn->obj_init, pc_addr, d_tn->vec_len);
407  }
408  else {
409  d_tn->pod_init = __kmp_init_common_data( data_addr, d_tn->cmn_size );
410  }
411  } else {
412  if (d_tn->ct.ctor != 0) {
413  /* Construct from scratch so no prototype exists */
414  d_tn->obj_init = 0;
415  }
416  else if (d_tn->cct.cctor != 0) {
417  /* Now data initialize the prototype since it was previously registered */
418  d_tn->obj_init = (void *) __kmp_allocate( d_tn->cmn_size );
419  (void) (*d_tn->cct.cctor) (d_tn->obj_init, pc_addr);
420  }
421  else {
422  d_tn->pod_init = __kmp_init_common_data( data_addr, d_tn->cmn_size );
423  }
424  }
425  }
426  }
427  else {
428  struct shared_common **lnk_tn;
429 
430  d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) );
431  d_tn->gbl_addr = pc_addr;
432  d_tn->cmn_size = pc_size;
433  d_tn->pod_init = __kmp_init_common_data( data_addr, pc_size );
434 /*
435  d_tn->obj_init = 0; // AC: commented out because __kmp_allocate zeroes the memory
436  d_tn->ct.ctor = 0;
437  d_tn->cct.cctor = 0;
438  d_tn->dt.dtor = 0;
439  d_tn->is_vec = FALSE;
440  d_tn->vec_len = 0L;
441 */
442  lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(pc_addr) ]);
443 
444  d_tn->next = *lnk_tn;
445  *lnk_tn = d_tn;
446  }
447 
448  tn->cmn_size = d_tn->cmn_size;
449 
450  if ( (__kmp_foreign_tp) ? (KMP_INITIAL_GTID (gtid)) : (KMP_UBER_GTID (gtid)) ) {
451  tn->par_addr = (void *) pc_addr;
452  }
453  else {
454  tn->par_addr = (void *) __kmp_allocate( tn->cmn_size );
455  }
456 
457  __kmp_release_lock( & __kmp_global_lock, gtid );
458 
459  /* +++++++++ END OF CRITICAL SECTION +++++++++ */
460 
461 #ifdef USE_CHECKS_COMMON
462  if (pc_size > d_tn->cmn_size) {
463  KC_TRACE( 10, ( "__kmp_threadprivate_insert: THREADPRIVATE: %p (%"
464  KMP_UINTPTR_SPEC " ,%" KMP_UINTPTR_SPEC ")\n",
465  pc_addr, pc_size, d_tn->cmn_size ) );
466  KMP_FATAL( TPCommonBlocksInconsist );
467  }
468 #endif /* USE_CHECKS_COMMON */
469 
470  tt = &(__kmp_threads[ gtid ]->th.th_pri_common->data[ KMP_HASH(pc_addr) ]);
471 
472 #ifdef KMP_TASK_COMMON_DEBUG
473  if (*tt != 0) {
474  KC_TRACE( 10, ( "__kmp_threadprivate_insert: WARNING! thread#%d: collision on %p\n",
475  gtid, pc_addr ) );
476  }
477 #endif
478  tn->next = *tt;
479  *tt = tn;
480 
481 #ifdef KMP_TASK_COMMON_DEBUG
482  KC_TRACE( 10, ( "__kmp_threadprivate_insert: thread#%d, inserted node %p on list\n",
483  gtid, pc_addr ) );
484  dump_list( );
485 #endif
486 
487  /* Link the node into a simple list */
488 
489  tn->link = __kmp_threads[ gtid ]->th.th_pri_head;
490  __kmp_threads[ gtid ]->th.th_pri_head = tn;
491 
492 #ifdef BUILD_TV
493  __kmp_tv_threadprivate_store( __kmp_threads[ gtid ], tn->gbl_addr, tn->par_addr );
494 #endif
495 
496  if( (__kmp_foreign_tp) ? (KMP_INITIAL_GTID (gtid)) : (KMP_UBER_GTID (gtid)) )
497  return tn;
498 
499  /*
500  * if C++ object with copy constructor, use it;
501  * else if C++ object with constructor, use it for the non-master copies only;
502  * else use pod_init and memcpy
503  *
504  * C++ constructors need to be called once for each non-master thread on allocate
505  * C++ copy constructors need to be called once for each thread on allocate
506  */
507 
508  /*
509  * C++ object with constructors/destructors;
510  * don't call constructors for master thread though
511  */
512  if (d_tn->is_vec) {
513  if ( d_tn->ct.ctorv != 0) {
514  (void) (*d_tn->ct.ctorv) (tn->par_addr, d_tn->vec_len);
515  } else if (d_tn->cct.cctorv != 0) {
516  (void) (*d_tn->cct.cctorv) (tn->par_addr, d_tn->obj_init, d_tn->vec_len);
517  } else if (tn->par_addr != tn->gbl_addr) {
518  __kmp_copy_common_data( tn->par_addr, d_tn->pod_init );
519  }
520  } else {
521  if ( d_tn->ct.ctor != 0 ) {
522  (void) (*d_tn->ct.ctor) (tn->par_addr);
523  } else if (d_tn->cct.cctor != 0) {
524  (void) (*d_tn->cct.cctor) (tn->par_addr, d_tn->obj_init);
525  } else if (tn->par_addr != tn->gbl_addr) {
526  __kmp_copy_common_data( tn->par_addr, d_tn->pod_init );
527  }
528  }
529 /* !BUILD_OPENMP_C
530  if (tn->par_addr != tn->gbl_addr)
531  __kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); */
532 
533  return tn;
534 }
535 
536 /* ------------------------------------------------------------------------ */
537 /* We are currently parallel, and we know the thread id. */
538 /* ------------------------------------------------------------------------ */
539 
552 void
554 {
555  struct shared_common *d_tn, **lnk_tn;
556 
557  KC_TRACE( 10, ("__kmpc_threadprivate_register: called\n" ) );
558 
559 #ifdef USE_CHECKS_COMMON
560  /* copy constructor must be zero for current code gen (Nov 2002 - jph) */
561  KMP_ASSERT( cctor == 0);
562 #endif /* USE_CHECKS_COMMON */
563 
564  /* Only the global data table exists. */
565  d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table, -1, data );
566 
567  if (d_tn == 0) {
568  d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) );
569  d_tn->gbl_addr = data;
570 
571  d_tn->ct.ctor = ctor;
572  d_tn->cct.cctor = cctor;
573  d_tn->dt.dtor = dtor;
574 /*
575  d_tn->is_vec = FALSE; // AC: commented out because __kmp_allocate zeroes the memory
576  d_tn->vec_len = 0L;
577  d_tn->obj_init = 0;
578  d_tn->pod_init = 0;
579 */
580  lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(data) ]);
581 
582  d_tn->next = *lnk_tn;
583  *lnk_tn = d_tn;
584  }
585 }
586 
587 void *
588 __kmpc_threadprivate(ident_t *loc, kmp_int32 global_tid, void *data, size_t size)
589 {
590  void *ret;
591  struct private_common *tn;
592 
593  KC_TRACE( 10, ("__kmpc_threadprivate: T#%d called\n", global_tid ) );
594 
595 #ifdef USE_CHECKS_COMMON
596  if (! __kmp_init_serial)
597  KMP_FATAL( RTLNotInitialized );
598 #endif /* USE_CHECKS_COMMON */
599 
600  if ( ! __kmp_threads[global_tid] -> th.th_root -> r.r_active && ! __kmp_foreign_tp ) {
601  /* The parallel address will NEVER overlap with the data_address */
602  /* dkp: 3rd arg to kmp_threadprivate_insert_private_data() is the data_address; use data_address = data */
603 
604  KC_TRACE( 20, ("__kmpc_threadprivate: T#%d inserting private data\n", global_tid ) );
605  kmp_threadprivate_insert_private_data( global_tid, data, data, size );
606 
607  ret = data;
608  }
609  else {
610  KC_TRACE( 50, ("__kmpc_threadprivate: T#%d try to find private data at address %p\n",
611  global_tid, data ) );
612  tn = __kmp_threadprivate_find_task_common( __kmp_threads[ global_tid ]->th.th_pri_common, global_tid, data );
613 
614  if ( tn ) {
615  KC_TRACE( 20, ("__kmpc_threadprivate: T#%d found data\n", global_tid ) );
616 #ifdef USE_CHECKS_COMMON
617  if ((size_t) size > tn->cmn_size) {
618  KC_TRACE( 10, ( "THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC " ,%" KMP_UINTPTR_SPEC ")\n",
619  data, size, tn->cmn_size ) );
620  KMP_FATAL( TPCommonBlocksInconsist );
621  }
622 #endif /* USE_CHECKS_COMMON */
623  }
624  else {
625  /* The parallel address will NEVER overlap with the data_address */
626  /* dkp: 3rd arg to kmp_threadprivate_insert() is the data_address; use data_address = data */
627  KC_TRACE( 20, ("__kmpc_threadprivate: T#%d inserting data\n", global_tid ) );
628  tn = kmp_threadprivate_insert( global_tid, data, data, size );
629  }
630 
631  ret = tn->par_addr;
632  }
633  KC_TRACE( 10, ("__kmpc_threadprivate: T#%d exiting; return value = %p\n",
634  global_tid, ret ) );
635 
636  return ret;
637 }
638 
650 void *
652  ident_t * loc,
653  kmp_int32 global_tid, // gtid.
654  void * data, // Pointer to original global variable.
655  size_t size, // Size of original global variable.
656  void *** cache
657 ) {
658  KC_TRACE( 10, ("__kmpc_threadprivate_cached: T#%d called with cache: %p, address: %p, size: %"
659  KMP_SIZE_T_SPEC "\n",
660  global_tid, *cache, data, size ) );
661 
662  if ( TCR_PTR(*cache) == 0) {
663  __kmp_acquire_lock( & __kmp_global_lock, global_tid );
664 
665  if ( TCR_PTR(*cache) == 0) {
666  __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
667  __kmp_tp_cached = 1;
668  __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
669  void ** my_cache;
670  KMP_ITT_IGNORE(
671  my_cache = (void**)
672  __kmp_allocate(sizeof( void * ) * __kmp_tp_capacity + sizeof ( kmp_cached_addr_t ));
673  );
674  // No need to zero the allocated memory; __kmp_allocate does that.
675  KC_TRACE( 50, ("__kmpc_threadprivate_cached: T#%d allocated cache at address %p\n",
676  global_tid, my_cache ) );
677 
678  /* TODO: free all this memory in __kmp_common_destroy using __kmp_threadpriv_cache_list */
679  /* Add address of mycache to linked list for cleanup later */
680  kmp_cached_addr_t *tp_cache_addr;
681 
682  tp_cache_addr = (kmp_cached_addr_t *) & my_cache[__kmp_tp_capacity];
683  tp_cache_addr -> addr = my_cache;
684  tp_cache_addr -> next = __kmp_threadpriv_cache_list;
685  __kmp_threadpriv_cache_list = tp_cache_addr;
686 
687  KMP_MB();
688 
689  TCW_PTR( *cache, my_cache);
690 
691  KMP_MB();
692  }
693 
694  __kmp_release_lock( & __kmp_global_lock, global_tid );
695  }
696 
697  void *ret;
698  if ((ret = TCR_PTR((*cache)[ global_tid ])) == 0) {
699  ret = __kmpc_threadprivate( loc, global_tid, data, (size_t) size);
700 
701  TCW_PTR( (*cache)[ global_tid ], ret);
702  }
703  KC_TRACE( 10, ("__kmpc_threadprivate_cached: T#%d exiting; return value = %p\n",
704  global_tid, ret ) );
705 
706  return ret;
707 }
708 
719 void
721  kmpc_cctor_vec cctor, kmpc_dtor_vec dtor,
722  size_t vector_length )
723 {
724  struct shared_common *d_tn, **lnk_tn;
725 
726  KC_TRACE( 10, ("__kmpc_threadprivate_register_vec: called\n" ) );
727 
728 #ifdef USE_CHECKS_COMMON
729  /* copy constructor must be zero for current code gen (Nov 2002 - jph) */
730  KMP_ASSERT( cctor == 0);
731 #endif /* USE_CHECKS_COMMON */
732 
733  d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table,
734  -1, data ); /* Only the global data table exists. */
735 
736  if (d_tn == 0) {
737  d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) );
738  d_tn->gbl_addr = data;
739 
740  d_tn->ct.ctorv = ctor;
741  d_tn->cct.cctorv = cctor;
742  d_tn->dt.dtorv = dtor;
743  d_tn->is_vec = TRUE;
744  d_tn->vec_len = (size_t) vector_length;
745 /*
746  d_tn->obj_init = 0; // AC: commented out because __kmp_allocate zeroes the memory
747  d_tn->pod_init = 0;
748 */
749  lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(data) ]);
750 
751  d_tn->next = *lnk_tn;
752  *lnk_tn = d_tn;
753  }
754 }