14 #include "kmp_affinity.h" 15 #include "kmp_atomic.h" 16 #include "kmp_environment.h" 17 #include "kmp_error.h" 21 #include "kmp_settings.h" 22 #include "kmp_stats.h" 24 #include "kmp_wait_release.h" 25 #include "kmp_wrapper_getpid.h" 26 #include "kmp_dispatch.h" 27 #if KMP_USE_HIER_SCHED 28 #include "kmp_dispatch_hier.h" 32 #include "ompt-specific.h" 36 #define KMP_USE_PRCTL 0 42 #include "tsan_annotations.h" 44 #if defined(KMP_GOMP_COMPAT) 45 char const __kmp_version_alt_comp[] =
46 KMP_VERSION_PREFIX
"alternative compiler support: yes";
49 char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX
"API version: " 61 char const __kmp_version_lock[] =
62 KMP_VERSION_PREFIX
"lock type: run time selectable";
65 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 70 kmp_info_t __kmp_monitor;
75 void __kmp_cleanup(
void);
77 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *,
int tid,
79 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
80 kmp_internal_control_t *new_icvs,
82 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 83 static void __kmp_partition_places(kmp_team_t *team,
84 int update_master_only = 0);
86 static void __kmp_do_serial_initialize(
void);
87 void __kmp_fork_barrier(
int gtid,
int tid);
88 void __kmp_join_barrier(
int gtid);
89 void __kmp_setup_icv_copy(kmp_team_t *team,
int new_nproc,
90 kmp_internal_control_t *new_icvs,
ident_t *loc);
92 #ifdef USE_LOAD_BALANCE 93 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc);
96 static int __kmp_expand_threads(
int nNeed);
98 static int __kmp_unregister_root_other_thread(
int gtid);
100 static void __kmp_unregister_library(
void);
101 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root);
102 kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
107 int __kmp_get_global_thread_id() {
109 kmp_info_t **other_threads;
117 (
"*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
118 __kmp_nth, __kmp_all_nth));
125 if (!TCR_4(__kmp_init_gtid))
128 #ifdef KMP_TDATA_GTID 129 if (TCR_4(__kmp_gtid_mode) >= 3) {
130 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using TDATA\n"));
134 if (TCR_4(__kmp_gtid_mode) >= 2) {
135 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using keyed TLS\n"));
136 return __kmp_gtid_get_specific();
138 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using internal alg.\n"));
140 stack_addr = (
char *)&stack_data;
141 other_threads = __kmp_threads;
154 for (i = 0; i < __kmp_threads_capacity; i++) {
156 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
160 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
161 stack_base = (
char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
165 if (stack_addr <= stack_base) {
166 size_t stack_diff = stack_base - stack_addr;
168 if (stack_diff <= stack_size) {
171 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
179 (
"*** __kmp_get_global_thread_id: internal alg. failed to find " 180 "thread, using TLS\n"));
181 i = __kmp_gtid_get_specific();
191 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
192 KMP_FATAL(StackOverflow, i);
195 stack_base = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
196 if (stack_addr > stack_base) {
197 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
199 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
202 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
203 stack_base - stack_addr);
207 if (__kmp_storage_map) {
208 char *stack_end = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
209 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
210 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
211 other_threads[i]->th.th_info.ds.ds_stacksize,
212 "th_%d stack (refinement)", i);
217 int __kmp_get_global_thread_id_reg() {
220 if (!__kmp_init_serial) {
223 #ifdef KMP_TDATA_GTID 224 if (TCR_4(__kmp_gtid_mode) >= 3) {
225 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using TDATA\n"));
229 if (TCR_4(__kmp_gtid_mode) >= 2) {
230 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
231 gtid = __kmp_gtid_get_specific();
234 (
"*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
235 gtid = __kmp_get_global_thread_id();
239 if (gtid == KMP_GTID_DNE) {
241 (
"__kmp_get_global_thread_id_reg: Encountered new root thread. " 242 "Registering a new gtid.\n"));
243 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
244 if (!__kmp_init_serial) {
245 __kmp_do_serial_initialize();
246 gtid = __kmp_gtid_get_specific();
248 gtid = __kmp_register_root(FALSE);
250 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
254 KMP_DEBUG_ASSERT(gtid >= 0);
260 void __kmp_check_stack_overlap(kmp_info_t *th) {
262 char *stack_beg = NULL;
263 char *stack_end = NULL;
266 KA_TRACE(10, (
"__kmp_check_stack_overlap: called\n"));
267 if (__kmp_storage_map) {
268 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
269 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
271 gtid = __kmp_gtid_from_thread(th);
273 if (gtid == KMP_GTID_MONITOR) {
274 __kmp_print_storage_map_gtid(
275 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
276 "th_%s stack (%s)",
"mon",
277 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
279 __kmp_print_storage_map_gtid(
280 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
281 "th_%d stack (%s)", gtid,
282 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
288 gtid = __kmp_gtid_from_thread(th);
289 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
291 (
"__kmp_check_stack_overlap: performing extensive checking\n"));
292 if (stack_beg == NULL) {
293 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
294 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
297 for (f = 0; f < __kmp_threads_capacity; f++) {
298 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
300 if (f_th && f_th != th) {
301 char *other_stack_end =
302 (
char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
303 char *other_stack_beg =
304 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
305 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
306 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
309 if (__kmp_storage_map)
310 __kmp_print_storage_map_gtid(
311 -1, other_stack_beg, other_stack_end,
312 (
size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
313 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
315 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
321 KA_TRACE(10, (
"__kmp_check_stack_overlap: returning\n"));
326 void __kmp_infinite_loop(
void) {
327 static int done = FALSE;
334 #define MAX_MESSAGE 512 336 void __kmp_print_storage_map_gtid(
int gtid,
void *p1,
void *p2,
size_t size,
337 char const *format, ...) {
338 char buffer[MAX_MESSAGE];
341 va_start(ap, format);
342 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP storage map: %p %p%8lu %s\n", p1,
343 p2, (
unsigned long)size, format);
344 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
345 __kmp_vprintf(kmp_err, buffer, ap);
346 #if KMP_PRINT_DATA_PLACEMENT 349 if (p1 <= p2 && (
char *)p2 - (
char *)p1 == size) {
350 if (__kmp_storage_map_verbose) {
351 node = __kmp_get_host_node(p1);
353 __kmp_storage_map_verbose = FALSE;
357 int localProc = __kmp_get_cpu_from_gtid(gtid);
359 const int page_size = KMP_GET_PAGE_SIZE();
361 p1 = (
void *)((
size_t)p1 & ~((size_t)page_size - 1));
362 p2 = (
void *)(((
size_t)p2 - 1) & ~((
size_t)page_size - 1));
364 __kmp_printf_no_lock(
" GTID %d localNode %d\n", gtid,
367 __kmp_printf_no_lock(
" GTID %d\n", gtid);
376 (
char *)p1 += page_size;
377 }
while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
378 __kmp_printf_no_lock(
" %p-%p memNode %d\n", last, (
char *)p1 - 1,
382 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p1,
383 (
char *)p1 + (page_size - 1),
384 __kmp_get_host_node(p1));
386 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p2,
387 (
char *)p2 + (page_size - 1),
388 __kmp_get_host_node(p2));
394 __kmp_printf_no_lock(
" %s\n", KMP_I18N_STR(StorageMapWarning));
397 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
400 void __kmp_warn(
char const *format, ...) {
401 char buffer[MAX_MESSAGE];
404 if (__kmp_generate_warnings == kmp_warnings_off) {
408 va_start(ap, format);
410 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP warning: %s\n", format);
411 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
412 __kmp_vprintf(kmp_err, buffer, ap);
413 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
418 void __kmp_abort_process() {
420 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
422 if (__kmp_debug_buf) {
423 __kmp_dump_debug_buffer();
426 if (KMP_OS_WINDOWS) {
429 __kmp_global.g.g_abort = SIGABRT;
446 __kmp_infinite_loop();
447 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
451 void __kmp_abort_thread(
void) {
454 __kmp_infinite_loop();
460 static void __kmp_print_thread_storage_map(kmp_info_t *thr,
int gtid) {
461 __kmp_print_storage_map_gtid(gtid, thr, thr + 1,
sizeof(kmp_info_t),
"th_%d",
464 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
465 sizeof(kmp_desc_t),
"th_%d.th_info", gtid);
467 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
468 sizeof(kmp_local_t),
"th_%d.th_local", gtid);
470 __kmp_print_storage_map_gtid(
471 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
472 sizeof(kmp_balign_t) * bs_last_barrier,
"th_%d.th_bar", gtid);
474 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
475 &thr->th.th_bar[bs_plain_barrier + 1],
476 sizeof(kmp_balign_t),
"th_%d.th_bar[plain]",
479 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
480 &thr->th.th_bar[bs_forkjoin_barrier + 1],
481 sizeof(kmp_balign_t),
"th_%d.th_bar[forkjoin]",
484 #if KMP_FAST_REDUCTION_BARRIER 485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
486 &thr->th.th_bar[bs_reduction_barrier + 1],
487 sizeof(kmp_balign_t),
"th_%d.th_bar[reduction]",
489 #endif // KMP_FAST_REDUCTION_BARRIER 495 static void __kmp_print_team_storage_map(
const char *header, kmp_team_t *team,
496 int team_id,
int num_thr) {
497 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
498 __kmp_print_storage_map_gtid(-1, team, team + 1,
sizeof(kmp_team_t),
"%s_%d",
501 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
502 &team->t.t_bar[bs_last_barrier],
503 sizeof(kmp_balign_team_t) * bs_last_barrier,
504 "%s_%d.t_bar", header, team_id);
506 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
507 &team->t.t_bar[bs_plain_barrier + 1],
508 sizeof(kmp_balign_team_t),
"%s_%d.t_bar[plain]",
511 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
512 &team->t.t_bar[bs_forkjoin_barrier + 1],
513 sizeof(kmp_balign_team_t),
514 "%s_%d.t_bar[forkjoin]", header, team_id);
516 #if KMP_FAST_REDUCTION_BARRIER 517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
518 &team->t.t_bar[bs_reduction_barrier + 1],
519 sizeof(kmp_balign_team_t),
520 "%s_%d.t_bar[reduction]", header, team_id);
521 #endif // KMP_FAST_REDUCTION_BARRIER 523 __kmp_print_storage_map_gtid(
524 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
525 sizeof(kmp_disp_t) * num_thr,
"%s_%d.t_dispatch", header, team_id);
527 __kmp_print_storage_map_gtid(
528 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
529 sizeof(kmp_info_t *) * num_thr,
"%s_%d.t_threads", header, team_id);
531 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
532 &team->t.t_disp_buffer[num_disp_buff],
533 sizeof(dispatch_shared_info_t) * num_disp_buff,
534 "%s_%d.t_disp_buffer", header, team_id);
537 static void __kmp_init_allocator() {
539 __kmp_init_memkind();
542 static void __kmp_fini_allocator() {
544 __kmp_fini_memkind();
553 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
555 __kmp_init_bootstrap_lock(lck);
558 static void __kmp_reset_locks_on_process_detach(
int gtid_req) {
576 for (i = 0; i < __kmp_threads_capacity; ++i) {
579 kmp_info_t *th = __kmp_threads[i];
582 int gtid = th->th.th_info.ds.ds_gtid;
583 if (gtid == gtid_req)
588 int alive = __kmp_is_thread_alive(th, &exit_val);
593 if (thread_count == 0)
599 __kmp_reset_lock(&__kmp_forkjoin_lock);
601 __kmp_reset_lock(&__kmp_stdio_lock);
605 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
610 case DLL_PROCESS_ATTACH:
611 KA_TRACE(10, (
"DllMain: PROCESS_ATTACH\n"));
615 case DLL_PROCESS_DETACH:
616 KA_TRACE(10, (
"DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
618 if (lpReserved != NULL) {
644 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
647 __kmp_internal_end_library(__kmp_gtid_get_specific());
651 case DLL_THREAD_ATTACH:
652 KA_TRACE(10, (
"DllMain: THREAD_ATTACH\n"));
658 case DLL_THREAD_DETACH:
659 KA_TRACE(10, (
"DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
661 __kmp_internal_end_thread(__kmp_gtid_get_specific());
672 void __kmp_parallel_deo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
673 int gtid = *gtid_ref;
674 #ifdef BUILD_PARALLEL_ORDERED 675 kmp_team_t *team = __kmp_team_from_gtid(gtid);
678 if (__kmp_env_consistency_check) {
679 if (__kmp_threads[gtid]->th.th_root->r.r_active)
680 #if KMP_USE_DYNAMIC_LOCK 681 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
683 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
686 #ifdef BUILD_PARALLEL_ORDERED 687 if (!team->t.t_serialized) {
689 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,
697 void __kmp_parallel_dxo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
698 int gtid = *gtid_ref;
699 #ifdef BUILD_PARALLEL_ORDERED 700 int tid = __kmp_tid_from_gtid(gtid);
701 kmp_team_t *team = __kmp_team_from_gtid(gtid);
704 if (__kmp_env_consistency_check) {
705 if (__kmp_threads[gtid]->th.th_root->r.r_active)
706 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
708 #ifdef BUILD_PARALLEL_ORDERED 709 if (!team->t.t_serialized) {
714 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
724 int __kmp_enter_single(
int gtid,
ident_t *id_ref,
int push_ws) {
729 if (!TCR_4(__kmp_init_parallel))
730 __kmp_parallel_initialize();
733 __kmp_resume_if_soft_paused();
736 th = __kmp_threads[gtid];
737 team = th->th.th_team;
740 th->th.th_ident = id_ref;
742 if (team->t.t_serialized) {
745 kmp_int32 old_this = th->th.th_local.this_construct;
747 ++th->th.th_local.this_construct;
751 if (team->t.t_construct == old_this) {
752 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
753 th->th.th_local.this_construct);
756 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
757 KMP_MASTER_GTID(gtid) &&
759 th->th.th_teams_microtask == NULL &&
761 team->t.t_active_level ==
763 __kmp_itt_metadata_single(id_ref);
768 if (__kmp_env_consistency_check) {
769 if (status && push_ws) {
770 __kmp_push_workshare(gtid, ct_psingle, id_ref);
772 __kmp_check_workshare(gtid, ct_psingle, id_ref);
777 __kmp_itt_single_start(gtid);
783 void __kmp_exit_single(
int gtid) {
785 __kmp_itt_single_end(gtid);
787 if (__kmp_env_consistency_check)
788 __kmp_pop_workshare(gtid, ct_psingle, NULL);
797 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
798 int master_tid,
int set_nthreads
806 KMP_DEBUG_ASSERT(__kmp_init_serial);
807 KMP_DEBUG_ASSERT(root && parent_team);
808 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
812 new_nthreads = set_nthreads;
813 if (!get__dynamic_2(parent_team, master_tid)) {
816 #ifdef USE_LOAD_BALANCE 817 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
818 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
819 if (new_nthreads == 1) {
820 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced " 821 "reservation to 1 thread\n",
825 if (new_nthreads < set_nthreads) {
826 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced " 827 "reservation to %d threads\n",
828 master_tid, new_nthreads));
832 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
833 new_nthreads = __kmp_avail_proc - __kmp_nth +
834 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
835 if (new_nthreads <= 1) {
836 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced " 837 "reservation to 1 thread\n",
841 if (new_nthreads < set_nthreads) {
842 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced " 843 "reservation to %d threads\n",
844 master_tid, new_nthreads));
846 new_nthreads = set_nthreads;
848 }
else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
849 if (set_nthreads > 2) {
850 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
851 new_nthreads = (new_nthreads % set_nthreads) + 1;
852 if (new_nthreads == 1) {
853 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced " 854 "reservation to 1 thread\n",
858 if (new_nthreads < set_nthreads) {
859 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced " 860 "reservation to %d threads\n",
861 master_tid, new_nthreads));
869 if (__kmp_nth + new_nthreads -
870 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
872 int tl_nthreads = __kmp_max_nth - __kmp_nth +
873 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
874 if (tl_nthreads <= 0) {
879 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
880 __kmp_reserve_warn = 1;
881 __kmp_msg(kmp_ms_warning,
882 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
883 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
885 if (tl_nthreads == 1) {
886 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 887 "reduced reservation to 1 thread\n",
891 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 892 "reservation to %d threads\n",
893 master_tid, tl_nthreads));
894 new_nthreads = tl_nthreads;
898 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
899 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
900 if (cg_nthreads + new_nthreads -
901 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
903 int tl_nthreads = max_cg_threads - cg_nthreads +
904 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
905 if (tl_nthreads <= 0) {
910 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
911 __kmp_reserve_warn = 1;
912 __kmp_msg(kmp_ms_warning,
913 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
914 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
916 if (tl_nthreads == 1) {
917 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 918 "reduced reservation to 1 thread\n",
922 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 923 "reservation to %d threads\n",
924 master_tid, tl_nthreads));
925 new_nthreads = tl_nthreads;
931 capacity = __kmp_threads_capacity;
932 if (TCR_PTR(__kmp_threads[0]) == NULL) {
935 if (__kmp_nth + new_nthreads -
936 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
939 int slotsRequired = __kmp_nth + new_nthreads -
940 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
942 int slotsAdded = __kmp_expand_threads(slotsRequired);
943 if (slotsAdded < slotsRequired) {
945 new_nthreads -= (slotsRequired - slotsAdded);
946 KMP_ASSERT(new_nthreads >= 1);
949 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
950 __kmp_reserve_warn = 1;
951 if (__kmp_tp_cached) {
952 __kmp_msg(kmp_ms_warning,
953 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
954 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
955 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
957 __kmp_msg(kmp_ms_warning,
958 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
959 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
966 if (new_nthreads == 1) {
968 (
"__kmp_reserve_threads: T#%d serializing team after reclaiming " 969 "dead roots and rechecking; requested %d threads\n",
970 __kmp_get_gtid(), set_nthreads));
972 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d allocating %d threads; requested" 974 __kmp_get_gtid(), new_nthreads, set_nthreads));
983 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
984 kmp_info_t *master_th,
int master_gtid) {
988 KA_TRACE(10, (
"__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
989 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
993 master_th->th.th_info.ds.ds_tid = 0;
994 master_th->th.th_team = team;
995 master_th->th.th_team_nproc = team->t.t_nproc;
996 master_th->th.th_team_master = master_th;
997 master_th->th.th_team_serialized = FALSE;
998 master_th->th.th_dispatch = &team->t.t_dispatch[0];
1001 #if KMP_NESTED_HOT_TEAMS 1003 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1006 int level = team->t.t_active_level - 1;
1007 if (master_th->th.th_teams_microtask) {
1008 if (master_th->th.th_teams_size.nteams > 1) {
1012 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1013 master_th->th.th_teams_level == team->t.t_level) {
1018 if (level < __kmp_hot_teams_max_level) {
1019 if (hot_teams[level].hot_team) {
1021 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
1025 hot_teams[level].hot_team = team;
1026 hot_teams[level].hot_team_nth = team->t.t_nproc;
1033 use_hot_team = team == root->r.r_hot_team;
1035 if (!use_hot_team) {
1038 team->t.t_threads[0] = master_th;
1039 __kmp_initialize_info(master_th, team, 0, master_gtid);
1042 for (i = 1; i < team->t.t_nproc; i++) {
1045 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1046 team->t.t_threads[i] = thr;
1047 KMP_DEBUG_ASSERT(thr);
1048 KMP_DEBUG_ASSERT(thr->th.th_team == team);
1050 KA_TRACE(20, (
"__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 1051 "T#%d(%d:%d) join =%llu, plain=%llu\n",
1052 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
1053 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
1054 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
1055 team->t.t_bar[bs_plain_barrier].b_arrived));
1057 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1058 thr->th.th_teams_level = master_th->th.th_teams_level;
1059 thr->th.th_teams_size = master_th->th.th_teams_size;
1063 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1064 for (b = 0; b < bs_last_barrier; ++b) {
1065 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1066 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
1068 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1074 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1075 __kmp_partition_places(team);
1080 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1081 for (i = 0; i < team->t.t_nproc; i++) {
1082 kmp_info_t *thr = team->t.t_threads[i];
1083 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1084 thr->th.th_prev_level != team->t.t_level) {
1085 team->t.t_display_affinity = 1;
1095 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1099 inline static void propagateFPControl(kmp_team_t *team) {
1100 if (__kmp_inherit_fp_control) {
1101 kmp_int16 x87_fpu_control_word;
1105 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1106 __kmp_store_mxcsr(&mxcsr);
1107 mxcsr &= KMP_X86_MXCSR_MASK;
1118 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1119 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1122 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1126 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1132 inline static void updateHWFPControl(kmp_team_t *team) {
1133 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1136 kmp_int16 x87_fpu_control_word;
1138 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1139 __kmp_store_mxcsr(&mxcsr);
1140 mxcsr &= KMP_X86_MXCSR_MASK;
1142 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1143 __kmp_clear_x87_fpu_status_word();
1144 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1147 if (team->t.t_mxcsr != mxcsr) {
1148 __kmp_load_mxcsr(&team->t.t_mxcsr);
1153 #define propagateFPControl(x) ((void)0) 1154 #define updateHWFPControl(x) ((void)0) 1157 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
1162 void __kmp_serialized_parallel(
ident_t *loc, kmp_int32 global_tid) {
1163 kmp_info_t *this_thr;
1164 kmp_team_t *serial_team;
1166 KC_TRACE(10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid));
1173 if (!TCR_4(__kmp_init_parallel))
1174 __kmp_parallel_initialize();
1177 __kmp_resume_if_soft_paused();
1180 this_thr = __kmp_threads[global_tid];
1181 serial_team = this_thr->th.th_serial_team;
1184 KMP_DEBUG_ASSERT(serial_team);
1187 if (__kmp_tasking_mode != tskm_immediate_exec) {
1189 this_thr->th.th_task_team ==
1190 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1191 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1193 KA_TRACE(20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1194 "team %p, new task_team = NULL\n",
1195 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1196 this_thr->th.th_task_team = NULL;
1200 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1201 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1202 proc_bind = proc_bind_false;
1203 }
else if (proc_bind == proc_bind_default) {
1206 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1209 this_thr->th.th_set_proc_bind = proc_bind_default;
1213 ompt_data_t ompt_parallel_data = ompt_data_none;
1214 ompt_data_t *implicit_task_data;
1215 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1216 if (ompt_enabled.enabled &&
1217 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1219 ompt_task_info_t *parent_task_info;
1220 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1222 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1223 if (ompt_enabled.ompt_callback_parallel_begin) {
1226 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1227 &(parent_task_info->task_data), &(parent_task_info->frame),
1228 &ompt_parallel_data, team_size, ompt_parallel_invoker_program,
1232 #endif // OMPT_SUPPORT 1234 if (this_thr->th.th_team != serial_team) {
1236 int level = this_thr->th.th_team->t.t_level;
1238 if (serial_team->t.t_serialized) {
1241 kmp_team_t *new_team;
1243 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1245 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1252 &this_thr->th.th_current_task->td_icvs,
1253 0 USE_NESTED_HOT_ARG(NULL));
1254 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1255 KMP_ASSERT(new_team);
1258 new_team->t.t_threads[0] = this_thr;
1259 new_team->t.t_parent = this_thr->th.th_team;
1260 serial_team = new_team;
1261 this_thr->th.th_serial_team = serial_team;
1265 (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1266 global_tid, serial_team));
1274 (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1275 global_tid, serial_team));
1279 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1280 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1281 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1282 serial_team->t.t_ident = loc;
1283 serial_team->t.t_serialized = 1;
1284 serial_team->t.t_nproc = 1;
1285 serial_team->t.t_parent = this_thr->th.th_team;
1286 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1287 this_thr->th.th_team = serial_team;
1288 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1290 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1291 this_thr->th.th_current_task));
1292 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1293 this_thr->th.th_current_task->td_flags.executing = 0;
1295 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1300 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1301 &this_thr->th.th_current_task->td_parent->td_icvs);
1305 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1306 this_thr->th.th_current_task->td_icvs.nproc =
1307 __kmp_nested_nth.nth[level + 1];
1311 if (__kmp_nested_proc_bind.used &&
1312 (level + 1 < __kmp_nested_proc_bind.used)) {
1313 this_thr->th.th_current_task->td_icvs.proc_bind =
1314 __kmp_nested_proc_bind.bind_types[level + 1];
1319 serial_team->t.t_pkfn = (microtask_t)(~0);
1321 this_thr->th.th_info.ds.ds_tid = 0;
1324 this_thr->th.th_team_nproc = 1;
1325 this_thr->th.th_team_master = this_thr;
1326 this_thr->th.th_team_serialized = 1;
1328 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1329 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1331 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator;
1334 propagateFPControl(serial_team);
1337 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1338 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1339 serial_team->t.t_dispatch->th_disp_buffer =
1340 (dispatch_private_info_t *)__kmp_allocate(
1341 sizeof(dispatch_private_info_t));
1343 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1350 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1351 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1352 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1353 ++serial_team->t.t_serialized;
1354 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1357 int level = this_thr->th.th_team->t.t_level;
1360 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1361 this_thr->th.th_current_task->td_icvs.nproc =
1362 __kmp_nested_nth.nth[level + 1];
1364 serial_team->t.t_level++;
1365 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level " 1366 "of serial team %p to %d\n",
1367 global_tid, serial_team, serial_team->t.t_level));
1370 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1372 dispatch_private_info_t *disp_buffer =
1373 (dispatch_private_info_t *)__kmp_allocate(
1374 sizeof(dispatch_private_info_t));
1375 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1376 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1378 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1383 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
1389 if (__kmp_display_affinity) {
1390 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1391 this_thr->th.th_prev_num_threads != 1) {
1393 __kmp_aux_display_affinity(global_tid, NULL);
1394 this_thr->th.th_prev_level = serial_team->t.t_level;
1395 this_thr->th.th_prev_num_threads = 1;
1400 if (__kmp_env_consistency_check)
1401 __kmp_push_parallel(global_tid, NULL);
1403 serial_team->t.ompt_team_info.master_return_address = codeptr;
1404 if (ompt_enabled.enabled &&
1405 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1406 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1408 ompt_lw_taskteam_t lw_taskteam;
1409 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1410 &ompt_parallel_data, codeptr);
1412 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1416 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr);
1417 if (ompt_enabled.ompt_callback_implicit_task) {
1418 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1419 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
1420 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), ompt_task_implicit);
1421 OMPT_CUR_TASK_INFO(this_thr)
1422 ->thread_num = __kmp_tid_from_gtid(global_tid);
1426 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1427 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1434 int __kmp_fork_call(
ident_t *loc,
int gtid,
1435 enum fork_context_e call_context,
1436 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1438 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1447 int master_this_cons;
1449 kmp_team_t *parent_team;
1450 kmp_info_t *master_th;
1454 int master_set_numthreads;
1460 #if KMP_NESTED_HOT_TEAMS 1461 kmp_hot_team_ptr_t **p_hot_teams;
1464 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
1467 KA_TRACE(20, (
"__kmp_fork_call: enter T#%d\n", gtid));
1468 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1471 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1473 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1474 __kmp_stkpadding += (short)((kmp_int64)dummy);
1480 if (!TCR_4(__kmp_init_parallel))
1481 __kmp_parallel_initialize();
1484 __kmp_resume_if_soft_paused();
1488 master_th = __kmp_threads[gtid];
1490 parent_team = master_th->th.th_team;
1491 master_tid = master_th->th.th_info.ds.ds_tid;
1492 master_this_cons = master_th->th.th_local.this_construct;
1493 root = master_th->th.th_root;
1494 master_active = root->r.r_active;
1495 master_set_numthreads = master_th->th.th_set_nproc;
1498 ompt_data_t ompt_parallel_data = ompt_data_none;
1499 ompt_data_t *parent_task_data;
1500 ompt_frame_t *ompt_frame;
1501 ompt_data_t *implicit_task_data;
1502 void *return_address = NULL;
1504 if (ompt_enabled.enabled) {
1505 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1507 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
1512 level = parent_team->t.t_level;
1514 active_level = parent_team->t.t_active_level;
1517 teams_level = master_th->th.th_teams_level;
1519 #if KMP_NESTED_HOT_TEAMS 1520 p_hot_teams = &master_th->th.th_hot_teams;
1521 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1522 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1523 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1524 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1526 (*p_hot_teams)[0].hot_team_nth = 1;
1531 if (ompt_enabled.enabled) {
1532 if (ompt_enabled.ompt_callback_parallel_begin) {
1533 int team_size = master_set_numthreads
1534 ? master_set_numthreads
1535 : get__nproc_2(parent_team, master_tid);
1536 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1537 parent_task_data, ompt_frame, &ompt_parallel_data, team_size,
1538 OMPT_INVOKER(call_context), return_address);
1540 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1544 master_th->th.th_ident = loc;
1547 if (master_th->th.th_teams_microtask && ap &&
1548 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1552 parent_team->t.t_ident = loc;
1553 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1554 parent_team->t.t_argc = argc;
1555 argv = (
void **)parent_team->t.t_argv;
1556 for (i = argc - 1; i >= 0; --i)
1558 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1559 *argv++ = va_arg(*ap,
void *);
1561 *argv++ = va_arg(ap,
void *);
1564 if (parent_team == master_th->th.th_serial_team) {
1567 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1570 parent_team->t.t_serialized--;
1573 void **exit_runtime_p;
1575 ompt_lw_taskteam_t lw_taskteam;
1577 if (ompt_enabled.enabled) {
1578 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1579 &ompt_parallel_data, return_address);
1580 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
1582 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1586 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1587 if (ompt_enabled.ompt_callback_implicit_task) {
1588 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1589 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1590 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit);
1591 OMPT_CUR_TASK_INFO(master_th)
1592 ->thread_num = __kmp_tid_from_gtid(gtid);
1596 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1598 exit_runtime_p = &dummy;
1603 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1604 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1605 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1614 *exit_runtime_p = NULL;
1615 if (ompt_enabled.enabled) {
1616 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;
1617 if (ompt_enabled.ompt_callback_implicit_task) {
1618 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1619 ompt_scope_end, NULL, implicit_task_data, 1,
1620 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1622 __ompt_lw_taskteam_unlink(master_th);
1624 if (ompt_enabled.ompt_callback_parallel_end) {
1625 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1626 OMPT_CUR_TEAM_DATA(master_th), OMPT_CUR_TASK_DATA(master_th),
1627 OMPT_INVOKER(call_context), return_address);
1629 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1635 parent_team->t.t_pkfn = microtask;
1636 parent_team->t.t_invoke = invoker;
1637 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1638 parent_team->t.t_active_level++;
1639 parent_team->t.t_level++;
1641 parent_team->t.t_def_allocator = master_th->th.th_def_allocator;
1645 if (master_set_numthreads) {
1646 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1648 kmp_info_t **other_threads = parent_team->t.t_threads;
1649 parent_team->t.t_nproc = master_set_numthreads;
1650 for (i = 0; i < master_set_numthreads; ++i) {
1651 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1655 master_th->th.th_set_nproc = 0;
1659 if (__kmp_debugging) {
1660 int nth = __kmp_omp_num_threads(loc);
1662 master_set_numthreads = nth;
1667 KF_TRACE(10, (
"__kmp_fork_call: before internal fork: root=%p, team=%p, " 1668 "master_th=%p, gtid=%d\n",
1669 root, parent_team, master_th, gtid));
1670 __kmp_internal_fork(loc, gtid, parent_team);
1671 KF_TRACE(10, (
"__kmp_fork_call: after internal fork: root=%p, team=%p, " 1672 "master_th=%p, gtid=%d\n",
1673 root, parent_team, master_th, gtid));
1676 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1677 parent_team->t.t_id, parent_team->t.t_pkfn));
1679 if (!parent_team->t.t_invoke(gtid)) {
1680 KMP_ASSERT2(0,
"cannot invoke microtask for MASTER thread");
1682 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1683 parent_team->t.t_id, parent_team->t.t_pkfn));
1686 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
1693 if (__kmp_tasking_mode != tskm_immediate_exec) {
1694 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1695 parent_team->t.t_task_team[master_th->th.th_task_state]);
1699 if (parent_team->t.t_active_level >=
1700 master_th->th.th_current_task->td_icvs.max_active_levels) {
1704 int enter_teams = ((ap == NULL && active_level == 0) ||
1705 (ap && teams_level > 0 && teams_level == level));
1708 master_set_numthreads
1709 ? master_set_numthreads
1718 if ((get__max_active_levels(master_th) == 1 && (root->r.r_in_parallel
1723 (__kmp_library == library_serial)) {
1724 KC_TRACE(10, (
"__kmp_fork_call: T#%d serializing team; requested %d" 1732 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1733 nthreads = __kmp_reserve_threads(
1734 root, parent_team, master_tid, nthreads
1745 if (nthreads == 1) {
1749 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1753 KMP_DEBUG_ASSERT(nthreads > 0);
1756 master_th->th.th_set_nproc = 0;
1759 if (nthreads == 1) {
1761 #if KMP_OS_LINUX && \ 1762 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1765 void **args = (
void **)KMP_ALLOCA(argc *
sizeof(
void *));
1770 (
"__kmp_fork_call: T#%d serializing parallel region\n", gtid));
1774 if (call_context == fork_context_intel) {
1776 master_th->th.th_serial_team->t.t_ident = loc;
1780 master_th->th.th_serial_team->t.t_level--;
1785 void **exit_runtime_p;
1786 ompt_task_info_t *task_info;
1788 ompt_lw_taskteam_t lw_taskteam;
1790 if (ompt_enabled.enabled) {
1791 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1792 &ompt_parallel_data, return_address);
1794 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1797 task_info = OMPT_CUR_TASK_INFO(master_th);
1798 exit_runtime_p = &(task_info->frame.exit_frame.ptr);
1799 if (ompt_enabled.ompt_callback_implicit_task) {
1800 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1801 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1802 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit);
1803 OMPT_CUR_TASK_INFO(master_th)
1804 ->thread_num = __kmp_tid_from_gtid(gtid);
1808 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1810 exit_runtime_p = &dummy;
1815 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1816 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1817 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1818 parent_team->t.t_argv
1827 if (ompt_enabled.enabled) {
1828 exit_runtime_p = NULL;
1829 if (ompt_enabled.ompt_callback_implicit_task) {
1830 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1831 ompt_scope_end, NULL, &(task_info->task_data), 1,
1832 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1835 __ompt_lw_taskteam_unlink(master_th);
1836 if (ompt_enabled.ompt_callback_parallel_end) {
1837 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1838 OMPT_CUR_TEAM_DATA(master_th), parent_task_data,
1839 OMPT_INVOKER(call_context), return_address);
1841 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1844 }
else if (microtask == (microtask_t)__kmp_teams_master) {
1845 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1846 master_th->th.th_serial_team);
1847 team = master_th->th.th_team;
1849 team->t.t_invoke = invoker;
1850 __kmp_alloc_argv_entries(argc, team, TRUE);
1851 team->t.t_argc = argc;
1852 argv = (
void **)team->t.t_argv;
1854 for (i = argc - 1; i >= 0; --i)
1856 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1857 *argv++ = va_arg(*ap,
void *);
1859 *argv++ = va_arg(ap,
void *);
1862 for (i = 0; i < argc; ++i)
1864 argv[i] = parent_team->t.t_argv[i];
1874 for (i = argc - 1; i >= 0; --i)
1876 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1877 *argv++ = va_arg(*ap,
void *);
1879 *argv++ = va_arg(ap,
void *);
1885 void **exit_runtime_p;
1886 ompt_task_info_t *task_info;
1888 ompt_lw_taskteam_t lw_taskteam;
1890 if (ompt_enabled.enabled) {
1891 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1892 &ompt_parallel_data, return_address);
1893 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1895 task_info = OMPT_CUR_TASK_INFO(master_th);
1896 exit_runtime_p = &(task_info->frame.exit_frame.ptr);
1899 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1900 if (ompt_enabled.ompt_callback_implicit_task) {
1901 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1902 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1903 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit);
1904 OMPT_CUR_TASK_INFO(master_th)
1905 ->thread_num = __kmp_tid_from_gtid(gtid);
1909 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1911 exit_runtime_p = &dummy;
1916 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1917 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1918 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1927 if (ompt_enabled.enabled) {
1928 *exit_runtime_p = NULL;
1929 if (ompt_enabled.ompt_callback_implicit_task) {
1930 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1931 ompt_scope_end, NULL, &(task_info->task_data), 1,
1932 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1935 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1936 __ompt_lw_taskteam_unlink(master_th);
1937 if (ompt_enabled.ompt_callback_parallel_end) {
1938 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1939 &ompt_parallel_data, parent_task_data,
1940 OMPT_INVOKER(call_context), return_address);
1942 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1948 }
else if (call_context == fork_context_gnu) {
1950 ompt_lw_taskteam_t lwt;
1951 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1954 lwt.ompt_task_info.frame.exit_frame = ompt_data_none;
1955 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1960 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1963 KMP_ASSERT2(call_context < fork_context_last,
1964 "__kmp_fork_call: unknown fork_context parameter");
1967 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1974 KF_TRACE(10, (
"__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1975 "curtask=%p, curtask_max_aclevel=%d\n",
1976 parent_team->t.t_active_level, master_th,
1977 master_th->th.th_current_task,
1978 master_th->th.th_current_task->td_icvs.max_active_levels));
1982 master_th->th.th_current_task->td_flags.executing = 0;
1985 if (!master_th->th.th_teams_microtask || level > teams_level)
1989 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1993 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1994 if ((level + 1 < __kmp_nested_nth.used) &&
1995 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1996 nthreads_icv = __kmp_nested_nth.nth[level + 1];
2003 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
2004 kmp_proc_bind_t proc_bind_icv =
2006 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
2007 proc_bind = proc_bind_false;
2009 if (proc_bind == proc_bind_default) {
2012 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
2018 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2019 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2020 master_th->th.th_current_task->td_icvs.proc_bind)) {
2021 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2026 master_th->th.th_set_proc_bind = proc_bind_default;
2029 if ((nthreads_icv > 0)
2031 || (proc_bind_icv != proc_bind_default)
2034 kmp_internal_control_t new_icvs;
2035 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2036 new_icvs.next = NULL;
2037 if (nthreads_icv > 0) {
2038 new_icvs.nproc = nthreads_icv;
2042 if (proc_bind_icv != proc_bind_default) {
2043 new_icvs.proc_bind = proc_bind_icv;
2048 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2049 team = __kmp_allocate_team(root, nthreads, nthreads,
2056 &new_icvs, argc USE_NESTED_HOT_ARG(master_th));
2059 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2060 team = __kmp_allocate_team(root, nthreads, nthreads,
2067 &master_th->th.th_current_task->td_icvs,
2068 argc USE_NESTED_HOT_ARG(master_th));
2071 10, (
"__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
2074 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2075 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2076 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2077 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2078 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
2080 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2083 KMP_CHECK_UPDATE(team->t.t_invoke, invoker);
2086 if (!master_th->th.th_teams_microtask || level > teams_level) {
2088 int new_level = parent_team->t.t_level + 1;
2089 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2090 new_level = parent_team->t.t_active_level + 1;
2091 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2095 int new_level = parent_team->t.t_level;
2096 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2097 new_level = parent_team->t.t_active_level;
2098 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2101 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
2103 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
2106 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
2109 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2113 propagateFPControl(team);
2115 if (__kmp_tasking_mode != tskm_immediate_exec) {
2118 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2119 parent_team->t.t_task_team[master_th->th.th_task_state]);
2120 KA_TRACE(20, (
"__kmp_fork_call: Master T#%d pushing task_team %p / team " 2121 "%p, new task_team %p / team %p\n",
2122 __kmp_gtid_from_thread(master_th),
2123 master_th->th.th_task_team, parent_team,
2124 team->t.t_task_team[master_th->th.th_task_state], team));
2126 if (active_level || master_th->th.th_task_team) {
2128 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2129 if (master_th->th.th_task_state_top >=
2130 master_th->th.th_task_state_stack_sz) {
2131 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2132 kmp_uint8 *old_stack, *new_stack;
2134 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2135 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2136 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2138 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2142 old_stack = master_th->th.th_task_state_memo_stack;
2143 master_th->th.th_task_state_memo_stack = new_stack;
2144 master_th->th.th_task_state_stack_sz = new_size;
2145 __kmp_free(old_stack);
2149 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2150 master_th->th.th_task_state;
2151 master_th->th.th_task_state_top++;
2152 #if KMP_NESTED_HOT_TEAMS 2153 if (master_th->th.th_hot_teams &&
2154 active_level < __kmp_hot_teams_max_level &&
2155 team == master_th->th.th_hot_teams[active_level].hot_team) {
2157 master_th->th.th_task_state =
2159 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2162 master_th->th.th_task_state = 0;
2163 #if KMP_NESTED_HOT_TEAMS 2167 #if !KMP_NESTED_HOT_TEAMS 2168 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2169 (team == root->r.r_hot_team));
2175 (
"__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2176 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2178 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2179 (team->t.t_master_tid == 0 &&
2180 (team->t.t_parent == root->r.r_root_team ||
2181 team->t.t_parent->t.t_serialized)));
2185 argv = (
void **)team->t.t_argv;
2189 for (i = argc - 1; i >= 0; --i) {
2191 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 2192 void *new_argv = va_arg(*ap,
void *);
2194 void *new_argv = va_arg(ap,
void *);
2196 KMP_CHECK_UPDATE(*argv, new_argv);
2201 for (i = 0; i < argc; ++i) {
2203 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2209 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
2210 if (!root->r.r_active)
2211 root->r.r_active = TRUE;
2213 __kmp_fork_team_threads(root, team, master_th, gtid);
2214 __kmp_setup_icv_copy(team, nthreads,
2215 &master_th->th.th_current_task->td_icvs, loc);
2218 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2221 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2224 if (team->t.t_active_level == 1
2226 && !master_th->th.th_teams_microtask
2230 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2231 (__kmp_forkjoin_frames_mode == 3 ||
2232 __kmp_forkjoin_frames_mode == 1)) {
2233 kmp_uint64 tmp_time = 0;
2234 if (__itt_get_timestamp_ptr)
2235 tmp_time = __itt_get_timestamp();
2237 master_th->th.th_frame_time = tmp_time;
2238 if (__kmp_forkjoin_frames_mode == 3)
2239 team->t.t_region_time = tmp_time;
2243 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2244 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2246 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2252 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
2255 (
"__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2256 root, team, master_th, gtid));
2259 if (__itt_stack_caller_create_ptr) {
2260 team->t.t_stack_id =
2261 __kmp_itt_stack_caller_create();
2272 __kmp_internal_fork(loc, gtid, team);
2273 KF_TRACE(10, (
"__kmp_internal_fork : after : root=%p, team=%p, " 2274 "master_th=%p, gtid=%d\n",
2275 root, team, master_th, gtid));
2278 if (call_context == fork_context_gnu) {
2279 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2284 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2285 team->t.t_id, team->t.t_pkfn));
2288 #if KMP_STATS_ENABLED && OMP_40_ENABLED 2292 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
2296 if (!team->t.t_invoke(gtid)) {
2297 KMP_ASSERT2(0,
"cannot invoke microtask for MASTER thread");
2300 #if KMP_STATS_ENABLED && OMP_40_ENABLED 2303 KMP_SET_THREAD_STATE(previous_state);
2307 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2308 team->t.t_id, team->t.t_pkfn));
2311 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2314 if (ompt_enabled.enabled) {
2315 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2323 static inline void __kmp_join_restore_state(kmp_info_t *thread,
2326 thread->th.ompt_thread_info.state =
2327 ((team->t.t_serialized) ? ompt_state_work_serial
2328 : ompt_state_work_parallel);
2331 static inline void __kmp_join_ompt(
int gtid, kmp_info_t *thread,
2332 kmp_team_t *team, ompt_data_t *parallel_data,
2333 fork_context_e fork_context,
void *codeptr) {
2334 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2335 if (ompt_enabled.ompt_callback_parallel_end) {
2336 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2337 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context),
2341 task_info->frame.enter_frame = ompt_data_none;
2342 __kmp_join_restore_state(thread, team);
2346 void __kmp_join_call(
ident_t *loc,
int gtid
2349 enum fork_context_e fork_context
2356 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2358 kmp_team_t *parent_team;
2359 kmp_info_t *master_th;
2363 KA_TRACE(20, (
"__kmp_join_call: enter T#%d\n", gtid));
2366 master_th = __kmp_threads[gtid];
2367 root = master_th->th.th_root;
2368 team = master_th->th.th_team;
2369 parent_team = team->t.t_parent;
2371 master_th->th.th_ident = loc;
2374 if (ompt_enabled.enabled) {
2375 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2380 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2381 KA_TRACE(20, (
"__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2382 "th_task_team = %p\n",
2383 __kmp_gtid_from_thread(master_th), team,
2384 team->t.t_task_team[master_th->th.th_task_state],
2385 master_th->th.th_task_team));
2386 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2387 team->t.t_task_team[master_th->th.th_task_state]);
2391 if (team->t.t_serialized) {
2393 if (master_th->th.th_teams_microtask) {
2395 int level = team->t.t_level;
2396 int tlevel = master_th->th.th_teams_level;
2397 if (level == tlevel) {
2401 }
else if (level == tlevel + 1) {
2405 team->t.t_serialized++;
2412 if (ompt_enabled.enabled) {
2413 __kmp_join_restore_state(master_th, parent_team);
2420 master_active = team->t.t_master_active;
2428 __kmp_internal_join(loc, gtid, team);
2432 master_th->th.th_task_state =
2440 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2441 void *codeptr = team->t.ompt_team_info.master_return_address;
2445 if (__itt_stack_caller_create_ptr) {
2446 __kmp_itt_stack_caller_destroy(
2447 (__itt_caller)team->t
2452 if (team->t.t_active_level == 1
2454 && !master_th->th.th_teams_microtask
2457 master_th->th.th_ident = loc;
2460 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2461 __kmp_forkjoin_frames_mode == 3)
2462 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2463 master_th->th.th_frame_time, 0, loc,
2464 master_th->th.th_team_nproc, 1);
2465 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2466 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2467 __kmp_itt_region_joined(gtid);
2472 if (master_th->th.th_teams_microtask && !exit_teams &&
2473 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2474 team->t.t_level == master_th->th.th_teams_level + 1) {
2481 team->t.t_active_level--;
2482 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2488 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2489 int old_num = master_th->th.th_team_nproc;
2490 int new_num = master_th->th.th_teams_size.nth;
2491 kmp_info_t **other_threads = team->t.t_threads;
2492 team->t.t_nproc = new_num;
2493 for (
int i = 0; i < old_num; ++i) {
2494 other_threads[i]->th.th_team_nproc = new_num;
2497 for (
int i = old_num; i < new_num; ++i) {
2499 KMP_DEBUG_ASSERT(other_threads[i]);
2500 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2501 for (
int b = 0; b < bs_last_barrier; ++b) {
2502 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2503 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2505 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2508 if (__kmp_tasking_mode != tskm_immediate_exec) {
2510 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2516 if (ompt_enabled.enabled) {
2517 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2527 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2528 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2530 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2535 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2538 if (!master_th->th.th_teams_microtask ||
2539 team->t.t_level > master_th->th.th_teams_level)
2543 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2545 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2548 if (ompt_enabled.enabled) {
2549 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2550 if (ompt_enabled.ompt_callback_implicit_task) {
2551 int ompt_team_size = team->t.t_nproc;
2552 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2553 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2554 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
2557 task_info->frame.exit_frame = ompt_data_none;
2558 task_info->task_data = ompt_data_none;
2562 KF_TRACE(10, (
"__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2564 __kmp_pop_current_task_from_thread(master_th);
2566 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2568 master_th->th.th_first_place = team->t.t_first_place;
2569 master_th->th.th_last_place = team->t.t_last_place;
2572 master_th->th.th_def_allocator = team->t.t_def_allocator;
2575 updateHWFPControl(team);
2577 if (root->r.r_active != master_active)
2578 root->r.r_active = master_active;
2580 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2588 master_th->th.th_team = parent_team;
2589 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2590 master_th->th.th_team_master = parent_team->t.t_threads[0];
2591 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2594 if (parent_team->t.t_serialized &&
2595 parent_team != master_th->th.th_serial_team &&
2596 parent_team != root->r.r_root_team) {
2597 __kmp_free_team(root,
2598 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2599 master_th->th.th_serial_team = parent_team;
2602 if (__kmp_tasking_mode != tskm_immediate_exec) {
2603 if (master_th->th.th_task_state_top >
2605 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2607 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2608 master_th->th.th_task_state;
2609 --master_th->th.th_task_state_top;
2611 master_th->th.th_task_state =
2613 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2616 master_th->th.th_task_team =
2617 parent_team->t.t_task_team[master_th->th.th_task_state];
2619 (
"__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
2620 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2627 master_th->th.th_current_task->td_flags.executing = 1;
2629 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2632 if (ompt_enabled.enabled) {
2633 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2639 KA_TRACE(20, (
"__kmp_join_call: exit T#%d\n", gtid));
2644 void __kmp_save_internal_controls(kmp_info_t *thread) {
2646 if (thread->th.th_team != thread->th.th_serial_team) {
2649 if (thread->th.th_team->t.t_serialized > 1) {
2652 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2655 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2656 thread->th.th_team->t.t_serialized) {
2661 kmp_internal_control_t *control =
2662 (kmp_internal_control_t *)__kmp_allocate(
2663 sizeof(kmp_internal_control_t));
2665 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2667 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2669 control->next = thread->th.th_team->t.t_control_stack_top;
2670 thread->th.th_team->t.t_control_stack_top = control;
2676 void __kmp_set_num_threads(
int new_nth,
int gtid) {
2680 KF_TRACE(10, (
"__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2681 KMP_DEBUG_ASSERT(__kmp_init_serial);
2685 else if (new_nth > __kmp_max_nth)
2686 new_nth = __kmp_max_nth;
2689 thread = __kmp_threads[gtid];
2690 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2693 __kmp_save_internal_controls(thread);
2695 set__nproc(thread, new_nth);
2700 root = thread->th.th_root;
2701 if (__kmp_init_parallel && (!root->r.r_active) &&
2702 (root->r.r_hot_team->t.t_nproc > new_nth)
2703 #
if KMP_NESTED_HOT_TEAMS
2704 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2707 kmp_team_t *hot_team = root->r.r_hot_team;
2710 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2713 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2714 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2715 if (__kmp_tasking_mode != tskm_immediate_exec) {
2718 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2720 __kmp_free_thread(hot_team->t.t_threads[f]);
2721 hot_team->t.t_threads[f] = NULL;
2723 hot_team->t.t_nproc = new_nth;
2724 #if KMP_NESTED_HOT_TEAMS 2725 if (thread->th.th_hot_teams) {
2726 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2727 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2731 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2734 for (f = 0; f < new_nth; f++) {
2735 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2736 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2739 hot_team->t.t_size_changed = -1;
2744 void __kmp_set_max_active_levels(
int gtid,
int max_active_levels) {
2747 KF_TRACE(10, (
"__kmp_set_max_active_levels: new max_active_levels for thread " 2749 gtid, max_active_levels));
2750 KMP_DEBUG_ASSERT(__kmp_init_serial);
2753 if (max_active_levels < 0) {
2754 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2759 KF_TRACE(10, (
"__kmp_set_max_active_levels: the call is ignored: new " 2760 "max_active_levels for thread %d = (%d)\n",
2761 gtid, max_active_levels));
2764 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2769 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2770 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2771 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2777 KF_TRACE(10, (
"__kmp_set_max_active_levels: after validation: new " 2778 "max_active_levels for thread %d = (%d)\n",
2779 gtid, max_active_levels));
2781 thread = __kmp_threads[gtid];
2783 __kmp_save_internal_controls(thread);
2785 set__max_active_levels(thread, max_active_levels);
2789 int __kmp_get_max_active_levels(
int gtid) {
2792 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d\n", gtid));
2793 KMP_DEBUG_ASSERT(__kmp_init_serial);
2795 thread = __kmp_threads[gtid];
2796 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2797 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d, curtask=%p, " 2798 "curtask_maxaclevel=%d\n",
2799 gtid, thread->th.th_current_task,
2800 thread->th.th_current_task->td_icvs.max_active_levels));
2801 return thread->th.th_current_task->td_icvs.max_active_levels;
2805 void __kmp_set_schedule(
int gtid, kmp_sched_t kind,
int chunk) {
2809 KF_TRACE(10, (
"__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2810 gtid, (
int)kind, chunk));
2811 KMP_DEBUG_ASSERT(__kmp_init_serial);
2817 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2818 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2820 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2821 KMP_HNT(DefaultScheduleKindUsed,
"static, no chunk"),
2823 kind = kmp_sched_default;
2827 thread = __kmp_threads[gtid];
2829 __kmp_save_internal_controls(thread);
2831 if (kind < kmp_sched_upper_std) {
2832 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2835 thread->th.th_current_task->td_icvs.sched.r_sched_type =
kmp_sch_static;
2837 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2838 __kmp_sch_map[kind - kmp_sched_lower - 1];
2843 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2844 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2845 kmp_sched_lower - 2];
2847 if (kind == kmp_sched_auto || chunk < 1) {
2849 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2851 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2856 void __kmp_get_schedule(
int gtid, kmp_sched_t *kind,
int *chunk) {
2860 KF_TRACE(10, (
"__kmp_get_schedule: thread %d\n", gtid));
2861 KMP_DEBUG_ASSERT(__kmp_init_serial);
2863 thread = __kmp_threads[gtid];
2865 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2869 case kmp_sch_static_greedy:
2870 case kmp_sch_static_balanced:
2871 *kind = kmp_sched_static;
2874 case kmp_sch_static_chunked:
2875 *kind = kmp_sched_static;
2877 case kmp_sch_dynamic_chunked:
2878 *kind = kmp_sched_dynamic;
2881 case kmp_sch_guided_iterative_chunked:
2882 case kmp_sch_guided_analytical_chunked:
2883 *kind = kmp_sched_guided;
2886 *kind = kmp_sched_auto;
2888 case kmp_sch_trapezoidal:
2889 *kind = kmp_sched_trapezoidal;
2891 #if KMP_STATIC_STEAL_ENABLED 2892 case kmp_sch_static_steal:
2893 *kind = kmp_sched_static_steal;
2897 KMP_FATAL(UnknownSchedulingType, th_type);
2900 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2903 int __kmp_get_ancestor_thread_num(
int gtid,
int level) {
2909 KF_TRACE(10, (
"__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2910 KMP_DEBUG_ASSERT(__kmp_init_serial);
2917 thr = __kmp_threads[gtid];
2918 team = thr->th.th_team;
2919 ii = team->t.t_level;
2924 if (thr->th.th_teams_microtask) {
2926 int tlevel = thr->th.th_teams_level;
2929 KMP_DEBUG_ASSERT(ii >= tlevel);
2942 return __kmp_tid_from_gtid(gtid);
2944 dd = team->t.t_serialized;
2946 while (ii > level) {
2947 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2949 if ((team->t.t_serialized) && (!dd)) {
2950 team = team->t.t_parent;
2954 team = team->t.t_parent;
2955 dd = team->t.t_serialized;
2960 return (dd > 1) ? (0) : (team->t.t_master_tid);
2963 int __kmp_get_team_size(
int gtid,
int level) {
2969 KF_TRACE(10, (
"__kmp_get_team_size: thread %d %d\n", gtid, level));
2970 KMP_DEBUG_ASSERT(__kmp_init_serial);
2977 thr = __kmp_threads[gtid];
2978 team = thr->th.th_team;
2979 ii = team->t.t_level;
2984 if (thr->th.th_teams_microtask) {
2986 int tlevel = thr->th.th_teams_level;
2989 KMP_DEBUG_ASSERT(ii >= tlevel);
3001 while (ii > level) {
3002 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
3004 if (team->t.t_serialized && (!dd)) {
3005 team = team->t.t_parent;
3009 team = team->t.t_parent;
3014 return team->t.t_nproc;
3017 kmp_r_sched_t __kmp_get_schedule_global() {
3022 kmp_r_sched_t r_sched;
3030 r_sched.r_sched_type = __kmp_static;
3033 r_sched.r_sched_type = __kmp_guided;
3035 r_sched.r_sched_type = __kmp_sched;
3038 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
3040 r_sched.chunk = KMP_DEFAULT_CHUNK;
3042 r_sched.chunk = __kmp_chunk;
3050 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
int realloc) {
3052 KMP_DEBUG_ASSERT(team);
3053 if (!realloc || argc > team->t.t_max_argc) {
3055 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: needed entries=%d, " 3056 "current entries=%d\n",
3057 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3059 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3060 __kmp_free((
void *)team->t.t_argv);
3062 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3064 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3065 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: inline allocate %d " 3067 team->t.t_id, team->t.t_max_argc));
3068 team->t.t_argv = &team->t.t_inline_argv[0];
3069 if (__kmp_storage_map) {
3070 __kmp_print_storage_map_gtid(
3071 -1, &team->t.t_inline_argv[0],
3072 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3073 (
sizeof(
void *) * KMP_INLINE_ARGV_ENTRIES),
"team_%d.t_inline_argv",
3078 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3079 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3081 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3083 team->t.t_id, team->t.t_max_argc));
3085 (
void **)__kmp_page_allocate(
sizeof(
void *) * team->t.t_max_argc);
3086 if (__kmp_storage_map) {
3087 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3088 &team->t.t_argv[team->t.t_max_argc],
3089 sizeof(
void *) * team->t.t_max_argc,
3090 "team_%d.t_argv", team->t.t_id);
3096 static void __kmp_allocate_team_arrays(kmp_team_t *team,
int max_nth) {
3098 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3100 (kmp_info_t **)__kmp_allocate(
sizeof(kmp_info_t *) * max_nth);
3101 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3102 sizeof(dispatch_shared_info_t) * num_disp_buff);
3103 team->t.t_dispatch =
3104 (kmp_disp_t *)__kmp_allocate(
sizeof(kmp_disp_t) * max_nth);
3105 team->t.t_implicit_task_taskdata =
3106 (kmp_taskdata_t *)__kmp_allocate(
sizeof(kmp_taskdata_t) * max_nth);
3107 team->t.t_max_nproc = max_nth;
3110 for (i = 0; i < num_disp_buff; ++i) {
3111 team->t.t_disp_buffer[i].buffer_index = i;
3113 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3118 static void __kmp_free_team_arrays(kmp_team_t *team) {
3121 for (i = 0; i < team->t.t_max_nproc; ++i) {
3122 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3123 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3124 team->t.t_dispatch[i].th_disp_buffer = NULL;
3127 #if KMP_USE_HIER_SCHED 3128 __kmp_dispatch_free_hierarchies(team);
3130 __kmp_free(team->t.t_threads);
3131 __kmp_free(team->t.t_disp_buffer);
3132 __kmp_free(team->t.t_dispatch);
3133 __kmp_free(team->t.t_implicit_task_taskdata);
3134 team->t.t_threads = NULL;
3135 team->t.t_disp_buffer = NULL;
3136 team->t.t_dispatch = NULL;
3137 team->t.t_implicit_task_taskdata = 0;
3140 static void __kmp_reallocate_team_arrays(kmp_team_t *team,
int max_nth) {
3141 kmp_info_t **oldThreads = team->t.t_threads;
3143 __kmp_free(team->t.t_disp_buffer);
3144 __kmp_free(team->t.t_dispatch);
3145 __kmp_free(team->t.t_implicit_task_taskdata);
3146 __kmp_allocate_team_arrays(team, max_nth);
3148 KMP_MEMCPY(team->t.t_threads, oldThreads,
3149 team->t.t_nproc *
sizeof(kmp_info_t *));
3151 __kmp_free(oldThreads);
3154 static kmp_internal_control_t __kmp_get_global_icvs(
void) {
3156 kmp_r_sched_t r_sched =
3157 __kmp_get_schedule_global();
3160 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
3163 kmp_internal_control_t g_icvs = {
3165 (kmp_int8)__kmp_global.g.g_dynamic,
3167 (kmp_int8)__kmp_env_blocktime,
3169 __kmp_dflt_blocktime,
3174 __kmp_dflt_team_nth,
3178 __kmp_dflt_max_active_levels,
3183 __kmp_nested_proc_bind.bind_types[0],
3184 __kmp_default_device,
3192 static kmp_internal_control_t __kmp_get_x_global_icvs(
const kmp_team_t *team) {
3194 kmp_internal_control_t gx_icvs;
3195 gx_icvs.serial_nesting_level =
3197 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3198 gx_icvs.next = NULL;
3203 static void __kmp_initialize_root(kmp_root_t *root) {
3205 kmp_team_t *root_team;
3206 kmp_team_t *hot_team;
3207 int hot_team_max_nth;
3208 kmp_r_sched_t r_sched =
3209 __kmp_get_schedule_global();
3210 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3211 KMP_DEBUG_ASSERT(root);
3212 KMP_ASSERT(!root->r.r_begin);
3215 __kmp_init_lock(&root->r.r_begin_lock);
3216 root->r.r_begin = FALSE;
3217 root->r.r_active = FALSE;
3218 root->r.r_in_parallel = 0;
3219 root->r.r_blocktime = __kmp_dflt_blocktime;
3223 KF_TRACE(10, (
"__kmp_initialize_root: before root_team\n"));
3226 __kmp_allocate_team(root,
3233 __kmp_nested_proc_bind.bind_types[0],
3237 USE_NESTED_HOT_ARG(NULL)
3242 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
3245 KF_TRACE(10, (
"__kmp_initialize_root: after root_team = %p\n", root_team));
3247 root->r.r_root_team = root_team;
3248 root_team->t.t_control_stack_top = NULL;
3251 root_team->t.t_threads[0] = NULL;
3252 root_team->t.t_nproc = 1;
3253 root_team->t.t_serialized = 1;
3255 root_team->t.t_sched.sched = r_sched.sched;
3258 (
"__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3259 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
3263 KF_TRACE(10, (
"__kmp_initialize_root: before hot_team\n"));
3266 __kmp_allocate_team(root,
3268 __kmp_dflt_team_nth_ub * 2,
3273 __kmp_nested_proc_bind.bind_types[0],
3277 USE_NESTED_HOT_ARG(NULL)
3279 KF_TRACE(10, (
"__kmp_initialize_root: after hot_team = %p\n", hot_team));
3281 root->r.r_hot_team = hot_team;
3282 root_team->t.t_control_stack_top = NULL;
3285 hot_team->t.t_parent = root_team;
3288 hot_team_max_nth = hot_team->t.t_max_nproc;
3289 for (f = 0; f < hot_team_max_nth; ++f) {
3290 hot_team->t.t_threads[f] = NULL;
3292 hot_team->t.t_nproc = 1;
3294 hot_team->t.t_sched.sched = r_sched.sched;
3295 hot_team->t.t_size_changed = 0;
3300 typedef struct kmp_team_list_item {
3301 kmp_team_p
const *entry;
3302 struct kmp_team_list_item *next;
3303 } kmp_team_list_item_t;
3304 typedef kmp_team_list_item_t *kmp_team_list_t;
3306 static void __kmp_print_structure_team_accum(
3307 kmp_team_list_t list,
3308 kmp_team_p
const *team
3318 KMP_DEBUG_ASSERT(list != NULL);
3323 __kmp_print_structure_team_accum(list, team->t.t_parent);
3324 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3328 while (l->next != NULL && l->entry != team) {
3331 if (l->next != NULL) {
3337 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3343 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3344 sizeof(kmp_team_list_item_t));
3351 static void __kmp_print_structure_team(
char const *title, kmp_team_p
const *team
3354 __kmp_printf(
"%s", title);
3356 __kmp_printf(
"%2x %p\n", team->t.t_id, team);
3358 __kmp_printf(
" - (nil)\n");
3362 static void __kmp_print_structure_thread(
char const *title,
3363 kmp_info_p
const *thread) {
3364 __kmp_printf(
"%s", title);
3365 if (thread != NULL) {
3366 __kmp_printf(
"%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3368 __kmp_printf(
" - (nil)\n");
3372 void __kmp_print_structure(
void) {
3374 kmp_team_list_t list;
3378 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
sizeof(kmp_team_list_item_t));
3382 __kmp_printf(
"\n------------------------------\nGlobal Thread " 3383 "Table\n------------------------------\n");
3386 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3387 __kmp_printf(
"%2d", gtid);
3388 if (__kmp_threads != NULL) {
3389 __kmp_printf(
" %p", __kmp_threads[gtid]);
3391 if (__kmp_root != NULL) {
3392 __kmp_printf(
" %p", __kmp_root[gtid]);
3399 __kmp_printf(
"\n------------------------------\nThreads\n--------------------" 3401 if (__kmp_threads != NULL) {
3403 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3404 kmp_info_t
const *thread = __kmp_threads[gtid];
3405 if (thread != NULL) {
3406 __kmp_printf(
"GTID %2d %p:\n", gtid, thread);
3407 __kmp_printf(
" Our Root: %p\n", thread->th.th_root);
3408 __kmp_print_structure_team(
" Our Team: ", thread->th.th_team);
3409 __kmp_print_structure_team(
" Serial Team: ",
3410 thread->th.th_serial_team);
3411 __kmp_printf(
" Threads: %2d\n", thread->th.th_team_nproc);
3412 __kmp_print_structure_thread(
" Master: ",
3413 thread->th.th_team_master);
3414 __kmp_printf(
" Serialized?: %2d\n", thread->th.th_team_serialized);
3415 __kmp_printf(
" Set NProc: %2d\n", thread->th.th_set_nproc);
3417 __kmp_printf(
" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3419 __kmp_print_structure_thread(
" Next in pool: ",
3420 thread->th.th_next_pool);
3422 __kmp_print_structure_team_accum(list, thread->th.th_team);
3423 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3427 __kmp_printf(
"Threads array is not allocated.\n");
3431 __kmp_printf(
"\n------------------------------\nUbers\n----------------------" 3433 if (__kmp_root != NULL) {
3435 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3436 kmp_root_t
const *root = __kmp_root[gtid];
3438 __kmp_printf(
"GTID %2d %p:\n", gtid, root);
3439 __kmp_print_structure_team(
" Root Team: ", root->r.r_root_team);
3440 __kmp_print_structure_team(
" Hot Team: ", root->r.r_hot_team);
3441 __kmp_print_structure_thread(
" Uber Thread: ",
3442 root->r.r_uber_thread);
3443 __kmp_printf(
" Active?: %2d\n", root->r.r_active);
3444 __kmp_printf(
" In Parallel: %2d\n",
3445 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
3447 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3448 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3452 __kmp_printf(
"Ubers array is not allocated.\n");
3455 __kmp_printf(
"\n------------------------------\nTeams\n----------------------" 3457 while (list->next != NULL) {
3458 kmp_team_p
const *team = list->entry;
3460 __kmp_printf(
"Team %2x %p:\n", team->t.t_id, team);
3461 __kmp_print_structure_team(
" Parent Team: ", team->t.t_parent);
3462 __kmp_printf(
" Master TID: %2d\n", team->t.t_master_tid);
3463 __kmp_printf(
" Max threads: %2d\n", team->t.t_max_nproc);
3464 __kmp_printf(
" Levels of serial: %2d\n", team->t.t_serialized);
3465 __kmp_printf(
" Number threads: %2d\n", team->t.t_nproc);
3466 for (i = 0; i < team->t.t_nproc; ++i) {
3467 __kmp_printf(
" Thread %2d: ", i);
3468 __kmp_print_structure_thread(
"", team->t.t_threads[i]);
3470 __kmp_print_structure_team(
" Next in pool: ", team->t.t_next_pool);
3476 __kmp_printf(
"\n------------------------------\nPools\n----------------------" 3478 __kmp_print_structure_thread(
"Thread pool: ",
3479 CCAST(kmp_info_t *, __kmp_thread_pool));
3480 __kmp_print_structure_team(
"Team pool: ",
3481 CCAST(kmp_team_t *, __kmp_team_pool));
3485 while (list != NULL) {
3486 kmp_team_list_item_t *item = list;
3488 KMP_INTERNAL_FREE(item);
3497 static const unsigned __kmp_primes[] = {
3498 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3499 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3500 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3501 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3502 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3503 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3504 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3505 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3506 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3507 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3508 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3512 unsigned short __kmp_get_random(kmp_info_t *thread) {
3513 unsigned x = thread->th.th_x;
3514 unsigned short r = x >> 16;
3516 thread->th.th_x = x * thread->th.th_a + 1;
3518 KA_TRACE(30, (
"__kmp_get_random: THREAD: %d, RETURN: %u\n",
3519 thread->th.th_info.ds.ds_tid, r));
3525 void __kmp_init_random(kmp_info_t *thread) {
3526 unsigned seed = thread->th.th_info.ds.ds_tid;
3529 __kmp_primes[seed % (
sizeof(__kmp_primes) /
sizeof(__kmp_primes[0]))];
3530 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3532 (
"__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
3538 static int __kmp_reclaim_dead_roots(
void) {
3541 for (i = 0; i < __kmp_threads_capacity; ++i) {
3542 if (KMP_UBER_GTID(i) &&
3543 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3546 r += __kmp_unregister_root_other_thread(i);
3571 static int __kmp_expand_threads(
int nNeed) {
3573 int minimumRequiredCapacity;
3575 kmp_info_t **newThreads;
3576 kmp_root_t **newRoot;
3582 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 3585 added = __kmp_reclaim_dead_roots();
3614 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
3617 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3621 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3623 newCapacity = __kmp_threads_capacity;
3625 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3626 : __kmp_sys_max_nth;
3627 }
while (newCapacity < minimumRequiredCapacity);
3628 newThreads = (kmp_info_t **)__kmp_allocate(
3629 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3631 (kmp_root_t **)((
char *)newThreads +
sizeof(kmp_info_t *) * newCapacity);
3632 KMP_MEMCPY(newThreads, __kmp_threads,
3633 __kmp_threads_capacity *
sizeof(kmp_info_t *));
3634 KMP_MEMCPY(newRoot, __kmp_root,
3635 __kmp_threads_capacity *
sizeof(kmp_root_t *));
3637 kmp_info_t **temp_threads = __kmp_threads;
3638 *(kmp_info_t * *
volatile *)&__kmp_threads = newThreads;
3639 *(kmp_root_t * *
volatile *)&__kmp_root = newRoot;
3640 __kmp_free(temp_threads);
3641 added += newCapacity - __kmp_threads_capacity;
3642 *(
volatile int *)&__kmp_threads_capacity = newCapacity;
3644 if (newCapacity > __kmp_tp_capacity) {
3645 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3646 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3647 __kmp_threadprivate_resize_cache(newCapacity);
3649 *(
volatile int *)&__kmp_tp_capacity = newCapacity;
3651 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3660 int __kmp_register_root(
int initial_thread) {
3661 kmp_info_t *root_thread;
3665 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3666 KA_TRACE(20, (
"__kmp_register_root: entered\n"));
3683 capacity = __kmp_threads_capacity;
3684 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3689 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3690 if (__kmp_tp_cached) {
3691 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3692 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3693 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
3695 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3703 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL;
3707 (
"__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3708 KMP_ASSERT(gtid < __kmp_threads_capacity);
3712 TCW_4(__kmp_nth, __kmp_nth + 1);
3716 if (__kmp_adjust_gtid_mode) {
3717 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3718 if (TCR_4(__kmp_gtid_mode) != 2) {
3719 TCW_4(__kmp_gtid_mode, 2);
3722 if (TCR_4(__kmp_gtid_mode) != 1) {
3723 TCW_4(__kmp_gtid_mode, 1);
3728 #ifdef KMP_ADJUST_BLOCKTIME 3731 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3732 if (__kmp_nth > __kmp_avail_proc) {
3733 __kmp_zero_bt = TRUE;
3739 if (!(root = __kmp_root[gtid])) {
3740 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(
sizeof(kmp_root_t));
3741 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3744 #if KMP_STATS_ENABLED 3746 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3747 __kmp_stats_thread_ptr->startLife();
3748 KMP_SET_THREAD_STATE(SERIAL_REGION);
3751 __kmp_initialize_root(root);
3754 if (root->r.r_uber_thread) {
3755 root_thread = root->r.r_uber_thread;
3757 root_thread = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
3758 if (__kmp_storage_map) {
3759 __kmp_print_thread_storage_map(root_thread, gtid);
3761 root_thread->th.th_info.ds.ds_gtid = gtid;
3763 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
3765 root_thread->th.th_root = root;
3766 if (__kmp_env_consistency_check) {
3767 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3770 __kmp_initialize_fast_memory(root_thread);
3774 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3775 __kmp_initialize_bget(root_thread);
3777 __kmp_init_random(root_thread);
3781 if (!root_thread->th.th_serial_team) {
3782 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3783 KF_TRACE(10, (
"__kmp_register_root: before serial_team\n"));
3784 root_thread->th.th_serial_team =
3785 __kmp_allocate_team(root, 1, 1,
3792 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3794 KMP_ASSERT(root_thread->th.th_serial_team);
3795 KF_TRACE(10, (
"__kmp_register_root: after serial_team = %p\n",
3796 root_thread->th.th_serial_team));
3799 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
3801 root->r.r_root_team->t.t_threads[0] = root_thread;
3802 root->r.r_hot_team->t.t_threads[0] = root_thread;
3803 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3805 root_thread->th.th_serial_team->t.t_serialized = 0;
3806 root->r.r_uber_thread = root_thread;
3809 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3810 TCW_4(__kmp_init_gtid, TRUE);
3813 __kmp_gtid_set_specific(gtid);
3816 __kmp_itt_thread_name(gtid);
3819 #ifdef KMP_TDATA_GTID 3822 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3823 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3825 KA_TRACE(20, (
"__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3827 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3828 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3829 KMP_INIT_BARRIER_STATE));
3832 for (b = 0; b < bs_last_barrier; ++b) {
3833 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3835 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3839 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3840 KMP_INIT_BARRIER_STATE);
3842 #if KMP_AFFINITY_SUPPORTED 3844 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3845 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3846 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3847 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3849 if (TCR_4(__kmp_init_middle)) {
3850 __kmp_affinity_set_init_mask(gtid, TRUE);
3854 root_thread->th.th_def_allocator = __kmp_def_allocator;
3855 root_thread->th.th_prev_level = 0;
3856 root_thread->th.th_prev_num_threads = 1;
3859 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
3860 tmp->cg_root = root_thread;
3861 tmp->cg_thread_limit = __kmp_cg_max_nth;
3862 tmp->cg_nthreads = 1;
3863 KA_TRACE(100, (
"__kmp_register_root: Thread %p created node %p with" 3864 " cg_nthreads init to 1\n",
3867 root_thread->th.th_cg_roots = tmp;
3869 __kmp_root_counter++;
3872 if (!initial_thread && ompt_enabled.enabled) {
3874 kmp_info_t *root_thread = ompt_get_thread();
3876 ompt_set_thread_state(root_thread, ompt_state_overhead);
3878 if (ompt_enabled.ompt_callback_thread_begin) {
3879 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3880 ompt_thread_initial, __ompt_get_thread_data_internal());
3882 ompt_data_t *task_data;
3883 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
3884 if (ompt_enabled.ompt_callback_task_create) {
3885 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
3886 NULL, NULL, task_data, ompt_task_initial, 0, NULL);
3890 ompt_set_thread_state(root_thread, ompt_state_work_serial);
3895 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3900 #if KMP_NESTED_HOT_TEAMS 3901 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr,
int level,
3902 const int max_level) {
3904 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3905 if (!hot_teams || !hot_teams[level].hot_team) {
3908 KMP_DEBUG_ASSERT(level < max_level);
3909 kmp_team_t *team = hot_teams[level].hot_team;
3910 nth = hot_teams[level].hot_team_nth;
3912 if (level < max_level - 1) {
3913 for (i = 0; i < nth; ++i) {
3914 kmp_info_t *th = team->t.t_threads[i];
3915 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3916 if (i > 0 && th->th.th_hot_teams) {
3917 __kmp_free(th->th.th_hot_teams);
3918 th->th.th_hot_teams = NULL;
3922 __kmp_free_team(root, team, NULL);
3929 static int __kmp_reset_root(
int gtid, kmp_root_t *root) {
3930 kmp_team_t *root_team = root->r.r_root_team;
3931 kmp_team_t *hot_team = root->r.r_hot_team;
3932 int n = hot_team->t.t_nproc;
3935 KMP_DEBUG_ASSERT(!root->r.r_active);
3937 root->r.r_root_team = NULL;
3938 root->r.r_hot_team = NULL;
3941 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
3942 #if KMP_NESTED_HOT_TEAMS 3943 if (__kmp_hot_teams_max_level >
3945 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3946 kmp_info_t *th = hot_team->t.t_threads[i];
3947 if (__kmp_hot_teams_max_level > 1) {
3948 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3950 if (th->th.th_hot_teams) {
3951 __kmp_free(th->th.th_hot_teams);
3952 th->th.th_hot_teams = NULL;
3957 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
3962 if (__kmp_tasking_mode != tskm_immediate_exec) {
3963 __kmp_wait_to_unref_task_teams();
3969 10, (
"__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3971 (LPVOID) & (root->r.r_uber_thread->th),
3972 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3973 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3977 if (ompt_enabled.ompt_callback_thread_end) {
3978 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3979 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
3985 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
3986 KA_TRACE(100, (
"__kmp_reset_root: Thread %p decrement cg_nthreads on node %p" 3988 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,
3989 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads));
3991 __kmp_reap_thread(root->r.r_uber_thread, 1);
3995 root->r.r_uber_thread = NULL;
3997 root->r.r_begin = FALSE;
4002 void __kmp_unregister_root_current_thread(
int gtid) {
4003 KA_TRACE(1, (
"__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
4007 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
4008 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
4009 KC_TRACE(10, (
"__kmp_unregister_root_current_thread: already finished, " 4012 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4015 kmp_root_t *root = __kmp_root[gtid];
4017 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4018 KMP_ASSERT(KMP_UBER_GTID(gtid));
4019 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4020 KMP_ASSERT(root->r.r_active == FALSE);
4025 kmp_info_t *thread = __kmp_threads[gtid];
4026 kmp_team_t *team = thread->th.th_team;
4027 kmp_task_team_t *task_team = thread->th.th_task_team;
4030 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
4033 thread->th.ompt_thread_info.state = ompt_state_undefined;
4035 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
4039 __kmp_reset_root(gtid, root);
4042 __kmp_gtid_set_specific(KMP_GTID_DNE);
4043 #ifdef KMP_TDATA_GTID 4044 __kmp_gtid = KMP_GTID_DNE;
4049 (
"__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
4051 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4058 static int __kmp_unregister_root_other_thread(
int gtid) {
4059 kmp_root_t *root = __kmp_root[gtid];
4062 KA_TRACE(1, (
"__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4063 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4064 KMP_ASSERT(KMP_UBER_GTID(gtid));
4065 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4066 KMP_ASSERT(root->r.r_active == FALSE);
4068 r = __kmp_reset_root(gtid, root);
4070 (
"__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4076 void __kmp_task_info() {
4078 kmp_int32 gtid = __kmp_entry_gtid();
4079 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4080 kmp_info_t *this_thr = __kmp_threads[gtid];
4081 kmp_team_t *steam = this_thr->th.th_serial_team;
4082 kmp_team_t *team = this_thr->th.th_team;
4085 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4087 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4088 team->t.t_implicit_task_taskdata[tid].td_parent);
4095 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4096 int tid,
int gtid) {
4100 kmp_info_t *master = team->t.t_threads[0];
4101 KMP_DEBUG_ASSERT(this_thr != NULL);
4102 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4103 KMP_DEBUG_ASSERT(team);
4104 KMP_DEBUG_ASSERT(team->t.t_threads);
4105 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4106 KMP_DEBUG_ASSERT(master);
4107 KMP_DEBUG_ASSERT(master->th.th_root);
4111 TCW_SYNC_PTR(this_thr->th.th_team, team);
4113 this_thr->th.th_info.ds.ds_tid = tid;
4114 this_thr->th.th_set_nproc = 0;
4115 if (__kmp_tasking_mode != tskm_immediate_exec)
4118 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4120 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
4122 this_thr->th.th_set_proc_bind = proc_bind_default;
4123 #if KMP_AFFINITY_SUPPORTED 4124 this_thr->th.th_new_place = this_thr->th.th_current_place;
4127 this_thr->th.th_root = master->th.th_root;
4130 this_thr->th.th_team_nproc = team->t.t_nproc;
4131 this_thr->th.th_team_master = master;
4132 this_thr->th.th_team_serialized = team->t.t_serialized;
4133 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
4135 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
4137 KF_TRACE(10, (
"__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4138 tid, gtid, this_thr, this_thr->th.th_current_task));
4140 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4143 KF_TRACE(10, (
"__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4144 tid, gtid, this_thr, this_thr->th.th_current_task));
4149 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4151 this_thr->th.th_local.this_construct = 0;
4153 if (!this_thr->th.th_pri_common) {
4154 this_thr->th.th_pri_common =
4155 (
struct common_table *)__kmp_allocate(
sizeof(
struct common_table));
4156 if (__kmp_storage_map) {
4157 __kmp_print_storage_map_gtid(
4158 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4159 sizeof(
struct common_table),
"th_%d.th_pri_common\n", gtid);
4161 this_thr->th.th_pri_head = NULL;
4164 if (this_thr != master &&
4165 this_thr->th.th_cg_roots != master->th.th_cg_roots) {
4167 KMP_DEBUG_ASSERT(master->th.th_cg_roots);
4168 this_thr->th.th_cg_roots = master->th.th_cg_roots;
4170 this_thr->th.th_cg_roots->cg_nthreads++;
4171 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p increment cg_nthreads on" 4172 " node %p of thread %p to %d\n",
4173 this_thr, this_thr->th.th_cg_roots,
4174 this_thr->th.th_cg_roots->cg_root,
4175 this_thr->th.th_cg_roots->cg_nthreads));
4176 this_thr->th.th_current_task->td_icvs.thread_limit =
4177 this_thr->th.th_cg_roots->cg_thread_limit;
4182 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4185 sizeof(dispatch_private_info_t) *
4186 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4187 KD_TRACE(10, (
"__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4188 team->t.t_max_nproc));
4189 KMP_ASSERT(dispatch);
4190 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4191 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
4193 dispatch->th_disp_index = 0;
4195 dispatch->th_doacross_buf_idx = 0;
4197 if (!dispatch->th_disp_buffer) {
4198 dispatch->th_disp_buffer =
4199 (dispatch_private_info_t *)__kmp_allocate(disp_size);
4201 if (__kmp_storage_map) {
4202 __kmp_print_storage_map_gtid(
4203 gtid, &dispatch->th_disp_buffer[0],
4204 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4206 : __kmp_dispatch_num_buffers],
4207 disp_size,
"th_%d.th_dispatch.th_disp_buffer " 4208 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4209 gtid, team->t.t_id, gtid);
4212 memset(&dispatch->th_disp_buffer[0],
'\0', disp_size);
4215 dispatch->th_dispatch_pr_current = 0;
4216 dispatch->th_dispatch_sh_current = 0;
4218 dispatch->th_deo_fcn = 0;
4219 dispatch->th_dxo_fcn = 0;
4222 this_thr->th.th_next_pool = NULL;
4224 if (!this_thr->th.th_task_state_memo_stack) {
4226 this_thr->th.th_task_state_memo_stack =
4227 (kmp_uint8 *)__kmp_allocate(4 *
sizeof(kmp_uint8));
4228 this_thr->th.th_task_state_top = 0;
4229 this_thr->th.th_task_state_stack_sz = 4;
4230 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4232 this_thr->th.th_task_state_memo_stack[i] = 0;
4235 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4236 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4246 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4248 kmp_team_t *serial_team;
4249 kmp_info_t *new_thr;
4252 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4253 KMP_DEBUG_ASSERT(root && team);
4254 #if !KMP_NESTED_HOT_TEAMS 4255 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
4260 if (__kmp_thread_pool) {
4261 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
4262 __kmp_thread_pool = (
volatile kmp_info_t *)new_thr->th.th_next_pool;
4263 if (new_thr == __kmp_thread_pool_insert_pt) {
4264 __kmp_thread_pool_insert_pt = NULL;
4266 TCW_4(new_thr->th.th_in_pool, FALSE);
4267 __kmp_suspend_initialize_thread(new_thr);
4268 __kmp_lock_suspend_mx(new_thr);
4269 if (new_thr->th.th_active_in_pool == TRUE) {
4270 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE);
4271 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
4272 new_thr->th.th_active_in_pool = FALSE;
4276 KMP_DEBUG_ASSERT(new_thr->th.th_active == FALSE);
4279 __kmp_unlock_suspend_mx(new_thr);
4281 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d using thread T#%d\n",
4282 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4283 KMP_ASSERT(!new_thr->th.th_team);
4284 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4287 __kmp_initialize_info(new_thr, team, new_tid,
4288 new_thr->th.th_info.ds.ds_gtid);
4289 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
4291 TCW_4(__kmp_nth, __kmp_nth + 1);
4293 new_thr->th.th_task_state = 0;
4294 new_thr->th.th_task_state_top = 0;
4295 new_thr->th.th_task_state_stack_sz = 4;
4297 #ifdef KMP_ADJUST_BLOCKTIME 4300 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4301 if (__kmp_nth > __kmp_avail_proc) {
4302 __kmp_zero_bt = TRUE;
4311 kmp_balign_t *balign = new_thr->th.th_bar;
4312 for (b = 0; b < bs_last_barrier; ++b)
4313 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
4316 KF_TRACE(10, (
"__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4317 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
4324 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4325 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4330 if (!TCR_4(__kmp_init_monitor)) {
4331 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4332 if (!TCR_4(__kmp_init_monitor)) {
4333 KF_TRACE(10, (
"before __kmp_create_monitor\n"));
4334 TCW_4(__kmp_init_monitor, 1);
4335 __kmp_create_monitor(&__kmp_monitor);
4336 KF_TRACE(10, (
"after __kmp_create_monitor\n"));
4347 while (TCR_4(__kmp_init_monitor) < 2) {
4350 KF_TRACE(10, (
"after monitor thread has started\n"));
4353 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4358 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) {
4359 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4363 new_thr = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
4365 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4367 if (__kmp_storage_map) {
4368 __kmp_print_thread_storage_map(new_thr, new_gtid);
4373 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4374 KF_TRACE(10, (
"__kmp_allocate_thread: before th_serial/serial_team\n"));
4375 new_thr->th.th_serial_team = serial_team =
4376 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4383 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
4385 KMP_ASSERT(serial_team);
4386 serial_team->t.t_serialized = 0;
4388 serial_team->t.t_threads[0] = new_thr;
4390 (
"__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4394 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4397 __kmp_initialize_fast_memory(new_thr);
4401 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4402 __kmp_initialize_bget(new_thr);
4405 __kmp_init_random(new_thr);
4409 (
"__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4410 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4413 kmp_balign_t *balign = new_thr->th.th_bar;
4414 for (b = 0; b < bs_last_barrier; ++b) {
4415 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4416 balign[b].bb.team = NULL;
4417 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4418 balign[b].bb.use_oncore_barrier = 0;
4421 new_thr->th.th_spin_here = FALSE;
4422 new_thr->th.th_next_waiting = 0;
4424 new_thr->th.th_blocking =
false;
4427 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4428 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4429 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4430 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4431 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4434 new_thr->th.th_def_allocator = __kmp_def_allocator;
4435 new_thr->th.th_prev_level = 0;
4436 new_thr->th.th_prev_num_threads = 1;
4439 TCW_4(new_thr->th.th_in_pool, FALSE);
4440 new_thr->th.th_active_in_pool = FALSE;
4441 TCW_4(new_thr->th.th_active, TRUE);
4449 if (__kmp_adjust_gtid_mode) {
4450 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4451 if (TCR_4(__kmp_gtid_mode) != 2) {
4452 TCW_4(__kmp_gtid_mode, 2);
4455 if (TCR_4(__kmp_gtid_mode) != 1) {
4456 TCW_4(__kmp_gtid_mode, 1);
4461 #ifdef KMP_ADJUST_BLOCKTIME 4464 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4465 if (__kmp_nth > __kmp_avail_proc) {
4466 __kmp_zero_bt = TRUE;
4473 10, (
"__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4474 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4476 (
"__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4478 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4489 static void __kmp_reinitialize_team(kmp_team_t *team,
4490 kmp_internal_control_t *new_icvs,
4492 KF_TRACE(10, (
"__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4493 team->t.t_threads[0], team));
4494 KMP_DEBUG_ASSERT(team && new_icvs);
4495 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4496 KMP_CHECK_UPDATE(team->t.t_ident, loc);
4498 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
4500 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4501 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4503 KF_TRACE(10, (
"__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4504 team->t.t_threads[0], team));
4510 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
4511 kmp_internal_control_t *new_icvs,
4513 KF_TRACE(10, (
"__kmp_initialize_team: enter: team=%p\n", team));
4516 KMP_DEBUG_ASSERT(team);
4517 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4518 KMP_DEBUG_ASSERT(team->t.t_threads);
4521 team->t.t_master_tid = 0;
4523 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4524 team->t.t_nproc = new_nproc;
4527 team->t.t_next_pool = NULL;
4531 TCW_SYNC_PTR(team->t.t_pkfn, NULL);
4532 team->t.t_invoke = NULL;
4535 team->t.t_sched.sched = new_icvs->sched.sched;
4537 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4538 team->t.t_fp_control_saved = FALSE;
4539 team->t.t_x87_fpu_control_word = 0;
4540 team->t.t_mxcsr = 0;
4543 team->t.t_construct = 0;
4545 team->t.t_ordered.dt.t_value = 0;
4546 team->t.t_master_active = FALSE;
4549 team->t.t_copypriv_data = NULL;
4552 team->t.t_copyin_counter = 0;
4555 team->t.t_control_stack_top = NULL;
4557 __kmp_reinitialize_team(team, new_icvs, loc);
4560 KF_TRACE(10, (
"__kmp_initialize_team: exit: team=%p\n", team));
4563 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 4566 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4567 if (KMP_AFFINITY_CAPABLE()) {
4569 if (old_mask != NULL) {
4570 status = __kmp_get_system_affinity(old_mask, TRUE);
4573 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4577 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4582 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4588 static void __kmp_partition_places(kmp_team_t *team,
int update_master_only) {
4590 kmp_info_t *master_th = team->t.t_threads[0];
4591 KMP_DEBUG_ASSERT(master_th != NULL);
4592 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4593 int first_place = master_th->th.th_first_place;
4594 int last_place = master_th->th.th_last_place;
4595 int masters_place = master_th->th.th_current_place;
4596 team->t.t_first_place = first_place;
4597 team->t.t_last_place = last_place;
4599 KA_TRACE(20, (
"__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4600 "bound to place %d partition = [%d,%d]\n",
4601 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4602 team->t.t_id, masters_place, first_place, last_place));
4604 switch (proc_bind) {
4606 case proc_bind_default:
4609 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4612 case proc_bind_master: {
4614 int n_th = team->t.t_nproc;
4615 for (f = 1; f < n_th; f++) {
4616 kmp_info_t *th = team->t.t_threads[f];
4617 KMP_DEBUG_ASSERT(th != NULL);
4618 th->th.th_first_place = first_place;
4619 th->th.th_last_place = last_place;
4620 th->th.th_new_place = masters_place;
4622 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4623 team->t.t_display_affinity != 1) {
4624 team->t.t_display_affinity = 1;
4628 KA_TRACE(100, (
"__kmp_partition_places: master: T#%d(%d:%d) place %d " 4629 "partition = [%d,%d]\n",
4630 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4631 f, masters_place, first_place, last_place));
4635 case proc_bind_close: {
4637 int n_th = team->t.t_nproc;
4639 if (first_place <= last_place) {
4640 n_places = last_place - first_place + 1;
4642 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4644 if (n_th <= n_places) {
4645 int place = masters_place;
4646 for (f = 1; f < n_th; f++) {
4647 kmp_info_t *th = team->t.t_threads[f];
4648 KMP_DEBUG_ASSERT(th != NULL);
4650 if (place == last_place) {
4651 place = first_place;
4652 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4657 th->th.th_first_place = first_place;
4658 th->th.th_last_place = last_place;
4659 th->th.th_new_place = place;
4661 if (__kmp_display_affinity && place != th->th.th_current_place &&
4662 team->t.t_display_affinity != 1) {
4663 team->t.t_display_affinity = 1;
4667 KA_TRACE(100, (
"__kmp_partition_places: close: T#%d(%d:%d) place %d " 4668 "partition = [%d,%d]\n",
4669 __kmp_gtid_from_thread(team->t.t_threads[f]),
4670 team->t.t_id, f, place, first_place, last_place));
4673 int S, rem, gap, s_count;
4674 S = n_th / n_places;
4676 rem = n_th - (S * n_places);
4677 gap = rem > 0 ? n_places / rem : n_places;
4678 int place = masters_place;
4680 for (f = 0; f < n_th; f++) {
4681 kmp_info_t *th = team->t.t_threads[f];
4682 KMP_DEBUG_ASSERT(th != NULL);
4684 th->th.th_first_place = first_place;
4685 th->th.th_last_place = last_place;
4686 th->th.th_new_place = place;
4688 if (__kmp_display_affinity && place != th->th.th_current_place &&
4689 team->t.t_display_affinity != 1) {
4690 team->t.t_display_affinity = 1;
4695 if ((s_count == S) && rem && (gap_ct == gap)) {
4697 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4699 if (place == last_place) {
4700 place = first_place;
4701 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4709 }
else if (s_count == S) {
4710 if (place == last_place) {
4711 place = first_place;
4712 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4722 (
"__kmp_partition_places: close: T#%d(%d:%d) place %d " 4723 "partition = [%d,%d]\n",
4724 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4725 th->th.th_new_place, first_place, last_place));
4727 KMP_DEBUG_ASSERT(place == masters_place);
4731 case proc_bind_spread: {
4733 int n_th = team->t.t_nproc;
4736 if (first_place <= last_place) {
4737 n_places = last_place - first_place + 1;
4739 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4741 if (n_th <= n_places) {
4744 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4745 int S = n_places / n_th;
4746 int s_count, rem, gap, gap_ct;
4748 place = masters_place;
4749 rem = n_places - n_th * S;
4750 gap = rem ? n_th / rem : 1;
4753 if (update_master_only == 1)
4755 for (f = 0; f < thidx; f++) {
4756 kmp_info_t *th = team->t.t_threads[f];
4757 KMP_DEBUG_ASSERT(th != NULL);
4759 th->th.th_first_place = place;
4760 th->th.th_new_place = place;
4762 if (__kmp_display_affinity && place != th->th.th_current_place &&
4763 team->t.t_display_affinity != 1) {
4764 team->t.t_display_affinity = 1;
4768 while (s_count < S) {
4769 if (place == last_place) {
4770 place = first_place;
4771 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4778 if (rem && (gap_ct == gap)) {
4779 if (place == last_place) {
4780 place = first_place;
4781 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4789 th->th.th_last_place = place;
4792 if (place == last_place) {
4793 place = first_place;
4794 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4801 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4802 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4803 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4804 f, th->th.th_new_place, th->th.th_first_place,
4805 th->th.th_last_place, __kmp_affinity_num_masks));
4811 double current =
static_cast<double>(masters_place);
4813 (
static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4818 if (update_master_only == 1)
4820 for (f = 0; f < thidx; f++) {
4821 first =
static_cast<int>(current);
4822 last =
static_cast<int>(current + spacing) - 1;
4823 KMP_DEBUG_ASSERT(last >= first);
4824 if (first >= n_places) {
4825 if (masters_place) {
4828 if (first == (masters_place + 1)) {
4829 KMP_DEBUG_ASSERT(f == n_th);
4832 if (last == masters_place) {
4833 KMP_DEBUG_ASSERT(f == (n_th - 1));
4837 KMP_DEBUG_ASSERT(f == n_th);
4842 if (last >= n_places) {
4843 last = (n_places - 1);
4848 KMP_DEBUG_ASSERT(0 <= first);
4849 KMP_DEBUG_ASSERT(n_places > first);
4850 KMP_DEBUG_ASSERT(0 <= last);
4851 KMP_DEBUG_ASSERT(n_places > last);
4852 KMP_DEBUG_ASSERT(last_place >= first_place);
4853 th = team->t.t_threads[f];
4854 KMP_DEBUG_ASSERT(th);
4855 th->th.th_first_place = first;
4856 th->th.th_new_place = place;
4857 th->th.th_last_place = last;
4859 if (__kmp_display_affinity && place != th->th.th_current_place &&
4860 team->t.t_display_affinity != 1) {
4861 team->t.t_display_affinity = 1;
4865 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4866 "partition = [%d,%d], spacing = %.4f\n",
4867 __kmp_gtid_from_thread(team->t.t_threads[f]),
4868 team->t.t_id, f, th->th.th_new_place,
4869 th->th.th_first_place, th->th.th_last_place, spacing));
4873 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4875 int S, rem, gap, s_count;
4876 S = n_th / n_places;
4878 rem = n_th - (S * n_places);
4879 gap = rem > 0 ? n_places / rem : n_places;
4880 int place = masters_place;
4883 if (update_master_only == 1)
4885 for (f = 0; f < thidx; f++) {
4886 kmp_info_t *th = team->t.t_threads[f];
4887 KMP_DEBUG_ASSERT(th != NULL);
4889 th->th.th_first_place = place;
4890 th->th.th_last_place = place;
4891 th->th.th_new_place = place;
4893 if (__kmp_display_affinity && place != th->th.th_current_place &&
4894 team->t.t_display_affinity != 1) {
4895 team->t.t_display_affinity = 1;
4900 if ((s_count == S) && rem && (gap_ct == gap)) {
4902 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4904 if (place == last_place) {
4905 place = first_place;
4906 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4914 }
else if (s_count == S) {
4915 if (place == last_place) {
4916 place = first_place;
4917 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4926 KA_TRACE(100, (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4927 "partition = [%d,%d]\n",
4928 __kmp_gtid_from_thread(team->t.t_threads[f]),
4929 team->t.t_id, f, th->th.th_new_place,
4930 th->th.th_first_place, th->th.th_last_place));
4932 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4940 KA_TRACE(20, (
"__kmp_partition_places: exit T#%d\n", team->t.t_id));
4948 __kmp_allocate_team(kmp_root_t *root,
int new_nproc,
int max_nproc,
4950 ompt_data_t ompt_parallel_data,
4953 kmp_proc_bind_t new_proc_bind,
4955 kmp_internal_control_t *new_icvs,
4956 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4957 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4960 int use_hot_team = !root->r.r_active;
4963 KA_TRACE(20, (
"__kmp_allocate_team: called\n"));
4964 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4965 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4968 #if KMP_NESTED_HOT_TEAMS 4969 kmp_hot_team_ptr_t *hot_teams;
4971 team = master->th.th_team;
4972 level = team->t.t_active_level;
4973 if (master->th.th_teams_microtask) {
4974 if (master->th.th_teams_size.nteams > 1 &&
4977 (microtask_t)__kmp_teams_master ||
4978 master->th.th_teams_level <
4984 hot_teams = master->th.th_hot_teams;
4985 if (level < __kmp_hot_teams_max_level && hot_teams &&
4995 if (use_hot_team && new_nproc > 1) {
4996 KMP_DEBUG_ASSERT(new_nproc <= max_nproc);
4997 #if KMP_NESTED_HOT_TEAMS 4998 team = hot_teams[level].hot_team;
5000 team = root->r.r_hot_team;
5003 if (__kmp_tasking_mode != tskm_immediate_exec) {
5004 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p " 5005 "task_team[1] = %p before reinit\n",
5006 team->t.t_task_team[0], team->t.t_task_team[1]));
5013 if (team->t.t_nproc == new_nproc) {
5014 KA_TRACE(20, (
"__kmp_allocate_team: reusing hot team\n"));
5017 if (team->t.t_size_changed == -1) {
5018 team->t.t_size_changed = 1;
5020 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
5024 kmp_r_sched_t new_sched = new_icvs->sched;
5026 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
5028 __kmp_reinitialize_team(team, new_icvs,
5029 root->r.r_uber_thread->th.th_ident);
5031 KF_TRACE(10, (
"__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
5032 team->t.t_threads[0], team));
5033 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5036 #if KMP_AFFINITY_SUPPORTED 5037 if ((team->t.t_size_changed == 0) &&
5038 (team->t.t_proc_bind == new_proc_bind)) {
5039 if (new_proc_bind == proc_bind_spread) {
5040 __kmp_partition_places(
5043 KA_TRACE(200, (
"__kmp_allocate_team: reusing hot team #%d bindings: " 5044 "proc_bind = %d, partition = [%d,%d]\n",
5045 team->t.t_id, new_proc_bind, team->t.t_first_place,
5046 team->t.t_last_place));
5048 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5049 __kmp_partition_places(team);
5052 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5055 }
else if (team->t.t_nproc > new_nproc) {
5057 (
"__kmp_allocate_team: decreasing hot team thread count to %d\n",
5060 team->t.t_size_changed = 1;
5061 #if KMP_NESTED_HOT_TEAMS 5062 if (__kmp_hot_teams_mode == 0) {
5065 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
5066 hot_teams[level].hot_team_nth = new_nproc;
5067 #endif // KMP_NESTED_HOT_TEAMS 5069 for (f = new_nproc; f < team->t.t_nproc; f++) {
5070 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5071 if (__kmp_tasking_mode != tskm_immediate_exec) {
5074 team->t.t_threads[f]->th.th_task_team = NULL;
5076 __kmp_free_thread(team->t.t_threads[f]);
5077 team->t.t_threads[f] = NULL;
5079 #if KMP_NESTED_HOT_TEAMS 5084 for (f = new_nproc; f < team->t.t_nproc; ++f) {
5085 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5086 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
5087 for (
int b = 0; b < bs_last_barrier; ++b) {
5088 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
5089 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5091 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
5095 #endif // KMP_NESTED_HOT_TEAMS 5096 team->t.t_nproc = new_nproc;
5098 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
5099 __kmp_reinitialize_team(team, new_icvs,
5100 root->r.r_uber_thread->th.th_ident);
5103 for (f = 0; f < new_nproc; ++f) {
5104 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5109 KF_TRACE(10, (
"__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5110 team->t.t_threads[0], team));
5112 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5115 for (f = 0; f < team->t.t_nproc; f++) {
5116 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5117 team->t.t_threads[f]->th.th_team_nproc ==
5123 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5124 #if KMP_AFFINITY_SUPPORTED 5125 __kmp_partition_places(team);
5129 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5130 kmp_affin_mask_t *old_mask;
5131 if (KMP_AFFINITY_CAPABLE()) {
5132 KMP_CPU_ALLOC(old_mask);
5137 (
"__kmp_allocate_team: increasing hot team thread count to %d\n",
5140 team->t.t_size_changed = 1;
5142 #if KMP_NESTED_HOT_TEAMS 5143 int avail_threads = hot_teams[level].hot_team_nth;
5144 if (new_nproc < avail_threads)
5145 avail_threads = new_nproc;
5146 kmp_info_t **other_threads = team->t.t_threads;
5147 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5151 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5152 for (b = 0; b < bs_last_barrier; ++b) {
5153 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5154 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5156 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5160 if (hot_teams[level].hot_team_nth >= new_nproc) {
5163 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5164 team->t.t_nproc = new_nproc;
5170 hot_teams[level].hot_team_nth = new_nproc;
5171 #endif // KMP_NESTED_HOT_TEAMS 5172 if (team->t.t_max_nproc < new_nproc) {
5174 __kmp_reallocate_team_arrays(team, new_nproc);
5175 __kmp_reinitialize_team(team, new_icvs, NULL);
5178 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5183 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5187 for (f = team->t.t_nproc; f < new_nproc; f++) {
5188 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5189 KMP_DEBUG_ASSERT(new_worker);
5190 team->t.t_threads[f] = new_worker;
5193 (
"__kmp_allocate_team: team %d init T#%d arrived: " 5194 "join=%llu, plain=%llu\n",
5195 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5196 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5197 team->t.t_bar[bs_plain_barrier].b_arrived));
5201 kmp_balign_t *balign = new_worker->th.th_bar;
5202 for (b = 0; b < bs_last_barrier; ++b) {
5203 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5204 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5205 KMP_BARRIER_PARENT_FLAG);
5207 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5213 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5214 if (KMP_AFFINITY_CAPABLE()) {
5216 __kmp_set_system_affinity(old_mask, TRUE);
5217 KMP_CPU_FREE(old_mask);
5220 #if KMP_NESTED_HOT_TEAMS 5222 #endif // KMP_NESTED_HOT_TEAMS 5224 int old_nproc = team->t.t_nproc;
5226 __kmp_initialize_team(team, new_nproc, new_icvs,
5227 root->r.r_uber_thread->th.th_ident);
5230 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5231 for (f = 0; f < team->t.t_nproc; ++f)
5232 __kmp_initialize_info(team->t.t_threads[f], team, f,
5233 __kmp_gtid_from_tid(f, team));
5241 for (f = old_nproc; f < team->t.t_nproc; ++f)
5242 team->t.t_threads[f]->th.th_task_state =
5243 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5246 team->t.t_threads[0]->th.th_task_state;
5247 for (f = old_nproc; f < team->t.t_nproc; ++f)
5248 team->t.t_threads[f]->th.th_task_state = old_state;
5252 for (f = 0; f < team->t.t_nproc; ++f) {
5253 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5254 team->t.t_threads[f]->th.th_team_nproc ==
5260 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5261 #if KMP_AFFINITY_SUPPORTED 5262 __kmp_partition_places(team);
5268 kmp_info_t *master = team->t.t_threads[0];
5269 if (master->th.th_teams_microtask) {
5270 for (f = 1; f < new_nproc; ++f) {
5272 kmp_info_t *thr = team->t.t_threads[f];
5273 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5274 thr->th.th_teams_level = master->th.th_teams_level;
5275 thr->th.th_teams_size = master->th.th_teams_size;
5279 #if KMP_NESTED_HOT_TEAMS 5283 for (f = 1; f < new_nproc; ++f) {
5284 kmp_info_t *thr = team->t.t_threads[f];
5286 kmp_balign_t *balign = thr->th.th_bar;
5287 for (b = 0; b < bs_last_barrier; ++b) {
5288 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5289 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5291 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5296 #endif // KMP_NESTED_HOT_TEAMS 5299 __kmp_alloc_argv_entries(argc, team, TRUE);
5300 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5304 KF_TRACE(10, (
" hot_team = %p\n", team));
5307 if (__kmp_tasking_mode != tskm_immediate_exec) {
5308 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p " 5309 "task_team[1] = %p after reinit\n",
5310 team->t.t_task_team[0], team->t.t_task_team[1]));
5315 __ompt_team_assign_id(team, ompt_parallel_data);
5325 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
5328 if (team->t.t_max_nproc >= max_nproc) {
5330 __kmp_team_pool = team->t.t_next_pool;
5333 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5335 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and " 5336 "task_team[1] %p to NULL\n",
5337 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5338 team->t.t_task_team[0] = NULL;
5339 team->t.t_task_team[1] = NULL;
5342 __kmp_alloc_argv_entries(argc, team, TRUE);
5343 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5346 20, (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5347 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5350 for (b = 0; b < bs_last_barrier; ++b) {
5351 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5353 team->t.t_bar[b].b_master_arrived = 0;
5354 team->t.t_bar[b].b_team_arrived = 0;
5360 team->t.t_proc_bind = new_proc_bind;
5363 KA_TRACE(20, (
"__kmp_allocate_team: using team from pool %d.\n",
5367 __ompt_team_assign_id(team, ompt_parallel_data);
5379 team = __kmp_reap_team(team);
5380 __kmp_team_pool = team;
5385 team = (kmp_team_t *)__kmp_allocate(
sizeof(kmp_team_t));
5388 team->t.t_max_nproc = max_nproc;
5391 __kmp_allocate_team_arrays(team, max_nproc);
5393 KA_TRACE(20, (
"__kmp_allocate_team: making a new team\n"));
5394 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5396 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5398 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5399 team->t.t_task_team[0] = NULL;
5401 team->t.t_task_team[1] = NULL;
5404 if (__kmp_storage_map) {
5405 __kmp_print_team_storage_map(
"team", team, team->t.t_id, new_nproc);
5409 __kmp_alloc_argv_entries(argc, team, FALSE);
5410 team->t.t_argc = argc;
5413 (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5414 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5417 for (b = 0; b < bs_last_barrier; ++b) {
5418 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5420 team->t.t_bar[b].b_master_arrived = 0;
5421 team->t.t_bar[b].b_team_arrived = 0;
5427 team->t.t_proc_bind = new_proc_bind;
5431 __ompt_team_assign_id(team, ompt_parallel_data);
5432 team->t.ompt_serialized_team_info = NULL;
5437 KA_TRACE(20, (
"__kmp_allocate_team: done creating a new team %d.\n",
5448 void __kmp_free_team(kmp_root_t *root,
5449 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5451 KA_TRACE(20, (
"__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5455 KMP_DEBUG_ASSERT(root);
5456 KMP_DEBUG_ASSERT(team);
5457 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5458 KMP_DEBUG_ASSERT(team->t.t_threads);
5460 int use_hot_team = team == root->r.r_hot_team;
5461 #if KMP_NESTED_HOT_TEAMS 5463 kmp_hot_team_ptr_t *hot_teams;
5465 level = team->t.t_active_level - 1;
5466 if (master->th.th_teams_microtask) {
5467 if (master->th.th_teams_size.nteams > 1) {
5471 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5472 master->th.th_teams_level == team->t.t_level) {
5477 hot_teams = master->th.th_hot_teams;
5478 if (level < __kmp_hot_teams_max_level) {
5479 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5483 #endif // KMP_NESTED_HOT_TEAMS 5486 TCW_SYNC_PTR(team->t.t_pkfn,
5489 team->t.t_copyin_counter = 0;
5494 if (!use_hot_team) {
5495 if (__kmp_tasking_mode != tskm_immediate_exec) {
5497 for (f = 1; f < team->t.t_nproc; ++f) {
5498 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5499 kmp_info_t *th = team->t.t_threads[f];
5500 volatile kmp_uint32 *state = &th->th.th_reap_state;
5501 while (*state != KMP_SAFE_TO_REAP) {
5505 if (!__kmp_is_thread_alive(th, &ecode)) {
5506 *state = KMP_SAFE_TO_REAP;
5511 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5512 if (fl.is_sleeping())
5513 fl.resume(__kmp_gtid_from_thread(th));
5520 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5521 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5522 if (task_team != NULL) {
5523 for (f = 0; f < team->t.t_nproc; ++f) {
5524 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5525 team->t.t_threads[f]->th.th_task_team = NULL;
5529 (
"__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5530 __kmp_get_gtid(), task_team, team->t.t_id));
5531 #if KMP_NESTED_HOT_TEAMS 5532 __kmp_free_task_team(master, task_team);
5534 team->t.t_task_team[tt_idx] = NULL;
5540 team->t.t_parent = NULL;
5541 team->t.t_level = 0;
5542 team->t.t_active_level = 0;
5545 for (f = 1; f < team->t.t_nproc; ++f) {
5546 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5547 __kmp_free_thread(team->t.t_threads[f]);
5548 team->t.t_threads[f] = NULL;
5553 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
5554 __kmp_team_pool = (
volatile kmp_team_t *)team;
5557 KMP_DEBUG_ASSERT(team->t.t_threads[1] &&
5558 team->t.t_threads[1]->th.th_cg_roots);
5559 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
5561 for (f = 1; f < team->t.t_nproc; ++f) {
5562 kmp_info_t *thr = team->t.t_threads[f];
5563 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&
5564 thr->th.th_cg_roots->cg_root == thr);
5566 kmp_cg_root_t *tmp = thr->th.th_cg_roots;
5567 thr->th.th_cg_roots = tmp->up;
5568 KA_TRACE(100, (
"__kmp_free_team: Thread %p popping node %p and moving" 5569 " up to node %p. cg_nthreads was %d\n",
5570 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads));
5573 if (thr->th.th_cg_roots)
5574 thr->th.th_current_task->td_icvs.thread_limit =
5575 thr->th.th_cg_roots->cg_thread_limit;
5584 kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5585 kmp_team_t *next_pool = team->t.t_next_pool;
5587 KMP_DEBUG_ASSERT(team);
5588 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5589 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5590 KMP_DEBUG_ASSERT(team->t.t_threads);
5591 KMP_DEBUG_ASSERT(team->t.t_argv);
5596 __kmp_free_team_arrays(team);
5597 if (team->t.t_argv != &team->t.t_inline_argv[0])
5598 __kmp_free((
void *)team->t.t_argv);
5630 void __kmp_free_thread(kmp_info_t *this_th) {
5634 KA_TRACE(20, (
"__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5635 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
5637 KMP_DEBUG_ASSERT(this_th);
5642 kmp_balign_t *balign = this_th->th.th_bar;
5643 for (b = 0; b < bs_last_barrier; ++b) {
5644 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5645 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5646 balign[b].bb.team = NULL;
5647 balign[b].bb.leaf_kids = 0;
5649 this_th->th.th_task_state = 0;
5650 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
5653 TCW_PTR(this_th->th.th_team, NULL);
5654 TCW_PTR(this_th->th.th_root, NULL);
5655 TCW_PTR(this_th->th.th_dispatch, NULL);
5657 while (this_th->th.th_cg_roots) {
5658 this_th->th.th_cg_roots->cg_nthreads--;
5659 KA_TRACE(100, (
"__kmp_free_thread: Thread %p decrement cg_nthreads on node" 5660 " %p of thread %p to %d\n",
5661 this_th, this_th->th.th_cg_roots,
5662 this_th->th.th_cg_roots->cg_root,
5663 this_th->th.th_cg_roots->cg_nthreads));
5664 kmp_cg_root_t *tmp = this_th->th.th_cg_roots;
5665 if (tmp->cg_root == this_th) {
5666 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0);
5668 5, (
"__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp));
5669 this_th->th.th_cg_roots = tmp->up;
5672 this_th->th.th_cg_roots = NULL;
5682 __kmp_free_implicit_task(this_th);
5683 this_th->th.th_current_task = NULL;
5687 gtid = this_th->th.th_info.ds.ds_gtid;
5688 if (__kmp_thread_pool_insert_pt != NULL) {
5689 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5690 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5691 __kmp_thread_pool_insert_pt = NULL;
5700 if (__kmp_thread_pool_insert_pt != NULL) {
5701 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5703 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
5705 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5706 scan = &((*scan)->th.th_next_pool))
5711 TCW_PTR(this_th->th.th_next_pool, *scan);
5712 __kmp_thread_pool_insert_pt = *scan = this_th;
5713 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5714 (this_th->th.th_info.ds.ds_gtid <
5715 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5716 TCW_4(this_th->th.th_in_pool, TRUE);
5717 __kmp_suspend_initialize_thread(this_th);
5718 __kmp_lock_suspend_mx(this_th);
5719 if (this_th->th.th_active == TRUE) {
5720 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
5721 this_th->th.th_active_in_pool = TRUE;
5725 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE);
5728 __kmp_unlock_suspend_mx(this_th);
5730 TCW_4(__kmp_nth, __kmp_nth - 1);
5732 #ifdef KMP_ADJUST_BLOCKTIME 5735 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5736 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5737 if (__kmp_nth <= __kmp_avail_proc) {
5738 __kmp_zero_bt = FALSE;
5748 void *__kmp_launch_thread(kmp_info_t *this_thr) {
5749 int gtid = this_thr->th.th_info.ds.ds_gtid;
5751 kmp_team_t *(*
volatile pteam);
5754 KA_TRACE(10, (
"__kmp_launch_thread: T#%d start\n", gtid));
5756 if (__kmp_env_consistency_check) {
5757 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid);
5761 ompt_data_t *thread_data;
5762 if (ompt_enabled.enabled) {
5763 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
5764 *thread_data = ompt_data_none;
5766 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5767 this_thr->th.ompt_thread_info.wait_id = 0;
5768 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5769 if (ompt_enabled.ompt_callback_thread_begin) {
5770 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5771 ompt_thread_worker, thread_data);
5777 if (ompt_enabled.enabled) {
5778 this_thr->th.ompt_thread_info.state = ompt_state_idle;
5782 while (!TCR_4(__kmp_global.g.g_done)) {
5783 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5787 KA_TRACE(20, (
"__kmp_launch_thread: T#%d waiting for work\n", gtid));
5790 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5793 if (ompt_enabled.enabled) {
5794 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5798 pteam = (kmp_team_t * (*))(&this_thr->th.th_team);
5801 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
5803 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5806 (
"__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5807 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5808 (*pteam)->t.t_pkfn));
5810 updateHWFPControl(*pteam);
5813 if (ompt_enabled.enabled) {
5814 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
5818 rc = (*pteam)->t.t_invoke(gtid);
5822 KA_TRACE(20, (
"__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5823 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5824 (*pteam)->t.t_pkfn));
5827 if (ompt_enabled.enabled) {
5829 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;
5831 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5835 __kmp_join_barrier(gtid);
5838 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
5841 if (ompt_enabled.ompt_callback_thread_end) {
5842 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
5846 this_thr->th.th_task_team = NULL;
5848 __kmp_common_destroy_gtid(gtid);
5850 KA_TRACE(10, (
"__kmp_launch_thread: T#%d done\n", gtid));
5857 void __kmp_internal_end_dest(
void *specific_gtid) {
5858 #if KMP_COMPILER_ICC 5859 #pragma warning(push) 5860 #pragma warning(disable : 810) // conversion from "void *" to "int" may lose 5864 int gtid = (kmp_intptr_t)specific_gtid - 1;
5865 #if KMP_COMPILER_ICC 5866 #pragma warning(pop) 5869 KA_TRACE(30, (
"__kmp_internal_end_dest: T#%d\n", gtid));
5882 if (gtid >= 0 && KMP_UBER_GTID(gtid))
5883 __kmp_gtid_set_specific(gtid);
5884 #ifdef KMP_TDATA_GTID 5887 __kmp_internal_end_thread(gtid);
5890 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5896 __attribute__((destructor))
void __kmp_internal_end_dtor(
void) {
5897 __kmp_internal_end_atexit();
5900 void __kmp_internal_end_fini(
void) { __kmp_internal_end_atexit(); }
5906 void __kmp_internal_end_atexit(
void) {
5907 KA_TRACE(30, (
"__kmp_internal_end_atexit\n"));
5931 __kmp_internal_end_library(-1);
5933 __kmp_close_console();
5937 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root) {
5942 KMP_DEBUG_ASSERT(thread != NULL);
5944 gtid = thread->th.th_info.ds.ds_gtid;
5947 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5950 20, (
"__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5954 ANNOTATE_HAPPENS_BEFORE(thread);
5955 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
5956 __kmp_release_64(&flag);
5960 __kmp_reap_worker(thread);
5972 if (thread->th.th_active_in_pool) {
5973 thread->th.th_active_in_pool = FALSE;
5974 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
5975 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
5979 __kmp_free_implicit_task(thread);
5983 __kmp_free_fast_memory(thread);
5986 __kmp_suspend_uninitialize_thread(thread);
5988 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5989 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5994 #ifdef KMP_ADJUST_BLOCKTIME 5997 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5998 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5999 if (__kmp_nth <= __kmp_avail_proc) {
6000 __kmp_zero_bt = FALSE;
6006 if (__kmp_env_consistency_check) {
6007 if (thread->th.th_cons) {
6008 __kmp_free_cons_stack(thread->th.th_cons);
6009 thread->th.th_cons = NULL;
6013 if (thread->th.th_pri_common != NULL) {
6014 __kmp_free(thread->th.th_pri_common);
6015 thread->th.th_pri_common = NULL;
6018 if (thread->th.th_task_state_memo_stack != NULL) {
6019 __kmp_free(thread->th.th_task_state_memo_stack);
6020 thread->th.th_task_state_memo_stack = NULL;
6024 if (thread->th.th_local.bget_data != NULL) {
6025 __kmp_finalize_bget(thread);
6029 #if KMP_AFFINITY_SUPPORTED 6030 if (thread->th.th_affin_mask != NULL) {
6031 KMP_CPU_FREE(thread->th.th_affin_mask);
6032 thread->th.th_affin_mask = NULL;
6036 #if KMP_USE_HIER_SCHED 6037 if (thread->th.th_hier_bar_data != NULL) {
6038 __kmp_free(thread->th.th_hier_bar_data);
6039 thread->th.th_hier_bar_data = NULL;
6043 __kmp_reap_team(thread->th.th_serial_team);
6044 thread->th.th_serial_team = NULL;
6051 static void __kmp_internal_end(
void) {
6055 __kmp_unregister_library();
6062 __kmp_reclaim_dead_roots();
6066 for (i = 0; i < __kmp_threads_capacity; i++)
6068 if (__kmp_root[i]->r.r_active)
6071 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6073 if (i < __kmp_threads_capacity) {
6085 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6086 if (TCR_4(__kmp_init_monitor)) {
6087 __kmp_reap_monitor(&__kmp_monitor);
6088 TCW_4(__kmp_init_monitor, 0);
6090 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6091 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6092 #endif // KMP_USE_MONITOR 6097 for (i = 0; i < __kmp_threads_capacity; i++) {
6098 if (__kmp_root[i]) {
6101 KMP_ASSERT(!__kmp_root[i]->r.r_active);
6110 while (__kmp_thread_pool != NULL) {
6112 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
6113 __kmp_thread_pool = thread->th.th_next_pool;
6115 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
6116 thread->th.th_next_pool = NULL;
6117 thread->th.th_in_pool = FALSE;
6118 thread->th.th_active_in_pool = FALSE;
6119 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
6120 __kmp_reap_thread(thread, 0);
6122 __kmp_thread_pool_insert_pt = NULL;
6125 while (__kmp_team_pool != NULL) {
6127 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
6128 __kmp_team_pool = team->t.t_next_pool;
6130 team->t.t_next_pool = NULL;
6131 __kmp_reap_team(team);
6134 __kmp_reap_task_teams();
6141 for (i = 0; i < __kmp_threads_capacity; i++) {
6142 kmp_info_t *thr = __kmp_threads[i];
6143 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
6148 for (i = 0; i < __kmp_threads_capacity; ++i) {
6155 TCW_SYNC_4(__kmp_init_common, FALSE);
6157 KA_TRACE(10, (
"__kmp_internal_end: all workers reaped\n"));
6165 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6166 if (TCR_4(__kmp_init_monitor)) {
6167 __kmp_reap_monitor(&__kmp_monitor);
6168 TCW_4(__kmp_init_monitor, 0);
6170 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6171 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6174 TCW_4(__kmp_init_gtid, FALSE);
6183 void __kmp_internal_end_library(
int gtid_req) {
6190 if (__kmp_global.g.g_abort) {
6191 KA_TRACE(11, (
"__kmp_internal_end_library: abort, exiting\n"));
6195 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6196 KA_TRACE(10, (
"__kmp_internal_end_library: already finished\n"));
6204 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6206 10, (
"__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6207 if (gtid == KMP_GTID_SHUTDOWN) {
6208 KA_TRACE(10, (
"__kmp_internal_end_library: !__kmp_init_runtime, system " 6209 "already shutdown\n"));
6211 }
else if (gtid == KMP_GTID_MONITOR) {
6212 KA_TRACE(10, (
"__kmp_internal_end_library: monitor thread, gtid not " 6213 "registered, or system shutdown\n"));
6215 }
else if (gtid == KMP_GTID_DNE) {
6216 KA_TRACE(10, (
"__kmp_internal_end_library: gtid not registered or system " 6219 }
else if (KMP_UBER_GTID(gtid)) {
6221 if (__kmp_root[gtid]->r.r_active) {
6222 __kmp_global.g.g_abort = -1;
6223 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6225 (
"__kmp_internal_end_library: root still active, abort T#%d\n",
6231 (
"__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6232 __kmp_unregister_root_current_thread(gtid);
6239 #ifdef DUMP_DEBUG_ON_EXIT 6240 if (__kmp_debug_buf)
6241 __kmp_dump_debug_buffer();
6247 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6250 if (__kmp_global.g.g_abort) {
6251 KA_TRACE(10, (
"__kmp_internal_end_library: abort, exiting\n"));
6253 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6256 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6257 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6266 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6269 __kmp_internal_end();
6271 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6272 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6274 KA_TRACE(10, (
"__kmp_internal_end_library: exit\n"));
6276 #ifdef DUMP_DEBUG_ON_EXIT 6277 if (__kmp_debug_buf)
6278 __kmp_dump_debug_buffer();
6282 __kmp_close_console();
6285 __kmp_fini_allocator();
6289 void __kmp_internal_end_thread(
int gtid_req) {
6298 if (__kmp_global.g.g_abort) {
6299 KA_TRACE(11, (
"__kmp_internal_end_thread: abort, exiting\n"));
6303 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6304 KA_TRACE(10, (
"__kmp_internal_end_thread: already finished\n"));
6312 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6314 (
"__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6315 if (gtid == KMP_GTID_SHUTDOWN) {
6316 KA_TRACE(10, (
"__kmp_internal_end_thread: !__kmp_init_runtime, system " 6317 "already shutdown\n"));
6319 }
else if (gtid == KMP_GTID_MONITOR) {
6320 KA_TRACE(10, (
"__kmp_internal_end_thread: monitor thread, gtid not " 6321 "registered, or system shutdown\n"));
6323 }
else if (gtid == KMP_GTID_DNE) {
6324 KA_TRACE(10, (
"__kmp_internal_end_thread: gtid not registered or system " 6328 }
else if (KMP_UBER_GTID(gtid)) {
6330 if (__kmp_root[gtid]->r.r_active) {
6331 __kmp_global.g.g_abort = -1;
6332 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6334 (
"__kmp_internal_end_thread: root still active, abort T#%d\n",
6338 KA_TRACE(10, (
"__kmp_internal_end_thread: unregistering sibling T#%d\n",
6340 __kmp_unregister_root_current_thread(gtid);
6344 KA_TRACE(10, (
"__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6347 __kmp_threads[gtid]->th.th_task_team = NULL;
6351 (
"__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6358 if (__kmp_pause_status != kmp_hard_paused)
6363 KA_TRACE(10, (
"__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6368 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6371 if (__kmp_global.g.g_abort) {
6372 KA_TRACE(10, (
"__kmp_internal_end_thread: abort, exiting\n"));
6374 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6377 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6378 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6389 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6391 for (i = 0; i < __kmp_threads_capacity; ++i) {
6392 if (KMP_UBER_GTID(i)) {
6395 (
"__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6396 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6397 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6404 __kmp_internal_end();
6406 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6407 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6409 KA_TRACE(10, (
"__kmp_internal_end_thread: exit T#%d\n", gtid_req));
6411 #ifdef DUMP_DEBUG_ON_EXIT 6412 if (__kmp_debug_buf)
6413 __kmp_dump_debug_buffer();
6420 static long __kmp_registration_flag = 0;
6422 static char *__kmp_registration_str = NULL;
6425 static inline char *__kmp_reg_status_name() {
6430 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d", (
int)getpid());
6433 void __kmp_register_library_startup(
void) {
6435 char *name = __kmp_reg_status_name();
6441 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6442 __kmp_initialize_system_tick();
6444 __kmp_read_system_time(&time.dtime);
6445 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6446 __kmp_registration_str =
6447 __kmp_str_format(
"%p-%lx-%s", &__kmp_registration_flag,
6448 __kmp_registration_flag, KMP_LIBRARY_FILE);
6450 KA_TRACE(50, (
"__kmp_register_library_startup: %s=\"%s\"\n", name,
6451 __kmp_registration_str));
6458 __kmp_env_set(name, __kmp_registration_str, 0);
6460 value = __kmp_env_get(name);
6461 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6471 char *flag_addr_str = NULL;
6472 char *flag_val_str = NULL;
6473 char const *file_name = NULL;
6474 __kmp_str_split(tail,
'-', &flag_addr_str, &tail);
6475 __kmp_str_split(tail,
'-', &flag_val_str, &tail);
6478 long *flag_addr = 0;
6480 KMP_SSCANF(flag_addr_str,
"%p", RCAST(
void**, &flag_addr));
6481 KMP_SSCANF(flag_val_str,
"%lx", &flag_val);
6482 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name,
"") != 0) {
6486 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6500 file_name =
"unknown library";
6505 char *duplicate_ok = __kmp_env_get(
"KMP_DUPLICATE_LIB_OK");
6506 if (!__kmp_str_match_true(duplicate_ok)) {
6508 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6509 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
6511 KMP_INTERNAL_FREE(duplicate_ok);
6512 __kmp_duplicate_library_ok = 1;
6517 __kmp_env_unset(name);
6519 default: { KMP_DEBUG_ASSERT(0); }
break;
6522 KMP_INTERNAL_FREE((
void *)value);
6524 KMP_INTERNAL_FREE((
void *)name);
6528 void __kmp_unregister_library(
void) {
6530 char *name = __kmp_reg_status_name();
6531 char *value = __kmp_env_get(name);
6533 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6534 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6535 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6537 __kmp_env_unset(name);
6540 KMP_INTERNAL_FREE(__kmp_registration_str);
6541 KMP_INTERNAL_FREE(value);
6542 KMP_INTERNAL_FREE(name);
6544 __kmp_registration_flag = 0;
6545 __kmp_registration_str = NULL;
6552 #if KMP_MIC_SUPPORTED 6554 static void __kmp_check_mic_type() {
6555 kmp_cpuid_t cpuid_state = {0};
6556 kmp_cpuid_t *cs_p = &cpuid_state;
6557 __kmp_x86_cpuid(1, 0, cs_p);
6559 if ((cs_p->eax & 0xff0) == 0xB10) {
6560 __kmp_mic_type = mic2;
6561 }
else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6562 __kmp_mic_type = mic3;
6564 __kmp_mic_type = non_mic;
6570 static void __kmp_do_serial_initialize(
void) {
6574 KA_TRACE(10, (
"__kmp_do_serial_initialize: enter\n"));
6576 KMP_DEBUG_ASSERT(
sizeof(kmp_int32) == 4);
6577 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) == 4);
6578 KMP_DEBUG_ASSERT(
sizeof(kmp_int64) == 8);
6579 KMP_DEBUG_ASSERT(
sizeof(kmp_uint64) == 8);
6580 KMP_DEBUG_ASSERT(
sizeof(kmp_intptr_t) ==
sizeof(
void *));
6586 __kmp_validate_locks();
6589 __kmp_init_allocator();
6594 __kmp_register_library_startup();
6597 if (TCR_4(__kmp_global.g.g_done)) {
6598 KA_TRACE(10, (
"__kmp_do_serial_initialize: reinitialization of library\n"));
6601 __kmp_global.g.g_abort = 0;
6602 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
6605 #if KMP_USE_ADAPTIVE_LOCKS 6606 #if KMP_DEBUG_ADAPTIVE_LOCKS 6607 __kmp_init_speculative_stats();
6610 #if KMP_STATS_ENABLED 6613 __kmp_init_lock(&__kmp_global_lock);
6614 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6615 __kmp_init_lock(&__kmp_debug_lock);
6616 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6617 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6618 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6619 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6620 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6621 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6622 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6623 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6624 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6625 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6626 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6627 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6628 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6629 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6630 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
6632 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
6634 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
6638 __kmp_runtime_initialize();
6640 #if KMP_MIC_SUPPORTED 6641 __kmp_check_mic_type();
6648 __kmp_abort_delay = 0;
6652 __kmp_dflt_team_nth_ub = __kmp_xproc;
6653 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6654 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6656 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6657 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6659 __kmp_max_nth = __kmp_sys_max_nth;
6660 __kmp_cg_max_nth = __kmp_sys_max_nth;
6661 __kmp_teams_max_nth = __kmp_xproc;
6662 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6663 __kmp_teams_max_nth = __kmp_sys_max_nth;
6668 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
6670 __kmp_monitor_wakeups =
6671 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6672 __kmp_bt_intervals =
6673 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6676 __kmp_library = library_throughput;
6678 __kmp_static = kmp_sch_static_balanced;
6685 #if KMP_FAST_REDUCTION_BARRIER 6686 #define kmp_reduction_barrier_gather_bb ((int)1) 6687 #define kmp_reduction_barrier_release_bb ((int)1) 6688 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6689 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6690 #endif // KMP_FAST_REDUCTION_BARRIER 6691 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6692 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6693 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6694 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6695 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6696 #if KMP_FAST_REDUCTION_BARRIER 6697 if (i == bs_reduction_barrier) {
6699 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6700 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6701 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6702 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
6704 #endif // KMP_FAST_REDUCTION_BARRIER 6706 #if KMP_FAST_REDUCTION_BARRIER 6707 #undef kmp_reduction_barrier_release_pat 6708 #undef kmp_reduction_barrier_gather_pat 6709 #undef kmp_reduction_barrier_release_bb 6710 #undef kmp_reduction_barrier_gather_bb 6711 #endif // KMP_FAST_REDUCTION_BARRIER 6712 #if KMP_MIC_SUPPORTED 6713 if (__kmp_mic_type == mic2) {
6715 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3;
6716 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6718 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6719 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6721 #if KMP_FAST_REDUCTION_BARRIER 6722 if (__kmp_mic_type == mic2) {
6723 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6724 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6726 #endif // KMP_FAST_REDUCTION_BARRIER 6727 #endif // KMP_MIC_SUPPORTED 6731 __kmp_env_checks = TRUE;
6733 __kmp_env_checks = FALSE;
6737 __kmp_foreign_tp = TRUE;
6739 __kmp_global.g.g_dynamic = FALSE;
6740 __kmp_global.g.g_dynamic_mode = dynamic_default;
6742 __kmp_env_initialize(NULL);
6746 char const *val = __kmp_env_get(
"KMP_DUMP_CATALOG");
6747 if (__kmp_str_match_true(val)) {
6748 kmp_str_buf_t buffer;
6749 __kmp_str_buf_init(&buffer);
6750 __kmp_i18n_dump_catalog(&buffer);
6751 __kmp_printf(
"%s", buffer.str);
6752 __kmp_str_buf_free(&buffer);
6754 __kmp_env_free(&val);
6757 __kmp_threads_capacity =
6758 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6760 __kmp_tp_capacity = __kmp_default_tp_capacity(
6761 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
6766 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6767 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6768 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6769 __kmp_thread_pool = NULL;
6770 __kmp_thread_pool_insert_pt = NULL;
6771 __kmp_team_pool = NULL;
6778 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6780 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6781 __kmp_root = (kmp_root_t **)((
char *)__kmp_threads +
6782 sizeof(kmp_info_t *) * __kmp_threads_capacity);
6785 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6787 KMP_DEBUG_ASSERT(__kmp_nth == 0);
6792 gtid = __kmp_register_root(TRUE);
6793 KA_TRACE(10, (
"__kmp_do_serial_initialize T#%d\n", gtid));
6794 KMP_ASSERT(KMP_UBER_GTID(gtid));
6795 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
6799 __kmp_common_initialize();
6803 __kmp_register_atfork();
6806 #if !KMP_DYNAMIC_LIB 6810 int rc = atexit(__kmp_internal_end_atexit);
6812 __kmp_fatal(KMP_MSG(FunctionError,
"atexit()"), KMP_ERR(rc),
6818 #if KMP_HANDLE_SIGNALS 6824 __kmp_install_signals(FALSE);
6827 __kmp_install_signals(TRUE);
6832 __kmp_init_counter++;
6834 __kmp_init_serial = TRUE;
6836 if (__kmp_settings) {
6841 if (__kmp_display_env || __kmp_display_env_verbose) {
6842 __kmp_env_print_2();
6844 #endif // OMP_40_ENABLED 6852 KA_TRACE(10, (
"__kmp_do_serial_initialize: exit\n"));
6855 void __kmp_serial_initialize(
void) {
6856 if (__kmp_init_serial) {
6859 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6860 if (__kmp_init_serial) {
6861 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6864 __kmp_do_serial_initialize();
6865 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6868 static void __kmp_do_middle_initialize(
void) {
6870 int prev_dflt_team_nth;
6872 if (!__kmp_init_serial) {
6873 __kmp_do_serial_initialize();
6876 KA_TRACE(10, (
"__kmp_middle_initialize: enter\n"));
6880 prev_dflt_team_nth = __kmp_dflt_team_nth;
6882 #if KMP_AFFINITY_SUPPORTED 6885 __kmp_affinity_initialize();
6889 for (i = 0; i < __kmp_threads_capacity; i++) {
6890 if (TCR_PTR(__kmp_threads[i]) != NULL) {
6891 __kmp_affinity_set_init_mask(i, TRUE);
6896 KMP_ASSERT(__kmp_xproc > 0);
6897 if (__kmp_avail_proc == 0) {
6898 __kmp_avail_proc = __kmp_xproc;
6904 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
6905 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
6910 if (__kmp_dflt_team_nth == 0) {
6911 #ifdef KMP_DFLT_NTH_CORES 6913 __kmp_dflt_team_nth = __kmp_ncores;
6914 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6915 "__kmp_ncores (%d)\n",
6916 __kmp_dflt_team_nth));
6919 __kmp_dflt_team_nth = __kmp_avail_proc;
6920 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6921 "__kmp_avail_proc(%d)\n",
6922 __kmp_dflt_team_nth));
6926 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
6927 __kmp_dflt_team_nth = KMP_MIN_NTH;
6929 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
6930 __kmp_dflt_team_nth = __kmp_sys_max_nth;
6935 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
6937 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
6942 for (i = 0; i < __kmp_threads_capacity; i++) {
6943 kmp_info_t *thread = __kmp_threads[i];
6946 if (thread->th.th_current_task->td_icvs.nproc != 0)
6949 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
6954 (
"__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
6955 __kmp_dflt_team_nth));
6957 #ifdef KMP_ADJUST_BLOCKTIME 6959 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6960 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
6961 if (__kmp_nth > __kmp_avail_proc) {
6962 __kmp_zero_bt = TRUE;
6968 TCW_SYNC_4(__kmp_init_middle, TRUE);
6970 KA_TRACE(10, (
"__kmp_do_middle_initialize: exit\n"));
6973 void __kmp_middle_initialize(
void) {
6974 if (__kmp_init_middle) {
6977 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6978 if (__kmp_init_middle) {
6979 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6982 __kmp_do_middle_initialize();
6983 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6986 void __kmp_parallel_initialize(
void) {
6987 int gtid = __kmp_entry_gtid();
6990 if (TCR_4(__kmp_init_parallel))
6992 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6993 if (TCR_4(__kmp_init_parallel)) {
6994 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6999 if (TCR_4(__kmp_global.g.g_done)) {
7002 (
"__kmp_parallel_initialize: attempt to init while shutting down\n"));
7003 __kmp_infinite_loop();
7009 if (!__kmp_init_middle) {
7010 __kmp_do_middle_initialize();
7014 __kmp_resume_if_hard_paused();
7018 KA_TRACE(10, (
"__kmp_parallel_initialize: enter\n"));
7019 KMP_ASSERT(KMP_UBER_GTID(gtid));
7021 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 7024 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
7025 __kmp_store_mxcsr(&__kmp_init_mxcsr);
7026 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
7030 #if KMP_HANDLE_SIGNALS 7032 __kmp_install_signals(TRUE);
7036 __kmp_suspend_initialize();
7038 #if defined(USE_LOAD_BALANCE) 7039 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7040 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
7043 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7044 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7048 if (__kmp_version) {
7049 __kmp_print_version_2();
7053 TCW_SYNC_4(__kmp_init_parallel, TRUE);
7056 KA_TRACE(10, (
"__kmp_parallel_initialize: exit\n"));
7058 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7063 void __kmp_run_before_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7065 kmp_disp_t *dispatch;
7070 this_thr->th.th_local.this_construct = 0;
7071 #if KMP_CACHE_MANAGE 7072 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
7074 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
7075 KMP_DEBUG_ASSERT(dispatch);
7076 KMP_DEBUG_ASSERT(team->t.t_dispatch);
7080 dispatch->th_disp_index = 0;
7082 dispatch->th_doacross_buf_idx =
7085 if (__kmp_env_consistency_check)
7086 __kmp_push_parallel(gtid, team->t.t_ident);
7091 void __kmp_run_after_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7093 if (__kmp_env_consistency_check)
7094 __kmp_pop_parallel(gtid, team->t.t_ident);
7096 __kmp_finish_implicit_task(this_thr);
7099 int __kmp_invoke_task_func(
int gtid) {
7101 int tid = __kmp_tid_from_gtid(gtid);
7102 kmp_info_t *this_thr = __kmp_threads[gtid];
7103 kmp_team_t *team = this_thr->th.th_team;
7105 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
7107 if (__itt_stack_caller_create_ptr) {
7108 __kmp_itt_stack_callee_enter(
7110 team->t.t_stack_id);
7113 #if INCLUDE_SSC_MARKS 7114 SSC_MARK_INVOKING();
7119 void **exit_runtime_p;
7120 ompt_data_t *my_task_data;
7121 ompt_data_t *my_parallel_data;
7124 if (ompt_enabled.enabled) {
7126 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame.ptr);
7128 exit_runtime_p = &dummy;
7132 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
7133 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
7134 if (ompt_enabled.ompt_callback_implicit_task) {
7135 ompt_team_size = team->t.t_nproc;
7136 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7137 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
7138 __kmp_tid_from_gtid(gtid), ompt_task_implicit);
7139 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
7143 #if KMP_STATS_ENABLED 7145 if (previous_state == stats_state_e::TEAMS_REGION) {
7146 KMP_PUSH_PARTITIONED_TIMER(OMP_teams);
7148 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel);
7150 KMP_SET_THREAD_STATE(IMPLICIT_TASK);
7153 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
7154 tid, (
int)team->t.t_argc, (
void **)team->t.t_argv
7161 *exit_runtime_p = NULL;
7164 #if KMP_STATS_ENABLED 7165 if (previous_state == stats_state_e::TEAMS_REGION) {
7166 KMP_SET_THREAD_STATE(previous_state);
7168 KMP_POP_PARTITIONED_TIMER();
7172 if (__itt_stack_caller_create_ptr) {
7173 __kmp_itt_stack_callee_leave(
7175 team->t.t_stack_id);
7178 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
7184 void __kmp_teams_master(
int gtid) {
7186 kmp_info_t *thr = __kmp_threads[gtid];
7187 kmp_team_t *team = thr->th.th_team;
7188 ident_t *loc = team->t.t_ident;
7189 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7190 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7191 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7192 KA_TRACE(20, (
"__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7193 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
7196 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
7199 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
7200 tmp->cg_nthreads = 1;
7201 KA_TRACE(100, (
"__kmp_teams_master: Thread %p created node %p and init" 7202 " cg_threads to 1\n",
7204 tmp->up = thr->th.th_cg_roots;
7205 thr->th.th_cg_roots = tmp;
7209 #if INCLUDE_SSC_MARKS 7212 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
7213 (microtask_t)thr->th.th_teams_microtask,
7214 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
7215 #if INCLUDE_SSC_MARKS 7219 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)
7220 thr->th.th_teams_size.nth = thr->th.th_team_nproc;
7223 __kmp_join_call(loc, gtid
7232 int __kmp_invoke_teams_master(
int gtid) {
7233 kmp_info_t *this_thr = __kmp_threads[gtid];
7234 kmp_team_t *team = this_thr->th.th_team;
7236 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7237 KMP_DEBUG_ASSERT((
void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7238 (
void *)__kmp_teams_master);
7240 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7241 __kmp_teams_master(gtid);
7242 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7252 void __kmp_push_num_threads(
ident_t *
id,
int gtid,
int num_threads) {
7253 kmp_info_t *thr = __kmp_threads[gtid];
7255 if (num_threads > 0)
7256 thr->th.th_set_nproc = num_threads;
7263 void __kmp_push_num_teams(
ident_t *
id,
int gtid,
int num_teams,
7265 kmp_info_t *thr = __kmp_threads[gtid];
7266 KMP_DEBUG_ASSERT(num_teams >= 0);
7267 KMP_DEBUG_ASSERT(num_threads >= 0);
7271 if (num_teams > __kmp_teams_max_nth) {
7272 if (!__kmp_reserve_warn) {
7273 __kmp_reserve_warn = 1;
7274 __kmp_msg(kmp_ms_warning,
7275 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7276 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7278 num_teams = __kmp_teams_max_nth;
7282 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7285 if (num_threads == 0) {
7286 if (!TCR_4(__kmp_init_middle))
7287 __kmp_middle_initialize();
7288 num_threads = __kmp_avail_proc / num_teams;
7289 if (num_teams * num_threads > __kmp_teams_max_nth) {
7291 num_threads = __kmp_teams_max_nth / num_teams;
7296 thr->th.th_current_task->td_icvs.thread_limit = num_threads;
7298 if (num_teams * num_threads > __kmp_teams_max_nth) {
7299 int new_threads = __kmp_teams_max_nth / num_teams;
7300 if (!__kmp_reserve_warn) {
7301 __kmp_reserve_warn = 1;
7302 __kmp_msg(kmp_ms_warning,
7303 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7304 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7306 num_threads = new_threads;
7309 thr->th.th_teams_size.nth = num_threads;
7313 void __kmp_push_proc_bind(
ident_t *
id,
int gtid, kmp_proc_bind_t proc_bind) {
7314 kmp_info_t *thr = __kmp_threads[gtid];
7315 thr->th.th_set_proc_bind = proc_bind;
7322 void __kmp_internal_fork(
ident_t *
id,
int gtid, kmp_team_t *team) {
7323 kmp_info_t *this_thr = __kmp_threads[gtid];
7329 KMP_DEBUG_ASSERT(team);
7330 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7331 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7334 team->t.t_construct = 0;
7335 team->t.t_ordered.dt.t_value =
7339 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7340 if (team->t.t_max_nproc > 1) {
7342 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7343 team->t.t_disp_buffer[i].buffer_index = i;
7345 team->t.t_disp_buffer[i].doacross_buf_idx = i;
7349 team->t.t_disp_buffer[0].buffer_index = 0;
7351 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7356 KMP_ASSERT(this_thr->th.th_team == team);
7359 for (f = 0; f < team->t.t_nproc; f++) {
7360 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7361 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7366 __kmp_fork_barrier(gtid, 0);
7369 void __kmp_internal_join(
ident_t *
id,
int gtid, kmp_team_t *team) {
7370 kmp_info_t *this_thr = __kmp_threads[gtid];
7372 KMP_DEBUG_ASSERT(team);
7373 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7374 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7380 if (__kmp_threads[gtid] &&
7381 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7382 __kmp_printf(
"GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7383 __kmp_threads[gtid]);
7384 __kmp_printf(
"__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7385 "team->t.t_nproc=%d\n",
7386 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7388 __kmp_print_structure();
7390 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7391 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
7394 __kmp_join_barrier(gtid);
7396 if (ompt_enabled.enabled &&
7397 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
7398 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7399 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
7400 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
7402 void *codeptr = NULL;
7403 if (KMP_MASTER_TID(ds_tid) &&
7404 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7405 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7406 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7408 if (ompt_enabled.ompt_callback_sync_region_wait) {
7409 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
7410 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7413 if (ompt_enabled.ompt_callback_sync_region) {
7414 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
7415 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7419 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7420 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7421 ompt_scope_end, NULL, task_data, 0, ds_tid, ompt_task_implicit);
7427 KMP_ASSERT(this_thr->th.th_team == team);
7432 #ifdef USE_LOAD_BALANCE 7436 static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7439 kmp_team_t *hot_team;
7441 if (root->r.r_active) {
7444 hot_team = root->r.r_hot_team;
7445 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7446 return hot_team->t.t_nproc - 1;
7451 for (i = 1; i < hot_team->t.t_nproc; i++) {
7452 if (hot_team->t.t_threads[i]->th.th_active) {
7461 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc) {
7464 int hot_team_active;
7465 int team_curr_active;
7468 KB_TRACE(20, (
"__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7470 KMP_DEBUG_ASSERT(root);
7471 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7472 ->th.th_current_task->td_icvs.dynamic == TRUE);
7473 KMP_DEBUG_ASSERT(set_nproc > 1);
7475 if (set_nproc == 1) {
7476 KB_TRACE(20, (
"__kmp_load_balance_nproc: serial execution.\n"));
7485 pool_active = __kmp_thread_pool_active_nth;
7486 hot_team_active = __kmp_active_hot_team_nproc(root);
7487 team_curr_active = pool_active + hot_team_active + 1;
7490 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7491 KB_TRACE(30, (
"__kmp_load_balance_nproc: system active = %d pool active = %d " 7492 "hot team active = %d\n",
7493 system_active, pool_active, hot_team_active));
7495 if (system_active < 0) {
7499 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7500 KMP_WARNING(CantLoadBalUsing,
"KMP_DYNAMIC_MODE=thread limit");
7503 retval = __kmp_avail_proc - __kmp_nth +
7504 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7505 if (retval > set_nproc) {
7508 if (retval < KMP_MIN_NTH) {
7509 retval = KMP_MIN_NTH;
7512 KB_TRACE(20, (
"__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7520 if (system_active < team_curr_active) {
7521 system_active = team_curr_active;
7523 retval = __kmp_avail_proc - system_active + team_curr_active;
7524 if (retval > set_nproc) {
7527 if (retval < KMP_MIN_NTH) {
7528 retval = KMP_MIN_NTH;
7531 KB_TRACE(20, (
"__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7540 void __kmp_cleanup(
void) {
7543 KA_TRACE(10, (
"__kmp_cleanup: enter\n"));
7545 if (TCR_4(__kmp_init_parallel)) {
7546 #if KMP_HANDLE_SIGNALS 7547 __kmp_remove_signals();
7549 TCW_4(__kmp_init_parallel, FALSE);
7552 if (TCR_4(__kmp_init_middle)) {
7553 #if KMP_AFFINITY_SUPPORTED 7554 __kmp_affinity_uninitialize();
7556 __kmp_cleanup_hierarchy();
7557 TCW_4(__kmp_init_middle, FALSE);
7560 KA_TRACE(10, (
"__kmp_cleanup: go serial cleanup\n"));
7562 if (__kmp_init_serial) {
7563 __kmp_runtime_destroy();
7564 __kmp_init_serial = FALSE;
7567 __kmp_cleanup_threadprivate_caches();
7569 for (f = 0; f < __kmp_threads_capacity; f++) {
7570 if (__kmp_root[f] != NULL) {
7571 __kmp_free(__kmp_root[f]);
7572 __kmp_root[f] = NULL;
7575 __kmp_free(__kmp_threads);
7578 __kmp_threads = NULL;
7580 __kmp_threads_capacity = 0;
7582 #if KMP_USE_DYNAMIC_LOCK 7583 __kmp_cleanup_indirect_user_locks();
7585 __kmp_cleanup_user_locks();
7588 #if KMP_AFFINITY_SUPPORTED 7589 KMP_INTERNAL_FREE(CCAST(
char *, __kmp_cpuinfo_file));
7590 __kmp_cpuinfo_file = NULL;
7593 #if KMP_USE_ADAPTIVE_LOCKS 7594 #if KMP_DEBUG_ADAPTIVE_LOCKS 7595 __kmp_print_speculative_stats();
7598 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7599 __kmp_nested_nth.nth = NULL;
7600 __kmp_nested_nth.size = 0;
7601 __kmp_nested_nth.used = 0;
7602 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7603 __kmp_nested_proc_bind.bind_types = NULL;
7604 __kmp_nested_proc_bind.size = 0;
7605 __kmp_nested_proc_bind.used = 0;
7607 if (__kmp_affinity_format) {
7608 KMP_INTERNAL_FREE(__kmp_affinity_format);
7609 __kmp_affinity_format = NULL;
7613 __kmp_i18n_catclose();
7615 #if KMP_USE_HIER_SCHED 7616 __kmp_hier_scheds.deallocate();
7619 #if KMP_STATS_ENABLED 7623 KA_TRACE(10, (
"__kmp_cleanup: exit\n"));
7628 int __kmp_ignore_mppbeg(
void) {
7631 if ((env = getenv(
"KMP_IGNORE_MPPBEG")) != NULL) {
7632 if (__kmp_str_match_false(env))
7639 int __kmp_ignore_mppend(
void) {
7642 if ((env = getenv(
"KMP_IGNORE_MPPEND")) != NULL) {
7643 if (__kmp_str_match_false(env))
7650 void __kmp_internal_begin(
void) {
7656 gtid = __kmp_entry_gtid();
7657 root = __kmp_threads[gtid]->th.th_root;
7658 KMP_ASSERT(KMP_UBER_GTID(gtid));
7660 if (root->r.r_begin)
7662 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7663 if (root->r.r_begin) {
7664 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7668 root->r.r_begin = TRUE;
7670 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7675 void __kmp_user_set_library(
enum library_type arg) {
7682 gtid = __kmp_entry_gtid();
7683 thread = __kmp_threads[gtid];
7685 root = thread->th.th_root;
7687 KA_TRACE(20, (
"__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7689 if (root->r.r_in_parallel) {
7691 KMP_WARNING(SetLibraryIncorrectCall);
7696 case library_serial:
7697 thread->th.th_set_nproc = 0;
7698 set__nproc(thread, 1);
7700 case library_turnaround:
7701 thread->th.th_set_nproc = 0;
7702 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7703 : __kmp_dflt_team_nth_ub);
7705 case library_throughput:
7706 thread->th.th_set_nproc = 0;
7707 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7708 : __kmp_dflt_team_nth_ub);
7711 KMP_FATAL(UnknownLibraryType, arg);
7714 __kmp_aux_set_library(arg);
7717 void __kmp_aux_set_stacksize(
size_t arg) {
7718 if (!__kmp_init_serial)
7719 __kmp_serial_initialize();
7722 if (arg & (0x1000 - 1)) {
7723 arg &= ~(0x1000 - 1);
7728 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7731 if (!TCR_4(__kmp_init_parallel)) {
7734 if (value < __kmp_sys_min_stksize)
7735 value = __kmp_sys_min_stksize;
7736 else if (value > KMP_MAX_STKSIZE)
7737 value = KMP_MAX_STKSIZE;
7739 __kmp_stksize = value;
7741 __kmp_env_stksize = TRUE;
7744 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7749 void __kmp_aux_set_library(
enum library_type arg) {
7750 __kmp_library = arg;
7752 switch (__kmp_library) {
7753 case library_serial: {
7754 KMP_INFORM(LibraryIsSerial);
7756 case library_turnaround:
7757 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set)
7758 __kmp_use_yield = 2;
7760 case library_throughput:
7761 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
7762 __kmp_dflt_blocktime = 200;
7765 KMP_FATAL(UnknownLibraryType, arg);
7771 static kmp_team_t *__kmp_aux_get_team_info(
int &teams_serialized) {
7772 kmp_info_t *thr = __kmp_entry_thread();
7773 teams_serialized = 0;
7774 if (thr->th.th_teams_microtask) {
7775 kmp_team_t *team = thr->th.th_team;
7776 int tlevel = thr->th.th_teams_level;
7777 int ii = team->t.t_level;
7778 teams_serialized = team->t.t_serialized;
7779 int level = tlevel + 1;
7780 KMP_DEBUG_ASSERT(ii >= tlevel);
7781 while (ii > level) {
7782 for (teams_serialized = team->t.t_serialized;
7783 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
7785 if (team->t.t_serialized && (!teams_serialized)) {
7786 team = team->t.t_parent;
7790 team = team->t.t_parent;
7799 int __kmp_aux_get_team_num() {
7801 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7803 if (serialized > 1) {
7806 return team->t.t_master_tid;
7812 int __kmp_aux_get_num_teams() {
7814 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7816 if (serialized > 1) {
7819 return team->t.t_parent->t.t_nproc;
7859 typedef struct kmp_affinity_format_field_t {
7861 const char *long_name;
7864 } kmp_affinity_format_field_t;
7866 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
7867 #if KMP_AFFINITY_SUPPORTED 7868 {
'A',
"thread_affinity",
's'},
7870 {
't',
"team_num",
'd'},
7871 {
'T',
"num_teams",
'd'},
7872 {
'L',
"nesting_level",
'd'},
7873 {
'n',
"thread_num",
'd'},
7874 {
'N',
"num_threads",
'd'},
7875 {
'a',
"ancestor_tnum",
'd'},
7877 {
'P',
"process_id",
'd'},
7878 {
'i',
"native_thread_id",
'd'}};
7881 static int __kmp_aux_capture_affinity_field(
int gtid,
const kmp_info_t *th,
7883 kmp_str_buf_t *field_buffer) {
7884 int rc, format_index, field_value;
7885 const char *width_left, *width_right;
7886 bool pad_zeros, right_justify, parse_long_name, found_valid_name;
7887 static const int FORMAT_SIZE = 20;
7888 char format[FORMAT_SIZE] = {0};
7889 char absolute_short_name = 0;
7891 KMP_DEBUG_ASSERT(gtid >= 0);
7892 KMP_DEBUG_ASSERT(th);
7893 KMP_DEBUG_ASSERT(**ptr ==
'%');
7894 KMP_DEBUG_ASSERT(field_buffer);
7896 __kmp_str_buf_clear(field_buffer);
7903 __kmp_str_buf_cat(field_buffer,
"%", 1);
7914 right_justify =
false;
7916 right_justify =
true;
7920 width_left = width_right = NULL;
7921 if (**ptr >=
'0' && **ptr <=
'9') {
7929 format[format_index++] =
'%';
7931 format[format_index++] =
'-';
7933 format[format_index++] =
'0';
7934 if (width_left && width_right) {
7938 while (i < 8 && width_left < width_right) {
7939 format[format_index++] = *width_left;
7947 found_valid_name =
false;
7948 parse_long_name = (**ptr ==
'{');
7949 if (parse_long_name)
7951 for (
size_t i = 0; i <
sizeof(__kmp_affinity_format_table) /
7952 sizeof(__kmp_affinity_format_table[0]);
7954 char short_name = __kmp_affinity_format_table[i].short_name;
7955 const char *long_name = __kmp_affinity_format_table[i].long_name;
7956 char field_format = __kmp_affinity_format_table[i].field_format;
7957 if (parse_long_name) {
7958 int length = KMP_STRLEN(long_name);
7959 if (strncmp(*ptr, long_name, length) == 0) {
7960 found_valid_name =
true;
7963 }
else if (**ptr == short_name) {
7964 found_valid_name =
true;
7967 if (found_valid_name) {
7968 format[format_index++] = field_format;
7969 format[format_index++] =
'\0';
7970 absolute_short_name = short_name;
7974 if (parse_long_name) {
7976 absolute_short_name = 0;
7984 switch (absolute_short_name) {
7986 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
7989 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
7992 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
7995 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
7998 static const int BUFFER_SIZE = 256;
7999 char buf[BUFFER_SIZE];
8000 __kmp_expand_host_name(buf, BUFFER_SIZE);
8001 rc = __kmp_str_buf_print(field_buffer, format, buf);
8004 rc = __kmp_str_buf_print(field_buffer, format, getpid());
8007 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid());
8010 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
8014 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
8015 rc = __kmp_str_buf_print(field_buffer, format, field_value);
8017 #if KMP_AFFINITY_SUPPORTED 8020 __kmp_str_buf_init(&buf);
8021 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
8022 rc = __kmp_str_buf_print(field_buffer, format, buf.str);
8023 __kmp_str_buf_free(&buf);
8029 rc = __kmp_str_buf_print(field_buffer,
"%s",
"undefined");
8031 if (parse_long_name) {
8040 KMP_ASSERT(format_index <= FORMAT_SIZE);
8050 size_t __kmp_aux_capture_affinity(
int gtid,
const char *format,
8051 kmp_str_buf_t *buffer) {
8052 const char *parse_ptr;
8054 const kmp_info_t *th;
8055 kmp_str_buf_t field;
8057 KMP_DEBUG_ASSERT(buffer);
8058 KMP_DEBUG_ASSERT(gtid >= 0);
8060 __kmp_str_buf_init(&field);
8061 __kmp_str_buf_clear(buffer);
8063 th = __kmp_threads[gtid];
8069 if (parse_ptr == NULL || *parse_ptr ==
'\0') {
8070 parse_ptr = __kmp_affinity_format;
8072 KMP_DEBUG_ASSERT(parse_ptr);
8074 while (*parse_ptr !=
'\0') {
8076 if (*parse_ptr ==
'%') {
8078 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
8079 __kmp_str_buf_catbuf(buffer, &field);
8083 __kmp_str_buf_cat(buffer, parse_ptr, 1);
8088 __kmp_str_buf_free(&field);
8093 void __kmp_aux_display_affinity(
int gtid,
const char *format) {
8095 __kmp_str_buf_init(&buf);
8096 __kmp_aux_capture_affinity(gtid, format, &buf);
8097 __kmp_fprintf(kmp_out,
"%s" KMP_END_OF_LINE, buf.str);
8098 __kmp_str_buf_free(&buf);
8100 #endif // OMP_50_ENABLED 8104 void __kmp_aux_set_blocktime(
int arg, kmp_info_t *thread,
int tid) {
8105 int blocktime = arg;
8111 __kmp_save_internal_controls(thread);
8114 if (blocktime < KMP_MIN_BLOCKTIME)
8115 blocktime = KMP_MIN_BLOCKTIME;
8116 else if (blocktime > KMP_MAX_BLOCKTIME)
8117 blocktime = KMP_MAX_BLOCKTIME;
8119 set__blocktime_team(thread->th.th_team, tid, blocktime);
8120 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
8124 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
8126 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
8127 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
8133 set__bt_set_team(thread->th.th_team, tid, bt_set);
8134 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
8136 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 8137 "bt_intervals=%d, monitor_updates=%d\n",
8138 __kmp_gtid_from_tid(tid, thread->th.th_team),
8139 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
8140 __kmp_monitor_wakeups));
8142 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
8143 __kmp_gtid_from_tid(tid, thread->th.th_team),
8144 thread->th.th_team->t.t_id, tid, blocktime));
8148 void __kmp_aux_set_defaults(
char const *str,
int len) {
8149 if (!__kmp_init_serial) {
8150 __kmp_serial_initialize();
8152 __kmp_env_initialize(str);
8156 || __kmp_display_env || __kmp_display_env_verbose
8166 PACKED_REDUCTION_METHOD_T
8167 __kmp_determine_reduction_method(
8168 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
size_t reduce_size,
8169 void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
8170 kmp_critical_name *lck) {
8181 PACKED_REDUCTION_METHOD_T retval;
8185 KMP_DEBUG_ASSERT(loc);
8186 KMP_DEBUG_ASSERT(lck);
8188 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 8189 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 8190 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 8192 retval = critical_reduce_block;
8195 team_size = __kmp_get_team_num_threads(global_tid);
8196 if (team_size == 1) {
8198 retval = empty_reduce_block;
8202 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8204 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 8206 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 8207 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD || KMP_OS_KFREEBSD 8209 int teamsize_cutoff = 4;
8211 #if KMP_MIC_SUPPORTED 8212 if (__kmp_mic_type != non_mic) {
8213 teamsize_cutoff = 8;
8216 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8217 if (tree_available) {
8218 if (team_size <= teamsize_cutoff) {
8219 if (atomic_available) {
8220 retval = atomic_reduce_block;
8223 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8225 }
else if (atomic_available) {
8226 retval = atomic_reduce_block;
8229 #error "Unknown or unsupported OS" 8230 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || 8233 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 8235 #if KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_HURD || KMP_OS_KFREEBSD 8239 if (atomic_available) {
8240 if (num_vars <= 2) {
8241 retval = atomic_reduce_block;
8247 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8248 if (atomic_available && (num_vars <= 3)) {
8249 retval = atomic_reduce_block;
8250 }
else if (tree_available) {
8251 if ((reduce_size > (9 *
sizeof(kmp_real64))) &&
8252 (reduce_size < (2000 *
sizeof(kmp_real64)))) {
8253 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
8258 #error "Unknown or unsupported OS" 8262 #error "Unknown or unsupported architecture" 8270 if (__kmp_force_reduction_method != reduction_method_not_defined &&
8273 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
8275 int atomic_available, tree_available;
8277 switch ((forced_retval = __kmp_force_reduction_method)) {
8278 case critical_reduce_block:
8282 case atomic_reduce_block:
8283 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8284 if (!atomic_available) {
8285 KMP_WARNING(RedMethodNotSupported,
"atomic");
8286 forced_retval = critical_reduce_block;
8290 case tree_reduce_block:
8291 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8292 if (!tree_available) {
8293 KMP_WARNING(RedMethodNotSupported,
"tree");
8294 forced_retval = critical_reduce_block;
8296 #if KMP_FAST_REDUCTION_BARRIER 8297 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8306 retval = forced_retval;
8309 KA_TRACE(10, (
"reduction method selected=%08x\n", retval));
8311 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 8312 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 8318 kmp_int32 __kmp_get_reduce_method(
void) {
8319 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
8326 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
8330 void __kmp_hard_pause() {
8331 __kmp_pause_status = kmp_hard_paused;
8332 __kmp_internal_end_thread(-1);
8336 void __kmp_resume_if_soft_paused() {
8337 if (__kmp_pause_status == kmp_soft_paused) {
8338 __kmp_pause_status = kmp_not_paused;
8340 for (
int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
8341 kmp_info_t *thread = __kmp_threads[gtid];
8343 kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
8344 if (fl.is_sleeping())
8346 else if (__kmp_try_suspend_mx(thread)) {
8347 __kmp_unlock_suspend_mx(thread);
8350 if (fl.is_sleeping()) {
8353 }
else if (__kmp_try_suspend_mx(thread)) {
8354 __kmp_unlock_suspend_mx(thread);
8366 int __kmp_pause_resource(kmp_pause_status_t level) {
8367 if (level == kmp_not_paused) {
8368 if (__kmp_pause_status == kmp_not_paused) {
8372 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
8373 __kmp_pause_status == kmp_hard_paused);
8374 __kmp_pause_status = kmp_not_paused;
8377 }
else if (level == kmp_soft_paused) {
8378 if (__kmp_pause_status != kmp_not_paused) {
8385 }
else if (level == kmp_hard_paused) {
8386 if (__kmp_pause_status != kmp_not_paused) {
8399 #endif // OMP_50_ENABLED
#define KMP_COUNT_VALUE(name, value)
Adds value to specified timer (name).
KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid)
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the paritioned timers to begin with name.
KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid)
stats_state_e
the states which a thread can be in