15 #include "kmp_affinity.h" 16 #include "kmp_atomic.h" 17 #include "kmp_environment.h" 18 #include "kmp_error.h" 22 #include "kmp_settings.h" 23 #include "kmp_stats.h" 25 #include "kmp_wait_release.h" 26 #include "kmp_wrapper_getpid.h" 27 #include "kmp_dispatch.h" 28 #if KMP_USE_HIER_SCHED 29 #include "kmp_dispatch_hier.h" 33 #include "ompt-specific.h" 37 #define KMP_USE_PRCTL 0 43 #include "tsan_annotations.h" 45 #if defined(KMP_GOMP_COMPAT) 46 char const __kmp_version_alt_comp[] =
47 KMP_VERSION_PREFIX
"alternative compiler support: yes";
50 char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX
"API version: " 62 char const __kmp_version_lock[] =
63 KMP_VERSION_PREFIX
"lock type: run time selectable";
66 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 71 kmp_info_t __kmp_monitor;
76 void __kmp_cleanup(
void);
78 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *,
int tid,
80 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
81 kmp_internal_control_t *new_icvs,
83 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 84 static void __kmp_partition_places(kmp_team_t *team,
85 int update_master_only = 0);
87 static void __kmp_do_serial_initialize(
void);
88 void __kmp_fork_barrier(
int gtid,
int tid);
89 void __kmp_join_barrier(
int gtid);
90 void __kmp_setup_icv_copy(kmp_team_t *team,
int new_nproc,
91 kmp_internal_control_t *new_icvs,
ident_t *loc);
93 #ifdef USE_LOAD_BALANCE 94 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc);
97 static int __kmp_expand_threads(
int nNeed);
99 static int __kmp_unregister_root_other_thread(
int gtid);
101 static void __kmp_unregister_library(
void);
102 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root);
103 kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
108 int __kmp_get_global_thread_id() {
110 kmp_info_t **other_threads;
118 (
"*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
119 __kmp_nth, __kmp_all_nth));
126 if (!TCR_4(__kmp_init_gtid))
129 #ifdef KMP_TDATA_GTID 130 if (TCR_4(__kmp_gtid_mode) >= 3) {
131 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using TDATA\n"));
135 if (TCR_4(__kmp_gtid_mode) >= 2) {
136 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using keyed TLS\n"));
137 return __kmp_gtid_get_specific();
139 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using internal alg.\n"));
141 stack_addr = (
char *)&stack_data;
142 other_threads = __kmp_threads;
155 for (i = 0; i < __kmp_threads_capacity; i++) {
157 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
161 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
162 stack_base = (
char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
166 if (stack_addr <= stack_base) {
167 size_t stack_diff = stack_base - stack_addr;
169 if (stack_diff <= stack_size) {
172 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
180 (
"*** __kmp_get_global_thread_id: internal alg. failed to find " 181 "thread, using TLS\n"));
182 i = __kmp_gtid_get_specific();
192 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
193 KMP_FATAL(StackOverflow, i);
196 stack_base = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
197 if (stack_addr > stack_base) {
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
200 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
204 stack_base - stack_addr);
208 if (__kmp_storage_map) {
209 char *stack_end = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
210 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
211 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
212 other_threads[i]->th.th_info.ds.ds_stacksize,
213 "th_%d stack (refinement)", i);
218 int __kmp_get_global_thread_id_reg() {
221 if (!__kmp_init_serial) {
224 #ifdef KMP_TDATA_GTID 225 if (TCR_4(__kmp_gtid_mode) >= 3) {
226 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using TDATA\n"));
230 if (TCR_4(__kmp_gtid_mode) >= 2) {
231 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
232 gtid = __kmp_gtid_get_specific();
235 (
"*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
236 gtid = __kmp_get_global_thread_id();
240 if (gtid == KMP_GTID_DNE) {
242 (
"__kmp_get_global_thread_id_reg: Encountered new root thread. " 243 "Registering a new gtid.\n"));
244 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
245 if (!__kmp_init_serial) {
246 __kmp_do_serial_initialize();
247 gtid = __kmp_gtid_get_specific();
249 gtid = __kmp_register_root(FALSE);
251 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
255 KMP_DEBUG_ASSERT(gtid >= 0);
261 void __kmp_check_stack_overlap(kmp_info_t *th) {
263 char *stack_beg = NULL;
264 char *stack_end = NULL;
267 KA_TRACE(10, (
"__kmp_check_stack_overlap: called\n"));
268 if (__kmp_storage_map) {
269 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
270 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
272 gtid = __kmp_gtid_from_thread(th);
274 if (gtid == KMP_GTID_MONITOR) {
275 __kmp_print_storage_map_gtid(
276 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
277 "th_%s stack (%s)",
"mon",
278 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
280 __kmp_print_storage_map_gtid(
281 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
282 "th_%d stack (%s)", gtid,
283 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
289 gtid = __kmp_gtid_from_thread(th);
290 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
292 (
"__kmp_check_stack_overlap: performing extensive checking\n"));
293 if (stack_beg == NULL) {
294 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
295 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
298 for (f = 0; f < __kmp_threads_capacity; f++) {
299 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
301 if (f_th && f_th != th) {
302 char *other_stack_end =
303 (
char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
304 char *other_stack_beg =
305 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
306 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
307 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
310 if (__kmp_storage_map)
311 __kmp_print_storage_map_gtid(
312 -1, other_stack_beg, other_stack_end,
313 (
size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
314 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
316 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
322 KA_TRACE(10, (
"__kmp_check_stack_overlap: returning\n"));
327 void __kmp_infinite_loop(
void) {
328 static int done = FALSE;
335 #define MAX_MESSAGE 512 337 void __kmp_print_storage_map_gtid(
int gtid,
void *p1,
void *p2,
size_t size,
338 char const *format, ...) {
339 char buffer[MAX_MESSAGE];
342 va_start(ap, format);
343 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP storage map: %p %p%8lu %s\n", p1,
344 p2, (
unsigned long)size, format);
345 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
346 __kmp_vprintf(kmp_err, buffer, ap);
347 #if KMP_PRINT_DATA_PLACEMENT 350 if (p1 <= p2 && (
char *)p2 - (
char *)p1 == size) {
351 if (__kmp_storage_map_verbose) {
352 node = __kmp_get_host_node(p1);
354 __kmp_storage_map_verbose = FALSE;
358 int localProc = __kmp_get_cpu_from_gtid(gtid);
360 const int page_size = KMP_GET_PAGE_SIZE();
362 p1 = (
void *)((
size_t)p1 & ~((size_t)page_size - 1));
363 p2 = (
void *)(((
size_t)p2 - 1) & ~((
size_t)page_size - 1));
365 __kmp_printf_no_lock(
" GTID %d localNode %d\n", gtid,
368 __kmp_printf_no_lock(
" GTID %d\n", gtid);
377 (
char *)p1 += page_size;
378 }
while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
379 __kmp_printf_no_lock(
" %p-%p memNode %d\n", last, (
char *)p1 - 1,
383 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p1,
384 (
char *)p1 + (page_size - 1),
385 __kmp_get_host_node(p1));
387 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p2,
388 (
char *)p2 + (page_size - 1),
389 __kmp_get_host_node(p2));
395 __kmp_printf_no_lock(
" %s\n", KMP_I18N_STR(StorageMapWarning));
398 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
401 void __kmp_warn(
char const *format, ...) {
402 char buffer[MAX_MESSAGE];
405 if (__kmp_generate_warnings == kmp_warnings_off) {
409 va_start(ap, format);
411 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP warning: %s\n", format);
412 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
413 __kmp_vprintf(kmp_err, buffer, ap);
414 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
419 void __kmp_abort_process() {
421 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
423 if (__kmp_debug_buf) {
424 __kmp_dump_debug_buffer();
427 if (KMP_OS_WINDOWS) {
430 __kmp_global.g.g_abort = SIGABRT;
447 __kmp_infinite_loop();
448 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
452 void __kmp_abort_thread(
void) {
455 __kmp_infinite_loop();
461 static void __kmp_print_thread_storage_map(kmp_info_t *thr,
int gtid) {
462 __kmp_print_storage_map_gtid(gtid, thr, thr + 1,
sizeof(kmp_info_t),
"th_%d",
465 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
466 sizeof(kmp_desc_t),
"th_%d.th_info", gtid);
468 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
469 sizeof(kmp_local_t),
"th_%d.th_local", gtid);
471 __kmp_print_storage_map_gtid(
472 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
473 sizeof(kmp_balign_t) * bs_last_barrier,
"th_%d.th_bar", gtid);
475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
476 &thr->th.th_bar[bs_plain_barrier + 1],
477 sizeof(kmp_balign_t),
"th_%d.th_bar[plain]",
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
481 &thr->th.th_bar[bs_forkjoin_barrier + 1],
482 sizeof(kmp_balign_t),
"th_%d.th_bar[forkjoin]",
485 #if KMP_FAST_REDUCTION_BARRIER 486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
487 &thr->th.th_bar[bs_reduction_barrier + 1],
488 sizeof(kmp_balign_t),
"th_%d.th_bar[reduction]",
490 #endif // KMP_FAST_REDUCTION_BARRIER 496 static void __kmp_print_team_storage_map(
const char *header, kmp_team_t *team,
497 int team_id,
int num_thr) {
498 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
499 __kmp_print_storage_map_gtid(-1, team, team + 1,
sizeof(kmp_team_t),
"%s_%d",
502 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
503 &team->t.t_bar[bs_last_barrier],
504 sizeof(kmp_balign_team_t) * bs_last_barrier,
505 "%s_%d.t_bar", header, team_id);
507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
508 &team->t.t_bar[bs_plain_barrier + 1],
509 sizeof(kmp_balign_team_t),
"%s_%d.t_bar[plain]",
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
513 &team->t.t_bar[bs_forkjoin_barrier + 1],
514 sizeof(kmp_balign_team_t),
515 "%s_%d.t_bar[forkjoin]", header, team_id);
517 #if KMP_FAST_REDUCTION_BARRIER 518 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
519 &team->t.t_bar[bs_reduction_barrier + 1],
520 sizeof(kmp_balign_team_t),
521 "%s_%d.t_bar[reduction]", header, team_id);
522 #endif // KMP_FAST_REDUCTION_BARRIER 524 __kmp_print_storage_map_gtid(
525 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
526 sizeof(kmp_disp_t) * num_thr,
"%s_%d.t_dispatch", header, team_id);
528 __kmp_print_storage_map_gtid(
529 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
530 sizeof(kmp_info_t *) * num_thr,
"%s_%d.t_threads", header, team_id);
532 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
533 &team->t.t_disp_buffer[num_disp_buff],
534 sizeof(dispatch_shared_info_t) * num_disp_buff,
535 "%s_%d.t_disp_buffer", header, team_id);
537 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data,
538 sizeof(kmp_taskq_t),
"%s_%d.t_taskq", header,
542 static void __kmp_init_allocator() {
544 __kmp_init_memkind();
547 static void __kmp_fini_allocator() {
549 __kmp_fini_memkind();
558 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
560 __kmp_init_bootstrap_lock(lck);
563 static void __kmp_reset_locks_on_process_detach(
int gtid_req) {
581 for (i = 0; i < __kmp_threads_capacity; ++i) {
584 kmp_info_t *th = __kmp_threads[i];
587 int gtid = th->th.th_info.ds.ds_gtid;
588 if (gtid == gtid_req)
593 int alive = __kmp_is_thread_alive(th, &exit_val);
598 if (thread_count == 0)
604 __kmp_reset_lock(&__kmp_forkjoin_lock);
606 __kmp_reset_lock(&__kmp_stdio_lock);
610 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
615 case DLL_PROCESS_ATTACH:
616 KA_TRACE(10, (
"DllMain: PROCESS_ATTACH\n"));
620 case DLL_PROCESS_DETACH:
621 KA_TRACE(10, (
"DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
623 if (lpReserved != NULL) {
649 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
652 __kmp_internal_end_library(__kmp_gtid_get_specific());
656 case DLL_THREAD_ATTACH:
657 KA_TRACE(10, (
"DllMain: THREAD_ATTACH\n"));
663 case DLL_THREAD_DETACH:
664 KA_TRACE(10, (
"DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
666 __kmp_internal_end_thread(__kmp_gtid_get_specific());
678 int __kmp_change_library(
int status) {
681 old_status = __kmp_yield_init &
685 __kmp_yield_init |= 1;
687 __kmp_yield_init &= ~1;
695 void __kmp_parallel_deo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
696 int gtid = *gtid_ref;
697 #ifdef BUILD_PARALLEL_ORDERED 698 kmp_team_t *team = __kmp_team_from_gtid(gtid);
701 if (__kmp_env_consistency_check) {
702 if (__kmp_threads[gtid]->th.th_root->r.r_active)
703 #if KMP_USE_DYNAMIC_LOCK 704 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
706 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
709 #ifdef BUILD_PARALLEL_ORDERED 710 if (!team->t.t_serialized) {
712 KMP_WAIT_YIELD(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid),
720 void __kmp_parallel_dxo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
721 int gtid = *gtid_ref;
722 #ifdef BUILD_PARALLEL_ORDERED 723 int tid = __kmp_tid_from_gtid(gtid);
724 kmp_team_t *team = __kmp_team_from_gtid(gtid);
727 if (__kmp_env_consistency_check) {
728 if (__kmp_threads[gtid]->th.th_root->r.r_active)
729 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
731 #ifdef BUILD_PARALLEL_ORDERED 732 if (!team->t.t_serialized) {
737 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
747 int __kmp_enter_single(
int gtid,
ident_t *id_ref,
int push_ws) {
752 if (!TCR_4(__kmp_init_parallel))
753 __kmp_parallel_initialize();
756 __kmp_resume_if_soft_paused();
759 th = __kmp_threads[gtid];
760 team = th->th.th_team;
763 th->th.th_ident = id_ref;
765 if (team->t.t_serialized) {
768 kmp_int32 old_this = th->th.th_local.this_construct;
770 ++th->th.th_local.this_construct;
774 if (team->t.t_construct == old_this) {
775 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
776 th->th.th_local.this_construct);
779 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
780 KMP_MASTER_GTID(gtid) &&
782 th->th.th_teams_microtask == NULL &&
784 team->t.t_active_level ==
786 __kmp_itt_metadata_single(id_ref);
791 if (__kmp_env_consistency_check) {
792 if (status && push_ws) {
793 __kmp_push_workshare(gtid, ct_psingle, id_ref);
795 __kmp_check_workshare(gtid, ct_psingle, id_ref);
800 __kmp_itt_single_start(gtid);
806 void __kmp_exit_single(
int gtid) {
808 __kmp_itt_single_end(gtid);
810 if (__kmp_env_consistency_check)
811 __kmp_pop_workshare(gtid, ct_psingle, NULL);
820 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
821 int master_tid,
int set_nthreads
829 KMP_DEBUG_ASSERT(__kmp_init_serial);
830 KMP_DEBUG_ASSERT(root && parent_team);
834 new_nthreads = set_nthreads;
835 if (!get__dynamic_2(parent_team, master_tid)) {
838 #ifdef USE_LOAD_BALANCE 839 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
840 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
841 if (new_nthreads == 1) {
842 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced " 843 "reservation to 1 thread\n",
847 if (new_nthreads < set_nthreads) {
848 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced " 849 "reservation to %d threads\n",
850 master_tid, new_nthreads));
854 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
855 new_nthreads = __kmp_avail_proc - __kmp_nth +
856 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
857 if (new_nthreads <= 1) {
858 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced " 859 "reservation to 1 thread\n",
863 if (new_nthreads < set_nthreads) {
864 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced " 865 "reservation to %d threads\n",
866 master_tid, new_nthreads));
868 new_nthreads = set_nthreads;
870 }
else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
871 if (set_nthreads > 2) {
872 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
873 new_nthreads = (new_nthreads % set_nthreads) + 1;
874 if (new_nthreads == 1) {
875 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced " 876 "reservation to 1 thread\n",
880 if (new_nthreads < set_nthreads) {
881 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced " 882 "reservation to %d threads\n",
883 master_tid, new_nthreads));
891 if (__kmp_nth + new_nthreads -
892 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
894 int tl_nthreads = __kmp_max_nth - __kmp_nth +
895 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
896 if (tl_nthreads <= 0) {
901 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
902 __kmp_reserve_warn = 1;
903 __kmp_msg(kmp_ms_warning,
904 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
905 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
907 if (tl_nthreads == 1) {
908 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 909 "reduced reservation to 1 thread\n",
913 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 914 "reservation to %d threads\n",
915 master_tid, tl_nthreads));
916 new_nthreads = tl_nthreads;
920 if (root->r.r_cg_nthreads + new_nthreads -
921 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
923 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads +
924 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
925 if (tl_nthreads <= 0) {
930 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
931 __kmp_reserve_warn = 1;
932 __kmp_msg(kmp_ms_warning,
933 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
934 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
936 if (tl_nthreads == 1) {
937 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 938 "reduced reservation to 1 thread\n",
942 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 943 "reservation to %d threads\n",
944 master_tid, tl_nthreads));
945 new_nthreads = tl_nthreads;
951 capacity = __kmp_threads_capacity;
952 if (TCR_PTR(__kmp_threads[0]) == NULL) {
955 if (__kmp_nth + new_nthreads -
956 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
959 int slotsRequired = __kmp_nth + new_nthreads -
960 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
962 int slotsAdded = __kmp_expand_threads(slotsRequired);
963 if (slotsAdded < slotsRequired) {
965 new_nthreads -= (slotsRequired - slotsAdded);
966 KMP_ASSERT(new_nthreads >= 1);
969 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
970 __kmp_reserve_warn = 1;
971 if (__kmp_tp_cached) {
972 __kmp_msg(kmp_ms_warning,
973 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
974 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
975 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
977 __kmp_msg(kmp_ms_warning,
978 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
979 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
986 if (new_nthreads == 1) {
988 (
"__kmp_reserve_threads: T#%d serializing team after reclaiming " 989 "dead roots and rechecking; requested %d threads\n",
990 __kmp_get_gtid(), set_nthreads));
992 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d allocating %d threads; requested" 994 __kmp_get_gtid(), new_nthreads, set_nthreads));
1003 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
1004 kmp_info_t *master_th,
int master_gtid) {
1008 KA_TRACE(10, (
"__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
1009 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
1013 master_th->th.th_info.ds.ds_tid = 0;
1014 master_th->th.th_team = team;
1015 master_th->th.th_team_nproc = team->t.t_nproc;
1016 master_th->th.th_team_master = master_th;
1017 master_th->th.th_team_serialized = FALSE;
1018 master_th->th.th_dispatch = &team->t.t_dispatch[0];
1021 #if KMP_NESTED_HOT_TEAMS 1023 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1026 int level = team->t.t_active_level - 1;
1027 if (master_th->th.th_teams_microtask) {
1028 if (master_th->th.th_teams_size.nteams > 1) {
1032 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1033 master_th->th.th_teams_level == team->t.t_level) {
1038 if (level < __kmp_hot_teams_max_level) {
1039 if (hot_teams[level].hot_team) {
1041 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
1045 hot_teams[level].hot_team = team;
1046 hot_teams[level].hot_team_nth = team->t.t_nproc;
1053 use_hot_team = team == root->r.r_hot_team;
1055 if (!use_hot_team) {
1058 team->t.t_threads[0] = master_th;
1059 __kmp_initialize_info(master_th, team, 0, master_gtid);
1062 for (i = 1; i < team->t.t_nproc; i++) {
1065 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1066 team->t.t_threads[i] = thr;
1067 KMP_DEBUG_ASSERT(thr);
1068 KMP_DEBUG_ASSERT(thr->th.th_team == team);
1070 KA_TRACE(20, (
"__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 1071 "T#%d(%d:%d) join =%llu, plain=%llu\n",
1072 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
1073 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
1074 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
1075 team->t.t_bar[bs_plain_barrier].b_arrived));
1077 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1078 thr->th.th_teams_level = master_th->th.th_teams_level;
1079 thr->th.th_teams_size = master_th->th.th_teams_size;
1083 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1084 for (b = 0; b < bs_last_barrier; ++b) {
1085 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1086 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
1088 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1094 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1095 __kmp_partition_places(team);
1100 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1101 for (i = 0; i < team->t.t_nproc; i++) {
1102 kmp_info_t *thr = team->t.t_threads[i];
1103 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1104 thr->th.th_prev_level != team->t.t_level) {
1105 team->t.t_display_affinity = 1;
1115 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1119 inline static void propagateFPControl(kmp_team_t *team) {
1120 if (__kmp_inherit_fp_control) {
1121 kmp_int16 x87_fpu_control_word;
1125 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1126 __kmp_store_mxcsr(&mxcsr);
1127 mxcsr &= KMP_X86_MXCSR_MASK;
1138 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1139 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1142 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1146 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1152 inline static void updateHWFPControl(kmp_team_t *team) {
1153 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1156 kmp_int16 x87_fpu_control_word;
1158 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1159 __kmp_store_mxcsr(&mxcsr);
1160 mxcsr &= KMP_X86_MXCSR_MASK;
1162 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1163 __kmp_clear_x87_fpu_status_word();
1164 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1167 if (team->t.t_mxcsr != mxcsr) {
1168 __kmp_load_mxcsr(&team->t.t_mxcsr);
1173 #define propagateFPControl(x) ((void)0) 1174 #define updateHWFPControl(x) ((void)0) 1177 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
1182 void __kmp_serialized_parallel(
ident_t *loc, kmp_int32 global_tid) {
1183 kmp_info_t *this_thr;
1184 kmp_team_t *serial_team;
1186 KC_TRACE(10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid));
1193 if (!TCR_4(__kmp_init_parallel))
1194 __kmp_parallel_initialize();
1197 __kmp_resume_if_soft_paused();
1200 this_thr = __kmp_threads[global_tid];
1201 serial_team = this_thr->th.th_serial_team;
1204 KMP_DEBUG_ASSERT(serial_team);
1207 if (__kmp_tasking_mode != tskm_immediate_exec) {
1209 this_thr->th.th_task_team ==
1210 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1211 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1213 KA_TRACE(20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1214 "team %p, new task_team = NULL\n",
1215 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1216 this_thr->th.th_task_team = NULL;
1220 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1221 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1222 proc_bind = proc_bind_false;
1223 }
else if (proc_bind == proc_bind_default) {
1226 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1229 this_thr->th.th_set_proc_bind = proc_bind_default;
1233 ompt_data_t ompt_parallel_data = ompt_data_none;
1234 ompt_data_t *implicit_task_data;
1235 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1236 if (ompt_enabled.enabled &&
1237 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1239 ompt_task_info_t *parent_task_info;
1240 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1242 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1243 if (ompt_enabled.ompt_callback_parallel_begin) {
1246 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1247 &(parent_task_info->task_data), &(parent_task_info->frame),
1248 &ompt_parallel_data, team_size, ompt_parallel_invoker_program,
1252 #endif // OMPT_SUPPORT 1254 if (this_thr->th.th_team != serial_team) {
1256 int level = this_thr->th.th_team->t.t_level;
1258 if (serial_team->t.t_serialized) {
1261 kmp_team_t *new_team;
1263 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1265 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1272 &this_thr->th.th_current_task->td_icvs,
1273 0 USE_NESTED_HOT_ARG(NULL));
1274 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1275 KMP_ASSERT(new_team);
1278 new_team->t.t_threads[0] = this_thr;
1279 new_team->t.t_parent = this_thr->th.th_team;
1280 serial_team = new_team;
1281 this_thr->th.th_serial_team = serial_team;
1285 (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1286 global_tid, serial_team));
1294 (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1295 global_tid, serial_team));
1299 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1300 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1301 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1302 serial_team->t.t_ident = loc;
1303 serial_team->t.t_serialized = 1;
1304 serial_team->t.t_nproc = 1;
1305 serial_team->t.t_parent = this_thr->th.th_team;
1306 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1307 this_thr->th.th_team = serial_team;
1308 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1310 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1311 this_thr->th.th_current_task));
1312 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1313 this_thr->th.th_current_task->td_flags.executing = 0;
1315 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1320 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1321 &this_thr->th.th_current_task->td_parent->td_icvs);
1325 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1326 this_thr->th.th_current_task->td_icvs.nproc =
1327 __kmp_nested_nth.nth[level + 1];
1331 if (__kmp_nested_proc_bind.used &&
1332 (level + 1 < __kmp_nested_proc_bind.used)) {
1333 this_thr->th.th_current_task->td_icvs.proc_bind =
1334 __kmp_nested_proc_bind.bind_types[level + 1];
1339 serial_team->t.t_pkfn = (microtask_t)(~0);
1341 this_thr->th.th_info.ds.ds_tid = 0;
1344 this_thr->th.th_team_nproc = 1;
1345 this_thr->th.th_team_master = this_thr;
1346 this_thr->th.th_team_serialized = 1;
1348 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1349 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1351 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator;
1354 propagateFPControl(serial_team);
1357 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1358 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1359 serial_team->t.t_dispatch->th_disp_buffer =
1360 (dispatch_private_info_t *)__kmp_allocate(
1361 sizeof(dispatch_private_info_t));
1363 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1370 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1371 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1372 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1373 ++serial_team->t.t_serialized;
1374 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1377 int level = this_thr->th.th_team->t.t_level;
1380 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1381 this_thr->th.th_current_task->td_icvs.nproc =
1382 __kmp_nested_nth.nth[level + 1];
1384 serial_team->t.t_level++;
1385 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level " 1386 "of serial team %p to %d\n",
1387 global_tid, serial_team, serial_team->t.t_level));
1390 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1392 dispatch_private_info_t *disp_buffer =
1393 (dispatch_private_info_t *)__kmp_allocate(
1394 sizeof(dispatch_private_info_t));
1395 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1396 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1398 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1403 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
1409 if (__kmp_display_affinity) {
1410 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1411 this_thr->th.th_prev_num_threads != 1) {
1413 __kmp_aux_display_affinity(global_tid, NULL);
1414 this_thr->th.th_prev_level = serial_team->t.t_level;
1415 this_thr->th.th_prev_num_threads = 1;
1420 if (__kmp_env_consistency_check)
1421 __kmp_push_parallel(global_tid, NULL);
1423 serial_team->t.ompt_team_info.master_return_address = codeptr;
1424 if (ompt_enabled.enabled &&
1425 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1426 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1428 ompt_lw_taskteam_t lw_taskteam;
1429 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1430 &ompt_parallel_data, codeptr);
1432 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1436 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr);
1437 if (ompt_enabled.ompt_callback_implicit_task) {
1438 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1439 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
1440 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), ompt_task_implicit);
1441 OMPT_CUR_TASK_INFO(this_thr)
1442 ->thread_num = __kmp_tid_from_gtid(global_tid);
1446 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1447 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1454 int __kmp_fork_call(
ident_t *loc,
int gtid,
1455 enum fork_context_e call_context,
1456 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1458 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1467 int master_this_cons;
1469 kmp_team_t *parent_team;
1470 kmp_info_t *master_th;
1474 int master_set_numthreads;
1480 #if KMP_NESTED_HOT_TEAMS 1481 kmp_hot_team_ptr_t **p_hot_teams;
1484 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
1487 KA_TRACE(20, (
"__kmp_fork_call: enter T#%d\n", gtid));
1488 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1491 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1493 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1494 __kmp_stkpadding += (short)((kmp_int64)dummy);
1500 if (!TCR_4(__kmp_init_parallel))
1501 __kmp_parallel_initialize();
1504 __kmp_resume_if_soft_paused();
1508 master_th = __kmp_threads[gtid];
1510 parent_team = master_th->th.th_team;
1511 master_tid = master_th->th.th_info.ds.ds_tid;
1512 master_this_cons = master_th->th.th_local.this_construct;
1513 root = master_th->th.th_root;
1514 master_active = root->r.r_active;
1515 master_set_numthreads = master_th->th.th_set_nproc;
1518 ompt_data_t ompt_parallel_data = ompt_data_none;
1519 ompt_data_t *parent_task_data;
1520 ompt_frame_t *ompt_frame;
1521 ompt_data_t *implicit_task_data;
1522 void *return_address = NULL;
1524 if (ompt_enabled.enabled) {
1525 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1527 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
1532 level = parent_team->t.t_level;
1534 active_level = parent_team->t.t_active_level;
1537 teams_level = master_th->th.th_teams_level;
1539 #if KMP_NESTED_HOT_TEAMS 1540 p_hot_teams = &master_th->th.th_hot_teams;
1541 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1542 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1543 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1544 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1546 (*p_hot_teams)[0].hot_team_nth = 1;
1551 if (ompt_enabled.enabled) {
1552 if (ompt_enabled.ompt_callback_parallel_begin) {
1553 int team_size = master_set_numthreads
1554 ? master_set_numthreads
1555 : get__nproc_2(parent_team, master_tid);
1556 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1557 parent_task_data, ompt_frame, &ompt_parallel_data, team_size,
1558 OMPT_INVOKER(call_context), return_address);
1560 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1564 master_th->th.th_ident = loc;
1567 if (master_th->th.th_teams_microtask && ap &&
1568 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1572 parent_team->t.t_ident = loc;
1573 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1574 parent_team->t.t_argc = argc;
1575 argv = (
void **)parent_team->t.t_argv;
1576 for (i = argc - 1; i >= 0; --i)
1578 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1579 *argv++ = va_arg(*ap,
void *);
1581 *argv++ = va_arg(ap,
void *);
1584 if (parent_team == master_th->th.th_serial_team) {
1587 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1590 parent_team->t.t_serialized--;
1593 void **exit_runtime_p;
1595 ompt_lw_taskteam_t lw_taskteam;
1597 if (ompt_enabled.enabled) {
1598 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1599 &ompt_parallel_data, return_address);
1600 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
1602 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1606 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1607 if (ompt_enabled.ompt_callback_implicit_task) {
1608 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1609 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1610 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit);
1611 OMPT_CUR_TASK_INFO(master_th)
1612 ->thread_num = __kmp_tid_from_gtid(gtid);
1616 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1618 exit_runtime_p = &dummy;
1623 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1624 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1625 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1634 *exit_runtime_p = NULL;
1635 if (ompt_enabled.enabled) {
1636 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;
1637 if (ompt_enabled.ompt_callback_implicit_task) {
1638 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1639 ompt_scope_end, NULL, implicit_task_data, 1,
1640 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1642 __ompt_lw_taskteam_unlink(master_th);
1644 if (ompt_enabled.ompt_callback_parallel_end) {
1645 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1646 OMPT_CUR_TEAM_DATA(master_th), OMPT_CUR_TASK_DATA(master_th),
1647 OMPT_INVOKER(call_context), return_address);
1649 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1655 parent_team->t.t_pkfn = microtask;
1656 parent_team->t.t_invoke = invoker;
1657 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1658 parent_team->t.t_active_level++;
1659 parent_team->t.t_level++;
1661 parent_team->t.t_def_allocator = master_th->th.th_def_allocator;
1665 if (master_set_numthreads) {
1666 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1668 kmp_info_t **other_threads = parent_team->t.t_threads;
1669 parent_team->t.t_nproc = master_set_numthreads;
1670 for (i = 0; i < master_set_numthreads; ++i) {
1671 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1675 master_th->th.th_set_nproc = 0;
1679 if (__kmp_debugging) {
1680 int nth = __kmp_omp_num_threads(loc);
1682 master_set_numthreads = nth;
1687 KF_TRACE(10, (
"__kmp_fork_call: before internal fork: root=%p, team=%p, " 1688 "master_th=%p, gtid=%d\n",
1689 root, parent_team, master_th, gtid));
1690 __kmp_internal_fork(loc, gtid, parent_team);
1691 KF_TRACE(10, (
"__kmp_fork_call: after internal fork: root=%p, team=%p, " 1692 "master_th=%p, gtid=%d\n",
1693 root, parent_team, master_th, gtid));
1696 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1697 parent_team->t.t_id, parent_team->t.t_pkfn));
1699 if (!parent_team->t.t_invoke(gtid)) {
1700 KMP_ASSERT2(0,
"cannot invoke microtask for MASTER thread");
1702 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1703 parent_team->t.t_id, parent_team->t.t_pkfn));
1706 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
1713 if (__kmp_tasking_mode != tskm_immediate_exec) {
1714 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1715 parent_team->t.t_task_team[master_th->th.th_task_state]);
1719 if (parent_team->t.t_active_level >=
1720 master_th->th.th_current_task->td_icvs.max_active_levels) {
1724 int enter_teams = ((ap == NULL && active_level == 0) ||
1725 (ap && teams_level > 0 && teams_level == level));
1728 master_set_numthreads
1729 ? master_set_numthreads
1738 if ((!get__nested(master_th) && (root->r.r_in_parallel
1743 (__kmp_library == library_serial)) {
1744 KC_TRACE(10, (
"__kmp_fork_call: T#%d serializing team; requested %d" 1752 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1753 nthreads = __kmp_reserve_threads(
1754 root, parent_team, master_tid, nthreads
1765 if (nthreads == 1) {
1769 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1773 KMP_DEBUG_ASSERT(nthreads > 0);
1776 master_th->th.th_set_nproc = 0;
1779 if (nthreads == 1) {
1781 #if KMP_OS_LINUX && \ 1782 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1785 void **args = (
void **)KMP_ALLOCA(argc *
sizeof(
void *));
1790 (
"__kmp_fork_call: T#%d serializing parallel region\n", gtid));
1794 if (call_context == fork_context_intel) {
1796 master_th->th.th_serial_team->t.t_ident = loc;
1800 master_th->th.th_serial_team->t.t_level--;
1805 void **exit_runtime_p;
1806 ompt_task_info_t *task_info;
1808 ompt_lw_taskteam_t lw_taskteam;
1810 if (ompt_enabled.enabled) {
1811 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1812 &ompt_parallel_data, return_address);
1814 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1817 task_info = OMPT_CUR_TASK_INFO(master_th);
1818 exit_runtime_p = &(task_info->frame.exit_frame.ptr);
1819 if (ompt_enabled.ompt_callback_implicit_task) {
1820 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1821 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1822 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit);
1823 OMPT_CUR_TASK_INFO(master_th)
1824 ->thread_num = __kmp_tid_from_gtid(gtid);
1828 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1830 exit_runtime_p = &dummy;
1835 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1836 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1837 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1838 parent_team->t.t_argv
1847 if (ompt_enabled.enabled) {
1848 exit_runtime_p = NULL;
1849 if (ompt_enabled.ompt_callback_implicit_task) {
1850 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1851 ompt_scope_end, NULL, &(task_info->task_data), 1,
1852 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1855 __ompt_lw_taskteam_unlink(master_th);
1856 if (ompt_enabled.ompt_callback_parallel_end) {
1857 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1858 OMPT_CUR_TEAM_DATA(master_th), parent_task_data,
1859 OMPT_INVOKER(call_context), return_address);
1861 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1864 }
else if (microtask == (microtask_t)__kmp_teams_master) {
1865 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1866 master_th->th.th_serial_team);
1867 team = master_th->th.th_team;
1869 team->t.t_invoke = invoker;
1870 __kmp_alloc_argv_entries(argc, team, TRUE);
1871 team->t.t_argc = argc;
1872 argv = (
void **)team->t.t_argv;
1874 for (i = argc - 1; i >= 0; --i)
1876 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1877 *argv++ = va_arg(*ap,
void *);
1879 *argv++ = va_arg(ap,
void *);
1882 for (i = 0; i < argc; ++i)
1884 argv[i] = parent_team->t.t_argv[i];
1894 for (i = argc - 1; i >= 0; --i)
1896 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1897 *argv++ = va_arg(*ap,
void *);
1899 *argv++ = va_arg(ap,
void *);
1905 void **exit_runtime_p;
1906 ompt_task_info_t *task_info;
1908 ompt_lw_taskteam_t lw_taskteam;
1910 if (ompt_enabled.enabled) {
1911 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1912 &ompt_parallel_data, return_address);
1913 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1915 task_info = OMPT_CUR_TASK_INFO(master_th);
1916 exit_runtime_p = &(task_info->frame.exit_frame.ptr);
1919 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1920 if (ompt_enabled.ompt_callback_implicit_task) {
1921 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1922 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1923 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit);
1924 OMPT_CUR_TASK_INFO(master_th)
1925 ->thread_num = __kmp_tid_from_gtid(gtid);
1929 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1931 exit_runtime_p = &dummy;
1936 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1937 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1938 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1947 if (ompt_enabled.enabled) {
1948 *exit_runtime_p = NULL;
1949 if (ompt_enabled.ompt_callback_implicit_task) {
1950 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1951 ompt_scope_end, NULL, &(task_info->task_data), 1,
1952 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1955 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1956 __ompt_lw_taskteam_unlink(master_th);
1957 if (ompt_enabled.ompt_callback_parallel_end) {
1958 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1959 &ompt_parallel_data, parent_task_data,
1960 OMPT_INVOKER(call_context), return_address);
1962 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1968 }
else if (call_context == fork_context_gnu) {
1970 ompt_lw_taskteam_t lwt;
1971 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1974 lwt.ompt_task_info.frame.exit_frame = ompt_data_none;
1975 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1980 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1983 KMP_ASSERT2(call_context < fork_context_last,
1984 "__kmp_fork_call: unknown fork_context parameter");
1987 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1994 KF_TRACE(10, (
"__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1995 "curtask=%p, curtask_max_aclevel=%d\n",
1996 parent_team->t.t_active_level, master_th,
1997 master_th->th.th_current_task,
1998 master_th->th.th_current_task->td_icvs.max_active_levels));
2002 master_th->th.th_current_task->td_flags.executing = 0;
2005 if (!master_th->th.th_teams_microtask || level > teams_level)
2009 KMP_ATOMIC_INC(&root->r.r_in_parallel);
2013 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
2014 if ((level + 1 < __kmp_nested_nth.used) &&
2015 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
2016 nthreads_icv = __kmp_nested_nth.nth[level + 1];
2023 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
2024 kmp_proc_bind_t proc_bind_icv =
2026 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
2027 proc_bind = proc_bind_false;
2029 if (proc_bind == proc_bind_default) {
2032 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
2038 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2039 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2040 master_th->th.th_current_task->td_icvs.proc_bind)) {
2041 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2046 master_th->th.th_set_proc_bind = proc_bind_default;
2049 if ((nthreads_icv > 0)
2051 || (proc_bind_icv != proc_bind_default)
2054 kmp_internal_control_t new_icvs;
2055 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2056 new_icvs.next = NULL;
2057 if (nthreads_icv > 0) {
2058 new_icvs.nproc = nthreads_icv;
2062 if (proc_bind_icv != proc_bind_default) {
2063 new_icvs.proc_bind = proc_bind_icv;
2068 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2069 team = __kmp_allocate_team(root, nthreads, nthreads,
2076 &new_icvs, argc USE_NESTED_HOT_ARG(master_th));
2079 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2080 team = __kmp_allocate_team(root, nthreads, nthreads,
2087 &master_th->th.th_current_task->td_icvs,
2088 argc USE_NESTED_HOT_ARG(master_th));
2091 10, (
"__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
2094 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2095 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2096 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2097 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2098 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
2100 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2103 KMP_CHECK_UPDATE(team->t.t_invoke, invoker);
2106 if (!master_th->th.th_teams_microtask || level > teams_level) {
2108 int new_level = parent_team->t.t_level + 1;
2109 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2110 new_level = parent_team->t.t_active_level + 1;
2111 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2115 int new_level = parent_team->t.t_level;
2116 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2117 new_level = parent_team->t.t_active_level;
2118 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2121 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
2123 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
2126 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
2129 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2133 propagateFPControl(team);
2135 if (__kmp_tasking_mode != tskm_immediate_exec) {
2138 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2139 parent_team->t.t_task_team[master_th->th.th_task_state]);
2140 KA_TRACE(20, (
"__kmp_fork_call: Master T#%d pushing task_team %p / team " 2141 "%p, new task_team %p / team %p\n",
2142 __kmp_gtid_from_thread(master_th),
2143 master_th->th.th_task_team, parent_team,
2144 team->t.t_task_team[master_th->th.th_task_state], team));
2146 if (active_level || master_th->th.th_task_team) {
2148 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2149 if (master_th->th.th_task_state_top >=
2150 master_th->th.th_task_state_stack_sz) {
2151 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2152 kmp_uint8 *old_stack, *new_stack;
2154 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2155 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2156 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2158 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2162 old_stack = master_th->th.th_task_state_memo_stack;
2163 master_th->th.th_task_state_memo_stack = new_stack;
2164 master_th->th.th_task_state_stack_sz = new_size;
2165 __kmp_free(old_stack);
2169 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2170 master_th->th.th_task_state;
2171 master_th->th.th_task_state_top++;
2172 #if KMP_NESTED_HOT_TEAMS 2173 if (master_th->th.th_hot_teams &&
2174 active_level < __kmp_hot_teams_max_level &&
2175 team == master_th->th.th_hot_teams[active_level].hot_team) {
2177 master_th->th.th_task_state =
2179 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2182 master_th->th.th_task_state = 0;
2183 #if KMP_NESTED_HOT_TEAMS 2187 #if !KMP_NESTED_HOT_TEAMS 2188 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2189 (team == root->r.r_hot_team));
2195 (
"__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2196 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2198 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2199 (team->t.t_master_tid == 0 &&
2200 (team->t.t_parent == root->r.r_root_team ||
2201 team->t.t_parent->t.t_serialized)));
2205 argv = (
void **)team->t.t_argv;
2209 for (i = argc - 1; i >= 0; --i) {
2211 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 2212 void *new_argv = va_arg(*ap,
void *);
2214 void *new_argv = va_arg(ap,
void *);
2216 KMP_CHECK_UPDATE(*argv, new_argv);
2221 for (i = 0; i < argc; ++i) {
2223 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2229 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
2230 if (!root->r.r_active)
2231 root->r.r_active = TRUE;
2233 __kmp_fork_team_threads(root, team, master_th, gtid);
2234 __kmp_setup_icv_copy(team, nthreads,
2235 &master_th->th.th_current_task->td_icvs, loc);
2238 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2241 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2244 if (team->t.t_active_level == 1
2246 && !master_th->th.th_teams_microtask
2250 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2251 (__kmp_forkjoin_frames_mode == 3 ||
2252 __kmp_forkjoin_frames_mode == 1)) {
2253 kmp_uint64 tmp_time = 0;
2254 if (__itt_get_timestamp_ptr)
2255 tmp_time = __itt_get_timestamp();
2257 master_th->th.th_frame_time = tmp_time;
2258 if (__kmp_forkjoin_frames_mode == 3)
2259 team->t.t_region_time = tmp_time;
2263 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2264 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2266 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2272 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
2275 (
"__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2276 root, team, master_th, gtid));
2279 if (__itt_stack_caller_create_ptr) {
2280 team->t.t_stack_id =
2281 __kmp_itt_stack_caller_create();
2292 __kmp_internal_fork(loc, gtid, team);
2293 KF_TRACE(10, (
"__kmp_internal_fork : after : root=%p, team=%p, " 2294 "master_th=%p, gtid=%d\n",
2295 root, team, master_th, gtid));
2298 if (call_context == fork_context_gnu) {
2299 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2304 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2305 team->t.t_id, team->t.t_pkfn));
2308 if (!team->t.t_invoke(gtid)) {
2309 KMP_ASSERT2(0,
"cannot invoke microtask for MASTER thread");
2311 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2312 team->t.t_id, team->t.t_pkfn));
2315 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2318 if (ompt_enabled.enabled) {
2319 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2327 static inline void __kmp_join_restore_state(kmp_info_t *thread,
2330 thread->th.ompt_thread_info.state =
2331 ((team->t.t_serialized) ? ompt_state_work_serial
2332 : ompt_state_work_parallel);
2335 static inline void __kmp_join_ompt(
int gtid, kmp_info_t *thread,
2336 kmp_team_t *team, ompt_data_t *parallel_data,
2337 fork_context_e fork_context,
void *codeptr) {
2338 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2339 if (ompt_enabled.ompt_callback_parallel_end) {
2340 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2341 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context),
2345 task_info->frame.enter_frame = ompt_data_none;
2346 __kmp_join_restore_state(thread, team);
2350 void __kmp_join_call(
ident_t *loc,
int gtid
2353 enum fork_context_e fork_context
2360 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2362 kmp_team_t *parent_team;
2363 kmp_info_t *master_th;
2368 KA_TRACE(20, (
"__kmp_join_call: enter T#%d\n", gtid));
2371 master_th = __kmp_threads[gtid];
2372 root = master_th->th.th_root;
2373 team = master_th->th.th_team;
2374 parent_team = team->t.t_parent;
2376 master_th->th.th_ident = loc;
2379 if (ompt_enabled.enabled) {
2380 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2385 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2386 KA_TRACE(20, (
"__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2387 "th_task_team = %p\n",
2388 __kmp_gtid_from_thread(master_th), team,
2389 team->t.t_task_team[master_th->th.th_task_state],
2390 master_th->th.th_task_team));
2391 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2392 team->t.t_task_team[master_th->th.th_task_state]);
2396 if (team->t.t_serialized) {
2398 if (master_th->th.th_teams_microtask) {
2400 int level = team->t.t_level;
2401 int tlevel = master_th->th.th_teams_level;
2402 if (level == tlevel) {
2406 }
else if (level == tlevel + 1) {
2410 team->t.t_serialized++;
2417 if (ompt_enabled.enabled) {
2418 __kmp_join_restore_state(master_th, parent_team);
2425 master_active = team->t.t_master_active;
2433 __kmp_internal_join(loc, gtid, team);
2437 master_th->th.th_task_state =
2445 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2446 void *codeptr = team->t.ompt_team_info.master_return_address;
2450 if (__itt_stack_caller_create_ptr) {
2451 __kmp_itt_stack_caller_destroy(
2452 (__itt_caller)team->t
2457 if (team->t.t_active_level == 1
2459 && !master_th->th.th_teams_microtask
2462 master_th->th.th_ident = loc;
2465 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2466 __kmp_forkjoin_frames_mode == 3)
2467 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2468 master_th->th.th_frame_time, 0, loc,
2469 master_th->th.th_team_nproc, 1);
2470 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2471 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2472 __kmp_itt_region_joined(gtid);
2477 if (master_th->th.th_teams_microtask && !exit_teams &&
2478 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2479 team->t.t_level == master_th->th.th_teams_level + 1) {
2486 team->t.t_active_level--;
2487 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2490 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2491 int old_num = master_th->th.th_team_nproc;
2492 int new_num = master_th->th.th_teams_size.nth;
2493 kmp_info_t **other_threads = team->t.t_threads;
2494 team->t.t_nproc = new_num;
2495 for (i = 0; i < old_num; ++i) {
2496 other_threads[i]->th.th_team_nproc = new_num;
2499 for (i = old_num; i < new_num; ++i) {
2502 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2503 for (b = 0; b < bs_last_barrier; ++b) {
2504 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2505 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2507 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2510 if (__kmp_tasking_mode != tskm_immediate_exec) {
2512 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2518 if (ompt_enabled.enabled) {
2519 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2529 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2530 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2532 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2537 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2540 if (!master_th->th.th_teams_microtask ||
2541 team->t.t_level > master_th->th.th_teams_level)
2545 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2547 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2550 if (ompt_enabled.enabled) {
2551 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2552 if (ompt_enabled.ompt_callback_implicit_task) {
2553 int ompt_team_size = team->t.t_nproc;
2554 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2555 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2556 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
2559 task_info->frame.exit_frame = ompt_data_none;
2560 task_info->task_data = ompt_data_none;
2564 KF_TRACE(10, (
"__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2566 __kmp_pop_current_task_from_thread(master_th);
2568 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2570 master_th->th.th_first_place = team->t.t_first_place;
2571 master_th->th.th_last_place = team->t.t_last_place;
2574 master_th->th.th_def_allocator = team->t.t_def_allocator;
2577 updateHWFPControl(team);
2579 if (root->r.r_active != master_active)
2580 root->r.r_active = master_active;
2582 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2590 master_th->th.th_team = parent_team;
2591 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2592 master_th->th.th_team_master = parent_team->t.t_threads[0];
2593 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2596 if (parent_team->t.t_serialized &&
2597 parent_team != master_th->th.th_serial_team &&
2598 parent_team != root->r.r_root_team) {
2599 __kmp_free_team(root,
2600 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2601 master_th->th.th_serial_team = parent_team;
2604 if (__kmp_tasking_mode != tskm_immediate_exec) {
2605 if (master_th->th.th_task_state_top >
2607 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2609 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2610 master_th->th.th_task_state;
2611 --master_th->th.th_task_state_top;
2613 master_th->th.th_task_state =
2615 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2618 master_th->th.th_task_team =
2619 parent_team->t.t_task_team[master_th->th.th_task_state];
2621 (
"__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
2622 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2629 master_th->th.th_current_task->td_flags.executing = 1;
2631 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2634 if (ompt_enabled.enabled) {
2635 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2641 KA_TRACE(20, (
"__kmp_join_call: exit T#%d\n", gtid));
2646 void __kmp_save_internal_controls(kmp_info_t *thread) {
2648 if (thread->th.th_team != thread->th.th_serial_team) {
2651 if (thread->th.th_team->t.t_serialized > 1) {
2654 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2657 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2658 thread->th.th_team->t.t_serialized) {
2663 kmp_internal_control_t *control =
2664 (kmp_internal_control_t *)__kmp_allocate(
2665 sizeof(kmp_internal_control_t));
2667 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2669 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2671 control->next = thread->th.th_team->t.t_control_stack_top;
2672 thread->th.th_team->t.t_control_stack_top = control;
2678 void __kmp_set_num_threads(
int new_nth,
int gtid) {
2682 KF_TRACE(10, (
"__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2683 KMP_DEBUG_ASSERT(__kmp_init_serial);
2687 else if (new_nth > __kmp_max_nth)
2688 new_nth = __kmp_max_nth;
2691 thread = __kmp_threads[gtid];
2692 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2695 __kmp_save_internal_controls(thread);
2697 set__nproc(thread, new_nth);
2702 root = thread->th.th_root;
2703 if (__kmp_init_parallel && (!root->r.r_active) &&
2704 (root->r.r_hot_team->t.t_nproc > new_nth)
2705 #
if KMP_NESTED_HOT_TEAMS
2706 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2709 kmp_team_t *hot_team = root->r.r_hot_team;
2712 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2715 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2716 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2717 if (__kmp_tasking_mode != tskm_immediate_exec) {
2720 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2722 __kmp_free_thread(hot_team->t.t_threads[f]);
2723 hot_team->t.t_threads[f] = NULL;
2725 hot_team->t.t_nproc = new_nth;
2726 #if KMP_NESTED_HOT_TEAMS 2727 if (thread->th.th_hot_teams) {
2728 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2729 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2733 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2736 for (f = 0; f < new_nth; f++) {
2737 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2738 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2741 hot_team->t.t_size_changed = -1;
2746 void __kmp_set_max_active_levels(
int gtid,
int max_active_levels) {
2749 KF_TRACE(10, (
"__kmp_set_max_active_levels: new max_active_levels for thread " 2751 gtid, max_active_levels));
2752 KMP_DEBUG_ASSERT(__kmp_init_serial);
2755 if (max_active_levels < 0) {
2756 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2761 KF_TRACE(10, (
"__kmp_set_max_active_levels: the call is ignored: new " 2762 "max_active_levels for thread %d = (%d)\n",
2763 gtid, max_active_levels));
2766 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2771 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2772 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2773 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2779 KF_TRACE(10, (
"__kmp_set_max_active_levels: after validation: new " 2780 "max_active_levels for thread %d = (%d)\n",
2781 gtid, max_active_levels));
2783 thread = __kmp_threads[gtid];
2785 __kmp_save_internal_controls(thread);
2787 set__max_active_levels(thread, max_active_levels);
2791 int __kmp_get_max_active_levels(
int gtid) {
2794 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d\n", gtid));
2795 KMP_DEBUG_ASSERT(__kmp_init_serial);
2797 thread = __kmp_threads[gtid];
2798 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2799 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d, curtask=%p, " 2800 "curtask_maxaclevel=%d\n",
2801 gtid, thread->th.th_current_task,
2802 thread->th.th_current_task->td_icvs.max_active_levels));
2803 return thread->th.th_current_task->td_icvs.max_active_levels;
2807 void __kmp_set_schedule(
int gtid, kmp_sched_t kind,
int chunk) {
2811 KF_TRACE(10, (
"__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2812 gtid, (
int)kind, chunk));
2813 KMP_DEBUG_ASSERT(__kmp_init_serial);
2819 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2820 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2822 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2823 KMP_HNT(DefaultScheduleKindUsed,
"static, no chunk"),
2825 kind = kmp_sched_default;
2829 thread = __kmp_threads[gtid];
2831 __kmp_save_internal_controls(thread);
2833 if (kind < kmp_sched_upper_std) {
2834 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2837 thread->th.th_current_task->td_icvs.sched.r_sched_type =
kmp_sch_static;
2839 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2840 __kmp_sch_map[kind - kmp_sched_lower - 1];
2845 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2846 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2847 kmp_sched_lower - 2];
2849 if (kind == kmp_sched_auto || chunk < 1) {
2851 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2853 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2858 void __kmp_get_schedule(
int gtid, kmp_sched_t *kind,
int *chunk) {
2862 KF_TRACE(10, (
"__kmp_get_schedule: thread %d\n", gtid));
2863 KMP_DEBUG_ASSERT(__kmp_init_serial);
2865 thread = __kmp_threads[gtid];
2867 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2871 case kmp_sch_static_greedy:
2872 case kmp_sch_static_balanced:
2873 *kind = kmp_sched_static;
2876 case kmp_sch_static_chunked:
2877 *kind = kmp_sched_static;
2879 case kmp_sch_dynamic_chunked:
2880 *kind = kmp_sched_dynamic;
2883 case kmp_sch_guided_iterative_chunked:
2884 case kmp_sch_guided_analytical_chunked:
2885 *kind = kmp_sched_guided;
2888 *kind = kmp_sched_auto;
2890 case kmp_sch_trapezoidal:
2891 *kind = kmp_sched_trapezoidal;
2893 #if KMP_STATIC_STEAL_ENABLED 2894 case kmp_sch_static_steal:
2895 *kind = kmp_sched_static_steal;
2899 KMP_FATAL(UnknownSchedulingType, th_type);
2902 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2905 int __kmp_get_ancestor_thread_num(
int gtid,
int level) {
2911 KF_TRACE(10, (
"__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2912 KMP_DEBUG_ASSERT(__kmp_init_serial);
2919 thr = __kmp_threads[gtid];
2920 team = thr->th.th_team;
2921 ii = team->t.t_level;
2926 if (thr->th.th_teams_microtask) {
2928 int tlevel = thr->th.th_teams_level;
2931 KMP_DEBUG_ASSERT(ii >= tlevel);
2944 return __kmp_tid_from_gtid(gtid);
2946 dd = team->t.t_serialized;
2948 while (ii > level) {
2949 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2951 if ((team->t.t_serialized) && (!dd)) {
2952 team = team->t.t_parent;
2956 team = team->t.t_parent;
2957 dd = team->t.t_serialized;
2962 return (dd > 1) ? (0) : (team->t.t_master_tid);
2965 int __kmp_get_team_size(
int gtid,
int level) {
2971 KF_TRACE(10, (
"__kmp_get_team_size: thread %d %d\n", gtid, level));
2972 KMP_DEBUG_ASSERT(__kmp_init_serial);
2979 thr = __kmp_threads[gtid];
2980 team = thr->th.th_team;
2981 ii = team->t.t_level;
2986 if (thr->th.th_teams_microtask) {
2988 int tlevel = thr->th.th_teams_level;
2991 KMP_DEBUG_ASSERT(ii >= tlevel);
3003 while (ii > level) {
3004 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
3006 if (team->t.t_serialized && (!dd)) {
3007 team = team->t.t_parent;
3011 team = team->t.t_parent;
3016 return team->t.t_nproc;
3019 kmp_r_sched_t __kmp_get_schedule_global() {
3024 kmp_r_sched_t r_sched;
3032 r_sched.r_sched_type = __kmp_static;
3035 r_sched.r_sched_type = __kmp_guided;
3037 r_sched.r_sched_type = __kmp_sched;
3040 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
3042 r_sched.chunk = KMP_DEFAULT_CHUNK;
3044 r_sched.chunk = __kmp_chunk;
3052 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
int realloc) {
3054 KMP_DEBUG_ASSERT(team);
3055 if (!realloc || argc > team->t.t_max_argc) {
3057 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: needed entries=%d, " 3058 "current entries=%d\n",
3059 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3061 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3062 __kmp_free((
void *)team->t.t_argv);
3064 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3066 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3067 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: inline allocate %d " 3069 team->t.t_id, team->t.t_max_argc));
3070 team->t.t_argv = &team->t.t_inline_argv[0];
3071 if (__kmp_storage_map) {
3072 __kmp_print_storage_map_gtid(
3073 -1, &team->t.t_inline_argv[0],
3074 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3075 (
sizeof(
void *) * KMP_INLINE_ARGV_ENTRIES),
"team_%d.t_inline_argv",
3080 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3081 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3083 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3085 team->t.t_id, team->t.t_max_argc));
3087 (
void **)__kmp_page_allocate(
sizeof(
void *) * team->t.t_max_argc);
3088 if (__kmp_storage_map) {
3089 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3090 &team->t.t_argv[team->t.t_max_argc],
3091 sizeof(
void *) * team->t.t_max_argc,
3092 "team_%d.t_argv", team->t.t_id);
3098 static void __kmp_allocate_team_arrays(kmp_team_t *team,
int max_nth) {
3100 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3102 (kmp_info_t **)__kmp_allocate(
sizeof(kmp_info_t *) * max_nth);
3103 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3104 sizeof(dispatch_shared_info_t) * num_disp_buff);
3105 team->t.t_dispatch =
3106 (kmp_disp_t *)__kmp_allocate(
sizeof(kmp_disp_t) * max_nth);
3107 team->t.t_implicit_task_taskdata =
3108 (kmp_taskdata_t *)__kmp_allocate(
sizeof(kmp_taskdata_t) * max_nth);
3109 team->t.t_max_nproc = max_nth;
3112 for (i = 0; i < num_disp_buff; ++i) {
3113 team->t.t_disp_buffer[i].buffer_index = i;
3115 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3120 static void __kmp_free_team_arrays(kmp_team_t *team) {
3123 for (i = 0; i < team->t.t_max_nproc; ++i) {
3124 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3125 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3126 team->t.t_dispatch[i].th_disp_buffer = NULL;
3129 #if KMP_USE_HIER_SCHED 3130 __kmp_dispatch_free_hierarchies(team);
3132 __kmp_free(team->t.t_threads);
3133 __kmp_free(team->t.t_disp_buffer);
3134 __kmp_free(team->t.t_dispatch);
3135 __kmp_free(team->t.t_implicit_task_taskdata);
3136 team->t.t_threads = NULL;
3137 team->t.t_disp_buffer = NULL;
3138 team->t.t_dispatch = NULL;
3139 team->t.t_implicit_task_taskdata = 0;
3142 static void __kmp_reallocate_team_arrays(kmp_team_t *team,
int max_nth) {
3143 kmp_info_t **oldThreads = team->t.t_threads;
3145 __kmp_free(team->t.t_disp_buffer);
3146 __kmp_free(team->t.t_dispatch);
3147 __kmp_free(team->t.t_implicit_task_taskdata);
3148 __kmp_allocate_team_arrays(team, max_nth);
3150 KMP_MEMCPY(team->t.t_threads, oldThreads,
3151 team->t.t_nproc *
sizeof(kmp_info_t *));
3153 __kmp_free(oldThreads);
3156 static kmp_internal_control_t __kmp_get_global_icvs(
void) {
3158 kmp_r_sched_t r_sched =
3159 __kmp_get_schedule_global();
3162 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
3165 kmp_internal_control_t g_icvs = {
3167 (kmp_int8)__kmp_dflt_nested,
3169 (kmp_int8)__kmp_global.g.g_dynamic,
3171 (kmp_int8)__kmp_env_blocktime,
3173 __kmp_dflt_blocktime,
3178 __kmp_dflt_team_nth,
3181 __kmp_dflt_max_active_levels,
3186 __kmp_nested_proc_bind.bind_types[0],
3187 __kmp_default_device,
3195 static kmp_internal_control_t __kmp_get_x_global_icvs(
const kmp_team_t *team) {
3197 kmp_internal_control_t gx_icvs;
3198 gx_icvs.serial_nesting_level =
3200 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3201 gx_icvs.next = NULL;
3206 static void __kmp_initialize_root(kmp_root_t *root) {
3208 kmp_team_t *root_team;
3209 kmp_team_t *hot_team;
3210 int hot_team_max_nth;
3211 kmp_r_sched_t r_sched =
3212 __kmp_get_schedule_global();
3213 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3214 KMP_DEBUG_ASSERT(root);
3215 KMP_ASSERT(!root->r.r_begin);
3218 __kmp_init_lock(&root->r.r_begin_lock);
3219 root->r.r_begin = FALSE;
3220 root->r.r_active = FALSE;
3221 root->r.r_in_parallel = 0;
3222 root->r.r_blocktime = __kmp_dflt_blocktime;
3223 root->r.r_nested = __kmp_dflt_nested;
3224 root->r.r_cg_nthreads = 1;
3228 KF_TRACE(10, (
"__kmp_initialize_root: before root_team\n"));
3231 __kmp_allocate_team(root,
3238 __kmp_nested_proc_bind.bind_types[0],
3242 USE_NESTED_HOT_ARG(NULL)
3247 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
3250 KF_TRACE(10, (
"__kmp_initialize_root: after root_team = %p\n", root_team));
3252 root->r.r_root_team = root_team;
3253 root_team->t.t_control_stack_top = NULL;
3256 root_team->t.t_threads[0] = NULL;
3257 root_team->t.t_nproc = 1;
3258 root_team->t.t_serialized = 1;
3260 root_team->t.t_sched.sched = r_sched.sched;
3263 (
"__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3264 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
3268 KF_TRACE(10, (
"__kmp_initialize_root: before hot_team\n"));
3271 __kmp_allocate_team(root,
3273 __kmp_dflt_team_nth_ub * 2,
3278 __kmp_nested_proc_bind.bind_types[0],
3282 USE_NESTED_HOT_ARG(NULL)
3284 KF_TRACE(10, (
"__kmp_initialize_root: after hot_team = %p\n", hot_team));
3286 root->r.r_hot_team = hot_team;
3287 root_team->t.t_control_stack_top = NULL;
3290 hot_team->t.t_parent = root_team;
3293 hot_team_max_nth = hot_team->t.t_max_nproc;
3294 for (f = 0; f < hot_team_max_nth; ++f) {
3295 hot_team->t.t_threads[f] = NULL;
3297 hot_team->t.t_nproc = 1;
3299 hot_team->t.t_sched.sched = r_sched.sched;
3300 hot_team->t.t_size_changed = 0;
3305 typedef struct kmp_team_list_item {
3306 kmp_team_p
const *entry;
3307 struct kmp_team_list_item *next;
3308 } kmp_team_list_item_t;
3309 typedef kmp_team_list_item_t *kmp_team_list_t;
3311 static void __kmp_print_structure_team_accum(
3312 kmp_team_list_t list,
3313 kmp_team_p
const *team
3323 KMP_DEBUG_ASSERT(list != NULL);
3328 __kmp_print_structure_team_accum(list, team->t.t_parent);
3329 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3333 while (l->next != NULL && l->entry != team) {
3336 if (l->next != NULL) {
3342 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3348 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3349 sizeof(kmp_team_list_item_t));
3356 static void __kmp_print_structure_team(
char const *title, kmp_team_p
const *team
3359 __kmp_printf(
"%s", title);
3361 __kmp_printf(
"%2x %p\n", team->t.t_id, team);
3363 __kmp_printf(
" - (nil)\n");
3367 static void __kmp_print_structure_thread(
char const *title,
3368 kmp_info_p
const *thread) {
3369 __kmp_printf(
"%s", title);
3370 if (thread != NULL) {
3371 __kmp_printf(
"%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3373 __kmp_printf(
" - (nil)\n");
3377 void __kmp_print_structure(
void) {
3379 kmp_team_list_t list;
3383 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
sizeof(kmp_team_list_item_t));
3387 __kmp_printf(
"\n------------------------------\nGlobal Thread " 3388 "Table\n------------------------------\n");
3391 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3392 __kmp_printf(
"%2d", gtid);
3393 if (__kmp_threads != NULL) {
3394 __kmp_printf(
" %p", __kmp_threads[gtid]);
3396 if (__kmp_root != NULL) {
3397 __kmp_printf(
" %p", __kmp_root[gtid]);
3404 __kmp_printf(
"\n------------------------------\nThreads\n--------------------" 3406 if (__kmp_threads != NULL) {
3408 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3409 kmp_info_t
const *thread = __kmp_threads[gtid];
3410 if (thread != NULL) {
3411 __kmp_printf(
"GTID %2d %p:\n", gtid, thread);
3412 __kmp_printf(
" Our Root: %p\n", thread->th.th_root);
3413 __kmp_print_structure_team(
" Our Team: ", thread->th.th_team);
3414 __kmp_print_structure_team(
" Serial Team: ",
3415 thread->th.th_serial_team);
3416 __kmp_printf(
" Threads: %2d\n", thread->th.th_team_nproc);
3417 __kmp_print_structure_thread(
" Master: ",
3418 thread->th.th_team_master);
3419 __kmp_printf(
" Serialized?: %2d\n", thread->th.th_team_serialized);
3420 __kmp_printf(
" Set NProc: %2d\n", thread->th.th_set_nproc);
3422 __kmp_printf(
" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3424 __kmp_print_structure_thread(
" Next in pool: ",
3425 thread->th.th_next_pool);
3427 __kmp_print_structure_team_accum(list, thread->th.th_team);
3428 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3432 __kmp_printf(
"Threads array is not allocated.\n");
3436 __kmp_printf(
"\n------------------------------\nUbers\n----------------------" 3438 if (__kmp_root != NULL) {
3440 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3441 kmp_root_t
const *root = __kmp_root[gtid];
3443 __kmp_printf(
"GTID %2d %p:\n", gtid, root);
3444 __kmp_print_structure_team(
" Root Team: ", root->r.r_root_team);
3445 __kmp_print_structure_team(
" Hot Team: ", root->r.r_hot_team);
3446 __kmp_print_structure_thread(
" Uber Thread: ",
3447 root->r.r_uber_thread);
3448 __kmp_printf(
" Active?: %2d\n", root->r.r_active);
3449 __kmp_printf(
" Nested?: %2d\n", root->r.r_nested);
3450 __kmp_printf(
" In Parallel: %2d\n",
3451 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
3453 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3454 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3458 __kmp_printf(
"Ubers array is not allocated.\n");
3461 __kmp_printf(
"\n------------------------------\nTeams\n----------------------" 3463 while (list->next != NULL) {
3464 kmp_team_p
const *team = list->entry;
3466 __kmp_printf(
"Team %2x %p:\n", team->t.t_id, team);
3467 __kmp_print_structure_team(
" Parent Team: ", team->t.t_parent);
3468 __kmp_printf(
" Master TID: %2d\n", team->t.t_master_tid);
3469 __kmp_printf(
" Max threads: %2d\n", team->t.t_max_nproc);
3470 __kmp_printf(
" Levels of serial: %2d\n", team->t.t_serialized);
3471 __kmp_printf(
" Number threads: %2d\n", team->t.t_nproc);
3472 for (i = 0; i < team->t.t_nproc; ++i) {
3473 __kmp_printf(
" Thread %2d: ", i);
3474 __kmp_print_structure_thread(
"", team->t.t_threads[i]);
3476 __kmp_print_structure_team(
" Next in pool: ", team->t.t_next_pool);
3482 __kmp_printf(
"\n------------------------------\nPools\n----------------------" 3484 __kmp_print_structure_thread(
"Thread pool: ",
3485 CCAST(kmp_info_t *, __kmp_thread_pool));
3486 __kmp_print_structure_team(
"Team pool: ",
3487 CCAST(kmp_team_t *, __kmp_team_pool));
3491 while (list != NULL) {
3492 kmp_team_list_item_t *item = list;
3494 KMP_INTERNAL_FREE(item);
3503 static const unsigned __kmp_primes[] = {
3504 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3505 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3506 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3507 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3508 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3509 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3510 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3511 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3512 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3513 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3514 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3518 unsigned short __kmp_get_random(kmp_info_t *thread) {
3519 unsigned x = thread->th.th_x;
3520 unsigned short r = x >> 16;
3522 thread->th.th_x = x * thread->th.th_a + 1;
3524 KA_TRACE(30, (
"__kmp_get_random: THREAD: %d, RETURN: %u\n",
3525 thread->th.th_info.ds.ds_tid, r));
3531 void __kmp_init_random(kmp_info_t *thread) {
3532 unsigned seed = thread->th.th_info.ds.ds_tid;
3535 __kmp_primes[seed % (
sizeof(__kmp_primes) /
sizeof(__kmp_primes[0]))];
3536 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3538 (
"__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
3544 static int __kmp_reclaim_dead_roots(
void) {
3547 for (i = 0; i < __kmp_threads_capacity; ++i) {
3548 if (KMP_UBER_GTID(i) &&
3549 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3552 r += __kmp_unregister_root_other_thread(i);
3577 static int __kmp_expand_threads(
int nNeed) {
3579 int minimumRequiredCapacity;
3581 kmp_info_t **newThreads;
3582 kmp_root_t **newRoot;
3588 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 3591 added = __kmp_reclaim_dead_roots();
3620 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
3623 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3627 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3629 newCapacity = __kmp_threads_capacity;
3631 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3632 : __kmp_sys_max_nth;
3633 }
while (newCapacity < minimumRequiredCapacity);
3634 newThreads = (kmp_info_t **)__kmp_allocate(
3635 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3637 (kmp_root_t **)((
char *)newThreads +
sizeof(kmp_info_t *) * newCapacity);
3638 KMP_MEMCPY(newThreads, __kmp_threads,
3639 __kmp_threads_capacity *
sizeof(kmp_info_t *));
3640 KMP_MEMCPY(newRoot, __kmp_root,
3641 __kmp_threads_capacity *
sizeof(kmp_root_t *));
3643 kmp_info_t **temp_threads = __kmp_threads;
3644 *(kmp_info_t * *
volatile *)&__kmp_threads = newThreads;
3645 *(kmp_root_t * *
volatile *)&__kmp_root = newRoot;
3646 __kmp_free(temp_threads);
3647 added += newCapacity - __kmp_threads_capacity;
3648 *(
volatile int *)&__kmp_threads_capacity = newCapacity;
3650 if (newCapacity > __kmp_tp_capacity) {
3651 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3652 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3653 __kmp_threadprivate_resize_cache(newCapacity);
3655 *(
volatile int *)&__kmp_tp_capacity = newCapacity;
3657 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3666 int __kmp_register_root(
int initial_thread) {
3667 kmp_info_t *root_thread;
3671 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3672 KA_TRACE(20, (
"__kmp_register_root: entered\n"));
3689 capacity = __kmp_threads_capacity;
3690 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3695 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3696 if (__kmp_tp_cached) {
3697 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3698 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3699 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
3701 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3709 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL;
3713 (
"__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3714 KMP_ASSERT(gtid < __kmp_threads_capacity);
3718 TCW_4(__kmp_nth, __kmp_nth + 1);
3722 if (__kmp_adjust_gtid_mode) {
3723 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3724 if (TCR_4(__kmp_gtid_mode) != 2) {
3725 TCW_4(__kmp_gtid_mode, 2);
3728 if (TCR_4(__kmp_gtid_mode) != 1) {
3729 TCW_4(__kmp_gtid_mode, 1);
3734 #ifdef KMP_ADJUST_BLOCKTIME 3737 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3738 if (__kmp_nth > __kmp_avail_proc) {
3739 __kmp_zero_bt = TRUE;
3745 if (!(root = __kmp_root[gtid])) {
3746 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(
sizeof(kmp_root_t));
3747 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3750 #if KMP_STATS_ENABLED 3752 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3753 __kmp_stats_thread_ptr->startLife();
3754 KMP_SET_THREAD_STATE(SERIAL_REGION);
3757 __kmp_initialize_root(root);
3760 if (root->r.r_uber_thread) {
3761 root_thread = root->r.r_uber_thread;
3763 root_thread = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
3764 if (__kmp_storage_map) {
3765 __kmp_print_thread_storage_map(root_thread, gtid);
3767 root_thread->th.th_info.ds.ds_gtid = gtid;
3769 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
3771 root_thread->th.th_root = root;
3772 if (__kmp_env_consistency_check) {
3773 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3776 __kmp_initialize_fast_memory(root_thread);
3780 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3781 __kmp_initialize_bget(root_thread);
3783 __kmp_init_random(root_thread);
3787 if (!root_thread->th.th_serial_team) {
3788 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3789 KF_TRACE(10, (
"__kmp_register_root: before serial_team\n"));
3790 root_thread->th.th_serial_team =
3791 __kmp_allocate_team(root, 1, 1,
3798 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3800 KMP_ASSERT(root_thread->th.th_serial_team);
3801 KF_TRACE(10, (
"__kmp_register_root: after serial_team = %p\n",
3802 root_thread->th.th_serial_team));
3805 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
3807 root->r.r_root_team->t.t_threads[0] = root_thread;
3808 root->r.r_hot_team->t.t_threads[0] = root_thread;
3809 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3811 root_thread->th.th_serial_team->t.t_serialized = 0;
3812 root->r.r_uber_thread = root_thread;
3815 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3816 TCW_4(__kmp_init_gtid, TRUE);
3819 __kmp_gtid_set_specific(gtid);
3822 __kmp_itt_thread_name(gtid);
3825 #ifdef KMP_TDATA_GTID 3828 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3829 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3831 KA_TRACE(20, (
"__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3833 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3834 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3835 KMP_INIT_BARRIER_STATE));
3838 for (b = 0; b < bs_last_barrier; ++b) {
3839 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3841 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3845 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3846 KMP_INIT_BARRIER_STATE);
3848 #if KMP_AFFINITY_SUPPORTED 3850 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3851 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3852 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3853 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3855 if (TCR_4(__kmp_init_middle)) {
3856 __kmp_affinity_set_init_mask(gtid, TRUE);
3860 root_thread->th.th_def_allocator = __kmp_def_allocator;
3861 root_thread->th.th_prev_level = 0;
3862 root_thread->th.th_prev_num_threads = 1;
3865 __kmp_root_counter++;
3868 if (!initial_thread && ompt_enabled.enabled) {
3870 kmp_info_t *root_thread = ompt_get_thread();
3872 ompt_set_thread_state(root_thread, ompt_state_overhead);
3874 if (ompt_enabled.ompt_callback_thread_begin) {
3875 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3876 ompt_thread_initial, __ompt_get_thread_data_internal());
3878 ompt_data_t *task_data;
3879 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
3880 if (ompt_enabled.ompt_callback_task_create) {
3881 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
3882 NULL, NULL, task_data, ompt_task_initial, 0, NULL);
3886 ompt_set_thread_state(root_thread, ompt_state_work_serial);
3891 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3896 #if KMP_NESTED_HOT_TEAMS 3897 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr,
int level,
3898 const int max_level) {
3900 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3901 if (!hot_teams || !hot_teams[level].hot_team) {
3904 KMP_DEBUG_ASSERT(level < max_level);
3905 kmp_team_t *team = hot_teams[level].hot_team;
3906 nth = hot_teams[level].hot_team_nth;
3908 if (level < max_level - 1) {
3909 for (i = 0; i < nth; ++i) {
3910 kmp_info_t *th = team->t.t_threads[i];
3911 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3912 if (i > 0 && th->th.th_hot_teams) {
3913 __kmp_free(th->th.th_hot_teams);
3914 th->th.th_hot_teams = NULL;
3918 __kmp_free_team(root, team, NULL);
3925 static int __kmp_reset_root(
int gtid, kmp_root_t *root) {
3926 kmp_team_t *root_team = root->r.r_root_team;
3927 kmp_team_t *hot_team = root->r.r_hot_team;
3928 int n = hot_team->t.t_nproc;
3931 KMP_DEBUG_ASSERT(!root->r.r_active);
3933 root->r.r_root_team = NULL;
3934 root->r.r_hot_team = NULL;
3937 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
3938 #if KMP_NESTED_HOT_TEAMS 3939 if (__kmp_hot_teams_max_level >
3941 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3942 kmp_info_t *th = hot_team->t.t_threads[i];
3943 if (__kmp_hot_teams_max_level > 1) {
3944 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3946 if (th->th.th_hot_teams) {
3947 __kmp_free(th->th.th_hot_teams);
3948 th->th.th_hot_teams = NULL;
3953 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
3958 if (__kmp_tasking_mode != tskm_immediate_exec) {
3959 __kmp_wait_to_unref_task_teams();
3965 10, (
"__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3967 (LPVOID) & (root->r.r_uber_thread->th),
3968 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3969 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3973 if (ompt_enabled.ompt_callback_thread_end) {
3974 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3975 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
3981 root->r.r_cg_nthreads--;
3983 __kmp_reap_thread(root->r.r_uber_thread, 1);
3987 root->r.r_uber_thread = NULL;
3989 root->r.r_begin = FALSE;
3994 void __kmp_unregister_root_current_thread(
int gtid) {
3995 KA_TRACE(1, (
"__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
3999 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
4000 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
4001 KC_TRACE(10, (
"__kmp_unregister_root_current_thread: already finished, " 4004 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4007 kmp_root_t *root = __kmp_root[gtid];
4009 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4010 KMP_ASSERT(KMP_UBER_GTID(gtid));
4011 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4012 KMP_ASSERT(root->r.r_active == FALSE);
4017 kmp_info_t *thread = __kmp_threads[gtid];
4018 kmp_team_t *team = thread->th.th_team;
4019 kmp_task_team_t *task_team = thread->th.th_task_team;
4022 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
4025 thread->th.ompt_thread_info.state = ompt_state_undefined;
4027 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
4031 __kmp_reset_root(gtid, root);
4034 __kmp_gtid_set_specific(KMP_GTID_DNE);
4035 #ifdef KMP_TDATA_GTID 4036 __kmp_gtid = KMP_GTID_DNE;
4041 (
"__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
4043 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4050 static int __kmp_unregister_root_other_thread(
int gtid) {
4051 kmp_root_t *root = __kmp_root[gtid];
4054 KA_TRACE(1, (
"__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4055 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4056 KMP_ASSERT(KMP_UBER_GTID(gtid));
4057 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4058 KMP_ASSERT(root->r.r_active == FALSE);
4060 r = __kmp_reset_root(gtid, root);
4062 (
"__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4068 void __kmp_task_info() {
4070 kmp_int32 gtid = __kmp_entry_gtid();
4071 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4072 kmp_info_t *this_thr = __kmp_threads[gtid];
4073 kmp_team_t *steam = this_thr->th.th_serial_team;
4074 kmp_team_t *team = this_thr->th.th_team;
4077 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4079 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4080 team->t.t_implicit_task_taskdata[tid].td_parent);
4087 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4088 int tid,
int gtid) {
4092 kmp_info_t *master = team->t.t_threads[0];
4093 KMP_DEBUG_ASSERT(this_thr != NULL);
4094 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4095 KMP_DEBUG_ASSERT(team);
4096 KMP_DEBUG_ASSERT(team->t.t_threads);
4097 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4098 KMP_DEBUG_ASSERT(master);
4099 KMP_DEBUG_ASSERT(master->th.th_root);
4103 TCW_SYNC_PTR(this_thr->th.th_team, team);
4105 this_thr->th.th_info.ds.ds_tid = tid;
4106 this_thr->th.th_set_nproc = 0;
4107 if (__kmp_tasking_mode != tskm_immediate_exec)
4110 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4112 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
4114 this_thr->th.th_set_proc_bind = proc_bind_default;
4115 #if KMP_AFFINITY_SUPPORTED 4116 this_thr->th.th_new_place = this_thr->th.th_current_place;
4119 this_thr->th.th_root = master->th.th_root;
4122 this_thr->th.th_team_nproc = team->t.t_nproc;
4123 this_thr->th.th_team_master = master;
4124 this_thr->th.th_team_serialized = team->t.t_serialized;
4125 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
4127 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
4129 KF_TRACE(10, (
"__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4130 tid, gtid, this_thr, this_thr->th.th_current_task));
4132 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4135 KF_TRACE(10, (
"__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4136 tid, gtid, this_thr, this_thr->th.th_current_task));
4141 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4143 this_thr->th.th_local.this_construct = 0;
4145 if (!this_thr->th.th_pri_common) {
4146 this_thr->th.th_pri_common =
4147 (
struct common_table *)__kmp_allocate(
sizeof(
struct common_table));
4148 if (__kmp_storage_map) {
4149 __kmp_print_storage_map_gtid(
4150 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4151 sizeof(
struct common_table),
"th_%d.th_pri_common\n", gtid);
4153 this_thr->th.th_pri_head = NULL;
4158 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4161 sizeof(dispatch_private_info_t) *
4162 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4163 KD_TRACE(10, (
"__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4164 team->t.t_max_nproc));
4165 KMP_ASSERT(dispatch);
4166 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4167 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
4169 dispatch->th_disp_index = 0;
4171 dispatch->th_doacross_buf_idx = 0;
4173 if (!dispatch->th_disp_buffer) {
4174 dispatch->th_disp_buffer =
4175 (dispatch_private_info_t *)__kmp_allocate(disp_size);
4177 if (__kmp_storage_map) {
4178 __kmp_print_storage_map_gtid(
4179 gtid, &dispatch->th_disp_buffer[0],
4180 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4182 : __kmp_dispatch_num_buffers],
4183 disp_size,
"th_%d.th_dispatch.th_disp_buffer " 4184 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4185 gtid, team->t.t_id, gtid);
4188 memset(&dispatch->th_disp_buffer[0],
'\0', disp_size);
4191 dispatch->th_dispatch_pr_current = 0;
4192 dispatch->th_dispatch_sh_current = 0;
4194 dispatch->th_deo_fcn = 0;
4195 dispatch->th_dxo_fcn = 0;
4198 this_thr->th.th_next_pool = NULL;
4200 if (!this_thr->th.th_task_state_memo_stack) {
4202 this_thr->th.th_task_state_memo_stack =
4203 (kmp_uint8 *)__kmp_allocate(4 *
sizeof(kmp_uint8));
4204 this_thr->th.th_task_state_top = 0;
4205 this_thr->th.th_task_state_stack_sz = 4;
4206 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4208 this_thr->th.th_task_state_memo_stack[i] = 0;
4211 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4212 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4222 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4224 kmp_team_t *serial_team;
4225 kmp_info_t *new_thr;
4228 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4229 KMP_DEBUG_ASSERT(root && team);
4230 #if !KMP_NESTED_HOT_TEAMS 4231 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
4236 if (__kmp_thread_pool) {
4238 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
4239 __kmp_thread_pool = (
volatile kmp_info_t *)new_thr->th.th_next_pool;
4240 if (new_thr == __kmp_thread_pool_insert_pt) {
4241 __kmp_thread_pool_insert_pt = NULL;
4243 TCW_4(new_thr->th.th_in_pool, FALSE);
4246 __kmp_thread_pool_nth--;
4248 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d using thread T#%d\n",
4249 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4250 KMP_ASSERT(!new_thr->th.th_team);
4251 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4252 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0);
4255 __kmp_initialize_info(new_thr, team, new_tid,
4256 new_thr->th.th_info.ds.ds_gtid);
4257 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
4259 TCW_4(__kmp_nth, __kmp_nth + 1);
4260 root->r.r_cg_nthreads++;
4262 new_thr->th.th_task_state = 0;
4263 new_thr->th.th_task_state_top = 0;
4264 new_thr->th.th_task_state_stack_sz = 4;
4266 #ifdef KMP_ADJUST_BLOCKTIME 4269 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4270 if (__kmp_nth > __kmp_avail_proc) {
4271 __kmp_zero_bt = TRUE;
4280 kmp_balign_t *balign = new_thr->th.th_bar;
4281 for (b = 0; b < bs_last_barrier; ++b)
4282 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
4285 KF_TRACE(10, (
"__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4286 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
4293 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4294 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4299 if (!TCR_4(__kmp_init_monitor)) {
4300 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4301 if (!TCR_4(__kmp_init_monitor)) {
4302 KF_TRACE(10, (
"before __kmp_create_monitor\n"));
4303 TCW_4(__kmp_init_monitor, 1);
4304 __kmp_create_monitor(&__kmp_monitor);
4305 KF_TRACE(10, (
"after __kmp_create_monitor\n"));
4316 while (TCR_4(__kmp_init_monitor) < 2) {
4319 KF_TRACE(10, (
"after monitor thread has started\n"));
4322 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4327 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) {
4328 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4332 new_thr = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
4334 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4336 if (__kmp_storage_map) {
4337 __kmp_print_thread_storage_map(new_thr, new_gtid);
4342 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4343 KF_TRACE(10, (
"__kmp_allocate_thread: before th_serial/serial_team\n"));
4344 new_thr->th.th_serial_team = serial_team =
4345 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4352 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
4354 KMP_ASSERT(serial_team);
4355 serial_team->t.t_serialized = 0;
4357 serial_team->t.t_threads[0] = new_thr;
4359 (
"__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4363 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4366 __kmp_initialize_fast_memory(new_thr);
4370 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4371 __kmp_initialize_bget(new_thr);
4374 __kmp_init_random(new_thr);
4378 (
"__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4379 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4382 kmp_balign_t *balign = new_thr->th.th_bar;
4383 for (b = 0; b < bs_last_barrier; ++b) {
4384 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4385 balign[b].bb.team = NULL;
4386 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4387 balign[b].bb.use_oncore_barrier = 0;
4390 new_thr->th.th_spin_here = FALSE;
4391 new_thr->th.th_next_waiting = 0;
4393 new_thr->th.th_blocking =
false;
4396 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4397 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4398 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4399 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4400 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4403 new_thr->th.th_def_allocator = __kmp_def_allocator;
4404 new_thr->th.th_prev_level = 0;
4405 new_thr->th.th_prev_num_threads = 1;
4408 TCW_4(new_thr->th.th_in_pool, FALSE);
4409 new_thr->th.th_active_in_pool = FALSE;
4410 TCW_4(new_thr->th.th_active, TRUE);
4416 root->r.r_cg_nthreads++;
4420 if (__kmp_adjust_gtid_mode) {
4421 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4422 if (TCR_4(__kmp_gtid_mode) != 2) {
4423 TCW_4(__kmp_gtid_mode, 2);
4426 if (TCR_4(__kmp_gtid_mode) != 1) {
4427 TCW_4(__kmp_gtid_mode, 1);
4432 #ifdef KMP_ADJUST_BLOCKTIME 4435 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4436 if (__kmp_nth > __kmp_avail_proc) {
4437 __kmp_zero_bt = TRUE;
4444 10, (
"__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4445 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4447 (
"__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4449 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4460 static void __kmp_reinitialize_team(kmp_team_t *team,
4461 kmp_internal_control_t *new_icvs,
4463 KF_TRACE(10, (
"__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4464 team->t.t_threads[0], team));
4465 KMP_DEBUG_ASSERT(team && new_icvs);
4466 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4467 KMP_CHECK_UPDATE(team->t.t_ident, loc);
4469 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
4471 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4472 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4474 KF_TRACE(10, (
"__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4475 team->t.t_threads[0], team));
4481 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
4482 kmp_internal_control_t *new_icvs,
4484 KF_TRACE(10, (
"__kmp_initialize_team: enter: team=%p\n", team));
4487 KMP_DEBUG_ASSERT(team);
4488 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4489 KMP_DEBUG_ASSERT(team->t.t_threads);
4492 team->t.t_master_tid = 0;
4494 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4495 team->t.t_nproc = new_nproc;
4498 team->t.t_next_pool = NULL;
4502 TCW_SYNC_PTR(team->t.t_pkfn, NULL);
4503 team->t.t_invoke = NULL;
4506 team->t.t_sched.sched = new_icvs->sched.sched;
4508 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4509 team->t.t_fp_control_saved = FALSE;
4510 team->t.t_x87_fpu_control_word = 0;
4511 team->t.t_mxcsr = 0;
4514 team->t.t_construct = 0;
4516 team->t.t_ordered.dt.t_value = 0;
4517 team->t.t_master_active = FALSE;
4519 memset(&team->t.t_taskq,
'\0',
sizeof(kmp_taskq_t));
4522 team->t.t_copypriv_data = NULL;
4525 team->t.t_copyin_counter = 0;
4528 team->t.t_control_stack_top = NULL;
4530 __kmp_reinitialize_team(team, new_icvs, loc);
4533 KF_TRACE(10, (
"__kmp_initialize_team: exit: team=%p\n", team));
4536 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 4539 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4540 if (KMP_AFFINITY_CAPABLE()) {
4542 if (old_mask != NULL) {
4543 status = __kmp_get_system_affinity(old_mask, TRUE);
4546 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4550 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4555 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4561 static void __kmp_partition_places(kmp_team_t *team,
int update_master_only) {
4563 kmp_info_t *master_th = team->t.t_threads[0];
4564 KMP_DEBUG_ASSERT(master_th != NULL);
4565 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4566 int first_place = master_th->th.th_first_place;
4567 int last_place = master_th->th.th_last_place;
4568 int masters_place = master_th->th.th_current_place;
4569 team->t.t_first_place = first_place;
4570 team->t.t_last_place = last_place;
4572 KA_TRACE(20, (
"__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4573 "bound to place %d partition = [%d,%d]\n",
4574 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4575 team->t.t_id, masters_place, first_place, last_place));
4577 switch (proc_bind) {
4579 case proc_bind_default:
4582 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4585 case proc_bind_master: {
4587 int n_th = team->t.t_nproc;
4588 for (f = 1; f < n_th; f++) {
4589 kmp_info_t *th = team->t.t_threads[f];
4590 KMP_DEBUG_ASSERT(th != NULL);
4591 th->th.th_first_place = first_place;
4592 th->th.th_last_place = last_place;
4593 th->th.th_new_place = masters_place;
4595 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4596 team->t.t_display_affinity != 1) {
4597 team->t.t_display_affinity = 1;
4601 KA_TRACE(100, (
"__kmp_partition_places: master: T#%d(%d:%d) place %d " 4602 "partition = [%d,%d]\n",
4603 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4604 f, masters_place, first_place, last_place));
4608 case proc_bind_close: {
4610 int n_th = team->t.t_nproc;
4612 if (first_place <= last_place) {
4613 n_places = last_place - first_place + 1;
4615 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4617 if (n_th <= n_places) {
4618 int place = masters_place;
4619 for (f = 1; f < n_th; f++) {
4620 kmp_info_t *th = team->t.t_threads[f];
4621 KMP_DEBUG_ASSERT(th != NULL);
4623 if (place == last_place) {
4624 place = first_place;
4625 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4630 th->th.th_first_place = first_place;
4631 th->th.th_last_place = last_place;
4632 th->th.th_new_place = place;
4634 if (__kmp_display_affinity && place != th->th.th_current_place &&
4635 team->t.t_display_affinity != 1) {
4636 team->t.t_display_affinity = 1;
4640 KA_TRACE(100, (
"__kmp_partition_places: close: T#%d(%d:%d) place %d " 4641 "partition = [%d,%d]\n",
4642 __kmp_gtid_from_thread(team->t.t_threads[f]),
4643 team->t.t_id, f, place, first_place, last_place));
4646 int S, rem, gap, s_count;
4647 S = n_th / n_places;
4649 rem = n_th - (S * n_places);
4650 gap = rem > 0 ? n_places / rem : n_places;
4651 int place = masters_place;
4653 for (f = 0; f < n_th; f++) {
4654 kmp_info_t *th = team->t.t_threads[f];
4655 KMP_DEBUG_ASSERT(th != NULL);
4657 th->th.th_first_place = first_place;
4658 th->th.th_last_place = last_place;
4659 th->th.th_new_place = place;
4661 if (__kmp_display_affinity && place != th->th.th_current_place &&
4662 team->t.t_display_affinity != 1) {
4663 team->t.t_display_affinity = 1;
4668 if ((s_count == S) && rem && (gap_ct == gap)) {
4670 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4672 if (place == last_place) {
4673 place = first_place;
4674 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4682 }
else if (s_count == S) {
4683 if (place == last_place) {
4684 place = first_place;
4685 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4695 (
"__kmp_partition_places: close: T#%d(%d:%d) place %d " 4696 "partition = [%d,%d]\n",
4697 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4698 th->th.th_new_place, first_place, last_place));
4700 KMP_DEBUG_ASSERT(place == masters_place);
4704 case proc_bind_spread: {
4706 int n_th = team->t.t_nproc;
4709 if (first_place <= last_place) {
4710 n_places = last_place - first_place + 1;
4712 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4714 if (n_th <= n_places) {
4717 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4718 int S = n_places / n_th;
4719 int s_count, rem, gap, gap_ct;
4721 place = masters_place;
4722 rem = n_places - n_th * S;
4723 gap = rem ? n_th / rem : 1;
4726 if (update_master_only == 1)
4728 for (f = 0; f < thidx; f++) {
4729 kmp_info_t *th = team->t.t_threads[f];
4730 KMP_DEBUG_ASSERT(th != NULL);
4732 th->th.th_first_place = place;
4733 th->th.th_new_place = place;
4735 if (__kmp_display_affinity && place != th->th.th_current_place &&
4736 team->t.t_display_affinity != 1) {
4737 team->t.t_display_affinity = 1;
4741 while (s_count < S) {
4742 if (place == last_place) {
4743 place = first_place;
4744 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4751 if (rem && (gap_ct == gap)) {
4752 if (place == last_place) {
4753 place = first_place;
4754 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4762 th->th.th_last_place = place;
4765 if (place == last_place) {
4766 place = first_place;
4767 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4774 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4775 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4776 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4777 f, th->th.th_new_place, th->th.th_first_place,
4778 th->th.th_last_place, __kmp_affinity_num_masks));
4784 double current =
static_cast<double>(masters_place);
4786 (
static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4791 if (update_master_only == 1)
4793 for (f = 0; f < thidx; f++) {
4794 first =
static_cast<int>(current);
4795 last =
static_cast<int>(current + spacing) - 1;
4796 KMP_DEBUG_ASSERT(last >= first);
4797 if (first >= n_places) {
4798 if (masters_place) {
4801 if (first == (masters_place + 1)) {
4802 KMP_DEBUG_ASSERT(f == n_th);
4805 if (last == masters_place) {
4806 KMP_DEBUG_ASSERT(f == (n_th - 1));
4810 KMP_DEBUG_ASSERT(f == n_th);
4815 if (last >= n_places) {
4816 last = (n_places - 1);
4821 KMP_DEBUG_ASSERT(0 <= first);
4822 KMP_DEBUG_ASSERT(n_places > first);
4823 KMP_DEBUG_ASSERT(0 <= last);
4824 KMP_DEBUG_ASSERT(n_places > last);
4825 KMP_DEBUG_ASSERT(last_place >= first_place);
4826 th = team->t.t_threads[f];
4827 KMP_DEBUG_ASSERT(th);
4828 th->th.th_first_place = first;
4829 th->th.th_new_place = place;
4830 th->th.th_last_place = last;
4832 if (__kmp_display_affinity && place != th->th.th_current_place &&
4833 team->t.t_display_affinity != 1) {
4834 team->t.t_display_affinity = 1;
4838 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4839 "partition = [%d,%d], spacing = %.4f\n",
4840 __kmp_gtid_from_thread(team->t.t_threads[f]),
4841 team->t.t_id, f, th->th.th_new_place,
4842 th->th.th_first_place, th->th.th_last_place, spacing));
4846 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4848 int S, rem, gap, s_count;
4849 S = n_th / n_places;
4851 rem = n_th - (S * n_places);
4852 gap = rem > 0 ? n_places / rem : n_places;
4853 int place = masters_place;
4856 if (update_master_only == 1)
4858 for (f = 0; f < thidx; f++) {
4859 kmp_info_t *th = team->t.t_threads[f];
4860 KMP_DEBUG_ASSERT(th != NULL);
4862 th->th.th_first_place = place;
4863 th->th.th_last_place = place;
4864 th->th.th_new_place = place;
4866 if (__kmp_display_affinity && place != th->th.th_current_place &&
4867 team->t.t_display_affinity != 1) {
4868 team->t.t_display_affinity = 1;
4873 if ((s_count == S) && rem && (gap_ct == gap)) {
4875 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4877 if (place == last_place) {
4878 place = first_place;
4879 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4887 }
else if (s_count == S) {
4888 if (place == last_place) {
4889 place = first_place;
4890 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4899 KA_TRACE(100, (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4900 "partition = [%d,%d]\n",
4901 __kmp_gtid_from_thread(team->t.t_threads[f]),
4902 team->t.t_id, f, th->th.th_new_place,
4903 th->th.th_first_place, th->th.th_last_place));
4905 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4913 KA_TRACE(20, (
"__kmp_partition_places: exit T#%d\n", team->t.t_id));
4921 __kmp_allocate_team(kmp_root_t *root,
int new_nproc,
int max_nproc,
4923 ompt_data_t ompt_parallel_data,
4926 kmp_proc_bind_t new_proc_bind,
4928 kmp_internal_control_t *new_icvs,
4929 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4930 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4933 int use_hot_team = !root->r.r_active;
4936 KA_TRACE(20, (
"__kmp_allocate_team: called\n"));
4937 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4938 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4941 #if KMP_NESTED_HOT_TEAMS 4942 kmp_hot_team_ptr_t *hot_teams;
4944 team = master->th.th_team;
4945 level = team->t.t_active_level;
4946 if (master->th.th_teams_microtask) {
4947 if (master->th.th_teams_size.nteams > 1 &&
4950 (microtask_t)__kmp_teams_master ||
4951 master->th.th_teams_level <
4957 hot_teams = master->th.th_hot_teams;
4958 if (level < __kmp_hot_teams_max_level && hot_teams &&
4968 if (use_hot_team && new_nproc > 1) {
4969 KMP_DEBUG_ASSERT(new_nproc == max_nproc);
4970 #if KMP_NESTED_HOT_TEAMS 4971 team = hot_teams[level].hot_team;
4973 team = root->r.r_hot_team;
4976 if (__kmp_tasking_mode != tskm_immediate_exec) {
4977 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p " 4978 "task_team[1] = %p before reinit\n",
4979 team->t.t_task_team[0], team->t.t_task_team[1]));
4986 if (team->t.t_nproc == new_nproc) {
4987 KA_TRACE(20, (
"__kmp_allocate_team: reusing hot team\n"));
4990 if (team->t.t_size_changed == -1) {
4991 team->t.t_size_changed = 1;
4993 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
4997 kmp_r_sched_t new_sched = new_icvs->sched;
4999 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
5001 __kmp_reinitialize_team(team, new_icvs,
5002 root->r.r_uber_thread->th.th_ident);
5004 KF_TRACE(10, (
"__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
5005 team->t.t_threads[0], team));
5006 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5009 #if KMP_AFFINITY_SUPPORTED 5010 if ((team->t.t_size_changed == 0) &&
5011 (team->t.t_proc_bind == new_proc_bind)) {
5012 if (new_proc_bind == proc_bind_spread) {
5013 __kmp_partition_places(
5016 KA_TRACE(200, (
"__kmp_allocate_team: reusing hot team #%d bindings: " 5017 "proc_bind = %d, partition = [%d,%d]\n",
5018 team->t.t_id, new_proc_bind, team->t.t_first_place,
5019 team->t.t_last_place));
5021 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5022 __kmp_partition_places(team);
5025 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5028 }
else if (team->t.t_nproc > new_nproc) {
5030 (
"__kmp_allocate_team: decreasing hot team thread count to %d\n",
5033 team->t.t_size_changed = 1;
5034 #if KMP_NESTED_HOT_TEAMS 5035 if (__kmp_hot_teams_mode == 0) {
5038 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
5039 hot_teams[level].hot_team_nth = new_nproc;
5040 #endif // KMP_NESTED_HOT_TEAMS 5042 for (f = new_nproc; f < team->t.t_nproc; f++) {
5043 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5044 if (__kmp_tasking_mode != tskm_immediate_exec) {
5047 team->t.t_threads[f]->th.th_task_team = NULL;
5049 __kmp_free_thread(team->t.t_threads[f]);
5050 team->t.t_threads[f] = NULL;
5052 #if KMP_NESTED_HOT_TEAMS 5057 for (f = new_nproc; f < team->t.t_nproc; ++f) {
5058 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5059 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
5060 for (
int b = 0; b < bs_last_barrier; ++b) {
5061 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
5062 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5064 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
5068 #endif // KMP_NESTED_HOT_TEAMS 5069 team->t.t_nproc = new_nproc;
5071 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
5072 __kmp_reinitialize_team(team, new_icvs,
5073 root->r.r_uber_thread->th.th_ident);
5076 for (f = 0; f < new_nproc; ++f) {
5077 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5081 KF_TRACE(10, (
"__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5082 team->t.t_threads[0], team));
5084 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5087 for (f = 0; f < team->t.t_nproc; f++) {
5088 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5089 team->t.t_threads[f]->th.th_team_nproc ==
5095 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5096 #if KMP_AFFINITY_SUPPORTED 5097 __kmp_partition_places(team);
5101 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5102 kmp_affin_mask_t *old_mask;
5103 if (KMP_AFFINITY_CAPABLE()) {
5104 KMP_CPU_ALLOC(old_mask);
5109 (
"__kmp_allocate_team: increasing hot team thread count to %d\n",
5112 team->t.t_size_changed = 1;
5114 #if KMP_NESTED_HOT_TEAMS 5115 int avail_threads = hot_teams[level].hot_team_nth;
5116 if (new_nproc < avail_threads)
5117 avail_threads = new_nproc;
5118 kmp_info_t **other_threads = team->t.t_threads;
5119 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5123 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5124 for (b = 0; b < bs_last_barrier; ++b) {
5125 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5126 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5128 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5132 if (hot_teams[level].hot_team_nth >= new_nproc) {
5135 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5136 team->t.t_nproc = new_nproc;
5142 hot_teams[level].hot_team_nth = new_nproc;
5143 #endif // KMP_NESTED_HOT_TEAMS 5144 if (team->t.t_max_nproc < new_nproc) {
5146 __kmp_reallocate_team_arrays(team, new_nproc);
5147 __kmp_reinitialize_team(team, new_icvs, NULL);
5150 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5155 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5159 for (f = team->t.t_nproc; f < new_nproc; f++) {
5160 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5161 KMP_DEBUG_ASSERT(new_worker);
5162 team->t.t_threads[f] = new_worker;
5165 (
"__kmp_allocate_team: team %d init T#%d arrived: " 5166 "join=%llu, plain=%llu\n",
5167 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5168 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5169 team->t.t_bar[bs_plain_barrier].b_arrived));
5173 kmp_balign_t *balign = new_worker->th.th_bar;
5174 for (b = 0; b < bs_last_barrier; ++b) {
5175 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5176 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5177 KMP_BARRIER_PARENT_FLAG);
5179 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5185 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5186 if (KMP_AFFINITY_CAPABLE()) {
5188 __kmp_set_system_affinity(old_mask, TRUE);
5189 KMP_CPU_FREE(old_mask);
5192 #if KMP_NESTED_HOT_TEAMS 5194 #endif // KMP_NESTED_HOT_TEAMS 5196 int old_nproc = team->t.t_nproc;
5198 __kmp_initialize_team(team, new_nproc, new_icvs,
5199 root->r.r_uber_thread->th.th_ident);
5202 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5203 for (f = 0; f < team->t.t_nproc; ++f)
5204 __kmp_initialize_info(team->t.t_threads[f], team, f,
5205 __kmp_gtid_from_tid(f, team));
5212 for (f = old_nproc; f < team->t.t_nproc; ++f)
5213 team->t.t_threads[f]->th.th_task_state =
5214 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5217 team->t.t_threads[0]->th.th_task_state;
5218 for (f = old_nproc; f < team->t.t_nproc; ++f)
5219 team->t.t_threads[f]->th.th_task_state = old_state;
5223 for (f = 0; f < team->t.t_nproc; ++f) {
5224 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5225 team->t.t_threads[f]->th.th_team_nproc ==
5231 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5232 #if KMP_AFFINITY_SUPPORTED 5233 __kmp_partition_places(team);
5239 kmp_info_t *master = team->t.t_threads[0];
5240 if (master->th.th_teams_microtask) {
5241 for (f = 1; f < new_nproc; ++f) {
5243 kmp_info_t *thr = team->t.t_threads[f];
5244 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5245 thr->th.th_teams_level = master->th.th_teams_level;
5246 thr->th.th_teams_size = master->th.th_teams_size;
5250 #if KMP_NESTED_HOT_TEAMS 5254 for (f = 1; f < new_nproc; ++f) {
5255 kmp_info_t *thr = team->t.t_threads[f];
5257 kmp_balign_t *balign = thr->th.th_bar;
5258 for (b = 0; b < bs_last_barrier; ++b) {
5259 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5260 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5262 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5267 #endif // KMP_NESTED_HOT_TEAMS 5270 __kmp_alloc_argv_entries(argc, team, TRUE);
5271 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5275 KF_TRACE(10, (
" hot_team = %p\n", team));
5278 if (__kmp_tasking_mode != tskm_immediate_exec) {
5279 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p " 5280 "task_team[1] = %p after reinit\n",
5281 team->t.t_task_team[0], team->t.t_task_team[1]));
5286 __ompt_team_assign_id(team, ompt_parallel_data);
5296 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
5299 if (team->t.t_max_nproc >= max_nproc) {
5301 __kmp_team_pool = team->t.t_next_pool;
5304 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5306 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and " 5307 "task_team[1] %p to NULL\n",
5308 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5309 team->t.t_task_team[0] = NULL;
5310 team->t.t_task_team[1] = NULL;
5313 __kmp_alloc_argv_entries(argc, team, TRUE);
5314 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5317 20, (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5318 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5321 for (b = 0; b < bs_last_barrier; ++b) {
5322 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5324 team->t.t_bar[b].b_master_arrived = 0;
5325 team->t.t_bar[b].b_team_arrived = 0;
5331 team->t.t_proc_bind = new_proc_bind;
5334 KA_TRACE(20, (
"__kmp_allocate_team: using team from pool %d.\n",
5338 __ompt_team_assign_id(team, ompt_parallel_data);
5350 team = __kmp_reap_team(team);
5351 __kmp_team_pool = team;
5356 team = (kmp_team_t *)__kmp_allocate(
sizeof(kmp_team_t));
5359 team->t.t_max_nproc = max_nproc;
5362 __kmp_allocate_team_arrays(team, max_nproc);
5364 KA_TRACE(20, (
"__kmp_allocate_team: making a new team\n"));
5365 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5367 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5369 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5370 team->t.t_task_team[0] = NULL;
5372 team->t.t_task_team[1] = NULL;
5375 if (__kmp_storage_map) {
5376 __kmp_print_team_storage_map(
"team", team, team->t.t_id, new_nproc);
5380 __kmp_alloc_argv_entries(argc, team, FALSE);
5381 team->t.t_argc = argc;
5384 (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5385 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5388 for (b = 0; b < bs_last_barrier; ++b) {
5389 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5391 team->t.t_bar[b].b_master_arrived = 0;
5392 team->t.t_bar[b].b_team_arrived = 0;
5398 team->t.t_proc_bind = new_proc_bind;
5402 __ompt_team_assign_id(team, ompt_parallel_data);
5403 team->t.ompt_serialized_team_info = NULL;
5408 KA_TRACE(20, (
"__kmp_allocate_team: done creating a new team %d.\n",
5419 void __kmp_free_team(kmp_root_t *root,
5420 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5422 KA_TRACE(20, (
"__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5426 KMP_DEBUG_ASSERT(root);
5427 KMP_DEBUG_ASSERT(team);
5428 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5429 KMP_DEBUG_ASSERT(team->t.t_threads);
5431 int use_hot_team = team == root->r.r_hot_team;
5432 #if KMP_NESTED_HOT_TEAMS 5434 kmp_hot_team_ptr_t *hot_teams;
5436 level = team->t.t_active_level - 1;
5437 if (master->th.th_teams_microtask) {
5438 if (master->th.th_teams_size.nteams > 1) {
5442 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5443 master->th.th_teams_level == team->t.t_level) {
5448 hot_teams = master->th.th_hot_teams;
5449 if (level < __kmp_hot_teams_max_level) {
5450 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5454 #endif // KMP_NESTED_HOT_TEAMS 5457 TCW_SYNC_PTR(team->t.t_pkfn,
5460 team->t.t_copyin_counter = 0;
5465 if (!use_hot_team) {
5466 if (__kmp_tasking_mode != tskm_immediate_exec) {
5468 for (f = 1; f < team->t.t_nproc; ++f) {
5469 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5470 kmp_info_t *th = team->t.t_threads[f];
5471 volatile kmp_uint32 *state = &th->th.th_reap_state;
5472 while (*state != KMP_SAFE_TO_REAP) {
5476 if (!__kmp_is_thread_alive(th, &ecode)) {
5477 *state = KMP_SAFE_TO_REAP;
5482 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5483 if (fl.is_sleeping())
5484 fl.resume(__kmp_gtid_from_thread(th));
5491 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5492 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5493 if (task_team != NULL) {
5494 for (f = 0; f < team->t.t_nproc;
5496 team->t.t_threads[f]->th.th_task_team = NULL;
5500 (
"__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5501 __kmp_get_gtid(), task_team, team->t.t_id));
5502 #if KMP_NESTED_HOT_TEAMS 5503 __kmp_free_task_team(master, task_team);
5505 team->t.t_task_team[tt_idx] = NULL;
5511 team->t.t_parent = NULL;
5512 team->t.t_level = 0;
5513 team->t.t_active_level = 0;
5516 for (f = 1; f < team->t.t_nproc; ++f) {
5517 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5518 __kmp_free_thread(team->t.t_threads[f]);
5519 team->t.t_threads[f] = NULL;
5524 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
5525 __kmp_team_pool = (
volatile kmp_team_t *)team;
5532 kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5533 kmp_team_t *next_pool = team->t.t_next_pool;
5535 KMP_DEBUG_ASSERT(team);
5536 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5537 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5538 KMP_DEBUG_ASSERT(team->t.t_threads);
5539 KMP_DEBUG_ASSERT(team->t.t_argv);
5544 __kmp_free_team_arrays(team);
5545 if (team->t.t_argv != &team->t.t_inline_argv[0])
5546 __kmp_free((
void *)team->t.t_argv);
5578 void __kmp_free_thread(kmp_info_t *this_th) {
5581 kmp_root_t *root = this_th->th.th_root;
5583 KA_TRACE(20, (
"__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5584 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
5586 KMP_DEBUG_ASSERT(this_th);
5591 kmp_balign_t *balign = this_th->th.th_bar;
5592 for (b = 0; b < bs_last_barrier; ++b) {
5593 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5594 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5595 balign[b].bb.team = NULL;
5596 balign[b].bb.leaf_kids = 0;
5598 this_th->th.th_task_state = 0;
5599 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
5602 TCW_PTR(this_th->th.th_team, NULL);
5603 TCW_PTR(this_th->th.th_root, NULL);
5604 TCW_PTR(this_th->th.th_dispatch, NULL);
5611 __kmp_free_implicit_task(this_th);
5612 this_th->th.th_current_task = NULL;
5616 gtid = this_th->th.th_info.ds.ds_gtid;
5617 if (__kmp_thread_pool_insert_pt != NULL) {
5618 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5619 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5620 __kmp_thread_pool_insert_pt = NULL;
5629 if (__kmp_thread_pool_insert_pt != NULL) {
5630 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5632 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
5634 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5635 scan = &((*scan)->th.th_next_pool))
5640 TCW_PTR(this_th->th.th_next_pool, *scan);
5641 __kmp_thread_pool_insert_pt = *scan = this_th;
5642 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5643 (this_th->th.th_info.ds.ds_gtid <
5644 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5645 TCW_4(this_th->th.th_in_pool, TRUE);
5646 __kmp_thread_pool_nth++;
5648 TCW_4(__kmp_nth, __kmp_nth - 1);
5649 root->r.r_cg_nthreads--;
5651 #ifdef KMP_ADJUST_BLOCKTIME 5654 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5655 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5656 if (__kmp_nth <= __kmp_avail_proc) {
5657 __kmp_zero_bt = FALSE;
5667 void *__kmp_launch_thread(kmp_info_t *this_thr) {
5668 int gtid = this_thr->th.th_info.ds.ds_gtid;
5670 kmp_team_t *(*
volatile pteam);
5673 KA_TRACE(10, (
"__kmp_launch_thread: T#%d start\n", gtid));
5675 if (__kmp_env_consistency_check) {
5676 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid);
5680 ompt_data_t *thread_data;
5681 if (ompt_enabled.enabled) {
5682 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
5683 *thread_data = ompt_data_none;
5685 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5686 this_thr->th.ompt_thread_info.wait_id = 0;
5687 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5688 if (ompt_enabled.ompt_callback_thread_begin) {
5689 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5690 ompt_thread_worker, thread_data);
5696 if (ompt_enabled.enabled) {
5697 this_thr->th.ompt_thread_info.state = ompt_state_idle;
5701 while (!TCR_4(__kmp_global.g.g_done)) {
5702 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5706 KA_TRACE(20, (
"__kmp_launch_thread: T#%d waiting for work\n", gtid));
5709 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5712 if (ompt_enabled.enabled) {
5713 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5717 pteam = (kmp_team_t * (*))(&this_thr->th.th_team);
5720 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
5722 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5725 (
"__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5726 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5727 (*pteam)->t.t_pkfn));
5729 updateHWFPControl(*pteam);
5732 if (ompt_enabled.enabled) {
5733 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
5737 rc = (*pteam)->t.t_invoke(gtid);
5741 KA_TRACE(20, (
"__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5742 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5743 (*pteam)->t.t_pkfn));
5746 if (ompt_enabled.enabled) {
5748 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;
5750 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5754 __kmp_join_barrier(gtid);
5757 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
5760 if (ompt_enabled.ompt_callback_thread_end) {
5761 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
5765 this_thr->th.th_task_team = NULL;
5767 __kmp_common_destroy_gtid(gtid);
5769 KA_TRACE(10, (
"__kmp_launch_thread: T#%d done\n", gtid));
5776 void __kmp_internal_end_dest(
void *specific_gtid) {
5777 #if KMP_COMPILER_ICC 5778 #pragma warning(push) 5779 #pragma warning(disable : 810) // conversion from "void *" to "int" may lose 5783 int gtid = (kmp_intptr_t)specific_gtid - 1;
5784 #if KMP_COMPILER_ICC 5785 #pragma warning(pop) 5788 KA_TRACE(30, (
"__kmp_internal_end_dest: T#%d\n", gtid));
5801 if (gtid >= 0 && KMP_UBER_GTID(gtid))
5802 __kmp_gtid_set_specific(gtid);
5803 #ifdef KMP_TDATA_GTID 5806 __kmp_internal_end_thread(gtid);
5809 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5815 __attribute__((destructor))
void __kmp_internal_end_dtor(
void) {
5816 __kmp_internal_end_atexit();
5819 void __kmp_internal_end_fini(
void) { __kmp_internal_end_atexit(); }
5825 void __kmp_internal_end_atexit(
void) {
5826 KA_TRACE(30, (
"__kmp_internal_end_atexit\n"));
5850 __kmp_internal_end_library(-1);
5852 __kmp_close_console();
5856 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root) {
5861 KMP_DEBUG_ASSERT(thread != NULL);
5863 gtid = thread->th.th_info.ds.ds_gtid;
5866 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5869 20, (
"__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5873 ANNOTATE_HAPPENS_BEFORE(thread);
5874 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
5875 __kmp_release_64(&flag);
5879 __kmp_reap_worker(thread);
5891 if (thread->th.th_active_in_pool) {
5892 thread->th.th_active_in_pool = FALSE;
5893 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
5894 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
5898 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0);
5899 --__kmp_thread_pool_nth;
5902 __kmp_free_implicit_task(thread);
5906 __kmp_free_fast_memory(thread);
5909 __kmp_suspend_uninitialize_thread(thread);
5911 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5912 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5917 #ifdef KMP_ADJUST_BLOCKTIME 5920 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5921 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5922 if (__kmp_nth <= __kmp_avail_proc) {
5923 __kmp_zero_bt = FALSE;
5929 if (__kmp_env_consistency_check) {
5930 if (thread->th.th_cons) {
5931 __kmp_free_cons_stack(thread->th.th_cons);
5932 thread->th.th_cons = NULL;
5936 if (thread->th.th_pri_common != NULL) {
5937 __kmp_free(thread->th.th_pri_common);
5938 thread->th.th_pri_common = NULL;
5941 if (thread->th.th_task_state_memo_stack != NULL) {
5942 __kmp_free(thread->th.th_task_state_memo_stack);
5943 thread->th.th_task_state_memo_stack = NULL;
5947 if (thread->th.th_local.bget_data != NULL) {
5948 __kmp_finalize_bget(thread);
5952 #if KMP_AFFINITY_SUPPORTED 5953 if (thread->th.th_affin_mask != NULL) {
5954 KMP_CPU_FREE(thread->th.th_affin_mask);
5955 thread->th.th_affin_mask = NULL;
5959 #if KMP_USE_HIER_SCHED 5960 if (thread->th.th_hier_bar_data != NULL) {
5961 __kmp_free(thread->th.th_hier_bar_data);
5962 thread->th.th_hier_bar_data = NULL;
5966 __kmp_reap_team(thread->th.th_serial_team);
5967 thread->th.th_serial_team = NULL;
5974 static void __kmp_internal_end(
void) {
5978 __kmp_unregister_library();
5985 __kmp_reclaim_dead_roots();
5989 for (i = 0; i < __kmp_threads_capacity; i++)
5991 if (__kmp_root[i]->r.r_active)
5994 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
5996 if (i < __kmp_threads_capacity) {
6008 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6009 if (TCR_4(__kmp_init_monitor)) {
6010 __kmp_reap_monitor(&__kmp_monitor);
6011 TCW_4(__kmp_init_monitor, 0);
6013 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6014 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6015 #endif // KMP_USE_MONITOR 6020 for (i = 0; i < __kmp_threads_capacity; i++) {
6021 if (__kmp_root[i]) {
6024 KMP_ASSERT(!__kmp_root[i]->r.r_active);
6033 while (__kmp_thread_pool != NULL) {
6035 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
6036 __kmp_thread_pool = thread->th.th_next_pool;
6038 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
6039 thread->th.th_next_pool = NULL;
6040 thread->th.th_in_pool = FALSE;
6041 __kmp_reap_thread(thread, 0);
6043 __kmp_thread_pool_insert_pt = NULL;
6046 while (__kmp_team_pool != NULL) {
6048 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
6049 __kmp_team_pool = team->t.t_next_pool;
6051 team->t.t_next_pool = NULL;
6052 __kmp_reap_team(team);
6055 __kmp_reap_task_teams();
6062 for (i = 0; i < __kmp_threads_capacity; i++) {
6063 kmp_info_t *thr = __kmp_threads[i];
6064 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
6069 for (i = 0; i < __kmp_threads_capacity; ++i) {
6076 TCW_SYNC_4(__kmp_init_common, FALSE);
6078 KA_TRACE(10, (
"__kmp_internal_end: all workers reaped\n"));
6086 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6087 if (TCR_4(__kmp_init_monitor)) {
6088 __kmp_reap_monitor(&__kmp_monitor);
6089 TCW_4(__kmp_init_monitor, 0);
6091 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6092 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6095 TCW_4(__kmp_init_gtid, FALSE);
6104 void __kmp_internal_end_library(
int gtid_req) {
6111 if (__kmp_global.g.g_abort) {
6112 KA_TRACE(11, (
"__kmp_internal_end_library: abort, exiting\n"));
6116 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6117 KA_TRACE(10, (
"__kmp_internal_end_library: already finished\n"));
6125 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6127 10, (
"__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6128 if (gtid == KMP_GTID_SHUTDOWN) {
6129 KA_TRACE(10, (
"__kmp_internal_end_library: !__kmp_init_runtime, system " 6130 "already shutdown\n"));
6132 }
else if (gtid == KMP_GTID_MONITOR) {
6133 KA_TRACE(10, (
"__kmp_internal_end_library: monitor thread, gtid not " 6134 "registered, or system shutdown\n"));
6136 }
else if (gtid == KMP_GTID_DNE) {
6137 KA_TRACE(10, (
"__kmp_internal_end_library: gtid not registered or system " 6140 }
else if (KMP_UBER_GTID(gtid)) {
6142 if (__kmp_root[gtid]->r.r_active) {
6143 __kmp_global.g.g_abort = -1;
6144 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6146 (
"__kmp_internal_end_library: root still active, abort T#%d\n",
6152 (
"__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6153 __kmp_unregister_root_current_thread(gtid);
6160 #ifdef DUMP_DEBUG_ON_EXIT 6161 if (__kmp_debug_buf)
6162 __kmp_dump_debug_buffer();
6168 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6171 if (__kmp_global.g.g_abort) {
6172 KA_TRACE(10, (
"__kmp_internal_end_library: abort, exiting\n"));
6174 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6177 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6178 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6187 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6190 __kmp_internal_end();
6192 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6193 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6195 KA_TRACE(10, (
"__kmp_internal_end_library: exit\n"));
6197 #ifdef DUMP_DEBUG_ON_EXIT 6198 if (__kmp_debug_buf)
6199 __kmp_dump_debug_buffer();
6203 __kmp_close_console();
6206 __kmp_fini_allocator();
6210 void __kmp_internal_end_thread(
int gtid_req) {
6219 if (__kmp_global.g.g_abort) {
6220 KA_TRACE(11, (
"__kmp_internal_end_thread: abort, exiting\n"));
6224 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6225 KA_TRACE(10, (
"__kmp_internal_end_thread: already finished\n"));
6233 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6235 (
"__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6236 if (gtid == KMP_GTID_SHUTDOWN) {
6237 KA_TRACE(10, (
"__kmp_internal_end_thread: !__kmp_init_runtime, system " 6238 "already shutdown\n"));
6240 }
else if (gtid == KMP_GTID_MONITOR) {
6241 KA_TRACE(10, (
"__kmp_internal_end_thread: monitor thread, gtid not " 6242 "registered, or system shutdown\n"));
6244 }
else if (gtid == KMP_GTID_DNE) {
6245 KA_TRACE(10, (
"__kmp_internal_end_thread: gtid not registered or system " 6249 }
else if (KMP_UBER_GTID(gtid)) {
6251 if (__kmp_root[gtid]->r.r_active) {
6252 __kmp_global.g.g_abort = -1;
6253 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6255 (
"__kmp_internal_end_thread: root still active, abort T#%d\n",
6259 KA_TRACE(10, (
"__kmp_internal_end_thread: unregistering sibling T#%d\n",
6261 __kmp_unregister_root_current_thread(gtid);
6265 KA_TRACE(10, (
"__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6268 __kmp_threads[gtid]->th.th_task_team = NULL;
6272 (
"__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6286 if (__kmp_pause_status != kmp_hard_paused) {
6287 KA_TRACE(10, (
"__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6292 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6295 if (__kmp_global.g.g_abort) {
6296 KA_TRACE(10, (
"__kmp_internal_end_thread: abort, exiting\n"));
6298 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6301 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6302 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6313 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6315 for (i = 0; i < __kmp_threads_capacity; ++i) {
6316 if (KMP_UBER_GTID(i)) {
6319 (
"__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6320 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6321 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6328 __kmp_internal_end();
6330 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6331 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6333 KA_TRACE(10, (
"__kmp_internal_end_thread: exit T#%d\n", gtid_req));
6335 #ifdef DUMP_DEBUG_ON_EXIT 6336 if (__kmp_debug_buf)
6337 __kmp_dump_debug_buffer();
6344 static long __kmp_registration_flag = 0;
6346 static char *__kmp_registration_str = NULL;
6349 static inline char *__kmp_reg_status_name() {
6354 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d", (
int)getpid());
6357 void __kmp_register_library_startup(
void) {
6359 char *name = __kmp_reg_status_name();
6365 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6366 __kmp_initialize_system_tick();
6368 __kmp_read_system_time(&time.dtime);
6369 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6370 __kmp_registration_str =
6371 __kmp_str_format(
"%p-%lx-%s", &__kmp_registration_flag,
6372 __kmp_registration_flag, KMP_LIBRARY_FILE);
6374 KA_TRACE(50, (
"__kmp_register_library_startup: %s=\"%s\"\n", name,
6375 __kmp_registration_str));
6382 __kmp_env_set(name, __kmp_registration_str, 0);
6384 value = __kmp_env_get(name);
6385 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6395 char *flag_addr_str = NULL;
6396 char *flag_val_str = NULL;
6397 char const *file_name = NULL;
6398 __kmp_str_split(tail,
'-', &flag_addr_str, &tail);
6399 __kmp_str_split(tail,
'-', &flag_val_str, &tail);
6402 long *flag_addr = 0;
6404 KMP_SSCANF(flag_addr_str,
"%p", RCAST(
void**, &flag_addr));
6405 KMP_SSCANF(flag_val_str,
"%lx", &flag_val);
6406 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name,
"") != 0) {
6410 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6424 file_name =
"unknown library";
6428 char *duplicate_ok = __kmp_env_get(
"KMP_DUPLICATE_LIB_OK");
6429 if (!__kmp_str_match_true(duplicate_ok)) {
6431 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6432 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
6434 KMP_INTERNAL_FREE(duplicate_ok);
6435 __kmp_duplicate_library_ok = 1;
6440 __kmp_env_unset(name);
6442 default: { KMP_DEBUG_ASSERT(0); }
break;
6445 KMP_INTERNAL_FREE((
void *)value);
6447 KMP_INTERNAL_FREE((
void *)name);
6451 void __kmp_unregister_library(
void) {
6453 char *name = __kmp_reg_status_name();
6454 char *value = __kmp_env_get(name);
6456 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6457 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6458 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6460 __kmp_env_unset(name);
6463 KMP_INTERNAL_FREE(__kmp_registration_str);
6464 KMP_INTERNAL_FREE(value);
6465 KMP_INTERNAL_FREE(name);
6467 __kmp_registration_flag = 0;
6468 __kmp_registration_str = NULL;
6475 #if KMP_MIC_SUPPORTED 6477 static void __kmp_check_mic_type() {
6478 kmp_cpuid_t cpuid_state = {0};
6479 kmp_cpuid_t *cs_p = &cpuid_state;
6480 __kmp_x86_cpuid(1, 0, cs_p);
6482 if ((cs_p->eax & 0xff0) == 0xB10) {
6483 __kmp_mic_type = mic2;
6484 }
else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6485 __kmp_mic_type = mic3;
6487 __kmp_mic_type = non_mic;
6493 static void __kmp_do_serial_initialize(
void) {
6497 KA_TRACE(10, (
"__kmp_do_serial_initialize: enter\n"));
6499 KMP_DEBUG_ASSERT(
sizeof(kmp_int32) == 4);
6500 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) == 4);
6501 KMP_DEBUG_ASSERT(
sizeof(kmp_int64) == 8);
6502 KMP_DEBUG_ASSERT(
sizeof(kmp_uint64) == 8);
6503 KMP_DEBUG_ASSERT(
sizeof(kmp_intptr_t) ==
sizeof(
void *));
6509 __kmp_validate_locks();
6512 __kmp_init_allocator();
6517 __kmp_register_library_startup();
6520 if (TCR_4(__kmp_global.g.g_done)) {
6521 KA_TRACE(10, (
"__kmp_do_serial_initialize: reinitialization of library\n"));
6524 __kmp_global.g.g_abort = 0;
6525 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
6528 #if KMP_USE_ADAPTIVE_LOCKS 6529 #if KMP_DEBUG_ADAPTIVE_LOCKS 6530 __kmp_init_speculative_stats();
6533 #if KMP_STATS_ENABLED 6536 __kmp_init_lock(&__kmp_global_lock);
6537 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6538 __kmp_init_lock(&__kmp_debug_lock);
6539 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6540 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6541 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6542 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6543 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6544 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6545 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6546 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6547 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6548 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6549 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6550 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6551 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6552 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6553 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
6555 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
6557 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
6561 __kmp_runtime_initialize();
6563 #if KMP_MIC_SUPPORTED 6564 __kmp_check_mic_type();
6571 __kmp_abort_delay = 0;
6575 __kmp_dflt_team_nth_ub = __kmp_xproc;
6576 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6577 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6579 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6580 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6582 __kmp_max_nth = __kmp_sys_max_nth;
6583 __kmp_cg_max_nth = __kmp_sys_max_nth;
6584 __kmp_teams_max_nth = __kmp_xproc;
6585 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6586 __kmp_teams_max_nth = __kmp_sys_max_nth;
6591 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
6593 __kmp_monitor_wakeups =
6594 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6595 __kmp_bt_intervals =
6596 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6599 __kmp_library = library_throughput;
6601 __kmp_static = kmp_sch_static_balanced;
6608 #if KMP_FAST_REDUCTION_BARRIER 6609 #define kmp_reduction_barrier_gather_bb ((int)1) 6610 #define kmp_reduction_barrier_release_bb ((int)1) 6611 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6612 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6613 #endif // KMP_FAST_REDUCTION_BARRIER 6614 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6615 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6616 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6617 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6618 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6619 #if KMP_FAST_REDUCTION_BARRIER 6620 if (i == bs_reduction_barrier) {
6622 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6623 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6624 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6625 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
6627 #endif // KMP_FAST_REDUCTION_BARRIER 6629 #if KMP_FAST_REDUCTION_BARRIER 6630 #undef kmp_reduction_barrier_release_pat 6631 #undef kmp_reduction_barrier_gather_pat 6632 #undef kmp_reduction_barrier_release_bb 6633 #undef kmp_reduction_barrier_gather_bb 6634 #endif // KMP_FAST_REDUCTION_BARRIER 6635 #if KMP_MIC_SUPPORTED 6636 if (__kmp_mic_type == mic2) {
6638 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3;
6639 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6641 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6642 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6644 #if KMP_FAST_REDUCTION_BARRIER 6645 if (__kmp_mic_type == mic2) {
6646 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6647 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6649 #endif // KMP_FAST_REDUCTION_BARRIER 6650 #endif // KMP_MIC_SUPPORTED 6654 __kmp_env_checks = TRUE;
6656 __kmp_env_checks = FALSE;
6660 __kmp_foreign_tp = TRUE;
6662 __kmp_global.g.g_dynamic = FALSE;
6663 __kmp_global.g.g_dynamic_mode = dynamic_default;
6665 __kmp_env_initialize(NULL);
6669 char const *val = __kmp_env_get(
"KMP_DUMP_CATALOG");
6670 if (__kmp_str_match_true(val)) {
6671 kmp_str_buf_t buffer;
6672 __kmp_str_buf_init(&buffer);
6673 __kmp_i18n_dump_catalog(&buffer);
6674 __kmp_printf(
"%s", buffer.str);
6675 __kmp_str_buf_free(&buffer);
6677 __kmp_env_free(&val);
6680 __kmp_threads_capacity =
6681 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6683 __kmp_tp_capacity = __kmp_default_tp_capacity(
6684 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
6689 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6690 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6691 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6692 __kmp_thread_pool = NULL;
6693 __kmp_thread_pool_insert_pt = NULL;
6694 __kmp_team_pool = NULL;
6701 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6703 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6704 __kmp_root = (kmp_root_t **)((
char *)__kmp_threads +
6705 sizeof(kmp_info_t *) * __kmp_threads_capacity);
6708 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6710 KMP_DEBUG_ASSERT(__kmp_nth == 0);
6715 gtid = __kmp_register_root(TRUE);
6716 KA_TRACE(10, (
"__kmp_do_serial_initialize T#%d\n", gtid));
6717 KMP_ASSERT(KMP_UBER_GTID(gtid));
6718 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
6722 __kmp_common_initialize();
6726 __kmp_register_atfork();
6729 #if !KMP_DYNAMIC_LIB 6733 int rc = atexit(__kmp_internal_end_atexit);
6735 __kmp_fatal(KMP_MSG(FunctionError,
"atexit()"), KMP_ERR(rc),
6741 #if KMP_HANDLE_SIGNALS 6747 __kmp_install_signals(FALSE);
6750 __kmp_install_signals(TRUE);
6755 __kmp_init_counter++;
6757 __kmp_init_serial = TRUE;
6759 if (__kmp_settings) {
6764 if (__kmp_display_env || __kmp_display_env_verbose) {
6765 __kmp_env_print_2();
6767 #endif // OMP_40_ENABLED 6775 KA_TRACE(10, (
"__kmp_do_serial_initialize: exit\n"));
6778 void __kmp_serial_initialize(
void) {
6779 if (__kmp_init_serial) {
6782 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6783 if (__kmp_init_serial) {
6784 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6787 __kmp_do_serial_initialize();
6788 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6791 static void __kmp_do_middle_initialize(
void) {
6793 int prev_dflt_team_nth;
6795 if (!__kmp_init_serial) {
6796 __kmp_do_serial_initialize();
6799 KA_TRACE(10, (
"__kmp_middle_initialize: enter\n"));
6803 prev_dflt_team_nth = __kmp_dflt_team_nth;
6805 #if KMP_AFFINITY_SUPPORTED 6808 __kmp_affinity_initialize();
6812 for (i = 0; i < __kmp_threads_capacity; i++) {
6813 if (TCR_PTR(__kmp_threads[i]) != NULL) {
6814 __kmp_affinity_set_init_mask(i, TRUE);
6819 KMP_ASSERT(__kmp_xproc > 0);
6820 if (__kmp_avail_proc == 0) {
6821 __kmp_avail_proc = __kmp_xproc;
6827 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
6828 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
6833 if (__kmp_dflt_team_nth == 0) {
6834 #ifdef KMP_DFLT_NTH_CORES 6836 __kmp_dflt_team_nth = __kmp_ncores;
6837 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6838 "__kmp_ncores (%d)\n",
6839 __kmp_dflt_team_nth));
6842 __kmp_dflt_team_nth = __kmp_avail_proc;
6843 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6844 "__kmp_avail_proc(%d)\n",
6845 __kmp_dflt_team_nth));
6849 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
6850 __kmp_dflt_team_nth = KMP_MIN_NTH;
6852 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
6853 __kmp_dflt_team_nth = __kmp_sys_max_nth;
6858 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
6860 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
6865 for (i = 0; i < __kmp_threads_capacity; i++) {
6866 kmp_info_t *thread = __kmp_threads[i];
6869 if (thread->th.th_current_task->td_icvs.nproc != 0)
6872 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
6877 (
"__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
6878 __kmp_dflt_team_nth));
6880 #ifdef KMP_ADJUST_BLOCKTIME 6882 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6883 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
6884 if (__kmp_nth > __kmp_avail_proc) {
6885 __kmp_zero_bt = TRUE;
6891 TCW_SYNC_4(__kmp_init_middle, TRUE);
6893 KA_TRACE(10, (
"__kmp_do_middle_initialize: exit\n"));
6896 void __kmp_middle_initialize(
void) {
6897 if (__kmp_init_middle) {
6900 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6901 if (__kmp_init_middle) {
6902 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6905 __kmp_do_middle_initialize();
6906 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6909 void __kmp_parallel_initialize(
void) {
6910 int gtid = __kmp_entry_gtid();
6913 if (TCR_4(__kmp_init_parallel))
6915 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6916 if (TCR_4(__kmp_init_parallel)) {
6917 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6922 if (TCR_4(__kmp_global.g.g_done)) {
6925 (
"__kmp_parallel_initialize: attempt to init while shutting down\n"));
6926 __kmp_infinite_loop();
6932 if (!__kmp_init_middle) {
6933 __kmp_do_middle_initialize();
6937 __kmp_resume_if_hard_paused();
6941 KA_TRACE(10, (
"__kmp_parallel_initialize: enter\n"));
6942 KMP_ASSERT(KMP_UBER_GTID(gtid));
6944 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6947 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
6948 __kmp_store_mxcsr(&__kmp_init_mxcsr);
6949 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
6953 #if KMP_HANDLE_SIGNALS 6955 __kmp_install_signals(TRUE);
6959 __kmp_suspend_initialize();
6961 #if defined(USE_LOAD_BALANCE) 6962 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6963 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
6966 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6967 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
6971 if (__kmp_version) {
6972 __kmp_print_version_2();
6976 TCW_SYNC_4(__kmp_init_parallel, TRUE);
6979 KA_TRACE(10, (
"__kmp_parallel_initialize: exit\n"));
6981 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6986 void __kmp_run_before_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
6988 kmp_disp_t *dispatch;
6993 this_thr->th.th_local.this_construct = 0;
6994 #if KMP_CACHE_MANAGE 6995 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
6997 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
6998 KMP_DEBUG_ASSERT(dispatch);
6999 KMP_DEBUG_ASSERT(team->t.t_dispatch);
7003 dispatch->th_disp_index = 0;
7005 dispatch->th_doacross_buf_idx =
7008 if (__kmp_env_consistency_check)
7009 __kmp_push_parallel(gtid, team->t.t_ident);
7014 void __kmp_run_after_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7016 if (__kmp_env_consistency_check)
7017 __kmp_pop_parallel(gtid, team->t.t_ident);
7019 __kmp_finish_implicit_task(this_thr);
7022 int __kmp_invoke_task_func(
int gtid) {
7024 int tid = __kmp_tid_from_gtid(gtid);
7025 kmp_info_t *this_thr = __kmp_threads[gtid];
7026 kmp_team_t *team = this_thr->th.th_team;
7028 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
7030 if (__itt_stack_caller_create_ptr) {
7031 __kmp_itt_stack_callee_enter(
7033 team->t.t_stack_id);
7036 #if INCLUDE_SSC_MARKS 7037 SSC_MARK_INVOKING();
7042 void **exit_runtime_p;
7043 ompt_data_t *my_task_data;
7044 ompt_data_t *my_parallel_data;
7047 if (ompt_enabled.enabled) {
7049 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame.ptr);
7051 exit_runtime_p = &dummy;
7055 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
7056 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
7057 if (ompt_enabled.ompt_callback_implicit_task) {
7058 ompt_team_size = team->t.t_nproc;
7059 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7060 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
7061 __kmp_tid_from_gtid(gtid), ompt_task_implicit);
7062 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
7067 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
7068 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
7070 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
7071 tid, (
int)team->t.t_argc, (
void **)team->t.t_argv
7078 *exit_runtime_p = NULL;
7083 if (__itt_stack_caller_create_ptr) {
7084 __kmp_itt_stack_callee_leave(
7086 team->t.t_stack_id);
7089 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
7095 void __kmp_teams_master(
int gtid) {
7097 kmp_info_t *thr = __kmp_threads[gtid];
7098 kmp_team_t *team = thr->th.th_team;
7099 ident_t *loc = team->t.t_ident;
7100 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7101 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7102 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7103 KA_TRACE(20, (
"__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7104 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
7107 #if INCLUDE_SSC_MARKS 7110 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
7111 (microtask_t)thr->th.th_teams_microtask,
7112 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
7113 #if INCLUDE_SSC_MARKS 7119 __kmp_join_call(loc, gtid
7128 int __kmp_invoke_teams_master(
int gtid) {
7129 kmp_info_t *this_thr = __kmp_threads[gtid];
7130 kmp_team_t *team = this_thr->th.th_team;
7132 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7133 KMP_DEBUG_ASSERT((
void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7134 (
void *)__kmp_teams_master);
7136 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7137 __kmp_teams_master(gtid);
7138 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7148 void __kmp_push_num_threads(
ident_t *
id,
int gtid,
int num_threads) {
7149 kmp_info_t *thr = __kmp_threads[gtid];
7151 if (num_threads > 0)
7152 thr->th.th_set_nproc = num_threads;
7159 void __kmp_push_num_teams(
ident_t *
id,
int gtid,
int num_teams,
7161 kmp_info_t *thr = __kmp_threads[gtid];
7162 KMP_DEBUG_ASSERT(num_teams >= 0);
7163 KMP_DEBUG_ASSERT(num_threads >= 0);
7167 if (num_teams > __kmp_teams_max_nth) {
7168 if (!__kmp_reserve_warn) {
7169 __kmp_reserve_warn = 1;
7170 __kmp_msg(kmp_ms_warning,
7171 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7172 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7174 num_teams = __kmp_teams_max_nth;
7178 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7181 if (num_threads == 0) {
7182 if (!TCR_4(__kmp_init_middle))
7183 __kmp_middle_initialize();
7184 num_threads = __kmp_avail_proc / num_teams;
7185 if (num_teams * num_threads > __kmp_teams_max_nth) {
7187 num_threads = __kmp_teams_max_nth / num_teams;
7190 if (num_teams * num_threads > __kmp_teams_max_nth) {
7191 int new_threads = __kmp_teams_max_nth / num_teams;
7192 if (!__kmp_reserve_warn) {
7193 __kmp_reserve_warn = 1;
7194 __kmp_msg(kmp_ms_warning,
7195 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7196 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7198 num_threads = new_threads;
7201 thr->th.th_teams_size.nth = num_threads;
7205 void __kmp_push_proc_bind(
ident_t *
id,
int gtid, kmp_proc_bind_t proc_bind) {
7206 kmp_info_t *thr = __kmp_threads[gtid];
7207 thr->th.th_set_proc_bind = proc_bind;
7214 void __kmp_internal_fork(
ident_t *
id,
int gtid, kmp_team_t *team) {
7215 kmp_info_t *this_thr = __kmp_threads[gtid];
7221 KMP_DEBUG_ASSERT(team);
7222 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7223 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7226 team->t.t_construct = 0;
7227 team->t.t_ordered.dt.t_value =
7231 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7232 if (team->t.t_max_nproc > 1) {
7234 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7235 team->t.t_disp_buffer[i].buffer_index = i;
7237 team->t.t_disp_buffer[i].doacross_buf_idx = i;
7241 team->t.t_disp_buffer[0].buffer_index = 0;
7243 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7248 KMP_ASSERT(this_thr->th.th_team == team);
7251 for (f = 0; f < team->t.t_nproc; f++) {
7252 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7253 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7258 __kmp_fork_barrier(gtid, 0);
7261 void __kmp_internal_join(
ident_t *
id,
int gtid, kmp_team_t *team) {
7262 kmp_info_t *this_thr = __kmp_threads[gtid];
7264 KMP_DEBUG_ASSERT(team);
7265 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7266 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7272 if (__kmp_threads[gtid] &&
7273 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7274 __kmp_printf(
"GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7275 __kmp_threads[gtid]);
7276 __kmp_printf(
"__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7277 "team->t.t_nproc=%d\n",
7278 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7280 __kmp_print_structure();
7282 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7283 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
7286 __kmp_join_barrier(gtid);
7288 if (ompt_enabled.enabled &&
7289 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
7290 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7291 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
7292 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
7294 void *codeptr = NULL;
7295 if (KMP_MASTER_TID(ds_tid) &&
7296 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7297 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7298 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7300 if (ompt_enabled.ompt_callback_sync_region_wait) {
7301 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
7302 ompt_sync_region_barrier, ompt_scope_end, NULL, task_data, codeptr);
7304 if (ompt_enabled.ompt_callback_sync_region) {
7305 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
7306 ompt_sync_region_barrier, ompt_scope_end, NULL, task_data, codeptr);
7309 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7310 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7311 ompt_scope_end, NULL, task_data, 0, ds_tid, ompt_task_implicit);
7317 KMP_ASSERT(this_thr->th.th_team == team);
7322 #ifdef USE_LOAD_BALANCE 7326 static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7329 kmp_team_t *hot_team;
7331 if (root->r.r_active) {
7334 hot_team = root->r.r_hot_team;
7335 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7336 return hot_team->t.t_nproc - 1;
7341 for (i = 1; i < hot_team->t.t_nproc; i++) {
7342 if (hot_team->t.t_threads[i]->th.th_active) {
7351 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc) {
7354 int hot_team_active;
7355 int team_curr_active;
7358 KB_TRACE(20, (
"__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7360 KMP_DEBUG_ASSERT(root);
7361 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7362 ->th.th_current_task->td_icvs.dynamic == TRUE);
7363 KMP_DEBUG_ASSERT(set_nproc > 1);
7365 if (set_nproc == 1) {
7366 KB_TRACE(20, (
"__kmp_load_balance_nproc: serial execution.\n"));
7375 pool_active = __kmp_thread_pool_active_nth;
7376 hot_team_active = __kmp_active_hot_team_nproc(root);
7377 team_curr_active = pool_active + hot_team_active + 1;
7380 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7381 KB_TRACE(30, (
"__kmp_load_balance_nproc: system active = %d pool active = %d " 7382 "hot team active = %d\n",
7383 system_active, pool_active, hot_team_active));
7385 if (system_active < 0) {
7389 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7390 KMP_WARNING(CantLoadBalUsing,
"KMP_DYNAMIC_MODE=thread limit");
7393 retval = __kmp_avail_proc - __kmp_nth +
7394 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7395 if (retval > set_nproc) {
7398 if (retval < KMP_MIN_NTH) {
7399 retval = KMP_MIN_NTH;
7402 KB_TRACE(20, (
"__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7410 if (system_active < team_curr_active) {
7411 system_active = team_curr_active;
7413 retval = __kmp_avail_proc - system_active + team_curr_active;
7414 if (retval > set_nproc) {
7417 if (retval < KMP_MIN_NTH) {
7418 retval = KMP_MIN_NTH;
7421 KB_TRACE(20, (
"__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7430 void __kmp_cleanup(
void) {
7433 KA_TRACE(10, (
"__kmp_cleanup: enter\n"));
7435 if (TCR_4(__kmp_init_parallel)) {
7436 #if KMP_HANDLE_SIGNALS 7437 __kmp_remove_signals();
7439 TCW_4(__kmp_init_parallel, FALSE);
7442 if (TCR_4(__kmp_init_middle)) {
7443 #if KMP_AFFINITY_SUPPORTED 7444 __kmp_affinity_uninitialize();
7446 __kmp_cleanup_hierarchy();
7447 TCW_4(__kmp_init_middle, FALSE);
7450 KA_TRACE(10, (
"__kmp_cleanup: go serial cleanup\n"));
7452 if (__kmp_init_serial) {
7453 __kmp_runtime_destroy();
7454 __kmp_init_serial = FALSE;
7457 __kmp_cleanup_threadprivate_caches();
7459 for (f = 0; f < __kmp_threads_capacity; f++) {
7460 if (__kmp_root[f] != NULL) {
7461 __kmp_free(__kmp_root[f]);
7462 __kmp_root[f] = NULL;
7465 __kmp_free(__kmp_threads);
7468 __kmp_threads = NULL;
7470 __kmp_threads_capacity = 0;
7472 #if KMP_USE_DYNAMIC_LOCK 7473 __kmp_cleanup_indirect_user_locks();
7475 __kmp_cleanup_user_locks();
7478 #if KMP_AFFINITY_SUPPORTED 7479 KMP_INTERNAL_FREE(CCAST(
char *, __kmp_cpuinfo_file));
7480 __kmp_cpuinfo_file = NULL;
7483 #if KMP_USE_ADAPTIVE_LOCKS 7484 #if KMP_DEBUG_ADAPTIVE_LOCKS 7485 __kmp_print_speculative_stats();
7488 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7489 __kmp_nested_nth.nth = NULL;
7490 __kmp_nested_nth.size = 0;
7491 __kmp_nested_nth.used = 0;
7492 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7493 __kmp_nested_proc_bind.bind_types = NULL;
7494 __kmp_nested_proc_bind.size = 0;
7495 __kmp_nested_proc_bind.used = 0;
7497 if (__kmp_affinity_format) {
7498 KMP_INTERNAL_FREE(__kmp_affinity_format);
7499 __kmp_affinity_format = NULL;
7503 __kmp_i18n_catclose();
7505 #if KMP_USE_HIER_SCHED 7506 __kmp_hier_scheds.deallocate();
7509 #if KMP_STATS_ENABLED 7513 KA_TRACE(10, (
"__kmp_cleanup: exit\n"));
7518 int __kmp_ignore_mppbeg(
void) {
7521 if ((env = getenv(
"KMP_IGNORE_MPPBEG")) != NULL) {
7522 if (__kmp_str_match_false(env))
7529 int __kmp_ignore_mppend(
void) {
7532 if ((env = getenv(
"KMP_IGNORE_MPPEND")) != NULL) {
7533 if (__kmp_str_match_false(env))
7540 void __kmp_internal_begin(
void) {
7546 gtid = __kmp_entry_gtid();
7547 root = __kmp_threads[gtid]->th.th_root;
7548 KMP_ASSERT(KMP_UBER_GTID(gtid));
7550 if (root->r.r_begin)
7552 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7553 if (root->r.r_begin) {
7554 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7558 root->r.r_begin = TRUE;
7560 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7565 void __kmp_user_set_library(
enum library_type arg) {
7572 gtid = __kmp_entry_gtid();
7573 thread = __kmp_threads[gtid];
7575 root = thread->th.th_root;
7577 KA_TRACE(20, (
"__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7579 if (root->r.r_in_parallel) {
7581 KMP_WARNING(SetLibraryIncorrectCall);
7586 case library_serial:
7587 thread->th.th_set_nproc = 0;
7588 set__nproc(thread, 1);
7590 case library_turnaround:
7591 thread->th.th_set_nproc = 0;
7592 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7593 : __kmp_dflt_team_nth_ub);
7595 case library_throughput:
7596 thread->th.th_set_nproc = 0;
7597 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7598 : __kmp_dflt_team_nth_ub);
7601 KMP_FATAL(UnknownLibraryType, arg);
7604 __kmp_aux_set_library(arg);
7607 void __kmp_aux_set_stacksize(
size_t arg) {
7608 if (!__kmp_init_serial)
7609 __kmp_serial_initialize();
7612 if (arg & (0x1000 - 1)) {
7613 arg &= ~(0x1000 - 1);
7618 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7621 if (!TCR_4(__kmp_init_parallel)) {
7624 if (value < __kmp_sys_min_stksize)
7625 value = __kmp_sys_min_stksize;
7626 else if (value > KMP_MAX_STKSIZE)
7627 value = KMP_MAX_STKSIZE;
7629 __kmp_stksize = value;
7631 __kmp_env_stksize = TRUE;
7634 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7639 void __kmp_aux_set_library(
enum library_type arg) {
7640 __kmp_library = arg;
7642 switch (__kmp_library) {
7643 case library_serial: {
7644 KMP_INFORM(LibraryIsSerial);
7645 (void)__kmp_change_library(TRUE);
7647 case library_turnaround:
7648 (void)__kmp_change_library(TRUE);
7650 case library_throughput:
7651 (void)__kmp_change_library(FALSE);
7654 KMP_FATAL(UnknownLibraryType, arg);
7660 static kmp_team_t *__kmp_aux_get_team_info(
int &teams_serialized) {
7661 kmp_info_t *thr = __kmp_entry_thread();
7662 teams_serialized = 0;
7663 if (thr->th.th_teams_microtask) {
7664 kmp_team_t *team = thr->th.th_team;
7665 int tlevel = thr->th.th_teams_level;
7666 int ii = team->t.t_level;
7667 teams_serialized = team->t.t_serialized;
7668 int level = tlevel + 1;
7669 KMP_DEBUG_ASSERT(ii >= tlevel);
7670 while (ii > level) {
7671 for (teams_serialized = team->t.t_serialized;
7672 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
7674 if (team->t.t_serialized && (!teams_serialized)) {
7675 team = team->t.t_parent;
7679 team = team->t.t_parent;
7688 int __kmp_aux_get_team_num() {
7690 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7692 if (serialized > 1) {
7695 return team->t.t_master_tid;
7701 int __kmp_aux_get_num_teams() {
7703 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7705 if (serialized > 1) {
7708 return team->t.t_parent->t.t_nproc;
7748 typedef struct kmp_affinity_format_field_t {
7750 const char *long_name;
7753 } kmp_affinity_format_field_t;
7755 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
7756 #if KMP_AFFINITY_SUPPORTED 7757 {
'A',
"thread_affinity",
's'},
7759 {
't',
"team_num",
'd'},
7760 {
'T',
"num_teams",
'd'},
7761 {
'L',
"nesting_level",
'd'},
7762 {
'n',
"thread_num",
'd'},
7763 {
'N',
"num_threads",
'd'},
7764 {
'a',
"ancestor_tnum",
'd'},
7766 {
'P',
"process_id",
'd'},
7767 {
'i',
"native_thread_id",
'd'}};
7770 static int __kmp_aux_capture_affinity_field(
int gtid,
const kmp_info_t *th,
7772 kmp_str_buf_t *field_buffer) {
7773 int rc, format_index, field_value;
7774 const char *width_left, *width_right;
7775 bool pad_zeros, right_justify, parse_long_name, found_valid_name;
7776 static const int FORMAT_SIZE = 20;
7777 char format[FORMAT_SIZE] = {0};
7778 char absolute_short_name = 0;
7780 KMP_DEBUG_ASSERT(gtid >= 0);
7781 KMP_DEBUG_ASSERT(th);
7782 KMP_DEBUG_ASSERT(**ptr ==
'%');
7783 KMP_DEBUG_ASSERT(field_buffer);
7785 __kmp_str_buf_clear(field_buffer);
7792 __kmp_str_buf_cat(field_buffer,
"%", 1);
7803 right_justify =
false;
7805 right_justify =
true;
7809 width_left = width_right = NULL;
7810 if (**ptr >=
'0' && **ptr <=
'9') {
7818 format[format_index++] =
'%';
7820 format[format_index++] =
'-';
7822 format[format_index++] =
'0';
7823 if (width_left && width_right) {
7827 while (i < 8 && width_left < width_right) {
7828 format[format_index++] = *width_left;
7836 found_valid_name =
false;
7837 parse_long_name = (**ptr ==
'{');
7838 if (parse_long_name)
7840 for (
size_t i = 0; i <
sizeof(__kmp_affinity_format_table) /
7841 sizeof(__kmp_affinity_format_table[0]);
7843 char short_name = __kmp_affinity_format_table[i].short_name;
7844 const char *long_name = __kmp_affinity_format_table[i].long_name;
7845 char field_format = __kmp_affinity_format_table[i].field_format;
7846 if (parse_long_name) {
7847 int length = KMP_STRLEN(long_name);
7848 if (strncmp(*ptr, long_name, length) == 0) {
7849 found_valid_name =
true;
7852 }
else if (**ptr == short_name) {
7853 found_valid_name =
true;
7856 if (found_valid_name) {
7857 format[format_index++] = field_format;
7858 format[format_index++] =
'\0';
7859 absolute_short_name = short_name;
7863 if (parse_long_name) {
7865 absolute_short_name = 0;
7873 switch (absolute_short_name) {
7875 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
7878 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
7881 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
7884 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
7887 static const int BUFFER_SIZE = 256;
7888 char buf[BUFFER_SIZE];
7889 __kmp_expand_host_name(buf, BUFFER_SIZE);
7890 rc = __kmp_str_buf_print(field_buffer, format, buf);
7893 rc = __kmp_str_buf_print(field_buffer, format, getpid());
7896 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid());
7899 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
7903 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
7904 rc = __kmp_str_buf_print(field_buffer, format, field_value);
7906 #if KMP_AFFINITY_SUPPORTED 7909 __kmp_str_buf_init(&buf);
7910 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
7911 rc = __kmp_str_buf_print(field_buffer, format, buf.str);
7912 __kmp_str_buf_free(&buf);
7918 rc = __kmp_str_buf_print(field_buffer,
"%s",
"undefined");
7920 if (parse_long_name) {
7929 KMP_ASSERT(format_index <= FORMAT_SIZE);
7939 size_t __kmp_aux_capture_affinity(
int gtid,
const char *format,
7940 kmp_str_buf_t *buffer) {
7941 const char *parse_ptr;
7943 const kmp_info_t *th;
7944 kmp_str_buf_t field;
7946 KMP_DEBUG_ASSERT(buffer);
7947 KMP_DEBUG_ASSERT(gtid >= 0);
7949 __kmp_str_buf_init(&field);
7950 __kmp_str_buf_clear(buffer);
7952 th = __kmp_threads[gtid];
7958 if (parse_ptr == NULL || *parse_ptr ==
'\0') {
7959 parse_ptr = __kmp_affinity_format;
7961 KMP_DEBUG_ASSERT(parse_ptr);
7963 while (*parse_ptr !=
'\0') {
7965 if (*parse_ptr ==
'%') {
7967 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
7968 __kmp_str_buf_catbuf(buffer, &field);
7972 __kmp_str_buf_cat(buffer, parse_ptr, 1);
7977 __kmp_str_buf_free(&field);
7982 void __kmp_aux_display_affinity(
int gtid,
const char *format) {
7984 __kmp_str_buf_init(&buf);
7985 __kmp_aux_capture_affinity(gtid, format, &buf);
7986 __kmp_fprintf(kmp_out,
"%s" KMP_END_OF_LINE, buf.str);
7987 __kmp_str_buf_free(&buf);
7989 #endif // OMP_50_ENABLED 7993 void __kmp_aux_set_blocktime(
int arg, kmp_info_t *thread,
int tid) {
7994 int blocktime = arg;
8000 __kmp_save_internal_controls(thread);
8003 if (blocktime < KMP_MIN_BLOCKTIME)
8004 blocktime = KMP_MIN_BLOCKTIME;
8005 else if (blocktime > KMP_MAX_BLOCKTIME)
8006 blocktime = KMP_MAX_BLOCKTIME;
8008 set__blocktime_team(thread->th.th_team, tid, blocktime);
8009 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
8013 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
8015 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
8016 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
8022 set__bt_set_team(thread->th.th_team, tid, bt_set);
8023 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
8025 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 8026 "bt_intervals=%d, monitor_updates=%d\n",
8027 __kmp_gtid_from_tid(tid, thread->th.th_team),
8028 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
8029 __kmp_monitor_wakeups));
8031 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
8032 __kmp_gtid_from_tid(tid, thread->th.th_team),
8033 thread->th.th_team->t.t_id, tid, blocktime));
8037 void __kmp_aux_set_defaults(
char const *str,
int len) {
8038 if (!__kmp_init_serial) {
8039 __kmp_serial_initialize();
8041 __kmp_env_initialize(str);
8045 || __kmp_display_env || __kmp_display_env_verbose
8055 PACKED_REDUCTION_METHOD_T
8056 __kmp_determine_reduction_method(
8057 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
size_t reduce_size,
8058 void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
8059 kmp_critical_name *lck) {
8070 PACKED_REDUCTION_METHOD_T retval;
8074 KMP_DEBUG_ASSERT(loc);
8075 KMP_DEBUG_ASSERT(lck);
8077 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 8078 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 8079 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 8081 retval = critical_reduce_block;
8084 team_size = __kmp_get_team_num_threads(global_tid);
8085 if (team_size == 1) {
8087 retval = empty_reduce_block;
8091 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8093 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 8095 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 8096 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8098 int teamsize_cutoff = 4;
8100 #if KMP_MIC_SUPPORTED 8101 if (__kmp_mic_type != non_mic) {
8102 teamsize_cutoff = 8;
8105 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8106 if (tree_available) {
8107 if (team_size <= teamsize_cutoff) {
8108 if (atomic_available) {
8109 retval = atomic_reduce_block;
8112 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8114 }
else if (atomic_available) {
8115 retval = atomic_reduce_block;
8118 #error "Unknown or unsupported OS" 8119 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || 8122 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 8124 #if KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_HURD 8128 if (atomic_available) {
8129 if (num_vars <= 2) {
8130 retval = atomic_reduce_block;
8136 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8137 if (atomic_available && (num_vars <= 3)) {
8138 retval = atomic_reduce_block;
8139 }
else if (tree_available) {
8140 if ((reduce_size > (9 *
sizeof(kmp_real64))) &&
8141 (reduce_size < (2000 *
sizeof(kmp_real64)))) {
8142 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
8147 #error "Unknown or unsupported OS" 8151 #error "Unknown or unsupported architecture" 8159 if (__kmp_force_reduction_method != reduction_method_not_defined &&
8162 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
8164 int atomic_available, tree_available;
8166 switch ((forced_retval = __kmp_force_reduction_method)) {
8167 case critical_reduce_block:
8171 case atomic_reduce_block:
8172 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8173 if (!atomic_available) {
8174 KMP_WARNING(RedMethodNotSupported,
"atomic");
8175 forced_retval = critical_reduce_block;
8179 case tree_reduce_block:
8180 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8181 if (!tree_available) {
8182 KMP_WARNING(RedMethodNotSupported,
"tree");
8183 forced_retval = critical_reduce_block;
8185 #if KMP_FAST_REDUCTION_BARRIER 8186 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8195 retval = forced_retval;
8198 KA_TRACE(10, (
"reduction method selected=%08x\n", retval));
8200 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 8201 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 8207 kmp_int32 __kmp_get_reduce_method(
void) {
8208 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
8215 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
8219 void __kmp_hard_pause() {
8220 __kmp_pause_status = kmp_hard_paused;
8221 __kmp_internal_end_thread(-1);
8225 void __kmp_resume_if_soft_paused() {
8226 if (__kmp_pause_status == kmp_soft_paused) {
8227 __kmp_pause_status = kmp_not_paused;
8229 for (
int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
8230 kmp_info_t *thread = __kmp_threads[gtid];
8232 kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
8233 if (fl.is_sleeping())
8235 else if (__kmp_try_suspend_mx(thread)) {
8236 __kmp_unlock_suspend_mx(thread);
8239 if (fl.is_sleeping()) {
8242 }
else if (__kmp_try_suspend_mx(thread)) {
8243 __kmp_unlock_suspend_mx(thread);
8255 int __kmp_pause_resource(kmp_pause_status_t level) {
8256 if (level == kmp_not_paused) {
8257 if (__kmp_pause_status == kmp_not_paused) {
8261 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
8262 __kmp_pause_status == kmp_hard_paused);
8263 __kmp_pause_status = kmp_not_paused;
8266 }
else if (level == kmp_soft_paused) {
8267 if (__kmp_pause_status != kmp_not_paused) {
8274 }
else if (level == kmp_hard_paused) {
8275 if (__kmp_pause_status != kmp_not_paused) {
8288 #endif // OMP_50_ENABLED
#define KMP_COUNT_VALUE(name, value)
Adds value to specified timer (name).
KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid)
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the paritioned timers to begin with name.
KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid)