17 #include "kmp_error.h" 21 #include "kmp_stats.h" 24 #include "ompt-specific.h" 27 #define MAX_MESSAGE 512 43 if ((env = getenv(
"KMP_INITIAL_THREAD_BIND")) != NULL &&
44 __kmp_str_match_true(env)) {
45 __kmp_middle_initialize();
46 KC_TRACE(10, (
"__kmpc_begin: middle initialization called\n"));
47 }
else if (__kmp_ignore_mppbeg() == FALSE) {
49 __kmp_internal_begin();
50 KC_TRACE(10, (
"__kmpc_begin: called\n"));
68 if (__kmp_ignore_mppend() == FALSE) {
69 KC_TRACE(10, (
"__kmpc_end: called\n"));
70 KA_TRACE(30, (
"__kmpc_end\n"));
72 __kmp_internal_end_thread(-1);
74 #if KMP_OS_WINDOWS && OMPT_SUPPORT 79 if (ompt_enabled.enabled)
80 __kmp_internal_end_library(__kmp_gtid_get_specific());
103 kmp_int32 gtid = __kmp_entry_gtid();
105 KC_TRACE(10, (
"__kmpc_global_thread_num: T#%d\n", gtid));
126 (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
128 return TCR_4(__kmp_all_nth);
138 KC_TRACE(10, (
"__kmpc_bound_thread_num: called\n"));
139 return __kmp_tid_from_gtid(__kmp_entry_gtid());
148 KC_TRACE(10, (
"__kmpc_bound_num_threads: called\n"));
150 return __kmp_entry_thread()->th.th_team->t.t_nproc;
170 if (__kmp_par_range == 0) {
177 semi2 = strchr(semi2,
';');
181 semi2 = strchr(semi2 + 1,
';');
185 if (__kmp_par_range_filename[0]) {
186 const char *name = semi2 - 1;
187 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
190 if ((*name ==
'/') || (*name ==
';')) {
193 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
194 return __kmp_par_range < 0;
197 semi3 = strchr(semi2 + 1,
';');
198 if (__kmp_par_range_routine[0]) {
199 if ((semi3 != NULL) && (semi3 > semi2) &&
200 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
201 return __kmp_par_range < 0;
204 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
205 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
206 return __kmp_par_range > 0;
208 return __kmp_par_range < 0;
222 return __kmp_entry_thread()->th.th_root->r.r_active;
235 kmp_int32 num_threads) {
236 KA_TRACE(20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
237 global_tid, num_threads));
239 __kmp_push_num_threads(loc, global_tid, num_threads);
242 void __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid) {
243 KA_TRACE(20, (
"__kmpc_pop_num_threads: enter\n"));
250 void __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid,
251 kmp_int32 proc_bind) {
252 KA_TRACE(20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
255 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
271 int gtid = __kmp_entry_gtid();
273 #if (KMP_STATS_ENABLED) 277 if (previous_state == stats_state_e::SERIAL_REGION) {
278 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
280 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
293 va_start(ap, microtask);
296 ompt_frame_t *ompt_frame;
297 if (ompt_enabled.enabled) {
298 kmp_info_t *master_th = __kmp_threads[gtid];
299 kmp_team_t *parent_team = master_th->th.th_team;
300 ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
302 ompt_frame = &(lwt->ompt_task_info.frame);
304 int tid = __kmp_tid_from_gtid(gtid);
306 parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame);
308 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
309 OMPT_STORE_RETURN_ADDRESS(gtid);
313 #if INCLUDE_SSC_MARKS 316 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
317 VOLATILE_CAST(microtask_t) microtask,
318 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
320 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
326 #if INCLUDE_SSC_MARKS 329 __kmp_join_call(loc, gtid
339 #if KMP_STATS_ENABLED 340 if (previous_state == stats_state_e::SERIAL_REGION) {
341 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
343 KMP_POP_PARTITIONED_TIMER();
345 #endif // KMP_STATS_ENABLED 361 kmp_int32 num_teams, kmp_int32 num_threads) {
363 (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
364 global_tid, num_teams, num_threads));
366 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
381 int gtid = __kmp_entry_gtid();
382 kmp_info_t *this_thr = __kmp_threads[gtid];
384 va_start(ap, microtask);
389 this_thr->th.th_teams_microtask = microtask;
390 this_thr->th.th_teams_level =
391 this_thr->th.th_team->t.t_level;
394 kmp_team_t *parent_team = this_thr->th.th_team;
395 int tid = __kmp_tid_from_gtid(gtid);
396 if (ompt_enabled.enabled) {
397 parent_team->t.t_implicit_task_taskdata[tid]
398 .ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
400 OMPT_STORE_RETURN_ADDRESS(gtid);
405 if (this_thr->th.th_teams_size.nteams == 0) {
406 __kmp_push_num_teams(loc, gtid, 0, 0);
408 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
409 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
410 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
412 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
413 VOLATILE_CAST(microtask_t)
415 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
416 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
422 __kmp_join_call(loc, gtid
429 this_thr->th.th_teams_microtask = NULL;
430 this_thr->th.th_teams_level = 0;
431 *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
440 int __kmpc_invoke_task_func(
int gtid) {
return __kmp_invoke_task_func(gtid); }
459 OMPT_STORE_RETURN_ADDRESS(global_tid);
461 __kmp_serialized_parallel(loc, global_tid);
472 kmp_internal_control_t *top;
473 kmp_info_t *this_thr;
474 kmp_team_t *serial_team;
477 (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
485 if (!TCR_4(__kmp_init_parallel))
486 __kmp_parallel_initialize();
489 __kmp_resume_if_soft_paused();
492 this_thr = __kmp_threads[global_tid];
493 serial_team = this_thr->th.th_serial_team;
496 kmp_task_team_t *task_team = this_thr->th.th_task_team;
499 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)
500 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
504 KMP_DEBUG_ASSERT(serial_team);
505 KMP_ASSERT(serial_team->t.t_serialized);
506 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
507 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
508 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
509 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
512 if (ompt_enabled.enabled &&
513 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
514 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none;
515 if (ompt_enabled.ompt_callback_implicit_task) {
516 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
517 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
518 OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit);
522 ompt_data_t *parent_task_data;
523 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
525 if (ompt_enabled.ompt_callback_parallel_end) {
526 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
527 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
528 ompt_parallel_invoker_program, OMPT_LOAD_RETURN_ADDRESS(global_tid));
530 __ompt_lw_taskteam_unlink(this_thr);
531 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
537 top = serial_team->t.t_control_stack_top;
538 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
539 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
540 serial_team->t.t_control_stack_top = top->next;
545 serial_team->t.t_level--;
548 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
550 dispatch_private_info_t *disp_buffer =
551 serial_team->t.t_dispatch->th_disp_buffer;
552 serial_team->t.t_dispatch->th_disp_buffer =
553 serial_team->t.t_dispatch->th_disp_buffer->next;
554 __kmp_free(disp_buffer);
557 this_thr->th.th_def_allocator = serial_team->t.t_def_allocator;
560 --serial_team->t.t_serialized;
561 if (serial_team->t.t_serialized == 0) {
565 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 566 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
567 __kmp_clear_x87_fpu_status_word();
568 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
569 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
573 this_thr->th.th_team = serial_team->t.t_parent;
574 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
577 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc;
578 this_thr->th.th_team_master =
579 serial_team->t.t_parent->t.t_threads[0];
580 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
583 this_thr->th.th_dispatch =
584 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
586 __kmp_pop_current_task_from_thread(this_thr);
588 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
589 this_thr->th.th_current_task->td_flags.executing = 1;
591 if (__kmp_tasking_mode != tskm_immediate_exec) {
593 this_thr->th.th_task_team =
594 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
596 (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / " 598 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
601 if (__kmp_tasking_mode != tskm_immediate_exec) {
602 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting " 603 "depth of serial team %p to %d\n",
604 global_tid, serial_team, serial_team->t.t_serialized));
608 if (__kmp_env_consistency_check)
609 __kmp_pop_parallel(global_tid, NULL);
611 if (ompt_enabled.enabled)
612 this_thr->th.ompt_thread_info.state =
613 ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
614 : ompt_state_work_parallel);
627 KC_TRACE(10, (
"__kmpc_flush: called\n"));
632 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 646 if (!__kmp_cpuinfo.initialized) {
647 __kmp_query_cpuid(&__kmp_cpuinfo);
649 if (!__kmp_cpuinfo.sse2) {
654 #elif KMP_COMPILER_MSVC 657 __sync_synchronize();
658 #endif // KMP_COMPILER_ICC 661 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) 677 #error Unknown or unsupported architecture 680 #if OMPT_SUPPORT && OMPT_OPTIONAL 681 if (ompt_enabled.ompt_callback_flush) {
682 ompt_callbacks.ompt_callback(ompt_callback_flush)(
683 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
698 KC_TRACE(10, (
"__kmpc_barrier: called T#%d\n", global_tid));
700 if (!TCR_4(__kmp_init_parallel))
701 __kmp_parallel_initialize();
704 __kmp_resume_if_soft_paused();
707 if (__kmp_env_consistency_check) {
709 KMP_WARNING(ConstructIdentInvalid);
712 __kmp_check_barrier(global_tid, ct_barrier, loc);
716 ompt_frame_t *ompt_frame;
717 if (ompt_enabled.enabled) {
718 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
719 if (ompt_frame->enter_frame.ptr == NULL)
720 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
721 OMPT_STORE_RETURN_ADDRESS(global_tid);
724 __kmp_threads[global_tid]->th.th_ident = loc;
732 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
733 #if OMPT_SUPPORT && OMPT_OPTIONAL 734 if (ompt_enabled.enabled) {
735 ompt_frame->enter_frame = ompt_data_none;
750 KC_TRACE(10, (
"__kmpc_master: called T#%d\n", global_tid));
752 if (!TCR_4(__kmp_init_parallel))
753 __kmp_parallel_initialize();
756 __kmp_resume_if_soft_paused();
759 if (KMP_MASTER_GTID(global_tid)) {
761 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
765 #if OMPT_SUPPORT && OMPT_OPTIONAL 767 if (ompt_enabled.ompt_callback_master) {
768 kmp_info_t *this_thr = __kmp_threads[global_tid];
769 kmp_team_t *team = this_thr->th.th_team;
771 int tid = __kmp_tid_from_gtid(global_tid);
772 ompt_callbacks.ompt_callback(ompt_callback_master)(
773 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
774 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
775 OMPT_GET_RETURN_ADDRESS(0));
780 if (__kmp_env_consistency_check) {
781 #if KMP_USE_DYNAMIC_LOCK 783 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
785 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
788 __kmp_push_sync(global_tid, ct_master, loc, NULL);
790 __kmp_check_sync(global_tid, ct_master, loc, NULL);
806 KC_TRACE(10, (
"__kmpc_end_master: called T#%d\n", global_tid));
808 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
809 KMP_POP_PARTITIONED_TIMER();
811 #if OMPT_SUPPORT && OMPT_OPTIONAL 812 kmp_info_t *this_thr = __kmp_threads[global_tid];
813 kmp_team_t *team = this_thr->th.th_team;
814 if (ompt_enabled.ompt_callback_master) {
815 int tid = __kmp_tid_from_gtid(global_tid);
816 ompt_callbacks.ompt_callback(ompt_callback_master)(
817 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
818 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
819 OMPT_GET_RETURN_ADDRESS(0));
823 if (__kmp_env_consistency_check) {
825 KMP_WARNING(ThreadIdentInvalid);
827 if (KMP_MASTER_GTID(global_tid))
828 __kmp_pop_sync(global_tid, ct_master, loc);
842 KMP_DEBUG_ASSERT(__kmp_init_serial);
844 KC_TRACE(10, (
"__kmpc_ordered: called T#%d\n", gtid));
846 if (!TCR_4(__kmp_init_parallel))
847 __kmp_parallel_initialize();
850 __kmp_resume_if_soft_paused();
854 __kmp_itt_ordered_prep(gtid);
858 th = __kmp_threads[gtid];
860 #if OMPT_SUPPORT && OMPT_OPTIONAL 864 if (ompt_enabled.enabled) {
865 OMPT_STORE_RETURN_ADDRESS(gtid);
866 team = __kmp_team_from_gtid(gtid);
867 lck = (ompt_wait_id_t)&team->t.t_ordered.dt.t_value;
869 th->th.ompt_thread_info.wait_id = lck;
870 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
873 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
874 if (ompt_enabled.ompt_callback_mutex_acquire) {
875 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
876 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin,
877 (ompt_wait_id_t)lck, codeptr_ra);
882 if (th->th.th_dispatch->th_deo_fcn != 0)
883 (*th->th.th_dispatch->th_deo_fcn)(>id, &cid, loc);
885 __kmp_parallel_deo(>id, &cid, loc);
887 #if OMPT_SUPPORT && OMPT_OPTIONAL 888 if (ompt_enabled.enabled) {
890 th->th.ompt_thread_info.state = ompt_state_work_parallel;
891 th->th.ompt_thread_info.wait_id = 0;
894 if (ompt_enabled.ompt_callback_mutex_acquired) {
895 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
896 ompt_mutex_ordered, (ompt_wait_id_t)lck, codeptr_ra);
902 __kmp_itt_ordered_start(gtid);
917 KC_TRACE(10, (
"__kmpc_end_ordered: called T#%d\n", gtid));
920 __kmp_itt_ordered_end(gtid);
924 th = __kmp_threads[gtid];
926 if (th->th.th_dispatch->th_dxo_fcn != 0)
927 (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid, loc);
929 __kmp_parallel_dxo(>id, &cid, loc);
931 #if OMPT_SUPPORT && OMPT_OPTIONAL 932 OMPT_STORE_RETURN_ADDRESS(gtid);
933 if (ompt_enabled.ompt_callback_mutex_released) {
934 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
936 (ompt_wait_id_t)&__kmp_team_from_gtid(gtid)->t.t_ordered.dt.t_value,
937 OMPT_LOAD_RETURN_ADDRESS(gtid));
942 #if KMP_USE_DYNAMIC_LOCK 944 static __forceinline
void 945 __kmp_init_indirect_csptr(kmp_critical_name *crit,
ident_t const *loc,
946 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
950 kmp_indirect_lock_t **lck;
951 lck = (kmp_indirect_lock_t **)crit;
952 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
953 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
954 KMP_SET_I_LOCK_LOCATION(ilk, loc);
955 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
957 (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
959 __kmp_itt_critical_creating(ilk->lock, loc);
961 int status = KMP_COMPARE_AND_STORE_PTR(lck,
nullptr, ilk);
964 __kmp_itt_critical_destroyed(ilk->lock);
970 KMP_DEBUG_ASSERT(*lck != NULL);
974 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \ 976 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 977 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 978 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 979 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 980 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 982 KMP_FSYNC_PREPARE(l); \ 983 KMP_INIT_YIELD(spins); \ 984 if (TCR_4(__kmp_nth) > \ 985 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 988 KMP_YIELD_SPIN(spins); \ 990 kmp_backoff_t backoff = __kmp_spin_backoff_params; \ 992 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 993 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 994 __kmp_spin_backoff(&backoff); \ 995 if (TCR_4(__kmp_nth) > \ 996 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 999 KMP_YIELD_SPIN(spins); \ 1003 KMP_FSYNC_ACQUIRED(l); \ 1007 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \ 1009 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 1010 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 1011 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 1012 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \ 1013 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \ 1017 #define KMP_RELEASE_TAS_LOCK(lock, gtid) \ 1018 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); } 1022 #include <sys/syscall.h> 1025 #define FUTEX_WAIT 0 1028 #define FUTEX_WAKE 1 1032 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \ 1034 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1035 kmp_int32 gtid_code = (gtid + 1) << 1; \ 1037 KMP_FSYNC_PREPARE(ftx); \ 1038 kmp_int32 poll_val; \ 1039 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \ 1040 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1041 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ 1042 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ 1044 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \ 1046 KMP_LOCK_BUSY(1, futex))) { \ 1049 poll_val |= KMP_LOCK_BUSY(1, futex); \ 1052 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \ 1053 NULL, NULL, 0)) != 0) { \ 1058 KMP_FSYNC_ACQUIRED(ftx); \ 1062 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \ 1064 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1065 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1066 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \ 1067 KMP_FSYNC_ACQUIRED(ftx); \ 1075 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \ 1077 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1079 KMP_FSYNC_RELEASING(ftx); \ 1080 kmp_int32 poll_val = \ 1081 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ 1082 if (KMP_LOCK_STRIP(poll_val) & 1) { \ 1083 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \ 1084 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ 1087 KMP_YIELD(TCR_4(__kmp_nth) > \ 1088 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ 1091 #endif // KMP_USE_FUTEX 1093 #else // KMP_USE_DYNAMIC_LOCK 1095 static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1098 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1101 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1108 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1109 __kmp_init_user_lock_with_checks(lck);
1110 __kmp_set_user_lock_location(lck, loc);
1112 __kmp_itt_critical_creating(lck);
1123 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1128 __kmp_itt_critical_destroyed(lck);
1132 __kmp_destroy_user_lock_with_checks(lck);
1133 __kmp_user_lock_free(&idx, gtid, lck);
1134 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1135 KMP_DEBUG_ASSERT(lck != NULL);
1141 #endif // KMP_USE_DYNAMIC_LOCK 1154 kmp_critical_name *crit) {
1155 #if KMP_USE_DYNAMIC_LOCK 1156 #if OMPT_SUPPORT && OMPT_OPTIONAL 1157 OMPT_STORE_RETURN_ADDRESS(global_tid);
1158 #endif // OMPT_SUPPORT 1159 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1162 #if OMPT_SUPPORT && OMPT_OPTIONAL 1163 ompt_state_t prev_state = ompt_state_undefined;
1164 ompt_thread_info_t ti;
1166 kmp_user_lock_p lck;
1168 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1172 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1173 KMP_CHECK_USER_LOCK_INIT();
1175 if ((__kmp_user_lock_kind == lk_tas) &&
1176 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1177 lck = (kmp_user_lock_p)crit;
1180 else if ((__kmp_user_lock_kind == lk_futex) &&
1181 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1182 lck = (kmp_user_lock_p)crit;
1186 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1189 if (__kmp_env_consistency_check)
1190 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1198 __kmp_itt_critical_acquiring(lck);
1200 #if OMPT_SUPPORT && OMPT_OPTIONAL 1201 OMPT_STORE_RETURN_ADDRESS(gtid);
1202 void *codeptr_ra = NULL;
1203 if (ompt_enabled.enabled) {
1204 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1206 prev_state = ti.state;
1207 ti.wait_id = (ompt_wait_id_t)lck;
1208 ti.state = ompt_state_wait_critical;
1211 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1212 if (ompt_enabled.ompt_callback_mutex_acquire) {
1213 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1214 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1215 (ompt_wait_id_t)crit, codeptr_ra);
1221 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1224 __kmp_itt_critical_acquired(lck);
1226 #if OMPT_SUPPORT && OMPT_OPTIONAL 1227 if (ompt_enabled.enabled) {
1229 ti.state = prev_state;
1233 if (ompt_enabled.ompt_callback_mutex_acquired) {
1234 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1235 ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr_ra);
1239 KMP_POP_PARTITIONED_TIMER();
1241 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1242 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1243 #endif // KMP_USE_DYNAMIC_LOCK 1246 #if KMP_USE_DYNAMIC_LOCK 1249 static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1251 #define KMP_TSX_LOCK(seq) lockseq_##seq 1253 #define KMP_TSX_LOCK(seq) __kmp_user_lock_seq 1256 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1257 #define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) 1259 #define KMP_CPUINFO_RTM 0 1263 if (hint & kmp_lock_hint_hle)
1264 return KMP_TSX_LOCK(hle);
1265 if (hint & kmp_lock_hint_rtm)
1266 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq;
1267 if (hint & kmp_lock_hint_adaptive)
1268 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1271 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1272 return __kmp_user_lock_seq;
1273 if ((hint & omp_lock_hint_speculative) &&
1274 (hint & omp_lock_hint_nonspeculative))
1275 return __kmp_user_lock_seq;
1278 if (hint & omp_lock_hint_contended)
1279 return lockseq_queuing;
1282 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1286 if (hint & omp_lock_hint_speculative)
1287 return KMP_TSX_LOCK(hle);
1289 return __kmp_user_lock_seq;
1292 #if OMPT_SUPPORT && OMPT_OPTIONAL 1293 #if KMP_USE_DYNAMIC_LOCK 1294 static kmp_mutex_impl_t
1295 __ompt_get_mutex_impl_type(
void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1297 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1302 return kmp_mutex_impl_queuing;
1305 return kmp_mutex_impl_spin;
1308 return kmp_mutex_impl_speculative;
1311 return kmp_mutex_impl_none;
1313 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1316 switch (ilock->type) {
1318 case locktag_adaptive:
1320 return kmp_mutex_impl_speculative;
1322 case locktag_nested_tas:
1323 return kmp_mutex_impl_spin;
1325 case locktag_nested_futex:
1327 case locktag_ticket:
1328 case locktag_queuing:
1330 case locktag_nested_ticket:
1331 case locktag_nested_queuing:
1332 case locktag_nested_drdpa:
1333 return kmp_mutex_impl_queuing;
1335 return kmp_mutex_impl_none;
1340 static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1341 switch (__kmp_user_lock_kind) {
1343 return kmp_mutex_impl_spin;
1350 return kmp_mutex_impl_queuing;
1355 return kmp_mutex_impl_speculative;
1358 return kmp_mutex_impl_none;
1361 #endif // KMP_USE_DYNAMIC_LOCK 1362 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1377 void __kmpc_critical_with_hint(
ident_t *loc, kmp_int32 global_tid,
1378 kmp_critical_name *crit, uint32_t hint) {
1380 kmp_user_lock_p lck;
1381 #if OMPT_SUPPORT && OMPT_OPTIONAL 1382 ompt_state_t prev_state = ompt_state_undefined;
1383 ompt_thread_info_t ti;
1385 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1387 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1390 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1392 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1394 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1396 kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
1397 if (KMP_IS_D_LOCK(lckseq)) {
1398 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
1399 KMP_GET_D_TAG(lckseq));
1401 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
1407 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1408 lck = (kmp_user_lock_p)lk;
1409 if (__kmp_env_consistency_check) {
1410 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1411 __kmp_map_hint_to_lock(hint));
1414 __kmp_itt_critical_acquiring(lck);
1416 #if OMPT_SUPPORT && OMPT_OPTIONAL 1417 if (ompt_enabled.enabled) {
1418 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1420 prev_state = ti.state;
1421 ti.wait_id = (ompt_wait_id_t)lck;
1422 ti.state = ompt_state_wait_critical;
1425 if (ompt_enabled.ompt_callback_mutex_acquire) {
1426 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1427 ompt_mutex_critical, (
unsigned int)hint,
1428 __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)crit, codeptr);
1432 #if KMP_USE_INLINED_TAS 1433 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1434 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1436 #elif KMP_USE_INLINED_FUTEX 1437 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1438 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1442 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
1445 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1447 if (__kmp_env_consistency_check) {
1448 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1449 __kmp_map_hint_to_lock(hint));
1452 __kmp_itt_critical_acquiring(lck);
1454 #if OMPT_SUPPORT && OMPT_OPTIONAL 1455 if (ompt_enabled.enabled) {
1456 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1458 prev_state = ti.state;
1459 ti.wait_id = (ompt_wait_id_t)lck;
1460 ti.state = ompt_state_wait_critical;
1463 if (ompt_enabled.ompt_callback_mutex_acquire) {
1464 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1465 ompt_mutex_critical, (
unsigned int)hint,
1466 __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)crit, codeptr);
1470 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
1472 KMP_POP_PARTITIONED_TIMER();
1475 __kmp_itt_critical_acquired(lck);
1477 #if OMPT_SUPPORT && OMPT_OPTIONAL 1478 if (ompt_enabled.enabled) {
1480 ti.state = prev_state;
1484 if (ompt_enabled.ompt_callback_mutex_acquired) {
1485 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1486 ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr);
1491 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1492 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1495 #endif // KMP_USE_DYNAMIC_LOCK 1507 kmp_critical_name *crit) {
1508 kmp_user_lock_p lck;
1510 KC_TRACE(10, (
"__kmpc_end_critical: called T#%d\n", global_tid));
1512 #if KMP_USE_DYNAMIC_LOCK 1513 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
1514 lck = (kmp_user_lock_p)crit;
1515 KMP_ASSERT(lck != NULL);
1516 if (__kmp_env_consistency_check) {
1517 __kmp_pop_sync(global_tid, ct_critical, loc);
1520 __kmp_itt_critical_releasing(lck);
1522 #if KMP_USE_INLINED_TAS 1523 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1524 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1526 #elif KMP_USE_INLINED_FUTEX 1527 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1528 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1532 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1535 kmp_indirect_lock_t *ilk =
1536 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1537 KMP_ASSERT(ilk != NULL);
1539 if (__kmp_env_consistency_check) {
1540 __kmp_pop_sync(global_tid, ct_critical, loc);
1543 __kmp_itt_critical_releasing(lck);
1545 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1548 #else // KMP_USE_DYNAMIC_LOCK 1550 if ((__kmp_user_lock_kind == lk_tas) &&
1551 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1552 lck = (kmp_user_lock_p)crit;
1555 else if ((__kmp_user_lock_kind == lk_futex) &&
1556 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1557 lck = (kmp_user_lock_p)crit;
1561 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1564 KMP_ASSERT(lck != NULL);
1566 if (__kmp_env_consistency_check)
1567 __kmp_pop_sync(global_tid, ct_critical, loc);
1570 __kmp_itt_critical_releasing(lck);
1574 __kmp_release_user_lock_with_checks(lck, global_tid);
1576 #endif // KMP_USE_DYNAMIC_LOCK 1578 #if OMPT_SUPPORT && OMPT_OPTIONAL 1581 OMPT_STORE_RETURN_ADDRESS(global_tid);
1582 if (ompt_enabled.ompt_callback_mutex_released) {
1583 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1584 ompt_mutex_critical, (ompt_wait_id_t)crit, OMPT_LOAD_RETURN_ADDRESS(0));
1588 KMP_POP_PARTITIONED_TIMER();
1589 KA_TRACE(15, (
"__kmpc_end_critical: done T#%d\n", global_tid));
1604 KC_TRACE(10, (
"__kmpc_barrier_master: called T#%d\n", global_tid));
1606 if (!TCR_4(__kmp_init_parallel))
1607 __kmp_parallel_initialize();
1610 __kmp_resume_if_soft_paused();
1613 if (__kmp_env_consistency_check)
1614 __kmp_check_barrier(global_tid, ct_barrier, loc);
1617 ompt_frame_t *ompt_frame;
1618 if (ompt_enabled.enabled) {
1619 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1620 if (ompt_frame->enter_frame.ptr == NULL)
1621 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1622 OMPT_STORE_RETURN_ADDRESS(global_tid);
1626 __kmp_threads[global_tid]->th.th_ident = loc;
1628 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1629 #if OMPT_SUPPORT && OMPT_OPTIONAL 1630 if (ompt_enabled.enabled) {
1631 ompt_frame->enter_frame = ompt_data_none;
1635 return (status != 0) ? 0 : 1;
1648 KC_TRACE(10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid));
1650 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1666 KC_TRACE(10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1668 if (!TCR_4(__kmp_init_parallel))
1669 __kmp_parallel_initialize();
1672 __kmp_resume_if_soft_paused();
1675 if (__kmp_env_consistency_check) {
1677 KMP_WARNING(ConstructIdentInvalid);
1679 __kmp_check_barrier(global_tid, ct_barrier, loc);
1683 ompt_frame_t *ompt_frame;
1684 if (ompt_enabled.enabled) {
1685 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1686 if (ompt_frame->enter_frame.ptr == NULL)
1687 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1688 OMPT_STORE_RETURN_ADDRESS(global_tid);
1692 __kmp_threads[global_tid]->th.th_ident = loc;
1694 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1695 #if OMPT_SUPPORT && OMPT_OPTIONAL 1696 if (ompt_enabled.enabled) {
1697 ompt_frame->enter_frame = ompt_data_none;
1703 if (__kmp_env_consistency_check) {
1707 if (global_tid < 0) {
1708 KMP_WARNING(ThreadIdentInvalid);
1714 __kmp_pop_sync(global_tid, ct_master, loc);
1734 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1739 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1742 #if OMPT_SUPPORT && OMPT_OPTIONAL 1743 kmp_info_t *this_thr = __kmp_threads[global_tid];
1744 kmp_team_t *team = this_thr->th.th_team;
1745 int tid = __kmp_tid_from_gtid(global_tid);
1747 if (ompt_enabled.enabled) {
1749 if (ompt_enabled.ompt_callback_work) {
1750 ompt_callbacks.ompt_callback(ompt_callback_work)(
1751 ompt_work_single_executor, ompt_scope_begin,
1752 &(team->t.ompt_team_info.parallel_data),
1753 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1754 1, OMPT_GET_RETURN_ADDRESS(0));
1757 if (ompt_enabled.ompt_callback_work) {
1758 ompt_callbacks.ompt_callback(ompt_callback_work)(
1759 ompt_work_single_other, ompt_scope_begin,
1760 &(team->t.ompt_team_info.parallel_data),
1761 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1762 1, OMPT_GET_RETURN_ADDRESS(0));
1763 ompt_callbacks.ompt_callback(ompt_callback_work)(
1764 ompt_work_single_other, ompt_scope_end,
1765 &(team->t.ompt_team_info.parallel_data),
1766 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1767 1, OMPT_GET_RETURN_ADDRESS(0));
1786 __kmp_exit_single(global_tid);
1787 KMP_POP_PARTITIONED_TIMER();
1789 #if OMPT_SUPPORT && OMPT_OPTIONAL 1790 kmp_info_t *this_thr = __kmp_threads[global_tid];
1791 kmp_team_t *team = this_thr->th.th_team;
1792 int tid = __kmp_tid_from_gtid(global_tid);
1794 if (ompt_enabled.ompt_callback_work) {
1795 ompt_callbacks.ompt_callback(ompt_callback_work)(
1796 ompt_work_single_executor, ompt_scope_end,
1797 &(team->t.ompt_team_info.parallel_data),
1798 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1799 OMPT_GET_RETURN_ADDRESS(0));
1812 KMP_POP_PARTITIONED_TIMER();
1813 KE_TRACE(10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1815 #if OMPT_SUPPORT && OMPT_OPTIONAL 1816 if (ompt_enabled.ompt_callback_work) {
1817 ompt_work_t ompt_work_type = ompt_work_loop;
1818 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1819 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
1823 ompt_work_type = ompt_work_loop;
1825 ompt_work_type = ompt_work_sections;
1827 ompt_work_type = ompt_work_distribute;
1832 KMP_DEBUG_ASSERT(ompt_work_type);
1834 ompt_callbacks.ompt_callback(ompt_callback_work)(
1835 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
1836 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
1839 if (__kmp_env_consistency_check)
1840 __kmp_pop_workshare(global_tid, ct_pdo, loc);
1846 void ompc_set_num_threads(
int arg) {
1848 __kmp_set_num_threads(arg, __kmp_entry_gtid());
1851 void ompc_set_dynamic(
int flag) {
1855 thread = __kmp_entry_thread();
1857 __kmp_save_internal_controls(thread);
1859 set__dynamic(thread, flag ? TRUE : FALSE);
1862 void ompc_set_nested(
int flag) {
1866 thread = __kmp_entry_thread();
1868 __kmp_save_internal_controls(thread);
1870 set__nested(thread, flag ? TRUE : FALSE);
1873 void ompc_set_max_active_levels(
int max_active_levels) {
1878 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
1881 void ompc_set_schedule(omp_sched_t kind,
int modifier) {
1883 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
1886 int ompc_get_ancestor_thread_num(
int level) {
1887 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
1890 int ompc_get_team_size(
int level) {
1891 return __kmp_get_team_size(__kmp_entry_gtid(), level);
1897 void ompc_set_affinity_format(
char const *format) {
1898 if (!__kmp_init_serial) {
1899 __kmp_serial_initialize();
1901 __kmp_strncpy_truncate(__kmp_affinity_format, KMP_AFFINITY_FORMAT_SIZE,
1902 format, KMP_STRLEN(format) + 1);
1905 size_t ompc_get_affinity_format(
char *buffer,
size_t size) {
1907 if (!__kmp_init_serial) {
1908 __kmp_serial_initialize();
1910 format_size = KMP_STRLEN(__kmp_affinity_format);
1911 if (buffer && size) {
1912 __kmp_strncpy_truncate(buffer, size, __kmp_affinity_format,
1918 void ompc_display_affinity(
char const *format) {
1920 if (!TCR_4(__kmp_init_middle)) {
1921 __kmp_middle_initialize();
1923 gtid = __kmp_get_gtid();
1924 __kmp_aux_display_affinity(gtid, format);
1927 size_t ompc_capture_affinity(
char *buffer,
size_t buf_size,
1928 char const *format) {
1930 size_t num_required;
1931 kmp_str_buf_t capture_buf;
1932 if (!TCR_4(__kmp_init_middle)) {
1933 __kmp_middle_initialize();
1935 gtid = __kmp_get_gtid();
1936 __kmp_str_buf_init(&capture_buf);
1937 num_required = __kmp_aux_capture_affinity(gtid, format, &capture_buf);
1938 if (buffer && buf_size) {
1939 __kmp_strncpy_truncate(buffer, buf_size, capture_buf.str,
1940 capture_buf.used + 1);
1942 __kmp_str_buf_free(&capture_buf);
1943 return num_required;
1947 void kmpc_set_stacksize(
int arg) {
1949 __kmp_aux_set_stacksize(arg);
1952 void kmpc_set_stacksize_s(
size_t arg) {
1954 __kmp_aux_set_stacksize(arg);
1957 void kmpc_set_blocktime(
int arg) {
1961 gtid = __kmp_entry_gtid();
1962 tid = __kmp_tid_from_gtid(gtid);
1963 thread = __kmp_thread_from_gtid(gtid);
1965 __kmp_aux_set_blocktime(arg, thread, tid);
1968 void kmpc_set_library(
int arg) {
1970 __kmp_user_set_library((
enum library_type)arg);
1973 void kmpc_set_defaults(
char const *str) {
1975 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
1978 void kmpc_set_disp_num_buffers(
int arg) {
1981 if (__kmp_init_serial == 0 && arg > 0)
1982 __kmp_dispatch_num_buffers = arg;
1985 int kmpc_set_affinity_mask_proc(
int proc,
void **mask) {
1986 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1989 if (!TCR_4(__kmp_init_middle)) {
1990 __kmp_middle_initialize();
1992 return __kmp_aux_set_affinity_mask_proc(proc, mask);
1996 int kmpc_unset_affinity_mask_proc(
int proc,
void **mask) {
1997 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 2000 if (!TCR_4(__kmp_init_middle)) {
2001 __kmp_middle_initialize();
2003 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
2007 int kmpc_get_affinity_mask_proc(
int proc,
void **mask) {
2008 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 2011 if (!TCR_4(__kmp_init_middle)) {
2012 __kmp_middle_initialize();
2014 return __kmp_aux_get_affinity_mask_proc(proc, mask);
2064 void *cpy_data,
void (*cpy_func)(
void *,
void *),
2068 KC_TRACE(10, (
"__kmpc_copyprivate: called T#%d\n", gtid));
2072 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
2074 if (__kmp_env_consistency_check) {
2076 KMP_WARNING(ConstructIdentInvalid);
2083 *data_ptr = cpy_data;
2086 ompt_frame_t *ompt_frame;
2087 if (ompt_enabled.enabled) {
2088 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2089 if (ompt_frame->enter_frame.ptr == NULL)
2090 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2091 OMPT_STORE_RETURN_ADDRESS(gtid);
2096 __kmp_threads[gtid]->th.th_ident = loc;
2098 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2101 (*cpy_func)(cpy_data, *data_ptr);
2107 if (ompt_enabled.enabled) {
2108 OMPT_STORE_RETURN_ADDRESS(gtid);
2112 __kmp_threads[gtid]->th.th_ident = loc;
2115 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2116 #if OMPT_SUPPORT && OMPT_OPTIONAL 2117 if (ompt_enabled.enabled) {
2118 ompt_frame->enter_frame = ompt_data_none;
2125 #define INIT_LOCK __kmp_init_user_lock_with_checks 2126 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks 2127 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks 2128 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed 2129 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks 2130 #define ACQUIRE_NESTED_LOCK_TIMED \ 2131 __kmp_acquire_nested_user_lock_with_checks_timed 2132 #define RELEASE_LOCK __kmp_release_user_lock_with_checks 2133 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks 2134 #define TEST_LOCK __kmp_test_user_lock_with_checks 2135 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks 2136 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks 2137 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks 2142 #if KMP_USE_DYNAMIC_LOCK 2145 static __forceinline
void __kmp_init_lock_with_hint(
ident_t *loc,
void **lock,
2146 kmp_dyna_lockseq_t seq) {
2147 if (KMP_IS_D_LOCK(seq)) {
2148 KMP_INIT_D_LOCK(lock, seq);
2150 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2153 KMP_INIT_I_LOCK(lock, seq);
2155 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2156 __kmp_itt_lock_creating(ilk->lock, loc);
2162 static __forceinline
void 2163 __kmp_init_nest_lock_with_hint(
ident_t *loc,
void **lock,
2164 kmp_dyna_lockseq_t seq) {
2167 if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
2168 seq = __kmp_user_lock_seq;
2172 seq = lockseq_nested_tas;
2176 seq = lockseq_nested_futex;
2179 case lockseq_ticket:
2180 seq = lockseq_nested_ticket;
2182 case lockseq_queuing:
2183 seq = lockseq_nested_queuing;
2186 seq = lockseq_nested_drdpa;
2189 seq = lockseq_nested_queuing;
2191 KMP_INIT_I_LOCK(lock, seq);
2193 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2194 __kmp_itt_lock_creating(ilk->lock, loc);
2199 void __kmpc_init_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock,
2201 KMP_DEBUG_ASSERT(__kmp_init_serial);
2202 if (__kmp_env_consistency_check && user_lock == NULL) {
2203 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
2206 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2208 #if OMPT_SUPPORT && OMPT_OPTIONAL 2210 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2212 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2213 if (ompt_enabled.ompt_callback_lock_init) {
2214 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2215 ompt_mutex_lock, (omp_lock_hint_t)hint,
2216 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2223 void __kmpc_init_nest_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
2224 void **user_lock, uintptr_t hint) {
2225 KMP_DEBUG_ASSERT(__kmp_init_serial);
2226 if (__kmp_env_consistency_check && user_lock == NULL) {
2227 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
2230 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2232 #if OMPT_SUPPORT && OMPT_OPTIONAL 2234 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2236 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2237 if (ompt_enabled.ompt_callback_lock_init) {
2238 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2239 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2240 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2246 #endif // KMP_USE_DYNAMIC_LOCK 2249 void __kmpc_init_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2250 #if KMP_USE_DYNAMIC_LOCK 2252 KMP_DEBUG_ASSERT(__kmp_init_serial);
2253 if (__kmp_env_consistency_check && user_lock == NULL) {
2254 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
2256 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2258 #if OMPT_SUPPORT && OMPT_OPTIONAL 2260 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2262 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2263 if (ompt_enabled.ompt_callback_lock_init) {
2264 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2265 ompt_mutex_lock, omp_lock_hint_none,
2266 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2271 #else // KMP_USE_DYNAMIC_LOCK 2273 static char const *
const func =
"omp_init_lock";
2274 kmp_user_lock_p lck;
2275 KMP_DEBUG_ASSERT(__kmp_init_serial);
2277 if (__kmp_env_consistency_check) {
2278 if (user_lock == NULL) {
2279 KMP_FATAL(LockIsUninitialized, func);
2283 KMP_CHECK_USER_LOCK_INIT();
2285 if ((__kmp_user_lock_kind == lk_tas) &&
2286 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2287 lck = (kmp_user_lock_p)user_lock;
2290 else if ((__kmp_user_lock_kind == lk_futex) &&
2291 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2292 lck = (kmp_user_lock_p)user_lock;
2296 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2299 __kmp_set_user_lock_location(lck, loc);
2301 #if OMPT_SUPPORT && OMPT_OPTIONAL 2303 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2305 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2306 if (ompt_enabled.ompt_callback_lock_init) {
2307 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2308 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2309 (ompt_wait_id_t)user_lock, codeptr);
2314 __kmp_itt_lock_creating(lck);
2317 #endif // KMP_USE_DYNAMIC_LOCK 2321 void __kmpc_init_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2322 #if KMP_USE_DYNAMIC_LOCK 2324 KMP_DEBUG_ASSERT(__kmp_init_serial);
2325 if (__kmp_env_consistency_check && user_lock == NULL) {
2326 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
2328 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2330 #if OMPT_SUPPORT && OMPT_OPTIONAL 2332 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2334 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2335 if (ompt_enabled.ompt_callback_lock_init) {
2336 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2337 ompt_mutex_nest_lock, omp_lock_hint_none,
2338 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2343 #else // KMP_USE_DYNAMIC_LOCK 2345 static char const *
const func =
"omp_init_nest_lock";
2346 kmp_user_lock_p lck;
2347 KMP_DEBUG_ASSERT(__kmp_init_serial);
2349 if (__kmp_env_consistency_check) {
2350 if (user_lock == NULL) {
2351 KMP_FATAL(LockIsUninitialized, func);
2355 KMP_CHECK_USER_LOCK_INIT();
2357 if ((__kmp_user_lock_kind == lk_tas) &&
2358 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2359 OMP_NEST_LOCK_T_SIZE)) {
2360 lck = (kmp_user_lock_p)user_lock;
2363 else if ((__kmp_user_lock_kind == lk_futex) &&
2364 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2365 OMP_NEST_LOCK_T_SIZE)) {
2366 lck = (kmp_user_lock_p)user_lock;
2370 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2373 INIT_NESTED_LOCK(lck);
2374 __kmp_set_user_lock_location(lck, loc);
2376 #if OMPT_SUPPORT && OMPT_OPTIONAL 2378 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2380 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2381 if (ompt_enabled.ompt_callback_lock_init) {
2382 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2383 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2384 (ompt_wait_id_t)user_lock, codeptr);
2389 __kmp_itt_lock_creating(lck);
2392 #endif // KMP_USE_DYNAMIC_LOCK 2395 void __kmpc_destroy_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2396 #if KMP_USE_DYNAMIC_LOCK 2399 kmp_user_lock_p lck;
2400 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2401 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2403 lck = (kmp_user_lock_p)user_lock;
2405 __kmp_itt_lock_destroyed(lck);
2407 #if OMPT_SUPPORT && OMPT_OPTIONAL 2409 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2411 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2412 if (ompt_enabled.ompt_callback_lock_destroy) {
2413 kmp_user_lock_p lck;
2414 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2415 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2417 lck = (kmp_user_lock_p)user_lock;
2419 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2420 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2423 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2425 kmp_user_lock_p lck;
2427 if ((__kmp_user_lock_kind == lk_tas) &&
2428 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2429 lck = (kmp_user_lock_p)user_lock;
2432 else if ((__kmp_user_lock_kind == lk_futex) &&
2433 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2434 lck = (kmp_user_lock_p)user_lock;
2438 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_lock");
2441 #if OMPT_SUPPORT && OMPT_OPTIONAL 2443 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2445 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2446 if (ompt_enabled.ompt_callback_lock_destroy) {
2447 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2448 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2453 __kmp_itt_lock_destroyed(lck);
2457 if ((__kmp_user_lock_kind == lk_tas) &&
2458 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2462 else if ((__kmp_user_lock_kind == lk_futex) &&
2463 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2468 __kmp_user_lock_free(user_lock, gtid, lck);
2470 #endif // KMP_USE_DYNAMIC_LOCK 2474 void __kmpc_destroy_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2475 #if KMP_USE_DYNAMIC_LOCK 2478 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2479 __kmp_itt_lock_destroyed(ilk->lock);
2481 #if OMPT_SUPPORT && OMPT_OPTIONAL 2483 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2485 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2486 if (ompt_enabled.ompt_callback_lock_destroy) {
2487 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2488 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2491 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2493 #else // KMP_USE_DYNAMIC_LOCK 2495 kmp_user_lock_p lck;
2497 if ((__kmp_user_lock_kind == lk_tas) &&
2498 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2499 OMP_NEST_LOCK_T_SIZE)) {
2500 lck = (kmp_user_lock_p)user_lock;
2503 else if ((__kmp_user_lock_kind == lk_futex) &&
2504 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2505 OMP_NEST_LOCK_T_SIZE)) {
2506 lck = (kmp_user_lock_p)user_lock;
2510 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_nest_lock");
2513 #if OMPT_SUPPORT && OMPT_OPTIONAL 2515 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2517 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2518 if (ompt_enabled.ompt_callback_lock_destroy) {
2519 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2520 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2525 __kmp_itt_lock_destroyed(lck);
2528 DESTROY_NESTED_LOCK(lck);
2530 if ((__kmp_user_lock_kind == lk_tas) &&
2531 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2532 OMP_NEST_LOCK_T_SIZE)) {
2536 else if ((__kmp_user_lock_kind == lk_futex) &&
2537 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2538 OMP_NEST_LOCK_T_SIZE)) {
2543 __kmp_user_lock_free(user_lock, gtid, lck);
2545 #endif // KMP_USE_DYNAMIC_LOCK 2548 void __kmpc_set_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2550 #if KMP_USE_DYNAMIC_LOCK 2551 int tag = KMP_EXTRACT_D_TAG(user_lock);
2553 __kmp_itt_lock_acquiring(
2557 #if OMPT_SUPPORT && OMPT_OPTIONAL 2559 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2561 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2562 if (ompt_enabled.ompt_callback_mutex_acquire) {
2563 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2564 ompt_mutex_lock, omp_lock_hint_none,
2565 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2569 #if KMP_USE_INLINED_TAS 2570 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2571 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2573 #elif KMP_USE_INLINED_FUTEX 2574 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2575 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2579 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2582 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2584 #if OMPT_SUPPORT && OMPT_OPTIONAL 2585 if (ompt_enabled.ompt_callback_mutex_acquired) {
2586 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2587 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2591 #else // KMP_USE_DYNAMIC_LOCK 2593 kmp_user_lock_p lck;
2595 if ((__kmp_user_lock_kind == lk_tas) &&
2596 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2597 lck = (kmp_user_lock_p)user_lock;
2600 else if ((__kmp_user_lock_kind == lk_futex) &&
2601 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2602 lck = (kmp_user_lock_p)user_lock;
2606 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_lock");
2610 __kmp_itt_lock_acquiring(lck);
2612 #if OMPT_SUPPORT && OMPT_OPTIONAL 2614 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2616 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2617 if (ompt_enabled.ompt_callback_mutex_acquire) {
2618 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2619 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2620 (ompt_wait_id_t)lck, codeptr);
2624 ACQUIRE_LOCK(lck, gtid);
2627 __kmp_itt_lock_acquired(lck);
2630 #if OMPT_SUPPORT && OMPT_OPTIONAL 2631 if (ompt_enabled.ompt_callback_mutex_acquired) {
2632 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2633 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2637 #endif // KMP_USE_DYNAMIC_LOCK 2640 void __kmpc_set_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2641 #if KMP_USE_DYNAMIC_LOCK 2644 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2646 #if OMPT_SUPPORT && OMPT_OPTIONAL 2648 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2650 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2651 if (ompt_enabled.enabled) {
2652 if (ompt_enabled.ompt_callback_mutex_acquire) {
2653 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2654 ompt_mutex_nest_lock, omp_lock_hint_none,
2655 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2660 int acquire_status =
2661 KMP_D_LOCK_FUNC(user_lock,
set)((kmp_dyna_lock_t *)user_lock, gtid);
2662 (void) acquire_status;
2664 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2667 #if OMPT_SUPPORT && OMPT_OPTIONAL 2668 if (ompt_enabled.enabled) {
2669 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2670 if (ompt_enabled.ompt_callback_mutex_acquired) {
2672 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2673 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2676 if (ompt_enabled.ompt_callback_nest_lock) {
2678 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2679 ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr);
2685 #else // KMP_USE_DYNAMIC_LOCK 2687 kmp_user_lock_p lck;
2689 if ((__kmp_user_lock_kind == lk_tas) &&
2690 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2691 OMP_NEST_LOCK_T_SIZE)) {
2692 lck = (kmp_user_lock_p)user_lock;
2695 else if ((__kmp_user_lock_kind == lk_futex) &&
2696 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2697 OMP_NEST_LOCK_T_SIZE)) {
2698 lck = (kmp_user_lock_p)user_lock;
2702 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_nest_lock");
2706 __kmp_itt_lock_acquiring(lck);
2708 #if OMPT_SUPPORT && OMPT_OPTIONAL 2710 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2712 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2713 if (ompt_enabled.enabled) {
2714 if (ompt_enabled.ompt_callback_mutex_acquire) {
2715 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2716 ompt_mutex_nest_lock, omp_lock_hint_none,
2717 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr);
2722 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
2725 __kmp_itt_lock_acquired(lck);
2728 #if OMPT_SUPPORT && OMPT_OPTIONAL 2729 if (ompt_enabled.enabled) {
2730 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2731 if (ompt_enabled.ompt_callback_mutex_acquired) {
2733 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2734 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2737 if (ompt_enabled.ompt_callback_nest_lock) {
2739 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2740 ompt_scope_begin, (ompt_wait_id_t)lck, codeptr);
2746 #endif // KMP_USE_DYNAMIC_LOCK 2749 void __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2750 #if KMP_USE_DYNAMIC_LOCK 2752 int tag = KMP_EXTRACT_D_TAG(user_lock);
2754 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2756 #if KMP_USE_INLINED_TAS 2757 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2758 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2760 #elif KMP_USE_INLINED_FUTEX 2761 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2762 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2766 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2769 #if OMPT_SUPPORT && OMPT_OPTIONAL 2771 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2773 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2774 if (ompt_enabled.ompt_callback_mutex_released) {
2775 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2776 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2780 #else // KMP_USE_DYNAMIC_LOCK 2782 kmp_user_lock_p lck;
2787 if ((__kmp_user_lock_kind == lk_tas) &&
2788 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2789 #if KMP_OS_LINUX && \ 2790 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2793 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2795 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2798 #if OMPT_SUPPORT && OMPT_OPTIONAL 2800 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2802 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2803 if (ompt_enabled.ompt_callback_mutex_released) {
2804 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2805 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2811 lck = (kmp_user_lock_p)user_lock;
2815 else if ((__kmp_user_lock_kind == lk_futex) &&
2816 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2817 lck = (kmp_user_lock_p)user_lock;
2821 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_lock");
2825 __kmp_itt_lock_releasing(lck);
2828 RELEASE_LOCK(lck, gtid);
2830 #if OMPT_SUPPORT && OMPT_OPTIONAL 2832 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2834 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2835 if (ompt_enabled.ompt_callback_mutex_released) {
2836 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2837 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2841 #endif // KMP_USE_DYNAMIC_LOCK 2845 void __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2846 #if KMP_USE_DYNAMIC_LOCK 2849 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2851 int release_status =
2852 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2853 (void) release_status;
2855 #if OMPT_SUPPORT && OMPT_OPTIONAL 2857 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2859 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2860 if (ompt_enabled.enabled) {
2861 if (release_status == KMP_LOCK_RELEASED) {
2862 if (ompt_enabled.ompt_callback_mutex_released) {
2864 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2865 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2867 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2869 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2870 ompt_scope_end, (ompt_wait_id_t)user_lock, codeptr);
2875 #else // KMP_USE_DYNAMIC_LOCK 2877 kmp_user_lock_p lck;
2881 if ((__kmp_user_lock_kind == lk_tas) &&
2882 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2883 OMP_NEST_LOCK_T_SIZE)) {
2884 #if KMP_OS_LINUX && \ 2885 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2887 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
2889 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2892 #if OMPT_SUPPORT && OMPT_OPTIONAL 2893 int release_status = KMP_LOCK_STILL_HELD;
2896 if (--(tl->lk.depth_locked) == 0) {
2897 TCW_4(tl->lk.poll, 0);
2898 #if OMPT_SUPPORT && OMPT_OPTIONAL 2899 release_status = KMP_LOCK_RELEASED;
2904 #if OMPT_SUPPORT && OMPT_OPTIONAL 2906 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2908 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2909 if (ompt_enabled.enabled) {
2910 if (release_status == KMP_LOCK_RELEASED) {
2911 if (ompt_enabled.ompt_callback_mutex_released) {
2913 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2914 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2916 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2918 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2919 ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr);
2926 lck = (kmp_user_lock_p)user_lock;
2930 else if ((__kmp_user_lock_kind == lk_futex) &&
2931 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2932 OMP_NEST_LOCK_T_SIZE)) {
2933 lck = (kmp_user_lock_p)user_lock;
2937 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_nest_lock");
2941 __kmp_itt_lock_releasing(lck);
2945 release_status = RELEASE_NESTED_LOCK(lck, gtid);
2946 #if OMPT_SUPPORT && OMPT_OPTIONAL 2948 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2950 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2951 if (ompt_enabled.enabled) {
2952 if (release_status == KMP_LOCK_RELEASED) {
2953 if (ompt_enabled.ompt_callback_mutex_released) {
2955 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2956 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2958 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2960 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2961 ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr);
2966 #endif // KMP_USE_DYNAMIC_LOCK 2970 int __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2973 #if KMP_USE_DYNAMIC_LOCK 2975 int tag = KMP_EXTRACT_D_TAG(user_lock);
2977 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2979 #if OMPT_SUPPORT && OMPT_OPTIONAL 2981 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2983 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2984 if (ompt_enabled.ompt_callback_mutex_acquire) {
2985 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2986 ompt_mutex_lock, omp_lock_hint_none,
2987 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2991 #if KMP_USE_INLINED_TAS 2992 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2993 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
2995 #elif KMP_USE_INLINED_FUTEX 2996 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2997 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
3001 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
3005 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3007 #if OMPT_SUPPORT && OMPT_OPTIONAL 3008 if (ompt_enabled.ompt_callback_mutex_acquired) {
3009 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3010 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
3016 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3021 #else // KMP_USE_DYNAMIC_LOCK 3023 kmp_user_lock_p lck;
3026 if ((__kmp_user_lock_kind == lk_tas) &&
3027 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
3028 lck = (kmp_user_lock_p)user_lock;
3031 else if ((__kmp_user_lock_kind == lk_futex) &&
3032 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
3033 lck = (kmp_user_lock_p)user_lock;
3037 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_lock");
3041 __kmp_itt_lock_acquiring(lck);
3043 #if OMPT_SUPPORT && OMPT_OPTIONAL 3045 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3047 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3048 if (ompt_enabled.ompt_callback_mutex_acquire) {
3049 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3050 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
3051 (ompt_wait_id_t)lck, codeptr);
3055 rc = TEST_LOCK(lck, gtid);
3058 __kmp_itt_lock_acquired(lck);
3060 __kmp_itt_lock_cancelled(lck);
3063 #if OMPT_SUPPORT && OMPT_OPTIONAL 3064 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
3065 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3066 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
3070 return (rc ? FTN_TRUE : FTN_FALSE);
3074 #endif // KMP_USE_DYNAMIC_LOCK 3078 int __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
3079 #if KMP_USE_DYNAMIC_LOCK 3082 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
3084 #if OMPT_SUPPORT && OMPT_OPTIONAL 3086 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3088 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3089 if (ompt_enabled.ompt_callback_mutex_acquire) {
3090 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3091 ompt_mutex_nest_lock, omp_lock_hint_none,
3092 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
3096 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
3099 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3101 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3104 #if OMPT_SUPPORT && OMPT_OPTIONAL 3105 if (ompt_enabled.enabled && rc) {
3107 if (ompt_enabled.ompt_callback_mutex_acquired) {
3109 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3110 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
3113 if (ompt_enabled.ompt_callback_nest_lock) {
3115 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3116 ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr);
3123 #else // KMP_USE_DYNAMIC_LOCK 3125 kmp_user_lock_p lck;
3128 if ((__kmp_user_lock_kind == lk_tas) &&
3129 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
3130 OMP_NEST_LOCK_T_SIZE)) {
3131 lck = (kmp_user_lock_p)user_lock;
3134 else if ((__kmp_user_lock_kind == lk_futex) &&
3135 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
3136 OMP_NEST_LOCK_T_SIZE)) {
3137 lck = (kmp_user_lock_p)user_lock;
3141 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_nest_lock");
3145 __kmp_itt_lock_acquiring(lck);
3148 #if OMPT_SUPPORT && OMPT_OPTIONAL 3150 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3152 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3153 if (ompt_enabled.enabled) &&
3154 ompt_enabled.ompt_callback_mutex_acquire) {
3155 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3156 ompt_mutex_nest_lock, omp_lock_hint_none,
3157 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr);
3161 rc = TEST_NESTED_LOCK(lck, gtid);
3164 __kmp_itt_lock_acquired(lck);
3166 __kmp_itt_lock_cancelled(lck);
3169 #if OMPT_SUPPORT && OMPT_OPTIONAL 3170 if (ompt_enabled.enabled && rc) {
3172 if (ompt_enabled.ompt_callback_mutex_acquired) {
3174 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3175 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
3178 if (ompt_enabled.ompt_callback_nest_lock) {
3180 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3181 ompt_mutex_scope_begin, (ompt_wait_id_t)lck, codeptr);
3190 #endif // KMP_USE_DYNAMIC_LOCK 3200 #define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \ 3201 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod)) 3203 #define __KMP_GET_REDUCTION_METHOD(gtid) \ 3204 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) 3210 static __forceinline
void 3211 __kmp_enter_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3212 kmp_critical_name *crit) {
3218 kmp_user_lock_p lck;
3220 #if KMP_USE_DYNAMIC_LOCK 3222 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3225 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3226 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
3227 KMP_GET_D_TAG(__kmp_user_lock_seq));
3229 __kmp_init_indirect_csptr(crit, loc, global_tid,
3230 KMP_GET_I_TAG(__kmp_user_lock_seq));
3236 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3237 lck = (kmp_user_lock_p)lk;
3238 KMP_DEBUG_ASSERT(lck != NULL);
3239 if (__kmp_env_consistency_check) {
3240 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3242 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
3244 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3246 KMP_DEBUG_ASSERT(lck != NULL);
3247 if (__kmp_env_consistency_check) {
3248 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3250 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
3253 #else // KMP_USE_DYNAMIC_LOCK 3258 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3259 lck = (kmp_user_lock_p)crit;
3261 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3263 KMP_DEBUG_ASSERT(lck != NULL);
3265 if (__kmp_env_consistency_check)
3266 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3268 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3270 #endif // KMP_USE_DYNAMIC_LOCK 3274 static __forceinline
void 3275 __kmp_end_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3276 kmp_critical_name *crit) {
3278 kmp_user_lock_p lck;
3280 #if KMP_USE_DYNAMIC_LOCK 3282 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3283 lck = (kmp_user_lock_p)crit;
3284 if (__kmp_env_consistency_check)
3285 __kmp_pop_sync(global_tid, ct_critical, loc);
3286 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3288 kmp_indirect_lock_t *ilk =
3289 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3290 if (__kmp_env_consistency_check)
3291 __kmp_pop_sync(global_tid, ct_critical, loc);
3292 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3295 #else // KMP_USE_DYNAMIC_LOCK 3300 if (__kmp_base_user_lock_size > 32) {
3301 lck = *((kmp_user_lock_p *)crit);
3302 KMP_ASSERT(lck != NULL);
3304 lck = (kmp_user_lock_p)crit;
3307 if (__kmp_env_consistency_check)
3308 __kmp_pop_sync(global_tid, ct_critical, loc);
3310 __kmp_release_user_lock_with_checks(lck, global_tid);
3312 #endif // KMP_USE_DYNAMIC_LOCK 3316 static __forceinline
int 3317 __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3322 if (th->th.th_teams_microtask) {
3323 *team_p = team = th->th.th_team;
3324 if (team->t.t_level == th->th.th_teams_level) {
3326 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
3328 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3329 th->th.th_team = team->t.t_parent;
3330 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3331 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3332 *task_state = th->th.th_task_state;
3333 th->th.th_task_state = 0;
3341 static __forceinline
void 3342 __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team,
int task_state) {
3344 th->th.th_info.ds.ds_tid = 0;
3345 th->th.th_team = team;
3346 th->th.th_team_nproc = team->t.t_nproc;
3347 th->th.th_task_team = team->t.t_task_team[task_state];
3348 th->th.th_task_state = task_state;
3370 size_t reduce_size,
void *reduce_data,
3371 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3372 kmp_critical_name *lck) {
3376 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3380 int teams_swapped = 0, task_state;
3382 KA_TRACE(10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3390 if (!TCR_4(__kmp_init_parallel))
3391 __kmp_parallel_initialize();
3394 __kmp_resume_if_soft_paused();
3398 #if KMP_USE_DYNAMIC_LOCK 3399 if (__kmp_env_consistency_check)
3400 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3402 if (__kmp_env_consistency_check)
3403 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3407 th = __kmp_thread_from_gtid(global_tid);
3408 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3409 #endif // OMP_40_ENABLED 3427 packed_reduction_method = __kmp_determine_reduction_method(
3428 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3429 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3431 if (packed_reduction_method == critical_reduce_block) {
3433 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3436 }
else if (packed_reduction_method == empty_reduce_block) {
3442 }
else if (packed_reduction_method == atomic_reduce_block) {
3452 if (__kmp_env_consistency_check)
3453 __kmp_pop_sync(global_tid, ct_reduce, loc);
3455 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3456 tree_reduce_block)) {
3476 ompt_frame_t *ompt_frame;
3477 if (ompt_enabled.enabled) {
3478 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3479 if (ompt_frame->enter_frame.ptr == NULL)
3480 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3481 OMPT_STORE_RETURN_ADDRESS(global_tid);
3485 __kmp_threads[global_tid]->th.th_ident = loc;
3488 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3489 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3490 retval = (retval != 0) ? (0) : (1);
3491 #if OMPT_SUPPORT && OMPT_OPTIONAL 3492 if (ompt_enabled.enabled) {
3493 ompt_frame->enter_frame = ompt_data_none;
3499 if (__kmp_env_consistency_check) {
3501 __kmp_pop_sync(global_tid, ct_reduce, loc);
3511 if (teams_swapped) {
3512 __kmp_restore_swapped_teams(th, team, task_state);
3517 (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3518 global_tid, packed_reduction_method, retval));
3532 kmp_critical_name *lck) {
3534 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3536 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3538 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3540 if (packed_reduction_method == critical_reduce_block) {
3542 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3544 }
else if (packed_reduction_method == empty_reduce_block) {
3549 }
else if (packed_reduction_method == atomic_reduce_block) {
3556 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3557 tree_reduce_block)) {
3567 if (__kmp_env_consistency_check)
3568 __kmp_pop_sync(global_tid, ct_reduce, loc);
3570 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3571 global_tid, packed_reduction_method));
3594 size_t reduce_size,
void *reduce_data,
3595 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3596 kmp_critical_name *lck) {
3599 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3603 int teams_swapped = 0, task_state;
3606 KA_TRACE(10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid));
3614 if (!TCR_4(__kmp_init_parallel))
3615 __kmp_parallel_initialize();
3618 __kmp_resume_if_soft_paused();
3622 #if KMP_USE_DYNAMIC_LOCK 3623 if (__kmp_env_consistency_check)
3624 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3626 if (__kmp_env_consistency_check)
3627 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3631 th = __kmp_thread_from_gtid(global_tid);
3632 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3633 #endif // OMP_40_ENABLED 3635 packed_reduction_method = __kmp_determine_reduction_method(
3636 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3637 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3639 if (packed_reduction_method == critical_reduce_block) {
3641 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3644 }
else if (packed_reduction_method == empty_reduce_block) {
3650 }
else if (packed_reduction_method == atomic_reduce_block) {
3654 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3655 tree_reduce_block)) {
3661 ompt_frame_t *ompt_frame;
3662 if (ompt_enabled.enabled) {
3663 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3664 if (ompt_frame->enter_frame.ptr == NULL)
3665 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3666 OMPT_STORE_RETURN_ADDRESS(global_tid);
3670 __kmp_threads[global_tid]->th.th_ident =
3674 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3675 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3676 retval = (retval != 0) ? (0) : (1);
3677 #if OMPT_SUPPORT && OMPT_OPTIONAL 3678 if (ompt_enabled.enabled) {
3679 ompt_frame->enter_frame = ompt_data_none;
3685 if (__kmp_env_consistency_check) {
3687 __kmp_pop_sync(global_tid, ct_reduce, loc);
3697 if (teams_swapped) {
3698 __kmp_restore_swapped_teams(th, team, task_state);
3703 (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3704 global_tid, packed_reduction_method, retval));
3720 kmp_critical_name *lck) {
3722 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3726 int teams_swapped = 0, task_state;
3729 KA_TRACE(10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3732 th = __kmp_thread_from_gtid(global_tid);
3733 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3734 #endif // OMP_40_ENABLED 3736 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3741 if (packed_reduction_method == critical_reduce_block) {
3743 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3747 ompt_frame_t *ompt_frame;
3748 if (ompt_enabled.enabled) {
3749 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3750 if (ompt_frame->enter_frame.ptr == NULL)
3751 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3752 OMPT_STORE_RETURN_ADDRESS(global_tid);
3756 __kmp_threads[global_tid]->th.th_ident = loc;
3758 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3759 #if OMPT_SUPPORT && OMPT_OPTIONAL 3760 if (ompt_enabled.enabled) {
3761 ompt_frame->enter_frame = ompt_data_none;
3765 }
else if (packed_reduction_method == empty_reduce_block) {
3771 ompt_frame_t *ompt_frame;
3772 if (ompt_enabled.enabled) {
3773 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3774 if (ompt_frame->enter_frame.ptr == NULL)
3775 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3776 OMPT_STORE_RETURN_ADDRESS(global_tid);
3780 __kmp_threads[global_tid]->th.th_ident = loc;
3782 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3783 #if OMPT_SUPPORT && OMPT_OPTIONAL 3784 if (ompt_enabled.enabled) {
3785 ompt_frame->enter_frame = ompt_data_none;
3789 }
else if (packed_reduction_method == atomic_reduce_block) {
3792 ompt_frame_t *ompt_frame;
3793 if (ompt_enabled.enabled) {
3794 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3795 if (ompt_frame->enter_frame.ptr == NULL)
3796 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3797 OMPT_STORE_RETURN_ADDRESS(global_tid);
3802 __kmp_threads[global_tid]->th.th_ident = loc;
3804 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3805 #if OMPT_SUPPORT && OMPT_OPTIONAL 3806 if (ompt_enabled.enabled) {
3807 ompt_frame->enter_frame = ompt_data_none;
3811 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3812 tree_reduce_block)) {
3815 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3824 if (teams_swapped) {
3825 __kmp_restore_swapped_teams(th, team, task_state);
3829 if (__kmp_env_consistency_check)
3830 __kmp_pop_sync(global_tid, ct_reduce, loc);
3832 KA_TRACE(10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n",
3833 global_tid, packed_reduction_method));
3838 #undef __KMP_GET_REDUCTION_METHOD 3839 #undef __KMP_SET_REDUCTION_METHOD 3843 kmp_uint64 __kmpc_get_taskid() {
3848 gtid = __kmp_get_gtid();
3852 thread = __kmp_thread_from_gtid(gtid);
3853 return thread->th.th_current_task->td_task_id;
3857 kmp_uint64 __kmpc_get_parent_taskid() {
3861 kmp_taskdata_t *parent_task;
3863 gtid = __kmp_get_gtid();
3867 thread = __kmp_thread_from_gtid(gtid);
3868 parent_task = thread->th.th_current_task->td_parent;
3869 return (parent_task == NULL ? 0 : parent_task->td_task_id);
3885 void __kmpc_doacross_init(
ident_t *loc,
int gtid,
int num_dims,
3886 const struct kmp_dim *dims) {
3888 kmp_int64 last, trace_count;
3889 kmp_info_t *th = __kmp_threads[gtid];
3890 kmp_team_t *team = th->th.th_team;
3892 kmp_disp_t *pr_buf = th->th.th_dispatch;
3893 dispatch_shared_info_t *sh_buf;
3897 (
"__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
3898 gtid, num_dims, !team->t.t_serialized));
3899 KMP_DEBUG_ASSERT(dims != NULL);
3900 KMP_DEBUG_ASSERT(num_dims > 0);
3902 if (team->t.t_serialized) {
3903 KA_TRACE(20, (
"__kmpc_doacross_init() exit: serialized team\n"));
3906 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
3907 idx = pr_buf->th_doacross_buf_idx++;
3909 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
3912 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
3913 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
3914 th,
sizeof(kmp_int64) * (4 * num_dims + 1));
3915 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3916 pr_buf->th_doacross_info[0] =
3917 (kmp_int64)num_dims;
3920 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
3921 pr_buf->th_doacross_info[2] = dims[0].lo;
3922 pr_buf->th_doacross_info[3] = dims[0].up;
3923 pr_buf->th_doacross_info[4] = dims[0].st;
3925 for (j = 1; j < num_dims; ++j) {
3928 if (dims[j].st == 1) {
3930 range_length = dims[j].up - dims[j].lo + 1;
3932 if (dims[j].st > 0) {
3933 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
3934 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
3936 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
3938 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
3941 pr_buf->th_doacross_info[last++] = range_length;
3942 pr_buf->th_doacross_info[last++] = dims[j].lo;
3943 pr_buf->th_doacross_info[last++] = dims[j].up;
3944 pr_buf->th_doacross_info[last++] = dims[j].st;
3949 if (dims[0].st == 1) {
3950 trace_count = dims[0].up - dims[0].lo + 1;
3951 }
else if (dims[0].st > 0) {
3952 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
3953 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
3955 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
3956 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
3958 for (j = 1; j < num_dims; ++j) {
3959 trace_count *= pr_buf->th_doacross_info[4 * j + 1];
3961 KMP_DEBUG_ASSERT(trace_count > 0);
3965 if (idx != sh_buf->doacross_buf_idx) {
3967 __kmp_wait_yield_4((
volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
3974 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
3975 (
volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
3977 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
3978 (
volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
3980 if (flags == NULL) {
3982 size_t size = trace_count / 8 + 8;
3983 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
3985 sh_buf->doacross_flags = flags;
3986 }
else if (flags == (kmp_uint32 *)1) {
3989 while (*(
volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
3991 while (*(
volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
3998 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1);
3999 pr_buf->th_doacross_flags =
4000 sh_buf->doacross_flags;
4002 KA_TRACE(20, (
"__kmpc_doacross_init() exit: T#%d\n", gtid));
4005 void __kmpc_doacross_wait(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
4006 kmp_int32 shft, num_dims, i;
4008 kmp_int64 iter_number;
4009 kmp_info_t *th = __kmp_threads[gtid];
4010 kmp_team_t *team = th->th.th_team;
4012 kmp_int64 lo, up, st;
4014 KA_TRACE(20, (
"__kmpc_doacross_wait() enter: called T#%d\n", gtid));
4015 if (team->t.t_serialized) {
4016 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: serialized team\n"));
4021 pr_buf = th->th.th_dispatch;
4022 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4023 num_dims = pr_buf->th_doacross_info[0];
4024 lo = pr_buf->th_doacross_info[2];
4025 up = pr_buf->th_doacross_info[3];
4026 st = pr_buf->th_doacross_info[4];
4028 if (vec[0] < lo || vec[0] > up) {
4029 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4030 "bounds [%lld,%lld]\n",
4031 gtid, vec[0], lo, up));
4034 iter_number = vec[0] - lo;
4035 }
else if (st > 0) {
4036 if (vec[0] < lo || vec[0] > up) {
4037 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4038 "bounds [%lld,%lld]\n",
4039 gtid, vec[0], lo, up));
4042 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4044 if (vec[0] > lo || vec[0] < up) {
4045 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4046 "bounds [%lld,%lld]\n",
4047 gtid, vec[0], lo, up));
4050 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4052 for (i = 1; i < num_dims; ++i) {
4054 kmp_int32 j = i * 4;
4055 ln = pr_buf->th_doacross_info[j + 1];
4056 lo = pr_buf->th_doacross_info[j + 2];
4057 up = pr_buf->th_doacross_info[j + 3];
4058 st = pr_buf->th_doacross_info[j + 4];
4060 if (vec[i] < lo || vec[i] > up) {
4061 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4062 "bounds [%lld,%lld]\n",
4063 gtid, vec[i], lo, up));
4067 }
else if (st > 0) {
4068 if (vec[i] < lo || vec[i] > up) {
4069 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4070 "bounds [%lld,%lld]\n",
4071 gtid, vec[i], lo, up));
4074 iter = (kmp_uint64)(vec[i] - lo) / st;
4076 if (vec[i] > lo || vec[i] < up) {
4077 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4078 "bounds [%lld,%lld]\n",
4079 gtid, vec[i], lo, up));
4082 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4084 iter_number = iter + ln * iter_number;
4086 shft = iter_number % 32;
4089 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
4094 (
"__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
4095 gtid, (iter_number << 5) + shft));
4098 void __kmpc_doacross_post(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
4099 kmp_int32 shft, num_dims, i;
4101 kmp_int64 iter_number;
4102 kmp_info_t *th = __kmp_threads[gtid];
4103 kmp_team_t *team = th->th.th_team;
4107 KA_TRACE(20, (
"__kmpc_doacross_post() enter: called T#%d\n", gtid));
4108 if (team->t.t_serialized) {
4109 KA_TRACE(20, (
"__kmpc_doacross_post() exit: serialized team\n"));
4115 pr_buf = th->th.th_dispatch;
4116 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4117 num_dims = pr_buf->th_doacross_info[0];
4118 lo = pr_buf->th_doacross_info[2];
4119 st = pr_buf->th_doacross_info[4];
4121 iter_number = vec[0] - lo;
4122 }
else if (st > 0) {
4123 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4125 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4127 for (i = 1; i < num_dims; ++i) {
4129 kmp_int32 j = i * 4;
4130 ln = pr_buf->th_doacross_info[j + 1];
4131 lo = pr_buf->th_doacross_info[j + 2];
4132 st = pr_buf->th_doacross_info[j + 4];
4135 }
else if (st > 0) {
4136 iter = (kmp_uint64)(vec[i] - lo) / st;
4138 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4140 iter_number = iter + ln * iter_number;
4142 shft = iter_number % 32;
4146 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4147 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4148 KA_TRACE(20, (
"__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4149 (iter_number << 5) + shft));
4152 void __kmpc_doacross_fini(
ident_t *loc,
int gtid) {
4154 kmp_info_t *th = __kmp_threads[gtid];
4155 kmp_team_t *team = th->th.th_team;
4156 kmp_disp_t *pr_buf = th->th.th_dispatch;
4158 KA_TRACE(20, (
"__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4159 if (team->t.t_serialized) {
4160 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: serialized team %p\n", team));
4163 num_done = KMP_TEST_THEN_INC32((kmp_int32 *)pr_buf->th_doacross_info[1]) + 1;
4164 if (num_done == th->th.th_team_nproc) {
4166 int idx = pr_buf->th_doacross_buf_idx - 1;
4167 dispatch_shared_info_t *sh_buf =
4168 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4169 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4170 (kmp_int64)&sh_buf->doacross_num_done);
4171 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4172 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4173 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4174 sh_buf->doacross_flags = NULL;
4175 sh_buf->doacross_num_done = 0;
4176 sh_buf->doacross_buf_idx +=
4177 __kmp_dispatch_num_buffers;
4180 pr_buf->th_doacross_flags = NULL;
4181 __kmp_thread_free(th, (
void *)pr_buf->th_doacross_info);
4182 pr_buf->th_doacross_info = NULL;
4183 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: T#%d\n", gtid));
4188 int __kmpc_get_target_offload(
void) {
4189 if (!__kmp_init_serial) {
4190 __kmp_serial_initialize();
4192 return __kmp_target_offload;
4195 int __kmpc_pause_resource(kmp_pause_status_t level) {
4196 if (!__kmp_init_serial) {
4199 return __kmp_pause_resource(level);
4201 #endif // OMP_50_ENABLED kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
stats_state_e
the states which a thread can be in
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)