14 #include "kmp_affinity.h" 18 #include "kmp_wait_release.h" 27 enum SYSTEM_INFORMATION_CLASS {
28 SystemProcessInformation = 5
48 SIZE_T PeakVirtualSize;
51 SIZE_T PeakWorkingSetSize;
52 SIZE_T WorkingSetSize;
53 SIZE_T QuotaPeakPagedPoolUsage;
54 SIZE_T QuotaPagedPoolUsage;
55 SIZE_T QuotaPeakNonPagedPoolUsage;
56 SIZE_T QuotaNonPagedPoolUsage;
58 SIZE_T PeakPagefileUsage;
59 SIZE_T PrivatePageCount;
62 struct SYSTEM_THREAD {
63 LARGE_INTEGER KernelTime;
64 LARGE_INTEGER UserTime;
65 LARGE_INTEGER CreateTime;
71 ULONG ContextSwitchCount;
76 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
78 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
85 struct SYSTEM_PROCESS_INFORMATION {
86 ULONG NextEntryOffset;
87 ULONG NumberOfThreads;
88 LARGE_INTEGER Reserved[3];
89 LARGE_INTEGER CreateTime;
90 LARGE_INTEGER UserTime;
91 LARGE_INTEGER KernelTime;
92 UNICODE_STRING ImageName;
95 HANDLE ParentProcessId;
98 VM_COUNTERS VMCounters;
99 IO_COUNTERS IOCounters;
100 SYSTEM_THREAD Threads[1];
102 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
104 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
121 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
122 PVOID, ULONG, PULONG);
123 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
125 HMODULE ntdll = NULL;
129 static HMODULE kernel32 = NULL;
131 #if KMP_HANDLE_SIGNALS 132 typedef void (*sig_func_t)(int);
133 static sig_func_t __kmp_sighldrs[NSIG];
134 static int __kmp_siginstalled[NSIG];
138 static HANDLE __kmp_monitor_ev;
140 static kmp_int64 __kmp_win32_time;
141 double __kmp_win32_tick;
143 int __kmp_init_runtime = FALSE;
144 CRITICAL_SECTION __kmp_win32_section;
146 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
147 InitializeCriticalSection(&mx->cs);
149 __kmp_itt_system_object_created(&mx->cs,
"Critical Section");
153 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
154 DeleteCriticalSection(&mx->cs);
157 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
158 EnterCriticalSection(&mx->cs);
161 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
162 return TryEnterCriticalSection(&mx->cs);
165 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
166 LeaveCriticalSection(&mx->cs);
169 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
170 cv->waiters_count_ = 0;
171 cv->wait_generation_count_ = 0;
172 cv->release_count_ = 0;
175 __kmp_win32_mutex_init(&cv->waiters_count_lock_);
178 cv->event_ = CreateEvent(NULL,
183 __kmp_itt_system_object_created(cv->event_,
"Event");
187 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
188 __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
189 __kmp_free_handle(cv->event_);
190 memset(cv,
'\0',
sizeof(*cv));
197 static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
198 kmp_info_t *th, C *flag) {
203 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
206 cv->waiters_count_++;
209 my_generation = cv->wait_generation_count_;
211 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
212 __kmp_win32_mutex_unlock(mx);
216 DWORD res, timeout = 5000;
218 res = WaitForSingleObject(cv->event_, timeout);
220 if (res == WAIT_OBJECT_0) {
222 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
226 wait_done = (cv->release_count_ > 0) &&
227 (cv->wait_generation_count_ != my_generation);
228 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
229 }
else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) {
232 __kmp_win32_mutex_lock(mx);
233 typename C::flag_t old_f = flag->set_sleeping();
234 if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) {
235 __kmp_win32_mutex_unlock(mx);
239 old_f = flag->unset_sleeping();
240 KMP_DEBUG_ASSERT(old_f & KMP_BARRIER_SLEEP_STATE);
241 TCW_PTR(th->th.th_sleep_loc, NULL);
242 KF_TRACE(50, (
"__kmp_win32_cond_wait: exiting, condition " 243 "fulfilled: flag's loc(%p): %u => %u\n",
244 flag->get(), old_f, *(flag->get())));
246 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
247 KMP_DEBUG_ASSERT(cv->waiters_count_ > 0);
248 cv->release_count_ = cv->waiters_count_;
249 cv->wait_generation_count_++;
251 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
253 __kmp_win32_mutex_unlock(mx);
261 __kmp_win32_mutex_lock(mx);
262 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
264 cv->waiters_count_--;
265 cv->release_count_--;
267 last_waiter = (cv->release_count_ == 0);
269 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
273 ResetEvent(cv->event_);
277 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
278 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
280 if (cv->waiters_count_ > 0) {
281 SetEvent(cv->event_);
284 cv->release_count_ = cv->waiters_count_;
287 cv->wait_generation_count_++;
290 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
293 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
294 __kmp_win32_cond_broadcast(cv);
297 void __kmp_enable(
int new_state) {
298 if (__kmp_init_runtime)
299 LeaveCriticalSection(&__kmp_win32_section);
302 void __kmp_disable(
int *old_state) {
305 if (__kmp_init_runtime)
306 EnterCriticalSection(&__kmp_win32_section);
309 void __kmp_suspend_initialize(
void) {
312 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
313 if (!TCR_4(th->th.th_suspend_init)) {
316 __kmp_win32_cond_init(&th->th.th_suspend_cv);
317 __kmp_win32_mutex_init(&th->th.th_suspend_mx);
318 TCW_4(th->th.th_suspend_init, TRUE);
322 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
323 if (TCR_4(th->th.th_suspend_init)) {
326 __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
327 __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
328 TCW_4(th->th.th_suspend_init, FALSE);
332 int __kmp_try_suspend_mx(kmp_info_t *th) {
333 return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
336 void __kmp_lock_suspend_mx(kmp_info_t *th) {
337 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
340 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
341 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
347 static inline void __kmp_suspend_template(
int th_gtid, C *flag) {
348 kmp_info_t *th = __kmp_threads[th_gtid];
350 typename C::flag_t old_spin;
352 KF_TRACE(30, (
"__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
353 th_gtid, flag->get()));
355 __kmp_suspend_initialize_thread(th);
356 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
358 KF_TRACE(10, (
"__kmp_suspend_template: T#%d setting sleep bit for flag's" 360 th_gtid, flag->get()));
364 old_spin = flag->set_sleeping();
366 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
367 __kmp_pause_status != kmp_soft_paused) {
368 flag->unset_sleeping();
369 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
374 KF_TRACE(5, (
"__kmp_suspend_template: T#%d set sleep bit for flag's" 376 th_gtid, flag->get(), *(flag->get())));
378 if (flag->done_check_val(old_spin)) {
379 old_spin = flag->unset_sleeping();
380 KF_TRACE(5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit " 381 "for flag's loc(%p)\n",
382 th_gtid, flag->get()));
385 __kmp_suspend_count++;
390 int deactivated = FALSE;
391 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
392 while (flag->is_sleeping()) {
393 KF_TRACE(15, (
"__kmp_suspend_template: T#%d about to perform " 394 "kmp_win32_cond_wait()\n",
399 th->th.th_active = FALSE;
400 if (th->th.th_active_in_pool) {
401 th->th.th_active_in_pool = FALSE;
402 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
403 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
406 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
409 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
414 if (flag->is_sleeping()) {
416 (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
424 th->th.th_active = TRUE;
425 if (TCR_4(th->th.th_in_pool)) {
426 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
427 th->th.th_active_in_pool = TRUE;
432 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
434 KF_TRACE(30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid));
437 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
438 __kmp_suspend_template(th_gtid, flag);
440 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
441 __kmp_suspend_template(th_gtid, flag);
443 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
444 __kmp_suspend_template(th_gtid, flag);
450 static inline void __kmp_resume_template(
int target_gtid, C *flag) {
451 kmp_info_t *th = __kmp_threads[target_gtid];
455 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
458 KF_TRACE(30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
461 __kmp_suspend_initialize_thread(th);
462 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
465 flag = (C *)th->th.th_sleep_loc;
470 if (!flag || flag->get_type() != flag->get_ptr_type()) {
473 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already " 474 "awake: flag's loc(%p)\n",
475 gtid, target_gtid, NULL));
476 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
479 typename C::flag_t old_spin = flag->unset_sleeping();
480 if (!flag->is_sleeping_val(old_spin)) {
481 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already " 482 "awake: flag's loc(%p): %u => %u\n",
483 gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
484 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
488 TCW_PTR(th->th.th_sleep_loc, NULL);
489 KF_TRACE(5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep " 490 "bit for flag's loc(%p)\n",
491 gtid, target_gtid, flag->get()));
493 __kmp_win32_cond_signal(&th->th.th_suspend_cv);
494 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
496 KF_TRACE(30, (
"__kmp_resume_template: T#%d exiting after signaling wake up" 501 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
502 __kmp_resume_template(target_gtid, flag);
504 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
505 __kmp_resume_template(target_gtid, flag);
507 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
508 __kmp_resume_template(target_gtid, flag);
511 void __kmp_yield() { Sleep(0); }
513 void __kmp_gtid_set_specific(
int gtid) {
514 if (__kmp_init_gtid) {
515 KA_TRACE(50, (
"__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
516 __kmp_gtid_threadprivate_key));
517 if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
518 KMP_FATAL(TLSSetValueFailed);
520 KA_TRACE(50, (
"__kmp_gtid_set_specific: runtime shutdown, returning\n"));
524 int __kmp_gtid_get_specific() {
526 if (!__kmp_init_gtid) {
527 KA_TRACE(50, (
"__kmp_gtid_get_specific: runtime shutdown, returning " 528 "KMP_GTID_SHUTDOWN\n"));
529 return KMP_GTID_SHUTDOWN;
531 gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
537 KA_TRACE(50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
538 __kmp_gtid_threadprivate_key, gtid));
542 void __kmp_affinity_bind_thread(
int proc) {
543 if (__kmp_num_proc_groups > 1) {
547 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
548 sizeof(DWORD_PTR))));
549 ga.Group = proc / (CHAR_BIT *
sizeof(DWORD_PTR));
550 ga.Mask = (
unsigned long long)1 << (proc % (CHAR_BIT *
sizeof(DWORD_PTR)));
551 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
553 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
554 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
555 DWORD error = GetLastError();
556 if (__kmp_affinity_verbose) {
557 kmp_msg_t err_code = KMP_ERR(error);
558 __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
560 if (__kmp_generate_warnings == kmp_warnings_off) {
561 __kmp_str_free(&err_code.str);
566 kmp_affin_mask_t *mask;
567 KMP_CPU_ALLOC_ON_STACK(mask);
569 KMP_CPU_SET(proc, mask);
570 __kmp_set_system_affinity(mask, TRUE);
571 KMP_CPU_FREE_FROM_STACK(mask);
575 void __kmp_affinity_determine_capable(
const char *env_var) {
578 #if KMP_GROUP_AFFINITY 579 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups *
sizeof(DWORD_PTR));
581 KMP_AFFINITY_ENABLE(
sizeof(DWORD_PTR));
584 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 585 "Windows* OS affinity interface functional (mask size = " 586 "%" KMP_SIZE_T_SPEC
").\n",
587 __kmp_affin_mask_size));
590 double __kmp_read_cpu_time(
void) {
591 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
597 status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
598 &KernelTime, &UserTime);
603 sec += KernelTime.dwHighDateTime;
604 sec += UserTime.dwHighDateTime;
607 sec *= (double)(1 << 16) * (double)(1 << 16);
609 sec += KernelTime.dwLowDateTime;
610 sec += UserTime.dwLowDateTime;
612 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
618 int __kmp_read_system_info(
struct kmp_sys_info *info) {
631 void __kmp_runtime_initialize(
void) {
636 if (__kmp_init_runtime) {
644 UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
646 BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
647 GET_MODULE_HANDLE_EX_FLAG_PIN,
648 (LPCTSTR)&__kmp_serial_initialize, &h);
649 KMP_DEBUG_ASSERT2(h && ret,
"OpenMP RTL cannot find itself loaded");
650 SetErrorMode(err_mode);
651 KA_TRACE(10, (
"__kmp_runtime_initialize: dynamic library pinned\n"));
655 InitializeCriticalSection(&__kmp_win32_section);
657 __kmp_itt_system_object_created(&__kmp_win32_section,
"Critical Section");
659 __kmp_initialize_system_tick();
661 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 662 if (!__kmp_cpuinfo.initialized) {
663 __kmp_query_cpuid(&__kmp_cpuinfo);
668 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 683 __kmp_tls_gtid_min = 0;
685 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
689 if (!__kmp_gtid_threadprivate_key) {
690 __kmp_gtid_threadprivate_key = TlsAlloc();
691 if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
692 KMP_FATAL(TLSOutOfIndexes);
700 __kmp_str_buf_init(&path);
701 path_size = GetSystemDirectory(path.str, path.size);
702 KMP_DEBUG_ASSERT(path_size > 0);
703 if (path_size >= path.size) {
705 __kmp_str_buf_reserve(&path, path_size);
706 path_size = GetSystemDirectory(path.str, path.size);
707 KMP_DEBUG_ASSERT(path_size > 0);
709 if (path_size > 0 && path_size < path.size) {
712 path.used = path_size;
713 __kmp_str_buf_print(&path,
"\\%s",
"ntdll.dll");
716 ntdll = GetModuleHandle(path.str);
719 KMP_DEBUG_ASSERT(ntdll != NULL);
721 NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
722 ntdll,
"NtQuerySystemInformation");
724 KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
726 #if KMP_GROUP_AFFINITY 729 if (path_size > 0 && path_size < path.size) {
732 path.used = path_size;
733 __kmp_str_buf_print(&path,
"\\%s",
"kernel32.dll");
736 kernel32 = GetModuleHandle(path.str);
737 KA_TRACE(10, (
"__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
741 if (kernel32 != NULL) {
742 __kmp_GetActiveProcessorCount =
743 (kmp_GetActiveProcessorCount_t)GetProcAddress(
744 kernel32,
"GetActiveProcessorCount");
745 __kmp_GetActiveProcessorGroupCount =
746 (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
747 kernel32,
"GetActiveProcessorGroupCount");
748 __kmp_GetThreadGroupAffinity =
749 (kmp_GetThreadGroupAffinity_t)GetProcAddress(
750 kernel32,
"GetThreadGroupAffinity");
751 __kmp_SetThreadGroupAffinity =
752 (kmp_SetThreadGroupAffinity_t)GetProcAddress(
753 kernel32,
"SetThreadGroupAffinity");
755 KA_TRACE(10, (
"__kmp_runtime_initialize: __kmp_GetActiveProcessorCount" 757 __kmp_GetActiveProcessorCount));
758 KA_TRACE(10, (
"__kmp_runtime_initialize: " 759 "__kmp_GetActiveProcessorGroupCount = %p\n",
760 __kmp_GetActiveProcessorGroupCount));
761 KA_TRACE(10, (
"__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity" 763 __kmp_GetThreadGroupAffinity));
764 KA_TRACE(10, (
"__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity" 766 __kmp_SetThreadGroupAffinity));
767 KA_TRACE(10, (
"__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
768 sizeof(kmp_affin_mask_t)));
775 if ((__kmp_GetActiveProcessorCount != NULL) &&
776 (__kmp_GetActiveProcessorGroupCount != NULL) &&
777 (__kmp_GetThreadGroupAffinity != NULL) &&
778 (__kmp_SetThreadGroupAffinity != NULL) &&
779 ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
784 KA_TRACE(10, (
"__kmp_runtime_initialize: %d processor groups" 786 __kmp_num_proc_groups));
790 for (i = 0; i < __kmp_num_proc_groups; i++) {
791 DWORD size = __kmp_GetActiveProcessorCount(i);
793 KA_TRACE(10, (
"__kmp_runtime_initialize: proc group %d size = %d\n",
797 KA_TRACE(10, (
"__kmp_runtime_initialize: %d processor groups" 799 __kmp_num_proc_groups));
803 if (__kmp_num_proc_groups <= 1) {
804 GetSystemInfo(&info);
805 __kmp_xproc = info.dwNumberOfProcessors;
808 GetSystemInfo(&info);
809 __kmp_xproc = info.dwNumberOfProcessors;
814 if (__kmp_xproc <= 0) {
819 (
"__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
821 __kmp_str_buf_free(&path);
824 __kmp_itt_initialize();
827 __kmp_init_runtime = TRUE;
830 void __kmp_runtime_destroy(
void) {
831 if (!__kmp_init_runtime) {
841 KA_TRACE(40, (
"__kmp_runtime_destroy\n"));
843 if (__kmp_gtid_threadprivate_key) {
844 TlsFree(__kmp_gtid_threadprivate_key);
845 __kmp_gtid_threadprivate_key = 0;
848 __kmp_affinity_uninitialize();
849 DeleteCriticalSection(&__kmp_win32_section);
852 NtQuerySystemInformation = NULL;
856 __kmp_GetActiveProcessorCount = NULL;
857 __kmp_GetActiveProcessorGroupCount = NULL;
858 __kmp_GetThreadGroupAffinity = NULL;
859 __kmp_SetThreadGroupAffinity = NULL;
860 #endif // KMP_ARCH_X86_64 862 __kmp_init_runtime = FALSE;
865 void __kmp_terminate_thread(
int gtid) {
866 kmp_info_t *th = __kmp_threads[gtid];
871 KA_TRACE(10, (
"__kmp_terminate_thread: kill (%d)\n", gtid));
873 if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
876 __kmp_free_handle(th->th.th_info.ds.ds_thread);
879 void __kmp_clear_system_time(
void) {
882 status = QueryPerformanceCounter(&time);
883 __kmp_win32_time = (kmp_int64)time.QuadPart;
886 void __kmp_initialize_system_tick(
void) {
891 status = QueryPerformanceFrequency(&freq);
893 DWORD error = GetLastError();
894 __kmp_fatal(KMP_MSG(FunctionError,
"QueryPerformanceFrequency()"),
895 KMP_ERR(error), __kmp_msg_null);
898 __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
905 void __kmp_elapsed(
double *t) {
908 status = QueryPerformanceCounter(&now);
909 *t = ((double)now.QuadPart) * __kmp_win32_tick;
914 void __kmp_elapsed_tick(
double *t) { *t = __kmp_win32_tick; }
916 void __kmp_read_system_time(
double *delta) {
921 status = QueryPerformanceCounter(&now);
923 *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
929 kmp_uint64 __kmp_now_nsec() {
931 QueryPerformanceCounter(&now);
932 return 1e9 * __kmp_win32_tick * now.QuadPart;
936 void *__stdcall __kmp_launch_worker(
void *arg) {
937 volatile void *stack_data;
940 kmp_info_t *this_thr = (kmp_info_t *)arg;
943 gtid = this_thr->th.th_info.ds.ds_gtid;
944 __kmp_gtid_set_specific(gtid);
945 #ifdef KMP_TDATA_GTID 946 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 947 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 948 "reference: http://support.microsoft.com/kb/118816" 953 __kmp_itt_thread_name(gtid);
956 __kmp_affinity_set_init_mask(gtid, FALSE);
958 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 960 __kmp_clear_x87_fpu_status_word();
961 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
962 __kmp_load_mxcsr(&__kmp_init_mxcsr);
965 if (__kmp_stkoffset > 0 && gtid > 0) {
966 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
969 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
970 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
971 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
973 if (TCR_4(__kmp_gtid_mode) <
975 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
976 KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
977 __kmp_check_stack_overlap(this_thr);
980 exit_val = __kmp_launch_thread(this_thr);
981 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
982 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
990 void *__stdcall __kmp_launch_monitor(
void *arg) {
992 kmp_thread_t monitor;
995 kmp_info_t *this_thr = (kmp_info_t *)arg;
997 KMP_DEBUG_ASSERT(__kmp_init_monitor);
998 TCW_4(__kmp_init_monitor, 2);
1000 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1001 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
1004 KA_TRACE(10, (
"__kmp_launch_monitor: launched\n"));
1006 monitor = GetCurrentThread();
1009 status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
1011 DWORD error = GetLastError();
1012 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1016 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
1017 #ifdef KMP_TDATA_GTID 1018 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 1019 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 1020 "reference: http://support.microsoft.com/kb/118816" 1025 __kmp_itt_thread_ignore();
1031 interval = (1000 / __kmp_monitor_wakeups);
1033 while (!TCR_4(__kmp_global.g.g_done)) {
1036 KA_TRACE(15, (
"__kmp_launch_monitor: update\n"));
1038 wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
1040 if (wait_status == WAIT_TIMEOUT) {
1041 TCW_4(__kmp_global.g.g_time.dt.t_value,
1042 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1048 KA_TRACE(10, (
"__kmp_launch_monitor: finished\n"));
1050 status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1052 DWORD error = GetLastError();
1053 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1056 if (__kmp_global.g.g_abort != 0) {
1061 KA_TRACE(10, (
"__kmp_launch_monitor: terminate sig=%d\n",
1062 (__kmp_global.g.g_abort)));
1067 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1068 __kmp_terminate_thread(gtid);
1075 (
"__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1077 if (__kmp_global.g.g_abort > 0) {
1078 raise(__kmp_global.g.g_abort);
1082 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1089 void __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size) {
1090 kmp_thread_t handle;
1093 KA_TRACE(10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid));
1095 th->th.th_info.ds.ds_gtid = gtid;
1097 if (KMP_UBER_GTID(gtid)) {
1105 rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1106 GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1107 FALSE, DUPLICATE_SAME_ACCESS);
1109 KA_TRACE(10, (
" __kmp_create_worker: ROOT Handle duplicated, th = %p, " 1110 "handle = %" KMP_UINTPTR_SPEC
"\n",
1111 (LPVOID)th, th->th.th_info.ds.ds_thread));
1112 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1114 if (TCR_4(__kmp_gtid_mode) < 2) {
1116 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1117 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1118 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1119 __kmp_check_stack_overlap(th);
1126 (
"__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
" bytes\n",
1129 stack_size += gtid * __kmp_stkoffset;
1131 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1132 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1135 (
"__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1136 " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1137 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1138 (LPVOID)th, &idThread));
1140 handle = CreateThread(
1141 NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1142 (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1145 (
"__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1146 " bytes, &__kmp_launch_worker = %p, th = %p, " 1147 "idThread = %u, handle = %" KMP_UINTPTR_SPEC
"\n",
1148 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1149 (LPVOID)th, idThread, handle));
1152 DWORD error = GetLastError();
1153 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1155 th->th.th_info.ds.ds_thread = handle;
1161 KA_TRACE(10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid));
1164 int __kmp_still_running(kmp_info_t *th) {
1165 return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1169 void __kmp_create_monitor(kmp_info_t *th) {
1170 kmp_thread_t handle;
1172 int ideal, new_ideal;
1174 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1176 KA_TRACE(10, (
"__kmp_create_monitor: skipping monitor thread because of " 1177 "MAX blocktime\n"));
1178 th->th.th_info.ds.ds_tid = 0;
1179 th->th.th_info.ds.ds_gtid = 0;
1180 TCW_4(__kmp_init_monitor, 2);
1183 KA_TRACE(10, (
"__kmp_create_monitor: try to create monitor\n"));
1187 __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1188 if (__kmp_monitor_ev == NULL) {
1189 DWORD error = GetLastError();
1190 __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1193 __kmp_itt_system_object_created(__kmp_monitor_ev,
"Event");
1196 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1197 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1201 if (__kmp_monitor_stksize == 0) {
1202 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1204 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1205 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1208 KA_TRACE(10, (
"__kmp_create_monitor: requested stacksize = %d bytes\n",
1209 (
int)__kmp_monitor_stksize));
1211 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1214 CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1215 (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1216 STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1218 DWORD error = GetLastError();
1219 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1221 th->th.th_info.ds.ds_thread = handle;
1225 KA_TRACE(10, (
"__kmp_create_monitor: monitor created %p\n",
1226 (
void *)th->th.th_info.ds.ds_thread));
1236 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1238 rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1240 DWORD error = GetLastError();
1241 __kmp_fatal(KMP_MSG(FunctionError,
"GetExitCodeThread()"), KMP_ERR(error),
1244 return (*exit_val == STILL_ACTIVE);
1247 void __kmp_exit_thread(
int exit_status) {
1248 ExitThread(exit_status);
1252 static void __kmp_reap_common(kmp_info_t *th) {
1258 10, (
"__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1275 KMP_FSYNC_SPIN_INIT(obj, (
void *)&th->th.th_info.ds.ds_alive);
1277 KMP_INIT_YIELD(spins);
1280 KMP_FSYNC_SPIN_PREPARE(obj);
1282 __kmp_is_thread_alive(th, &exit_val);
1283 KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
1284 }
while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1286 if (exit_val == STILL_ACTIVE) {
1287 KMP_FSYNC_CANCEL(obj);
1289 KMP_FSYNC_SPIN_ACQUIRED(obj);
1294 __kmp_free_handle(th->th.th_info.ds.ds_thread);
1299 if (exit_val == STILL_ACTIVE) {
1300 KA_TRACE(1, (
"__kmp_reap_common: thread still active.\n"));
1301 }
else if ((
void *)exit_val != (
void *)th) {
1302 KA_TRACE(1, (
"__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1306 (
"__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1308 th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1310 th->th.th_info.ds.ds_thread = 0;
1311 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1312 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1313 th->th.th_info.ds.ds_thread_id = 0;
1319 void __kmp_reap_monitor(kmp_info_t *th) {
1322 KA_TRACE(10, (
"__kmp_reap_monitor: try to reap %p\n",
1323 (
void *)th->th.th_info.ds.ds_thread));
1328 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1329 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1330 KA_TRACE(10, (
"__kmp_reap_monitor: monitor did not start, returning\n"));
1336 status = SetEvent(__kmp_monitor_ev);
1337 if (status == FALSE) {
1338 DWORD error = GetLastError();
1339 __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1341 KA_TRACE(10, (
"__kmp_reap_monitor: reaping thread (%d)\n",
1342 th->th.th_info.ds.ds_gtid));
1343 __kmp_reap_common(th);
1345 __kmp_free_handle(__kmp_monitor_ev);
1351 void __kmp_reap_worker(kmp_info_t *th) {
1352 KA_TRACE(10, (
"__kmp_reap_worker: reaping thread (%d)\n",
1353 th->th.th_info.ds.ds_gtid));
1354 __kmp_reap_common(th);
1357 #if KMP_HANDLE_SIGNALS 1359 static void __kmp_team_handler(
int signo) {
1360 if (__kmp_global.g.g_abort == 0) {
1362 if (__kmp_debug_buf) {
1363 __kmp_dump_debug_buffer();
1366 TCW_4(__kmp_global.g.g_abort, signo);
1368 TCW_4(__kmp_global.g.g_done, TRUE);
1373 static sig_func_t __kmp_signal(
int signum, sig_func_t handler) {
1374 sig_func_t old = signal(signum, handler);
1375 if (old == SIG_ERR) {
1377 __kmp_fatal(KMP_MSG(FunctionError,
"signal"), KMP_ERR(error),
1383 static void __kmp_install_one_handler(
int sig, sig_func_t handler,
1384 int parallel_init) {
1387 KB_TRACE(60, (
"__kmp_install_one_handler: called: sig=%d\n", sig));
1388 if (parallel_init) {
1389 old = __kmp_signal(sig, handler);
1391 if (old == __kmp_sighldrs[sig]) {
1392 __kmp_siginstalled[sig] = 1;
1394 old = __kmp_signal(sig, old);
1400 old = __kmp_signal(sig, SIG_DFL);
1401 __kmp_sighldrs[sig] = old;
1402 __kmp_signal(sig, old);
1407 static void __kmp_remove_one_handler(
int sig) {
1408 if (__kmp_siginstalled[sig]) {
1411 KB_TRACE(60, (
"__kmp_remove_one_handler: called: sig=%d\n", sig));
1412 old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1413 if (old != __kmp_team_handler) {
1414 KB_TRACE(10, (
"__kmp_remove_one_handler: oops, not our handler, " 1415 "restoring: sig=%d\n",
1417 old = __kmp_signal(sig, old);
1419 __kmp_sighldrs[sig] = NULL;
1420 __kmp_siginstalled[sig] = 0;
1425 void __kmp_install_signals(
int parallel_init) {
1426 KB_TRACE(10, (
"__kmp_install_signals: called\n"));
1427 if (!__kmp_handle_signals) {
1428 KB_TRACE(10, (
"__kmp_install_signals: KMP_HANDLE_SIGNALS is false - " 1429 "handlers not installed\n"));
1432 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1433 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1434 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1435 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1436 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1437 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1440 void __kmp_remove_signals(
void) {
1442 KB_TRACE(10, (
"__kmp_remove_signals: called\n"));
1443 for (sig = 1; sig < NSIG; ++sig) {
1444 __kmp_remove_one_handler(sig);
1448 #endif // KMP_HANDLE_SIGNALS 1451 void __kmp_thread_sleep(
int millis) {
1454 status = SleepEx((DWORD)millis, FALSE);
1456 DWORD error = GetLastError();
1457 __kmp_fatal(KMP_MSG(FunctionError,
"SleepEx()"), KMP_ERR(error),
1463 int __kmp_is_address_mapped(
void *addr) {
1465 MEMORY_BASIC_INFORMATION lpBuffer;
1468 dwLength =
sizeof(MEMORY_BASIC_INFORMATION);
1470 status = VirtualQuery(addr, &lpBuffer, dwLength);
1472 return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1473 ((lpBuffer.Protect == PAGE_NOACCESS) ||
1474 (lpBuffer.Protect == PAGE_EXECUTE)));
1477 kmp_uint64 __kmp_hardware_timestamp(
void) {
1480 QueryPerformanceCounter((LARGE_INTEGER *)&r);
1485 void __kmp_free_handle(kmp_thread_t tHandle) {
1489 rc = CloseHandle(tHandle);
1491 DWORD error = GetLastError();
1492 __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1496 int __kmp_get_load_balance(
int max) {
1497 static ULONG glb_buff_size = 100 * 1024;
1500 static int glb_running_threads = 0;
1501 static double glb_call_time = 0;
1503 int running_threads = 0;
1504 NTSTATUS status = 0;
1505 ULONG buff_size = 0;
1506 ULONG info_size = 0;
1507 void *buffer = NULL;
1508 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1511 double call_time = 0.0;
1513 __kmp_elapsed(&call_time);
1515 if (glb_call_time &&
1516 (call_time - glb_call_time < __kmp_load_balance_interval)) {
1517 running_threads = glb_running_threads;
1520 glb_call_time = call_time;
1523 if (NtQuerySystemInformation == NULL) {
1524 running_threads = -1;
1535 buff_size = glb_buff_size;
1537 buff_size = 2 * buff_size;
1540 buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1541 if (buffer == NULL) {
1542 running_threads = -1;
1545 status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1546 buff_size, &info_size);
1549 }
while (status == STATUS_INFO_LENGTH_MISMATCH);
1550 glb_buff_size = buff_size;
1552 #define CHECK(cond) \ 1554 KMP_DEBUG_ASSERT(cond); \ 1556 running_threads = -1; \ 1561 CHECK(buff_size >= info_size);
1562 spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1564 ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1565 CHECK(0 <= offset &&
1566 offset +
sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1567 HANDLE pid = spi->ProcessId;
1568 ULONG num = spi->NumberOfThreads;
1571 sizeof(SYSTEM_PROCESS_INFORMATION) +
sizeof(SYSTEM_THREAD) * (num - 1);
1572 CHECK(offset + spi_size <
1574 if (spi->NextEntryOffset != 0) {
1576 spi->NextEntryOffset);
1582 for (
int i = 0; i < num; ++i) {
1583 THREAD_STATE state = spi->Threads[i].State;
1586 if (state == StateRunning) {
1590 if (running_threads >= max) {
1596 if (spi->NextEntryOffset == 0) {
1599 spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1606 if (buffer != NULL) {
1607 KMP_INTERNAL_FREE(buffer);
1610 glb_running_threads = running_threads;
1612 return running_threads;