LLVM OpenMP* Runtime Library
z_Windows_NT_util.cpp
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_affinity.h"
18 #include "kmp_i18n.h"
19 #include "kmp_io.h"
20 #include "kmp_itt.h"
21 #include "kmp_wait_release.h"
22 
23 /* This code is related to NtQuerySystemInformation() function. This function
24  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
25  number of running threads in the system. */
26 
27 #include <ntsecapi.h> // UNICODE_STRING
28 #include <ntstatus.h>
29 
30 enum SYSTEM_INFORMATION_CLASS {
31  SystemProcessInformation = 5
32 }; // SYSTEM_INFORMATION_CLASS
33 
34 struct CLIENT_ID {
35  HANDLE UniqueProcess;
36  HANDLE UniqueThread;
37 }; // struct CLIENT_ID
38 
39 enum THREAD_STATE {
40  StateInitialized,
41  StateReady,
42  StateRunning,
43  StateStandby,
44  StateTerminated,
45  StateWait,
46  StateTransition,
47  StateUnknown
48 }; // enum THREAD_STATE
49 
50 struct VM_COUNTERS {
51  SIZE_T PeakVirtualSize;
52  SIZE_T VirtualSize;
53  ULONG PageFaultCount;
54  SIZE_T PeakWorkingSetSize;
55  SIZE_T WorkingSetSize;
56  SIZE_T QuotaPeakPagedPoolUsage;
57  SIZE_T QuotaPagedPoolUsage;
58  SIZE_T QuotaPeakNonPagedPoolUsage;
59  SIZE_T QuotaNonPagedPoolUsage;
60  SIZE_T PagefileUsage;
61  SIZE_T PeakPagefileUsage;
62  SIZE_T PrivatePageCount;
63 }; // struct VM_COUNTERS
64 
65 struct SYSTEM_THREAD {
66  LARGE_INTEGER KernelTime;
67  LARGE_INTEGER UserTime;
68  LARGE_INTEGER CreateTime;
69  ULONG WaitTime;
70  LPVOID StartAddress;
71  CLIENT_ID ClientId;
72  DWORD Priority;
73  LONG BasePriority;
74  ULONG ContextSwitchCount;
75  THREAD_STATE State;
76  ULONG WaitReason;
77 }; // SYSTEM_THREAD
78 
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
80 #if KMP_ARCH_X86
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
83 #else
84 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
85 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
86 #endif
87 
88 struct SYSTEM_PROCESS_INFORMATION {
89  ULONG NextEntryOffset;
90  ULONG NumberOfThreads;
91  LARGE_INTEGER Reserved[3];
92  LARGE_INTEGER CreateTime;
93  LARGE_INTEGER UserTime;
94  LARGE_INTEGER KernelTime;
95  UNICODE_STRING ImageName;
96  DWORD BasePriority;
97  HANDLE ProcessId;
98  HANDLE ParentProcessId;
99  ULONG HandleCount;
100  ULONG Reserved2[2];
101  VM_COUNTERS VMCounters;
102  IO_COUNTERS IOCounters;
103  SYSTEM_THREAD Threads[1];
104 }; // SYSTEM_PROCESS_INFORMATION
105 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
106 
107 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
110 #if KMP_ARCH_X86
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
113 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
116 #else
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
119 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
120 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
121 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
122 #endif
123 
124 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
125  PVOID, ULONG, PULONG);
126 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
127 
128 HMODULE ntdll = NULL;
129 
130 /* End of NtQuerySystemInformation()-related code */
131 
132 static HMODULE kernel32 = NULL;
133 
134 #if KMP_HANDLE_SIGNALS
135 typedef void (*sig_func_t)(int);
136 static sig_func_t __kmp_sighldrs[NSIG];
137 static int __kmp_siginstalled[NSIG];
138 #endif
139 
140 #if KMP_USE_MONITOR
141 static HANDLE __kmp_monitor_ev;
142 #endif
143 static kmp_int64 __kmp_win32_time;
144 double __kmp_win32_tick;
145 
146 int __kmp_init_runtime = FALSE;
147 CRITICAL_SECTION __kmp_win32_section;
148 
149 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
150  InitializeCriticalSection(&mx->cs);
151 #if USE_ITT_BUILD
152  __kmp_itt_system_object_created(&mx->cs, "Critical Section");
153 #endif /* USE_ITT_BUILD */
154 }
155 
156 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
157  DeleteCriticalSection(&mx->cs);
158 }
159 
160 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
161  EnterCriticalSection(&mx->cs);
162 }
163 
164 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
165  LeaveCriticalSection(&mx->cs);
166 }
167 
168 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
169  cv->waiters_count_ = 0;
170  cv->wait_generation_count_ = 0;
171  cv->release_count_ = 0;
172 
173  /* Initialize the critical section */
174  __kmp_win32_mutex_init(&cv->waiters_count_lock_);
175 
176  /* Create a manual-reset event. */
177  cv->event_ = CreateEvent(NULL, // no security
178  TRUE, // manual-reset
179  FALSE, // non-signaled initially
180  NULL); // unnamed
181 #if USE_ITT_BUILD
182  __kmp_itt_system_object_created(cv->event_, "Event");
183 #endif /* USE_ITT_BUILD */
184 }
185 
186 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
187  __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
188  __kmp_free_handle(cv->event_);
189  memset(cv, '\0', sizeof(*cv));
190 }
191 
192 /* TODO associate cv with a team instead of a thread so as to optimize
193  the case where we wake up a whole team */
194 
195 void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
196  kmp_info_t *th, int need_decrease_load) {
197  int my_generation;
198  int last_waiter;
199 
200  /* Avoid race conditions */
201  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
202 
203  /* Increment count of waiters */
204  cv->waiters_count_++;
205 
206  /* Store current generation in our activation record. */
207  my_generation = cv->wait_generation_count_;
208 
209  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
210  __kmp_win32_mutex_unlock(mx);
211 
212  for (;;) {
213  int wait_done;
214 
215  /* Wait until the event is signaled */
216  WaitForSingleObject(cv->event_, INFINITE);
217 
218  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
219 
220  /* Exit the loop when the <cv->event_> is signaled and there are still
221  waiting threads from this <wait_generation> that haven't been released
222  from this wait yet. */
223  wait_done = (cv->release_count_ > 0) &&
224  (cv->wait_generation_count_ != my_generation);
225 
226  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
227 
228  /* there used to be a semicolon after the if statement, it looked like a
229  bug, so i removed it */
230  if (wait_done)
231  break;
232  }
233 
234  __kmp_win32_mutex_lock(mx);
235  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
236 
237  cv->waiters_count_--;
238  cv->release_count_--;
239 
240  last_waiter = (cv->release_count_ == 0);
241 
242  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
243 
244  if (last_waiter) {
245  /* We're the last waiter to be notified, so reset the manual event. */
246  ResetEvent(cv->event_);
247  }
248 }
249 
250 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
251  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
252 
253  if (cv->waiters_count_ > 0) {
254  SetEvent(cv->event_);
255  /* Release all the threads in this generation. */
256 
257  cv->release_count_ = cv->waiters_count_;
258 
259  /* Start a new generation. */
260  cv->wait_generation_count_++;
261  }
262 
263  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
264 }
265 
266 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
267  __kmp_win32_cond_broadcast(cv);
268 }
269 
270 void __kmp_enable(int new_state) {
271  if (__kmp_init_runtime)
272  LeaveCriticalSection(&__kmp_win32_section);
273 }
274 
275 void __kmp_disable(int *old_state) {
276  *old_state = 0;
277 
278  if (__kmp_init_runtime)
279  EnterCriticalSection(&__kmp_win32_section);
280 }
281 
282 void __kmp_suspend_initialize(void) { /* do nothing */
283 }
284 
285 static void __kmp_suspend_initialize_thread(kmp_info_t *th) {
286  if (!TCR_4(th->th.th_suspend_init)) {
287  /* this means we haven't initialized the suspension pthread objects for this
288  thread in this instance of the process */
289  __kmp_win32_cond_init(&th->th.th_suspend_cv);
290  __kmp_win32_mutex_init(&th->th.th_suspend_mx);
291  TCW_4(th->th.th_suspend_init, TRUE);
292  }
293 }
294 
295 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
296  if (TCR_4(th->th.th_suspend_init)) {
297  /* this means we have initialize the suspension pthread objects for this
298  thread in this instance of the process */
299  __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
300  __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
301  TCW_4(th->th.th_suspend_init, FALSE);
302  }
303 }
304 
305 /* This routine puts the calling thread to sleep after setting the
306  sleep bit for the indicated flag variable to true. */
307 template <class C>
308 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
309  kmp_info_t *th = __kmp_threads[th_gtid];
310  int status;
311  typename C::flag_t old_spin;
312 
313  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
314  th_gtid, flag->get()));
315 
316  __kmp_suspend_initialize_thread(th);
317  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
318 
319  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
320  " loc(%p)\n",
321  th_gtid, flag->get()));
322 
323  /* TODO: shouldn't this use release semantics to ensure that
324  __kmp_suspend_initialize_thread gets called first? */
325  old_spin = flag->set_sleeping();
326 
327  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
328  " loc(%p)==%d\n",
329  th_gtid, flag->get(), *(flag->get())));
330 
331  if (flag->done_check_val(old_spin)) {
332  old_spin = flag->unset_sleeping();
333  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
334  "for flag's loc(%p)\n",
335  th_gtid, flag->get()));
336  } else {
337 #ifdef DEBUG_SUSPEND
338  __kmp_suspend_count++;
339 #endif
340  /* Encapsulate in a loop as the documentation states that this may "with
341  low probability" return when the condition variable has not been signaled
342  or broadcast */
343  int deactivated = FALSE;
344  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
345  while (flag->is_sleeping()) {
346  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
347  "kmp_win32_cond_wait()\n",
348  th_gtid));
349  // Mark the thread as no longer active (only in the first iteration of the
350  // loop).
351  if (!deactivated) {
352  th->th.th_active = FALSE;
353  if (th->th.th_active_in_pool) {
354  th->th.th_active_in_pool = FALSE;
355  KMP_TEST_THEN_DEC32((kmp_int32 *)&__kmp_thread_pool_active_nth);
356  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
357  }
358  deactivated = TRUE;
359 
360  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
361  0);
362  } else {
363  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
364  0);
365  }
366 
367 #ifdef KMP_DEBUG
368  if (flag->is_sleeping()) {
369  KF_TRACE(100,
370  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
371  }
372 #endif /* KMP_DEBUG */
373 
374  } // while
375 
376  // Mark the thread as active again (if it was previous marked as inactive)
377  if (deactivated) {
378  th->th.th_active = TRUE;
379  if (TCR_4(th->th.th_in_pool)) {
380  KMP_TEST_THEN_INC32((kmp_int32 *)&__kmp_thread_pool_active_nth);
381  th->th.th_active_in_pool = TRUE;
382  }
383  }
384  }
385 
386  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
387 
388  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
389 }
390 
391 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
392  __kmp_suspend_template(th_gtid, flag);
393 }
394 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
395  __kmp_suspend_template(th_gtid, flag);
396 }
397 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
398  __kmp_suspend_template(th_gtid, flag);
399 }
400 
401 /* This routine signals the thread specified by target_gtid to wake up
402  after setting the sleep bit indicated by the flag argument to FALSE */
403 template <class C>
404 static inline void __kmp_resume_template(int target_gtid, C *flag) {
405  kmp_info_t *th = __kmp_threads[target_gtid];
406  int status;
407 
408 #ifdef KMP_DEBUG
409  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
410 #endif
411 
412  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
413  gtid, target_gtid));
414 
415  __kmp_suspend_initialize_thread(th);
416  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
417 
418  if (!flag) { // coming from __kmp_null_resume_wrapper
419  flag = (C *)th->th.th_sleep_loc;
420  }
421 
422  // First, check if the flag is null or its type has changed. If so, someone
423  // else woke it up.
424  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
425  // simply shows what
426  // flag was cast to
427  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
428  "awake: flag's loc(%p)\n",
429  gtid, target_gtid, NULL));
430  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
431  return;
432  } else {
433  typename C::flag_t old_spin = flag->unset_sleeping();
434  if (!flag->is_sleeping_val(old_spin)) {
435  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
436  "awake: flag's loc(%p): %u => %u\n",
437  gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
438  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
439  return;
440  }
441  }
442  TCW_PTR(th->th.th_sleep_loc, NULL);
443  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
444  "bit for flag's loc(%p)\n",
445  gtid, target_gtid, flag->get()));
446 
447  __kmp_win32_cond_signal(&th->th.th_suspend_cv);
448  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
449 
450  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
451  " for T#%d\n",
452  gtid, target_gtid));
453 }
454 
455 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
456  __kmp_resume_template(target_gtid, flag);
457 }
458 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
459  __kmp_resume_template(target_gtid, flag);
460 }
461 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
462  __kmp_resume_template(target_gtid, flag);
463 }
464 
465 void __kmp_yield(int cond) {
466  if (cond)
467  Sleep(0);
468 }
469 
470 void __kmp_gtid_set_specific(int gtid) {
471  if (__kmp_init_gtid) {
472  KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
473  __kmp_gtid_threadprivate_key));
474  if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
475  KMP_FATAL(TLSSetValueFailed);
476  } else {
477  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
478  }
479 }
480 
481 int __kmp_gtid_get_specific() {
482  int gtid;
483  if (!__kmp_init_gtid) {
484  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
485  "KMP_GTID_SHUTDOWN\n"));
486  return KMP_GTID_SHUTDOWN;
487  }
488  gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
489  if (gtid == 0) {
490  gtid = KMP_GTID_DNE;
491  } else {
492  gtid--;
493  }
494  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
495  __kmp_gtid_threadprivate_key, gtid));
496  return gtid;
497 }
498 
499 void __kmp_affinity_bind_thread(int proc) {
500  if (__kmp_num_proc_groups > 1) {
501  // Form the GROUP_AFFINITY struct directly, rather than filling
502  // out a bit vector and calling __kmp_set_system_affinity().
503  GROUP_AFFINITY ga;
504  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
505  sizeof(DWORD_PTR))));
506  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
507  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
508  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
509 
510  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
511  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
512  DWORD error = GetLastError();
513  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
514  kmp_msg_t err_code = KMP_ERR(error);
515  __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
516  __kmp_msg_null);
517  if (__kmp_generate_warnings == kmp_warnings_off) {
518  __kmp_str_free(&err_code.str);
519  }
520  }
521  }
522  } else {
523  kmp_affin_mask_t *mask;
524  KMP_CPU_ALLOC_ON_STACK(mask);
525  KMP_CPU_ZERO(mask);
526  KMP_CPU_SET(proc, mask);
527  __kmp_set_system_affinity(mask, TRUE);
528  KMP_CPU_FREE_FROM_STACK(mask);
529  }
530 }
531 
532 void __kmp_affinity_determine_capable(const char *env_var) {
533 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
534 
535 #if KMP_GROUP_AFFINITY
536  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
537 #else
538  KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
539 #endif
540 
541  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
542  "Windows* OS affinity interface functional (mask size = "
543  "%" KMP_SIZE_T_SPEC ").\n",
544  __kmp_affin_mask_size));
545 }
546 
547 double __kmp_read_cpu_time(void) {
548  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
549  int status;
550  double cpu_time;
551 
552  cpu_time = 0;
553 
554  status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
555  &KernelTime, &UserTime);
556 
557  if (status) {
558  double sec = 0;
559 
560  sec += KernelTime.dwHighDateTime;
561  sec += UserTime.dwHighDateTime;
562 
563  /* Shift left by 32 bits */
564  sec *= (double)(1 << 16) * (double)(1 << 16);
565 
566  sec += KernelTime.dwLowDateTime;
567  sec += UserTime.dwLowDateTime;
568 
569  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
570  }
571 
572  return cpu_time;
573 }
574 
575 int __kmp_read_system_info(struct kmp_sys_info *info) {
576  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
577  info->minflt = 0; /* the number of page faults serviced without any I/O */
578  info->majflt = 0; /* the number of page faults serviced that required I/O */
579  info->nswap = 0; // the number of times a process was "swapped" out of memory
580  info->inblock = 0; // the number of times the file system had to perform input
581  info->oublock = 0; // number of times the file system had to perform output
582  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
583  info->nivcsw = 0; /* the number of times a context switch was forced */
584 
585  return 1;
586 }
587 
588 void __kmp_runtime_initialize(void) {
589  SYSTEM_INFO info;
590  kmp_str_buf_t path;
591  UINT path_size;
592 
593  if (__kmp_init_runtime) {
594  return;
595  };
596 
597 #if KMP_DYNAMIC_LIB
598  /* Pin dynamic library for the lifetime of application */
599  {
600  // First, turn off error message boxes
601  UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
602  HMODULE h;
603  BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
604  GET_MODULE_HANDLE_EX_FLAG_PIN,
605  (LPCTSTR)&__kmp_serial_initialize, &h);
606  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
607  SetErrorMode(err_mode); // Restore error mode
608  KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
609  }
610 #endif
611 
612  InitializeCriticalSection(&__kmp_win32_section);
613 #if USE_ITT_BUILD
614  __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
615 #endif /* USE_ITT_BUILD */
616  __kmp_initialize_system_tick();
617 
618 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
619  if (!__kmp_cpuinfo.initialized) {
620  __kmp_query_cpuid(&__kmp_cpuinfo);
621  }; // if
622 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
623 
624 /* Set up minimum number of threads to switch to TLS gtid */
625 #if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB
626  // Windows* OS, static library.
627  /* New thread may use stack space previously used by another thread,
628  currently terminated. On Windows* OS, in case of static linking, we do not
629  know the moment of thread termination, and our structures (__kmp_threads
630  and __kmp_root arrays) are still keep info about dead threads. This leads
631  to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
632  (by searching through stack addresses of all known threads) for
633  unregistered foreign tread.
634 
635  Setting __kmp_tls_gtid_min to 0 workarounds this problem:
636  __kmp_get_global_thread_id() does not search through stacks, but get gtid
637  from TLS immediately.
638  --ln
639  */
640  __kmp_tls_gtid_min = 0;
641 #else
642  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
643 #endif
644 
645  /* for the static library */
646  if (!__kmp_gtid_threadprivate_key) {
647  __kmp_gtid_threadprivate_key = TlsAlloc();
648  if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
649  KMP_FATAL(TLSOutOfIndexes);
650  }
651  }
652 
653  // Load ntdll.dll.
654  /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
655  (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
656  have to specify full path to the library. */
657  __kmp_str_buf_init(&path);
658  path_size = GetSystemDirectory(path.str, path.size);
659  KMP_DEBUG_ASSERT(path_size > 0);
660  if (path_size >= path.size) {
661  // Buffer is too short. Expand the buffer and try again.
662  __kmp_str_buf_reserve(&path, path_size);
663  path_size = GetSystemDirectory(path.str, path.size);
664  KMP_DEBUG_ASSERT(path_size > 0);
665  }; // if
666  if (path_size > 0 && path_size < path.size) {
667  // Now we have system directory name in the buffer.
668  // Append backslash and name of dll to form full path,
669  path.used = path_size;
670  __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
671 
672  // Now load ntdll using full path.
673  ntdll = GetModuleHandle(path.str);
674  }
675 
676  KMP_DEBUG_ASSERT(ntdll != NULL);
677  if (ntdll != NULL) {
678  NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
679  ntdll, "NtQuerySystemInformation");
680  }
681  KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
682 
683 #if KMP_GROUP_AFFINITY
684  // Load kernel32.dll.
685  // Same caveat - must use full system path name.
686  if (path_size > 0 && path_size < path.size) {
687  // Truncate the buffer back to just the system path length,
688  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
689  path.used = path_size;
690  __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
691 
692  // Load kernel32.dll using full path.
693  kernel32 = GetModuleHandle(path.str);
694  KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
695 
696  // Load the function pointers to kernel32.dll routines
697  // that may or may not exist on this system.
698  if (kernel32 != NULL) {
699  __kmp_GetActiveProcessorCount =
700  (kmp_GetActiveProcessorCount_t)GetProcAddress(
701  kernel32, "GetActiveProcessorCount");
702  __kmp_GetActiveProcessorGroupCount =
703  (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
704  kernel32, "GetActiveProcessorGroupCount");
705  __kmp_GetThreadGroupAffinity =
706  (kmp_GetThreadGroupAffinity_t)GetProcAddress(
707  kernel32, "GetThreadGroupAffinity");
708  __kmp_SetThreadGroupAffinity =
709  (kmp_SetThreadGroupAffinity_t)GetProcAddress(
710  kernel32, "SetThreadGroupAffinity");
711 
712  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
713  " = %p\n",
714  __kmp_GetActiveProcessorCount));
715  KA_TRACE(10, ("__kmp_runtime_initialize: "
716  "__kmp_GetActiveProcessorGroupCount = %p\n",
717  __kmp_GetActiveProcessorGroupCount));
718  KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
719  " = %p\n",
720  __kmp_GetThreadGroupAffinity));
721  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
722  " = %p\n",
723  __kmp_SetThreadGroupAffinity));
724  KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
725  sizeof(kmp_affin_mask_t)));
726 
727  // See if group affinity is supported on this system.
728  // If so, calculate the #groups and #procs.
729  //
730  // Group affinity was introduced with Windows* 7 OS and
731  // Windows* Server 2008 R2 OS.
732  if ((__kmp_GetActiveProcessorCount != NULL) &&
733  (__kmp_GetActiveProcessorGroupCount != NULL) &&
734  (__kmp_GetThreadGroupAffinity != NULL) &&
735  (__kmp_SetThreadGroupAffinity != NULL) &&
736  ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
737  1)) {
738  // Calculate the total number of active OS procs.
739  int i;
740 
741  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
742  " detected\n",
743  __kmp_num_proc_groups));
744 
745  __kmp_xproc = 0;
746 
747  for (i = 0; i < __kmp_num_proc_groups; i++) {
748  DWORD size = __kmp_GetActiveProcessorCount(i);
749  __kmp_xproc += size;
750  KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
751  i, size));
752  }
753  } else {
754  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
755  " detected\n",
756  __kmp_num_proc_groups));
757  }
758  }
759  }
760  if (__kmp_num_proc_groups <= 1) {
761  GetSystemInfo(&info);
762  __kmp_xproc = info.dwNumberOfProcessors;
763  }
764 #else
765  GetSystemInfo(&info);
766  __kmp_xproc = info.dwNumberOfProcessors;
767 #endif /* KMP_GROUP_AFFINITY */
768 
769  // If the OS said there were 0 procs, take a guess and use a value of 2.
770  // This is done for Linux* OS, also. Do we need error / warning?
771  if (__kmp_xproc <= 0) {
772  __kmp_xproc = 2;
773  }
774 
775  KA_TRACE(5,
776  ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
777 
778  __kmp_str_buf_free(&path);
779 
780 #if USE_ITT_BUILD
781  __kmp_itt_initialize();
782 #endif /* USE_ITT_BUILD */
783 
784  __kmp_init_runtime = TRUE;
785 } // __kmp_runtime_initialize
786 
787 void __kmp_runtime_destroy(void) {
788  if (!__kmp_init_runtime) {
789  return;
790  }
791 
792 #if USE_ITT_BUILD
793  __kmp_itt_destroy();
794 #endif /* USE_ITT_BUILD */
795 
796  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
797  /* due to the KX_TRACE() commands */
798  KA_TRACE(40, ("__kmp_runtime_destroy\n"));
799 
800  if (__kmp_gtid_threadprivate_key) {
801  TlsFree(__kmp_gtid_threadprivate_key);
802  __kmp_gtid_threadprivate_key = 0;
803  }
804 
805  __kmp_affinity_uninitialize();
806  DeleteCriticalSection(&__kmp_win32_section);
807 
808  ntdll = NULL;
809  NtQuerySystemInformation = NULL;
810 
811 #if KMP_ARCH_X86_64
812  kernel32 = NULL;
813  __kmp_GetActiveProcessorCount = NULL;
814  __kmp_GetActiveProcessorGroupCount = NULL;
815  __kmp_GetThreadGroupAffinity = NULL;
816  __kmp_SetThreadGroupAffinity = NULL;
817 #endif // KMP_ARCH_X86_64
818 
819  __kmp_init_runtime = FALSE;
820 }
821 
822 void __kmp_terminate_thread(int gtid) {
823  kmp_info_t *th = __kmp_threads[gtid];
824 
825  if (!th)
826  return;
827 
828  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
829 
830  if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
831  /* It's OK, the thread may have exited already */
832  }
833  __kmp_free_handle(th->th.th_info.ds.ds_thread);
834 }
835 
836 void __kmp_clear_system_time(void) {
837  BOOL status;
838  LARGE_INTEGER time;
839  status = QueryPerformanceCounter(&time);
840  __kmp_win32_time = (kmp_int64)time.QuadPart;
841 }
842 
843 void __kmp_initialize_system_tick(void) {
844  {
845  BOOL status;
846  LARGE_INTEGER freq;
847 
848  status = QueryPerformanceFrequency(&freq);
849  if (!status) {
850  DWORD error = GetLastError();
851  __kmp_msg(kmp_ms_fatal,
852  KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
853  KMP_ERR(error), __kmp_msg_null);
854 
855  } else {
856  __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
857  }
858  }
859 }
860 
861 /* Calculate the elapsed wall clock time for the user */
862 
863 void __kmp_elapsed(double *t) {
864  BOOL status;
865  LARGE_INTEGER now;
866  status = QueryPerformanceCounter(&now);
867  *t = ((double)now.QuadPart) * __kmp_win32_tick;
868 }
869 
870 /* Calculate the elapsed wall clock tick for the user */
871 
872 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
873 
874 void __kmp_read_system_time(double *delta) {
875  if (delta != NULL) {
876  BOOL status;
877  LARGE_INTEGER now;
878 
879  status = QueryPerformanceCounter(&now);
880 
881  *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
882  __kmp_win32_tick;
883  }
884 }
885 
886 /* Return the current time stamp in nsec */
887 kmp_uint64 __kmp_now_nsec() {
888  LARGE_INTEGER now;
889  QueryPerformanceCounter(&now);
890  return 1e9 * __kmp_win32_tick * now.QuadPart;
891 }
892 
893 void *__stdcall __kmp_launch_worker(void *arg) {
894  volatile void *stack_data;
895  void *exit_val;
896  void *padding = 0;
897  kmp_info_t *this_thr = (kmp_info_t *)arg;
898  int gtid;
899 
900  gtid = this_thr->th.th_info.ds.ds_gtid;
901  __kmp_gtid_set_specific(gtid);
902 #ifdef KMP_TDATA_GTID
903 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
904  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
905  "reference: http://support.microsoft.com/kb/118816"
906 //__kmp_gtid = gtid;
907 #endif
908 
909 #if USE_ITT_BUILD
910  __kmp_itt_thread_name(gtid);
911 #endif /* USE_ITT_BUILD */
912 
913  __kmp_affinity_set_init_mask(gtid, FALSE);
914 
915 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
916  // Set FP control regs to be a copy of the parallel initialization thread's.
917  __kmp_clear_x87_fpu_status_word();
918  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
919  __kmp_load_mxcsr(&__kmp_init_mxcsr);
920 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
921 
922  if (__kmp_stkoffset > 0 && gtid > 0) {
923  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
924  }
925 
926  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
927  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
928  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
929 
930  if (TCR_4(__kmp_gtid_mode) <
931  2) { // check stack only if it is used to get gtid
932  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
933  KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
934  __kmp_check_stack_overlap(this_thr);
935  }
936  KMP_MB();
937  exit_val = __kmp_launch_thread(this_thr);
938  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
939  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
940  KMP_MB();
941  return exit_val;
942 }
943 
944 #if KMP_USE_MONITOR
945 /* The monitor thread controls all of the threads in the complex */
946 
947 void *__stdcall __kmp_launch_monitor(void *arg) {
948  DWORD wait_status;
949  kmp_thread_t monitor;
950  int status;
951  int interval;
952  kmp_info_t *this_thr = (kmp_info_t *)arg;
953 
954  KMP_DEBUG_ASSERT(__kmp_init_monitor);
955  TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
956  // TODO: hide "2" in enum (like {true,false,started})
957  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
958  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
959 
960  KMP_MB(); /* Flush all pending memory write invalidates. */
961  KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
962 
963  monitor = GetCurrentThread();
964 
965  /* set thread priority */
966  status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
967  if (!status) {
968  DWORD error = GetLastError();
969  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantSetThreadPriority), KMP_ERR(error),
970  __kmp_msg_null);
971  }
972 
973  /* register us as monitor */
974  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
975 #ifdef KMP_TDATA_GTID
976 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
977  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
978  "reference: http://support.microsoft.com/kb/118816"
979 //__kmp_gtid = KMP_GTID_MONITOR;
980 #endif
981 
982 #if USE_ITT_BUILD
983  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
984 // monitor thread.
985 #endif /* USE_ITT_BUILD */
986 
987  KMP_MB(); /* Flush all pending memory write invalidates. */
988 
989  interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
990 
991  while (!TCR_4(__kmp_global.g.g_done)) {
992  /* This thread monitors the state of the system */
993 
994  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
995 
996  wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
997 
998  if (wait_status == WAIT_TIMEOUT) {
999  TCW_4(__kmp_global.g.g_time.dt.t_value,
1000  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1001  }
1002 
1003  KMP_MB(); /* Flush all pending memory write invalidates. */
1004  }
1005 
1006  KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1007 
1008  status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1009  if (!status) {
1010  DWORD error = GetLastError();
1011  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantSetThreadPriority), KMP_ERR(error),
1012  __kmp_msg_null);
1013  }
1014 
1015  if (__kmp_global.g.g_abort != 0) {
1016  /* now we need to terminate the worker threads */
1017  /* the value of t_abort is the signal we caught */
1018  int gtid;
1019 
1020  KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1021  (__kmp_global.g.g_abort)));
1022 
1023  /* terminate the OpenMP worker threads */
1024  /* TODO this is not valid for sibling threads!!
1025  * the uber master might not be 0 anymore.. */
1026  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1027  __kmp_terminate_thread(gtid);
1028 
1029  __kmp_cleanup();
1030 
1031  Sleep(0);
1032 
1033  KA_TRACE(10,
1034  ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1035 
1036  if (__kmp_global.g.g_abort > 0) {
1037  raise(__kmp_global.g.g_abort);
1038  }
1039  }
1040 
1041  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1042 
1043  KMP_MB();
1044  return arg;
1045 }
1046 #endif
1047 
1048 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1049  kmp_thread_t handle;
1050  DWORD idThread;
1051 
1052  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1053 
1054  th->th.th_info.ds.ds_gtid = gtid;
1055 
1056  if (KMP_UBER_GTID(gtid)) {
1057  int stack_data;
1058 
1059  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1060  other threads to use. Is it appropriate to just use GetCurrentThread?
1061  When should we close this handle? When unregistering the root? */
1062  {
1063  BOOL rc;
1064  rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1065  GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1066  FALSE, DUPLICATE_SAME_ACCESS);
1067  KMP_ASSERT(rc);
1068  KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1069  "handle = %" KMP_UINTPTR_SPEC "\n",
1070  (LPVOID)th, th->th.th_info.ds.ds_thread));
1071  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1072  }
1073  if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1074  /* we will dynamically update the stack range if gtid_mode == 1 */
1075  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1076  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1077  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1078  __kmp_check_stack_overlap(th);
1079  }
1080  } else {
1081  KMP_MB(); /* Flush all pending memory write invalidates. */
1082 
1083  /* Set stack size for this thread now. */
1084  KA_TRACE(10,
1085  ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1086  stack_size));
1087 
1088  stack_size += gtid * __kmp_stkoffset;
1089 
1090  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1091  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1092 
1093  KA_TRACE(10,
1094  ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1095  " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1096  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1097  (LPVOID)th, &idThread));
1098 
1099  handle = CreateThread(
1100  NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1101  (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1102 
1103  KA_TRACE(10,
1104  ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1105  " bytes, &__kmp_launch_worker = %p, th = %p, "
1106  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1107  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1108  (LPVOID)th, idThread, handle));
1109 
1110  if (handle == 0) {
1111  DWORD error = GetLastError();
1112  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCreateThread), KMP_ERR(error),
1113  __kmp_msg_null);
1114  } else {
1115  th->th.th_info.ds.ds_thread = handle;
1116  }
1117 
1118  KMP_MB(); /* Flush all pending memory write invalidates. */
1119  }
1120 
1121  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1122 }
1123 
1124 int __kmp_still_running(kmp_info_t *th) {
1125  return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1126 }
1127 
1128 #if KMP_USE_MONITOR
1129 void __kmp_create_monitor(kmp_info_t *th) {
1130  kmp_thread_t handle;
1131  DWORD idThread;
1132  int ideal, new_ideal;
1133 
1134  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1135  // We don't need monitor thread in case of MAX_BLOCKTIME
1136  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1137  "MAX blocktime\n"));
1138  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1139  th->th.th_info.ds.ds_gtid = 0;
1140  TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1141  return;
1142  }
1143  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1144 
1145  KMP_MB(); /* Flush all pending memory write invalidates. */
1146 
1147  __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1148  if (__kmp_monitor_ev == NULL) {
1149  DWORD error = GetLastError();
1150  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCreateEvent), KMP_ERR(error),
1151  __kmp_msg_null);
1152  }; // if
1153 #if USE_ITT_BUILD
1154  __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1155 #endif /* USE_ITT_BUILD */
1156 
1157  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1158  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1159 
1160  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1161  // to automatically expand stacksize based on CreateThread error code.
1162  if (__kmp_monitor_stksize == 0) {
1163  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1164  }
1165  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1166  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1167  }
1168 
1169  KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1170  (int)__kmp_monitor_stksize));
1171 
1172  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1173 
1174  handle =
1175  CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1176  (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1177  STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1178  if (handle == 0) {
1179  DWORD error = GetLastError();
1180  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCreateThread), KMP_ERR(error),
1181  __kmp_msg_null);
1182  } else
1183  th->th.th_info.ds.ds_thread = handle;
1184 
1185  KMP_MB(); /* Flush all pending memory write invalidates. */
1186 
1187  KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1188  (void *)th->th.th_info.ds.ds_thread));
1189 }
1190 #endif
1191 
1192 /* Check to see if thread is still alive.
1193  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1194  with a exit_val = code. Because of this we can not rely on exit_val having
1195  any particular value. So this routine may return STILL_ALIVE in exit_val
1196  even after the thread is dead. */
1197 
1198 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1199  DWORD rc;
1200  rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1201  if (rc == 0) {
1202  DWORD error = GetLastError();
1203  __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "GetExitCodeThread()"),
1204  KMP_ERR(error), __kmp_msg_null);
1205  }; // if
1206  return (*exit_val == STILL_ACTIVE);
1207 }
1208 
1209 void __kmp_exit_thread(int exit_status) {
1210  ExitThread(exit_status);
1211 } // __kmp_exit_thread
1212 
1213 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1214 static void __kmp_reap_common(kmp_info_t *th) {
1215  DWORD exit_val;
1216 
1217  KMP_MB(); /* Flush all pending memory write invalidates. */
1218 
1219  KA_TRACE(
1220  10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1221 
1222  /* 2006-10-19:
1223  There are two opposite situations:
1224  1. Windows* OS keep thread alive after it resets ds_alive flag and
1225  exits from thread function. (For example, see C70770/Q394281 "unloading of
1226  dll based on OMP is very slow".)
1227  2. Windows* OS may kill thread before it resets ds_alive flag.
1228 
1229  Right solution seems to be waiting for *either* thread termination *or*
1230  ds_alive resetting. */
1231  {
1232  // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize
1233  // KMP_WAIT_YIELD to cover this usage also.
1234  void *obj = NULL;
1235  kmp_uint32 spins;
1236 #if USE_ITT_BUILD
1237  KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1238 #endif /* USE_ITT_BUILD */
1239  KMP_INIT_YIELD(spins);
1240  do {
1241 #if USE_ITT_BUILD
1242  KMP_FSYNC_SPIN_PREPARE(obj);
1243 #endif /* USE_ITT_BUILD */
1244  __kmp_is_thread_alive(th, &exit_val);
1245  KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
1246  KMP_YIELD_SPIN(spins);
1247  } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1248 #if USE_ITT_BUILD
1249  if (exit_val == STILL_ACTIVE) {
1250  KMP_FSYNC_CANCEL(obj);
1251  } else {
1252  KMP_FSYNC_SPIN_ACQUIRED(obj);
1253  }; // if
1254 #endif /* USE_ITT_BUILD */
1255  }
1256 
1257  __kmp_free_handle(th->th.th_info.ds.ds_thread);
1258 
1259  /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1260  with a exit_val = code. Because of this we can not rely on exit_val having
1261  any particular value. */
1262  if (exit_val == STILL_ACTIVE) {
1263  KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1264  } else if ((void *)exit_val != (void *)th) {
1265  KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1266  }; // if
1267 
1268  KA_TRACE(10,
1269  ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1270  "\n",
1271  th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1272 
1273  th->th.th_info.ds.ds_thread = 0;
1274  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1275  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1276  th->th.th_info.ds.ds_thread_id = 0;
1277 
1278  KMP_MB(); /* Flush all pending memory write invalidates. */
1279 }
1280 
1281 #if KMP_USE_MONITOR
1282 void __kmp_reap_monitor(kmp_info_t *th) {
1283  int status;
1284 
1285  KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1286  (void *)th->th.th_info.ds.ds_thread));
1287 
1288  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1289  // If both tid and gtid are 0, it means the monitor did not ever start.
1290  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1291  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1292  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1293  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1294  return;
1295  }; // if
1296 
1297  KMP_MB(); /* Flush all pending memory write invalidates. */
1298 
1299  status = SetEvent(__kmp_monitor_ev);
1300  if (status == FALSE) {
1301  DWORD error = GetLastError();
1302  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantSetEvent), KMP_ERR(error),
1303  __kmp_msg_null);
1304  }
1305  KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1306  th->th.th_info.ds.ds_gtid));
1307  __kmp_reap_common(th);
1308 
1309  __kmp_free_handle(__kmp_monitor_ev);
1310 
1311  KMP_MB(); /* Flush all pending memory write invalidates. */
1312 }
1313 #endif
1314 
1315 void __kmp_reap_worker(kmp_info_t *th) {
1316  KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1317  th->th.th_info.ds.ds_gtid));
1318  __kmp_reap_common(th);
1319 }
1320 
1321 #if KMP_HANDLE_SIGNALS
1322 
1323 static void __kmp_team_handler(int signo) {
1324  if (__kmp_global.g.g_abort == 0) {
1325  // Stage 1 signal handler, let's shut down all of the threads.
1326  if (__kmp_debug_buf) {
1327  __kmp_dump_debug_buffer();
1328  }; // if
1329  KMP_MB(); // Flush all pending memory write invalidates.
1330  TCW_4(__kmp_global.g.g_abort, signo);
1331  KMP_MB(); // Flush all pending memory write invalidates.
1332  TCW_4(__kmp_global.g.g_done, TRUE);
1333  KMP_MB(); // Flush all pending memory write invalidates.
1334  }
1335 } // __kmp_team_handler
1336 
1337 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1338  sig_func_t old = signal(signum, handler);
1339  if (old == SIG_ERR) {
1340  int error = errno;
1341  __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1342  __kmp_msg_null);
1343  }; // if
1344  return old;
1345 }
1346 
1347 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1348  int parallel_init) {
1349  sig_func_t old;
1350  KMP_MB(); /* Flush all pending memory write invalidates. */
1351  KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1352  if (parallel_init) {
1353  old = __kmp_signal(sig, handler);
1354  // SIG_DFL on Windows* OS in NULL or 0.
1355  if (old == __kmp_sighldrs[sig]) {
1356  __kmp_siginstalled[sig] = 1;
1357  } else { // Restore/keep user's handler if one previously installed.
1358  old = __kmp_signal(sig, old);
1359  }; // if
1360  } else {
1361  // Save initial/system signal handlers to see if user handlers installed.
1362  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1363  // called once with parallel_init == TRUE.
1364  old = __kmp_signal(sig, SIG_DFL);
1365  __kmp_sighldrs[sig] = old;
1366  __kmp_signal(sig, old);
1367  }; // if
1368  KMP_MB(); /* Flush all pending memory write invalidates. */
1369 } // __kmp_install_one_handler
1370 
1371 static void __kmp_remove_one_handler(int sig) {
1372  if (__kmp_siginstalled[sig]) {
1373  sig_func_t old;
1374  KMP_MB(); // Flush all pending memory write invalidates.
1375  KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1376  old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1377  if (old != __kmp_team_handler) {
1378  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1379  "restoring: sig=%d\n",
1380  sig));
1381  old = __kmp_signal(sig, old);
1382  }; // if
1383  __kmp_sighldrs[sig] = NULL;
1384  __kmp_siginstalled[sig] = 0;
1385  KMP_MB(); // Flush all pending memory write invalidates.
1386  }; // if
1387 } // __kmp_remove_one_handler
1388 
1389 void __kmp_install_signals(int parallel_init) {
1390  KB_TRACE(10, ("__kmp_install_signals: called\n"));
1391  if (!__kmp_handle_signals) {
1392  KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1393  "handlers not installed\n"));
1394  return;
1395  }; // if
1396  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1397  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1398  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1399  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1400  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1401  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1402 } // __kmp_install_signals
1403 
1404 void __kmp_remove_signals(void) {
1405  int sig;
1406  KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1407  for (sig = 1; sig < NSIG; ++sig) {
1408  __kmp_remove_one_handler(sig);
1409  }; // for sig
1410 } // __kmp_remove_signals
1411 
1412 #endif // KMP_HANDLE_SIGNALS
1413 
1414 /* Put the thread to sleep for a time period */
1415 void __kmp_thread_sleep(int millis) {
1416  DWORD status;
1417 
1418  status = SleepEx((DWORD)millis, FALSE);
1419  if (status) {
1420  DWORD error = GetLastError();
1421  __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1422  __kmp_msg_null);
1423  }
1424 }
1425 
1426 // Determine whether the given address is mapped into the current address space.
1427 int __kmp_is_address_mapped(void *addr) {
1428  DWORD status;
1429  MEMORY_BASIC_INFORMATION lpBuffer;
1430  SIZE_T dwLength;
1431 
1432  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1433 
1434  status = VirtualQuery(addr, &lpBuffer, dwLength);
1435 
1436  return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1437  ((lpBuffer.Protect == PAGE_NOACCESS) ||
1438  (lpBuffer.Protect == PAGE_EXECUTE)));
1439 }
1440 
1441 kmp_uint64 __kmp_hardware_timestamp(void) {
1442  kmp_uint64 r = 0;
1443 
1444  QueryPerformanceCounter((LARGE_INTEGER *)&r);
1445  return r;
1446 }
1447 
1448 /* Free handle and check the error code */
1449 void __kmp_free_handle(kmp_thread_t tHandle) {
1450  /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1451  * as HANDLE */
1452  BOOL rc;
1453  rc = CloseHandle(tHandle);
1454  if (!rc) {
1455  DWORD error = GetLastError();
1456  __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCloseHandle), KMP_ERR(error),
1457  __kmp_msg_null);
1458  }
1459 }
1460 
1461 int __kmp_get_load_balance(int max) {
1462  static ULONG glb_buff_size = 100 * 1024;
1463 
1464  // Saved count of the running threads for the thread balance algortihm
1465  static int glb_running_threads = 0;
1466  static double glb_call_time = 0; /* Thread balance algorithm call time */
1467 
1468  int running_threads = 0; // Number of running threads in the system.
1469  NTSTATUS status = 0;
1470  ULONG buff_size = 0;
1471  ULONG info_size = 0;
1472  void *buffer = NULL;
1473  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1474  int first_time = 1;
1475 
1476  double call_time = 0.0; // start, finish;
1477 
1478  __kmp_elapsed(&call_time);
1479 
1480  if (glb_call_time &&
1481  (call_time - glb_call_time < __kmp_load_balance_interval)) {
1482  running_threads = glb_running_threads;
1483  goto finish;
1484  }
1485  glb_call_time = call_time;
1486 
1487  // Do not spend time on running algorithm if we have a permanent error.
1488  if (NtQuerySystemInformation == NULL) {
1489  running_threads = -1;
1490  goto finish;
1491  }; // if
1492 
1493  if (max <= 0) {
1494  max = INT_MAX;
1495  }; // if
1496 
1497  do {
1498 
1499  if (first_time) {
1500  buff_size = glb_buff_size;
1501  } else {
1502  buff_size = 2 * buff_size;
1503  }
1504 
1505  buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1506  if (buffer == NULL) {
1507  running_threads = -1;
1508  goto finish;
1509  }; // if
1510  status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1511  buff_size, &info_size);
1512  first_time = 0;
1513 
1514  } while (status == STATUS_INFO_LENGTH_MISMATCH);
1515  glb_buff_size = buff_size;
1516 
1517 #define CHECK(cond) \
1518  { \
1519  KMP_DEBUG_ASSERT(cond); \
1520  if (!(cond)) { \
1521  running_threads = -1; \
1522  goto finish; \
1523  } \
1524  }
1525 
1526  CHECK(buff_size >= info_size);
1527  spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1528  for (;;) {
1529  ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1530  CHECK(0 <= offset &&
1531  offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1532  HANDLE pid = spi->ProcessId;
1533  ULONG num = spi->NumberOfThreads;
1534  CHECK(num >= 1);
1535  size_t spi_size =
1536  sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1537  CHECK(offset + spi_size <
1538  info_size); // Make sure process info record fits the buffer.
1539  if (spi->NextEntryOffset != 0) {
1540  CHECK(spi_size <=
1541  spi->NextEntryOffset); // And do not overlap with the next record.
1542  }; // if
1543  // pid == 0 corresponds to the System Idle Process. It always has running
1544  // threads on all cores. So, we don't consider the running threads of this
1545  // process.
1546  if (pid != 0) {
1547  for (int i = 0; i < num; ++i) {
1548  THREAD_STATE state = spi->Threads[i].State;
1549  // Count threads that have Ready or Running state.
1550  // !!! TODO: Why comment does not match the code???
1551  if (state == StateRunning) {
1552  ++running_threads;
1553  // Stop counting running threads if the number is already greater than
1554  // the number of available cores
1555  if (running_threads >= max) {
1556  goto finish;
1557  }
1558  } // if
1559  }; // for i
1560  } // if
1561  if (spi->NextEntryOffset == 0) {
1562  break;
1563  }; // if
1564  spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1565  }; // forever
1566 
1567 #undef CHECK
1568 
1569 finish: // Clean up and exit.
1570 
1571  if (buffer != NULL) {
1572  KMP_INTERNAL_FREE(buffer);
1573  }; // if
1574 
1575  glb_running_threads = running_threads;
1576 
1577  return running_threads;
1578 } //__kmp_get_load_balance()