18 #include "kmp_wrapper_malloc.h" 25 typedef int (*bget_compact_t)(size_t, int);
26 typedef void *(*bget_acquire_t)(size_t);
27 typedef void (*bget_release_t)(
void *);
32 #if KMP_ARCH_X86 || KMP_ARCH_ARM 33 typedef kmp_int32 bufsize;
35 typedef kmp_int64 bufsize;
38 typedef ssize_t bufsize;
43 typedef enum bget_mode {
49 static void bpool(kmp_info_t *th,
void *buffer, bufsize len);
50 static void *bget(kmp_info_t *th, bufsize size);
51 static void *bgetz(kmp_info_t *th, bufsize size);
52 static void *bgetr(kmp_info_t *th,
void *buffer, bufsize newsize);
53 static void brel(kmp_info_t *th,
void *buf);
54 static void bectl(kmp_info_t *th, bget_compact_t compact,
55 bget_acquire_t acquire, bget_release_t release,
59 static void bstats(kmp_info_t *th, bufsize *curalloc, bufsize *totfree,
60 bufsize *maxfree,
long *nget,
long *nrel);
61 static void bstatse(kmp_info_t *th, bufsize *pool_incr,
long *npool,
62 long *npget,
long *nprel,
long *ndget,
long *ndrel);
63 static void bufdump(kmp_info_t *th,
void *buf);
64 static void bpoold(kmp_info_t *th,
void *pool,
int dumpalloc,
int dumpfree);
65 static int bpoolv(kmp_info_t *th,
void *pool);
75 #if KMP_ARCH_X86 || !KMP_HAVE_QUAD 78 #define AlignType double 83 #define AlignType _Quad 119 static bufsize bget_bin_size[] = {
129 1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20,
137 #define MAX_BGET_BINS (int)(sizeof(bget_bin_size) / sizeof(bufsize)) 144 typedef struct qlinks {
145 struct bfhead *flink;
146 struct bfhead *blink;
150 typedef struct bhead2 {
158 typedef union bhead {
161 char b_pad[
sizeof(bhead2_t) + (SizeQuant - (
sizeof(bhead2_t) % SizeQuant))];
164 #define BH(p) ((bhead_t *)(p)) 167 typedef struct bdhead {
171 #define BDH(p) ((bdhead_t *)(p)) 174 typedef struct bfhead {
178 #define BFH(p) ((bfhead_t *)(p)) 180 typedef struct thr_data {
181 bfhead_t freelist[MAX_BGET_BINS];
186 long numpget, numprel;
187 long numdget, numdrel;
191 bget_compact_t compfcn;
192 bget_acquire_t acqfcn;
193 bget_release_t relfcn;
206 #define QLSize (sizeof(qlinks_t)) 207 #define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize) 210 ~(((bufsize)(1) << (sizeof(bufsize) * CHAR_BIT - 1)) | (SizeQuant - 1))) 218 ((bufsize)(-(((((bufsize)1) << ((int)sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2)) 221 static int bget_get_bin(bufsize size) {
223 int lo = 0, hi = MAX_BGET_BINS - 1;
225 KMP_DEBUG_ASSERT(size > 0);
227 while ((hi - lo) > 1) {
228 int mid = (lo + hi) >> 1;
229 if (size < bget_bin_size[mid])
235 KMP_DEBUG_ASSERT((lo >= 0) && (lo < MAX_BGET_BINS));
240 static void set_thr_data(kmp_info_t *th) {
244 data = (thr_data_t *)((!th->th.th_local.bget_data)
245 ? __kmp_allocate(
sizeof(*data))
246 : th->th.th_local.bget_data);
248 memset(data,
'\0',
sizeof(*data));
250 for (i = 0; i < MAX_BGET_BINS; ++i) {
251 data->freelist[i].ql.flink = &data->freelist[i];
252 data->freelist[i].ql.blink = &data->freelist[i];
255 th->th.th_local.bget_data = data;
256 th->th.th_local.bget_list = 0;
257 #if !USE_CMP_XCHG_FOR_BGET 258 #ifdef USE_QUEUING_LOCK_FOR_BGET 259 __kmp_init_lock(&th->th.th_local.bget_lock);
261 __kmp_init_bootstrap_lock(&th->th.th_local.bget_lock);
266 static thr_data_t *get_thr_data(kmp_info_t *th) {
269 data = (thr_data_t *)th->th.th_local.bget_data;
271 KMP_DEBUG_ASSERT(data != 0);
278 static void __kmp_bget_validate_queue(kmp_info_t *th) {
281 void *p = (
void *)th->th.th_local.bget_list;
284 bfhead_t *b = BFH(((
char *)p) -
sizeof(bhead_t));
286 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
287 p = (
void *)b->ql.flink;
294 static void __kmp_bget_dequeue(kmp_info_t *th) {
295 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list);
298 #if USE_CMP_XCHG_FOR_BGET 300 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
301 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list,
302 CCAST(
void *, old_value),
nullptr)) {
304 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
306 p = CCAST(
void *, old_value);
309 #ifdef USE_QUEUING_LOCK_FOR_BGET 310 __kmp_acquire_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th));
312 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock);
315 p = (
void *)th->th.th_local.bget_list;
316 th->th.th_local.bget_list = 0;
318 #ifdef USE_QUEUING_LOCK_FOR_BGET
319 __kmp_release_lock(&th->th.th_local.bget_lock, __kmp_gtid_from_thread(th));
321 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock);
328 bfhead_t *b = BFH(((
char *)p) -
sizeof(bhead_t));
330 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
331 KMP_DEBUG_ASSERT(((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1) ==
333 KMP_DEBUG_ASSERT(b->ql.blink == 0);
335 p = (
void *)b->ql.flink;
343 static void __kmp_bget_enqueue(kmp_info_t *th,
void *buf
344 #ifdef USE_QUEUING_LOCK_FOR_BGET
349 bfhead_t *b = BFH(((
char *)buf) -
sizeof(bhead_t));
351 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
352 KMP_DEBUG_ASSERT(((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1) ==
357 KC_TRACE(10, (
"__kmp_bget_enqueue: moving buffer to T#%d list\n",
358 __kmp_gtid_from_thread(th)));
360 #if USE_CMP_XCHG_FOR_BGET 362 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list);
365 b->ql.flink = BFH(CCAST(
void *, old_value));
367 while (!KMP_COMPARE_AND_STORE_PTR(&th->th.th_local.bget_list,
368 CCAST(
void *, old_value), buf)) {
370 old_value = TCR_PTR(th->th.th_local.bget_list);
373 b->ql.flink = BFH(CCAST(
void *, old_value));
377 #ifdef USE_QUEUING_LOCK_FOR_BGET 378 __kmp_acquire_lock(&th->th.th_local.bget_lock, rel_gtid);
380 __kmp_acquire_bootstrap_lock(&th->th.th_local.bget_lock);
383 b->ql.flink = BFH(th->th.th_local.bget_list);
384 th->th.th_local.bget_list = (
void *)buf;
386 #ifdef USE_QUEUING_LOCK_FOR_BGET 387 __kmp_release_lock(&th->th.th_local.bget_lock, rel_gtid);
389 __kmp_release_bootstrap_lock(&th->th.th_local.bget_lock);
395 static void __kmp_bget_insert_into_freelist(thr_data_t *thr, bfhead_t *b) {
398 KMP_DEBUG_ASSERT(((
size_t)b) % SizeQuant == 0);
399 KMP_DEBUG_ASSERT(b->bh.bb.bsize % SizeQuant == 0);
401 bin = bget_get_bin(b->bh.bb.bsize);
403 KMP_DEBUG_ASSERT(thr->freelist[bin].ql.blink->ql.flink ==
404 &thr->freelist[bin]);
405 KMP_DEBUG_ASSERT(thr->freelist[bin].ql.flink->ql.blink ==
406 &thr->freelist[bin]);
408 b->ql.flink = &thr->freelist[bin];
409 b->ql.blink = thr->freelist[bin].ql.blink;
411 thr->freelist[bin].ql.blink = b;
412 b->ql.blink->ql.flink = b;
416 static void __kmp_bget_remove_from_freelist(bfhead_t *b) {
417 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b);
418 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b);
420 b->ql.blink->ql.flink = b->ql.flink;
421 b->ql.flink->ql.blink = b->ql.blink;
425 static void bcheck(kmp_info_t *th, bufsize *max_free, bufsize *total_free) {
426 thr_data_t *thr = get_thr_data(th);
429 *total_free = *max_free = 0;
431 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
434 best = &thr->freelist[bin];
437 while (b != &thr->freelist[bin]) {
438 *total_free += (b->bh.bb.bsize -
sizeof(bhead_t));
439 if ((best == &thr->freelist[bin]) || (b->bh.bb.bsize < best->bh.bb.bsize))
446 if (*max_free < best->bh.bb.bsize)
447 *max_free = best->bh.bb.bsize;
450 if (*max_free > (bufsize)
sizeof(bhead_t))
451 *max_free -=
sizeof(bhead_t);
455 static void *bget(kmp_info_t *th, bufsize requested_size) {
456 thr_data_t *thr = get_thr_data(th);
457 bufsize size = requested_size;
465 if (size < 0 || size +
sizeof(bhead_t) > MaxSize) {
469 __kmp_bget_dequeue(th);
471 if (size < (bufsize)SizeQ) {
474 #if defined(SizeQuant) && (SizeQuant > 1) 475 size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
478 size +=
sizeof(bhead_t);
479 KMP_DEBUG_ASSERT(size >= 0);
480 KMP_DEBUG_ASSERT(size % SizeQuant == 0);
482 use_blink = (thr->mode == bget_mode_lifo);
491 for (bin = bget_get_bin(size); bin < MAX_BGET_BINS; ++bin) {
493 b = (use_blink ? thr->freelist[bin].ql.blink
494 : thr->freelist[bin].ql.flink);
496 if (thr->mode == bget_mode_best) {
497 best = &thr->freelist[bin];
501 while (b != &thr->freelist[bin]) {
502 if (b->bh.bb.bsize >= (bufsize)size) {
503 if ((best == &thr->freelist[bin]) ||
504 (b->bh.bb.bsize < best->bh.bb.bsize)) {
510 b = (use_blink ? b->ql.blink : b->ql.flink);
515 while (b != &thr->freelist[bin]) {
516 if ((bufsize)b->bh.bb.bsize >= (bufsize)size) {
525 if ((b->bh.bb.bsize - (bufsize)size) >
526 (bufsize)(SizeQ + (
sizeof(bhead_t)))) {
529 ba = BH(((
char *)b) + (b->bh.bb.bsize - (bufsize)size));
530 bn = BH(((
char *)ba) + size);
532 KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize);
535 b->bh.bb.bsize -= (bufsize)size;
538 ba->bb.prevfree = b->bh.bb.bsize;
541 ba->bb.bsize = -size;
550 __kmp_bget_remove_from_freelist(b);
551 __kmp_bget_insert_into_freelist(thr, b);
553 thr->totalloc += (size_t)size;
556 buf = (
void *)((((
char *)ba) +
sizeof(bhead_t)));
557 KMP_DEBUG_ASSERT(((
size_t)buf) % SizeQuant == 0);
562 ba = BH(((
char *)b) + b->bh.bb.bsize);
564 KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize);
569 __kmp_bget_remove_from_freelist(b);
571 thr->totalloc += (size_t)b->bh.bb.bsize;
575 b->bh.bb.bsize = -(b->bh.bb.bsize);
578 TCW_PTR(ba->bb.bthr, th);
584 buf = (
void *)&(b->ql);
585 KMP_DEBUG_ASSERT(((
size_t)buf) % SizeQuant == 0);
591 b = (use_blink ? b->ql.blink : b->ql.flink);
599 if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) {
607 if (thr->acqfcn != 0) {
608 if (size > (bufsize)(thr->exp_incr -
sizeof(bhead_t))) {
613 size +=
sizeof(bdhead_t) -
sizeof(bhead_t);
615 KE_TRACE(10, (
"%%%%%% MALLOC( %d )\n", (
int)size));
618 bdh = BDH((*thr->acqfcn)((bufsize)size));
622 bdh->bh.bb.bsize = 0;
625 TCW_PTR(bdh->bh.bb.bthr, th);
627 bdh->bh.bb.prevfree = 0;
630 thr->totalloc += (size_t)size;
634 buf = (
void *)(bdh + 1);
635 KMP_DEBUG_ASSERT(((
size_t)buf) % SizeQuant == 0);
644 KE_TRACE(10, (
"%%%%%% MALLOCB( %d )\n", (
int)thr->exp_incr));
647 newpool = (*thr->acqfcn)((bufsize)thr->exp_incr);
648 KMP_DEBUG_ASSERT(((
size_t)newpool) % SizeQuant == 0);
649 if (newpool != NULL) {
650 bpool(th, newpool, thr->exp_incr);
667 static void *bgetz(kmp_info_t *th, bufsize size) {
668 char *buf = (
char *)bget(th, size);
674 b = BH(buf -
sizeof(bhead_t));
675 rsize = -(b->bb.bsize);
679 bd = BDH(buf -
sizeof(bdhead_t));
680 rsize = bd->tsize - (bufsize)
sizeof(bdhead_t);
682 rsize -=
sizeof(bhead_t);
685 KMP_DEBUG_ASSERT(rsize >= size);
687 (void)memset(buf, 0, (bufsize)rsize);
689 return ((
void *)buf);
697 static void *bgetr(kmp_info_t *th,
void *buf, bufsize size) {
702 nbuf = bget(th, size);
709 b = BH(((
char *)buf) -
sizeof(bhead_t));
710 osize = -b->bb.bsize;
715 bd = BDH(((
char *)buf) -
sizeof(bdhead_t));
716 osize = bd->tsize - (bufsize)
sizeof(bdhead_t);
718 osize -=
sizeof(bhead_t);
721 KMP_DEBUG_ASSERT(osize > 0);
723 (void)KMP_MEMCPY((
char *)nbuf, (
char *)buf,
724 (
size_t)((size < osize) ? size : osize));
731 static void brel(kmp_info_t *th,
void *buf) {
732 thr_data_t *thr = get_thr_data(th);
736 KMP_DEBUG_ASSERT(buf != NULL);
737 KMP_DEBUG_ASSERT(((
size_t)buf) % SizeQuant == 0);
739 b = BFH(((
char *)buf) -
sizeof(bhead_t));
741 if (b->bh.bb.bsize == 0) {
744 bdh = BDH(((
char *)buf) -
sizeof(bdhead_t));
745 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
747 thr->totalloc -= (size_t)bdh->tsize;
752 (
void)memset((
char *)buf, 0x55, (
size_t)(bdh->tsize -
sizeof(bdhead_t)));
755 KE_TRACE(10, (
"%%%%%% FREE( %p )\n", (
void *)bdh));
757 KMP_DEBUG_ASSERT(thr->relfcn != 0);
758 (*thr->relfcn)((
void *)bdh);
762 bth = (kmp_info_t *)((kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) &
766 __kmp_bget_enqueue(bth, buf
767 #ifdef USE_QUEUING_LOCK_FOR_BGET
769 __kmp_gtid_from_thread(th)
776 if (b->bh.bb.bsize >= 0) {
779 KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0);
783 KMP_DEBUG_ASSERT(BH((
char *)b - b->bh.bb.bsize)->bb.prevfree == 0);
787 thr->totalloc += (size_t)b->bh.bb.bsize;
792 if (b->bh.bb.prevfree != 0) {
797 bufsize size = b->bh.bb.bsize;
800 KMP_DEBUG_ASSERT(BH((
char *)b - b->bh.bb.prevfree)->bb.bsize ==
802 b = BFH(((
char *)b) - b->bh.bb.prevfree);
803 b->bh.bb.bsize -= size;
806 __kmp_bget_remove_from_freelist(b);
811 b->bh.bb.bsize = -b->bh.bb.bsize;
815 __kmp_bget_insert_into_freelist(thr, b);
821 bn = BFH(((
char *)b) + b->bh.bb.bsize);
822 if (bn->bh.bb.bsize > 0) {
826 KMP_DEBUG_ASSERT(BH((
char *)bn + bn->bh.bb.bsize)->bb.prevfree ==
829 __kmp_bget_remove_from_freelist(bn);
831 b->bh.bb.bsize += bn->bh.bb.bsize;
835 __kmp_bget_remove_from_freelist(b);
836 __kmp_bget_insert_into_freelist(thr, b);
844 bn = BFH(((
char *)b) + b->bh.bb.bsize);
847 (void)memset(((
char *)b) +
sizeof(bfhead_t), 0x55,
848 (
size_t)(b->bh.bb.bsize -
sizeof(bfhead_t)));
850 KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0);
855 bn->bh.bb.prevfree = b->bh.bb.bsize;
861 if (thr->relfcn != 0 &&
862 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t))) {
868 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
869 KMP_DEBUG_ASSERT(BH((
char *)b + b->bh.bb.bsize)->bb.bsize == ESent);
870 KMP_DEBUG_ASSERT(BH((
char *)b + b->bh.bb.bsize)->bb.prevfree ==
874 __kmp_bget_remove_from_freelist(b);
876 KE_TRACE(10, (
"%%%%%% FREE( %p )\n", (
void *)b));
882 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
885 if (thr->last_pool == b)
895 static void bectl(kmp_info_t *th, bget_compact_t compact,
896 bget_acquire_t acquire, bget_release_t release,
898 thr_data_t *thr = get_thr_data(th);
900 thr->compfcn = compact;
901 thr->acqfcn = acquire;
902 thr->relfcn = release;
903 thr->exp_incr = pool_incr;
907 static void bpool(kmp_info_t *th,
void *buf, bufsize len) {
909 thr_data_t *thr = get_thr_data(th);
910 bfhead_t *b = BFH(buf);
913 __kmp_bget_dequeue(th);
916 len &= ~(SizeQuant - 1);
918 if (thr->pool_len == 0) {
920 }
else if (len != thr->pool_len) {
926 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
932 KMP_DEBUG_ASSERT(len -
sizeof(bhead_t) <= -((bufsize)ESent + 1));
937 b->bh.bb.prevfree = 0;
946 len -=
sizeof(bhead_t);
947 b->bh.bb.bsize = (bufsize)len;
949 TCW_PTR(b->bh.bb.bthr,
950 (kmp_info_t *)((kmp_uintptr_t)th |
954 __kmp_bget_insert_into_freelist(thr, b);
957 (void)memset(((
char *)b) +
sizeof(bfhead_t), 0x55,
958 (
size_t)(len -
sizeof(bfhead_t)));
960 bn = BH(((
char *)b) + len);
961 bn->bb.prevfree = (bufsize)len;
963 KMP_DEBUG_ASSERT((~0) == -1 && (bn != 0));
965 bn->bb.bsize = ESent;
969 static void bfreed(kmp_info_t *th) {
970 int bin = 0, count = 0;
971 int gtid = __kmp_gtid_from_thread(th);
972 thr_data_t *thr = get_thr_data(th);
975 __kmp_printf_no_lock(
"__kmp_printpool: T#%d total=%" KMP_UINT64_SPEC
976 " get=%" KMP_INT64_SPEC
" rel=%" KMP_INT64_SPEC
977 " pblk=%" KMP_INT64_SPEC
" pget=%" KMP_INT64_SPEC
978 " prel=%" KMP_INT64_SPEC
" dget=%" KMP_INT64_SPEC
979 " drel=%" KMP_INT64_SPEC
"\n",
980 gtid, (kmp_uint64)thr->totalloc, (kmp_int64)thr->numget,
981 (kmp_int64)thr->numrel, (kmp_int64)thr->numpblk,
982 (kmp_int64)thr->numpget, (kmp_int64)thr->numprel,
983 (kmp_int64)thr->numdget, (kmp_int64)thr->numdrel);
986 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
989 for (b = thr->freelist[bin].ql.flink; b != &thr->freelist[bin];
991 bufsize bs = b->bh.bb.bsize;
993 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b);
994 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b);
995 KMP_DEBUG_ASSERT(bs > 0);
999 __kmp_printf_no_lock(
1000 "__kmp_printpool: T#%d Free block: 0x%p size %6ld bytes.\n", gtid, b,
1004 char *lerr = ((
char *)b) +
sizeof(bfhead_t);
1005 if ((bs >
sizeof(bfhead_t)) &&
1007 (memcmp(lerr, lerr + 1, (
size_t)(bs - (
sizeof(bfhead_t) + 1))) !=
1009 __kmp_printf_no_lock(
"__kmp_printpool: T#%d (Contents of above " 1010 "free block have been overstored.)\n",
1019 __kmp_printf_no_lock(
"__kmp_printpool: T#%d No free blocks\n", gtid);
1027 static void bstats(kmp_info_t *th, bufsize *curalloc, bufsize *totfree,
1028 bufsize *maxfree,
long *nget,
long *nrel) {
1030 thr_data_t *thr = get_thr_data(th);
1032 *nget = thr->numget;
1033 *nrel = thr->numrel;
1034 *curalloc = (bufsize)thr->totalloc;
1038 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1039 bfhead_t *b = thr->freelist[bin].ql.flink;
1041 while (b != &thr->freelist[bin]) {
1042 KMP_DEBUG_ASSERT(b->bh.bb.bsize > 0);
1043 *totfree += b->bh.bb.bsize;
1044 if (b->bh.bb.bsize > *maxfree) {
1045 *maxfree = b->bh.bb.bsize;
1053 static void bstatse(kmp_info_t *th, bufsize *pool_incr,
long *npool,
1054 long *npget,
long *nprel,
long *ndget,
long *ndrel) {
1055 thr_data_t *thr = get_thr_data(th);
1057 *pool_incr = (thr->pool_len < 0) ? -thr->exp_incr : thr->exp_incr;
1058 *npool = thr->numpblk;
1059 *npget = thr->numpget;
1060 *nprel = thr->numprel;
1061 *ndget = thr->numdget;
1062 *ndrel = thr->numdrel;
1070 static void bufdump(kmp_info_t *th,
void *buf) {
1072 unsigned char *bdump;
1075 b = BFH(((
char *)buf) -
sizeof(bhead_t));
1076 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
1077 if (b->bh.bb.bsize < 0) {
1078 bdump = (
unsigned char *)buf;
1079 bdlen = (-b->bh.bb.bsize) - (bufsize)
sizeof(bhead_t);
1081 bdump = (
unsigned char *)(((
char *)b) +
sizeof(bfhead_t));
1082 bdlen = b->bh.bb.bsize - (bufsize)
sizeof(bfhead_t);
1088 char bhex[50], bascii[20];
1094 for (i = 0; i < l; i++) {
1095 (void)KMP_SNPRINTF(bhex + i * 3,
sizeof(bhex) - i * 3,
"%02X ", bdump[i]);
1096 if (bdump[i] > 0x20 && bdump[i] < 0x7F)
1097 bascii[i] = bdump[i];
1102 (void)__kmp_printf_no_lock(
"%-48s %s\n", bhex, bascii);
1105 while ((bdlen > 16) &&
1106 (memcmp((
char *)(bdump - 16), (
char *)bdump, 16) == 0)) {
1112 (void)__kmp_printf_no_lock(
1113 " (%d lines [%d bytes] identical to above line skipped)\n", dupes,
1115 }
else if (dupes == 1) {
1127 static void bpoold(kmp_info_t *th,
void *buf,
int dumpalloc,
int dumpfree) {
1128 bfhead_t *b = BFH((
char *)buf -
sizeof(bhead_t));
1130 while (b->bh.bb.bsize != ESent) {
1131 bufsize bs = b->bh.bb.bsize;
1135 (void)__kmp_printf_no_lock(
"Allocated buffer: size %6ld bytes.\n",
1138 bufdump(th, (
void *)(((
char *)b) +
sizeof(bhead_t)));
1141 const char *lerr =
"";
1143 KMP_DEBUG_ASSERT(bs > 0);
1144 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1145 lerr =
" (Bad free list links)";
1147 (void)__kmp_printf_no_lock(
"Free block: size %6ld bytes.%s\n",
1150 lerr = ((
char *)b) +
sizeof(bfhead_t);
1151 if ((bs >
sizeof(bfhead_t)) &&
1153 (memcmp(lerr, lerr + 1, (
size_t)(bs - (
sizeof(bfhead_t) + 1))) !=
1155 (void)__kmp_printf_no_lock(
1156 "(Contents of above free block have been overstored.)\n");
1157 bufdump(th, (
void *)(((
char *)b) +
sizeof(bhead_t)));
1161 bufdump(th, (
void *)(((
char *)b) +
sizeof(bhead_t)));
1164 b = BFH(((
char *)b) + bs);
1169 static int bpoolv(kmp_info_t *th,
void *buf) {
1170 bfhead_t *b = BFH(buf);
1172 while (b->bh.bb.bsize != ESent) {
1173 bufsize bs = b->bh.bb.bsize;
1182 KMP_DEBUG_ASSERT(bs > 0);
1186 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1187 (void)__kmp_printf_no_lock(
1188 "Free block: size %6ld bytes. (Bad free list links)\n", (
long)bs);
1189 KMP_DEBUG_ASSERT(0);
1193 lerr = ((
char *)b) +
sizeof(bfhead_t);
1194 if ((bs >
sizeof(bfhead_t)) &&
1196 (memcmp(lerr, lerr + 1, (
size_t)(bs - (
sizeof(bfhead_t) + 1))) !=
1198 (void)__kmp_printf_no_lock(
1199 "(Contents of above free block have been overstored.)\n");
1200 bufdump(th, (
void *)(((
char *)b) +
sizeof(bhead_t)));
1201 KMP_DEBUG_ASSERT(0);
1206 b = BFH(((
char *)b) + bs);
1213 void __kmp_initialize_bget(kmp_info_t *th) {
1214 KMP_DEBUG_ASSERT(SizeQuant >=
sizeof(
void *) && (th != 0));
1218 bectl(th, (bget_compact_t)0, (bget_acquire_t)malloc, (bget_release_t)free,
1219 (bufsize)__kmp_malloc_pool_incr);
1222 void __kmp_finalize_bget(kmp_info_t *th) {
1226 KMP_DEBUG_ASSERT(th != 0);
1229 thr = (thr_data_t *)th->th.th_local.bget_data;
1230 KMP_DEBUG_ASSERT(thr != NULL);
1238 if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 &&
1239 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t))) {
1240 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
1241 KMP_DEBUG_ASSERT(BH((
char *)b + b->bh.bb.bsize)->bb.bsize == ESent);
1242 KMP_DEBUG_ASSERT(BH((
char *)b + b->bh.bb.bsize)->bb.prevfree ==
1246 __kmp_bget_remove_from_freelist(b);
1248 KE_TRACE(10, (
"%%%%%% FREE( %p )\n", (
void *)b));
1253 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1258 if (th->th.th_local.bget_data != NULL) {
1259 __kmp_free(th->th.th_local.bget_data);
1260 th->th.th_local.bget_data = NULL;
1264 void kmpc_set_poolsize(
size_t size) {
1265 bectl(__kmp_get_thread(), (bget_compact_t)0, (bget_acquire_t)malloc,
1266 (bget_release_t)free, (bufsize)size);
1269 size_t kmpc_get_poolsize(
void) {
1272 p = get_thr_data(__kmp_get_thread());
1277 void kmpc_set_poolmode(
int mode) {
1280 if (mode == bget_mode_fifo || mode == bget_mode_lifo ||
1281 mode == bget_mode_best) {
1282 p = get_thr_data(__kmp_get_thread());
1283 p->mode = (bget_mode_t)mode;
1287 int kmpc_get_poolmode(
void) {
1290 p = get_thr_data(__kmp_get_thread());
1295 void kmpc_get_poolstat(
size_t *maxmem,
size_t *allmem) {
1296 kmp_info_t *th = __kmp_get_thread();
1299 __kmp_bget_dequeue(th);
1307 void kmpc_poolprint(
void) {
1308 kmp_info_t *th = __kmp_get_thread();
1310 __kmp_bget_dequeue(th);
1315 #endif // #if KMP_USE_BGET 1317 void *kmpc_malloc(
size_t size) {
1319 ptr = bget(__kmp_entry_thread(), (bufsize)(size +
sizeof(ptr)));
1322 *(
void **)ptr = ptr;
1323 ptr = (
void **)ptr + 1;
1328 #define IS_POWER_OF_TWO(n) (((n) & ((n)-1)) == 0) 1330 void *kmpc_aligned_malloc(
size_t size,
size_t alignment) {
1332 void *ptr_allocated;
1333 KMP_DEBUG_ASSERT(alignment < 32 * 1024);
1334 if (!IS_POWER_OF_TWO(alignment)) {
1339 size = size +
sizeof(
void *) + alignment;
1340 ptr_allocated = bget(__kmp_entry_thread(), (bufsize)size);
1341 if (ptr_allocated != NULL) {
1343 ptr = (
void *)(((kmp_uintptr_t)ptr_allocated +
sizeof(
void *) + alignment) &
1345 *((
void **)ptr - 1) = ptr_allocated;
1352 void *kmpc_calloc(
size_t nelem,
size_t elsize) {
1354 ptr = bgetz(__kmp_entry_thread(), (bufsize)(nelem * elsize +
sizeof(ptr)));
1357 *(
void **)ptr = ptr;
1358 ptr = (
void **)ptr + 1;
1363 void *kmpc_realloc(
void *ptr,
size_t size) {
1364 void *result = NULL;
1367 result = bget(__kmp_entry_thread(), (bufsize)(size +
sizeof(ptr)));
1369 if (result != NULL) {
1370 *(
void **)result = result;
1371 result = (
void **)result + 1;
1373 }
else if (size == 0) {
1379 KMP_ASSERT(*((
void **)ptr - 1));
1380 brel(__kmp_get_thread(), *((
void **)ptr - 1));
1382 result = bgetr(__kmp_entry_thread(), *((
void **)ptr - 1),
1383 (bufsize)(size +
sizeof(ptr)));
1384 if (result != NULL) {
1385 *(
void **)result = result;
1386 result = (
void **)result + 1;
1393 void kmpc_free(
void *ptr) {
1394 if (!__kmp_init_serial) {
1398 kmp_info_t *th = __kmp_get_thread();
1399 __kmp_bget_dequeue(th);
1401 KMP_ASSERT(*((
void **)ptr - 1));
1402 brel(th, *((
void **)ptr - 1));
1406 void *___kmp_thread_malloc(kmp_info_t *th,
size_t size KMP_SRC_LOC_DECL) {
1408 KE_TRACE(30, (
"-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n", th,
1409 (
int)size KMP_SRC_LOC_PARM));
1410 ptr = bget(th, (bufsize)size);
1411 KE_TRACE(30, (
"<- __kmp_thread_malloc() returns %p\n", ptr));
1415 void *___kmp_thread_calloc(kmp_info_t *th,
size_t nelem,
1416 size_t elsize KMP_SRC_LOC_DECL) {
1418 KE_TRACE(30, (
"-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n", th,
1419 (
int)nelem, (
int)elsize KMP_SRC_LOC_PARM));
1420 ptr = bgetz(th, (bufsize)(nelem * elsize));
1421 KE_TRACE(30, (
"<- __kmp_thread_calloc() returns %p\n", ptr));
1425 void *___kmp_thread_realloc(kmp_info_t *th,
void *ptr,
1426 size_t size KMP_SRC_LOC_DECL) {
1427 KE_TRACE(30, (
"-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n", th,
1428 ptr, (
int)size KMP_SRC_LOC_PARM));
1429 ptr = bgetr(th, ptr, (bufsize)size);
1430 KE_TRACE(30, (
"<- __kmp_thread_realloc() returns %p\n", ptr));
1434 void ___kmp_thread_free(kmp_info_t *th,
void *ptr KMP_SRC_LOC_DECL) {
1435 KE_TRACE(30, (
"-> __kmp_thread_free( %p, %p ) called from %s:%d\n", th,
1436 ptr KMP_SRC_LOC_PARM));
1438 __kmp_bget_dequeue(th);
1441 KE_TRACE(30, (
"<- __kmp_thread_free()\n"));
1448 struct kmp_mem_descr {
1449 void *ptr_allocated;
1450 size_t size_allocated;
1452 size_t size_aligned;
1454 typedef struct kmp_mem_descr kmp_mem_descr_t;
1459 static void *___kmp_allocate_align(
size_t size,
1460 size_t alignment KMP_SRC_LOC_DECL) {
1477 kmp_mem_descr_t descr;
1478 kmp_uintptr_t addr_allocated;
1479 kmp_uintptr_t addr_aligned;
1480 kmp_uintptr_t addr_descr;
1482 KE_TRACE(25, (
"-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n",
1483 (
int)size, (
int)alignment KMP_SRC_LOC_PARM));
1485 KMP_DEBUG_ASSERT(alignment < 32 * 1024);
1486 KMP_DEBUG_ASSERT(
sizeof(
void *) <=
sizeof(kmp_uintptr_t));
1489 descr.size_aligned = size;
1490 descr.size_allocated =
1491 descr.size_aligned +
sizeof(kmp_mem_descr_t) + alignment;
1494 descr.ptr_allocated = _malloc_src_loc(descr.size_allocated, _file_, _line_);
1496 descr.ptr_allocated = malloc_src_loc(descr.size_allocated KMP_SRC_LOC_PARM);
1498 KE_TRACE(10, (
" malloc( %d ) returned %p\n", (
int)descr.size_allocated,
1499 descr.ptr_allocated));
1500 if (descr.ptr_allocated == NULL) {
1501 KMP_FATAL(OutOfHeapMemory);
1504 addr_allocated = (kmp_uintptr_t)descr.ptr_allocated;
1506 (addr_allocated +
sizeof(kmp_mem_descr_t) + alignment) & ~(alignment - 1);
1507 addr_descr = addr_aligned -
sizeof(kmp_mem_descr_t);
1509 descr.ptr_aligned = (
void *)addr_aligned;
1511 KE_TRACE(26, (
" ___kmp_allocate_align: " 1512 "ptr_allocated=%p, size_allocated=%d, " 1513 "ptr_aligned=%p, size_aligned=%d\n",
1514 descr.ptr_allocated, (
int)descr.size_allocated,
1515 descr.ptr_aligned, (
int)descr.size_aligned));
1517 KMP_DEBUG_ASSERT(addr_allocated <= addr_descr);
1518 KMP_DEBUG_ASSERT(addr_descr +
sizeof(kmp_mem_descr_t) == addr_aligned);
1519 KMP_DEBUG_ASSERT(addr_aligned + descr.size_aligned <=
1520 addr_allocated + descr.size_allocated);
1521 KMP_DEBUG_ASSERT(addr_aligned % alignment == 0);
1523 memset(descr.ptr_allocated, 0xEF, descr.size_allocated);
1526 memset(descr.ptr_aligned, 0x00, descr.size_aligned);
1532 *((kmp_mem_descr_t *)addr_descr) = descr;
1536 KE_TRACE(25, (
"<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned));
1537 return descr.ptr_aligned;
1544 void *___kmp_allocate(
size_t size KMP_SRC_LOC_DECL) {
1546 KE_TRACE(25, (
"-> __kmp_allocate( %d ) called from %s:%d\n",
1547 (
int)size KMP_SRC_LOC_PARM));
1548 ptr = ___kmp_allocate_align(size, __kmp_align_alloc KMP_SRC_LOC_PARM);
1549 KE_TRACE(25, (
"<- __kmp_allocate() returns %p\n", ptr));
1553 #if (BUILD_MEMORY == FIRST_TOUCH) 1554 void *__kmp_ft_page_allocate(
size_t size) {
1557 const int page_size = KMP_GET_PAGE_SIZE();
1559 adr = (
void *)__kmp_thread_malloc(__kmp_get_thread(),
1560 size + page_size + KMP_PTR_SKIP);
1562 KMP_FATAL(OutOfHeapMemory);
1565 if (((kmp_uintptr_t)adr & (page_size - 1)) == 0)
1570 aadr = (
void *)(((kmp_uintptr_t)adr + page_size) & ~(page_size - 1));
1573 *((
void **)aadr) = adr;
1576 return (
void *)((
char *)aadr + KMP_PTR_SKIP);
1584 void *___kmp_page_allocate(
size_t size KMP_SRC_LOC_DECL) {
1585 int page_size = 8 * 1024;
1588 KE_TRACE(25, (
"-> __kmp_page_allocate( %d ) called from %s:%d\n",
1589 (
int)size KMP_SRC_LOC_PARM));
1590 ptr = ___kmp_allocate_align(size, page_size KMP_SRC_LOC_PARM);
1591 KE_TRACE(25, (
"<- __kmp_page_allocate( %d ) returns %p\n", (
int)size, ptr));
1597 void ___kmp_free(
void *ptr KMP_SRC_LOC_DECL) {
1598 kmp_mem_descr_t descr;
1599 kmp_uintptr_t addr_allocated;
1600 kmp_uintptr_t addr_aligned;
1603 (
"-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM));
1604 KMP_ASSERT(ptr != NULL);
1606 descr = *(kmp_mem_descr_t *)((kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t));
1608 KE_TRACE(26, (
" __kmp_free: " 1609 "ptr_allocated=%p, size_allocated=%d, " 1610 "ptr_aligned=%p, size_aligned=%d\n",
1611 descr.ptr_allocated, (
int)descr.size_allocated,
1612 descr.ptr_aligned, (
int)descr.size_aligned));
1614 addr_allocated = (kmp_uintptr_t)descr.ptr_allocated;
1615 addr_aligned = (kmp_uintptr_t)descr.ptr_aligned;
1617 KMP_DEBUG_ASSERT(addr_aligned % CACHE_LINE == 0);
1618 KMP_DEBUG_ASSERT(descr.ptr_aligned == ptr);
1619 KMP_DEBUG_ASSERT(addr_allocated +
sizeof(kmp_mem_descr_t) <= addr_aligned);
1620 KMP_DEBUG_ASSERT(descr.size_aligned < descr.size_allocated);
1621 KMP_DEBUG_ASSERT(addr_aligned + descr.size_aligned <=
1622 addr_allocated + descr.size_allocated);
1625 memset(descr.ptr_allocated, 0xEF, descr.size_allocated);
1630 KE_TRACE(10, (
" free( %p )\n", descr.ptr_allocated));
1632 _free_src_loc(descr.ptr_allocated, _file_, _line_);
1634 free_src_loc(descr.ptr_allocated KMP_SRC_LOC_PARM);
1638 KE_TRACE(25, (
"<- __kmp_free() returns\n"));
1641 #if USE_FAST_MEMORY == 3 1647 #define KMP_FREE_LIST_LIMIT 16 1650 #define DCACHE_LINE 128 1652 void *___kmp_fast_allocate(kmp_info_t *this_thr,
size_t size KMP_SRC_LOC_DECL) {
1659 kmp_mem_descr_t *descr;
1661 KE_TRACE(25, (
"-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n",
1662 __kmp_gtid_from_thread(this_thr), (
int)size KMP_SRC_LOC_PARM));
1664 num_lines = (size + DCACHE_LINE - 1) / DCACHE_LINE;
1665 idx = num_lines - 1;
1666 KMP_DEBUG_ASSERT(idx >= 0);
1670 }
else if ((idx >>= 2) == 0) {
1673 }
else if ((idx >>= 2) == 0) {
1676 }
else if ((idx >>= 2) == 0) {
1683 ptr = this_thr->th.th_free_lists[index].th_free_list_self;
1686 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1689 ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t)))
1693 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync);
1698 while (!KMP_COMPARE_AND_STORE_PTR(
1699 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr,
nullptr)) {
1701 ptr = TCR_SYNC_PTR(this_thr->th.th_free_lists[index].th_free_list_sync);
1705 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1708 ((kmp_mem_descr_t *)((kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t)))
1715 size = num_lines * DCACHE_LINE;
1717 alloc_size = size +
sizeof(kmp_mem_descr_t) + DCACHE_LINE;
1718 KE_TRACE(25, (
"__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with " 1720 __kmp_gtid_from_thread(this_thr), alloc_size));
1721 alloc_ptr = bget(this_thr, (bufsize)alloc_size);
1724 ptr = (
void *)((((kmp_uintptr_t)alloc_ptr) +
sizeof(kmp_mem_descr_t) +
1726 ~(DCACHE_LINE - 1));
1727 descr = (kmp_mem_descr_t *)(((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t));
1729 descr->ptr_allocated = alloc_ptr;
1731 descr->ptr_aligned = (
void *)this_thr;
1734 descr->size_aligned = size;
1737 KE_TRACE(25, (
"<- __kmp_fast_allocate( T#%d ) returns %p\n",
1738 __kmp_gtid_from_thread(this_thr), ptr));
1744 void ___kmp_fast_free(kmp_info_t *this_thr,
void *ptr KMP_SRC_LOC_DECL) {
1745 kmp_mem_descr_t *descr;
1746 kmp_info_t *alloc_thr;
1751 KE_TRACE(25, (
"-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n",
1752 __kmp_gtid_from_thread(this_thr), ptr KMP_SRC_LOC_PARM));
1753 KMP_ASSERT(ptr != NULL);
1755 descr = (kmp_mem_descr_t *)(((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t));
1757 KE_TRACE(26, (
" __kmp_fast_free: size_aligned=%d\n",
1758 (
int)descr->size_aligned));
1760 size = descr->size_aligned;
1762 idx = DCACHE_LINE * 2;
1765 }
else if ((idx <<= 1) == size) {
1767 }
else if ((idx <<= 2) == size) {
1769 }
else if ((idx <<= 2) == size) {
1772 KMP_DEBUG_ASSERT(size > DCACHE_LINE * 64);
1776 alloc_thr = (kmp_info_t *)descr->ptr_aligned;
1777 if (alloc_thr == this_thr) {
1779 *((
void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self;
1780 this_thr->th.th_free_lists[index].th_free_list_self = ptr;
1782 void *head = this_thr->th.th_free_lists[index].th_free_list_other;
1785 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1786 *((
void **)ptr) = NULL;
1787 descr->size_allocated = (size_t)1;
1790 kmp_mem_descr_t *dsc =
1791 (kmp_mem_descr_t *)((
char *)head -
sizeof(kmp_mem_descr_t));
1793 kmp_info_t *q_th = (kmp_info_t *)(dsc->ptr_aligned);
1795 dsc->size_allocated + 1;
1796 if (q_th == alloc_thr && q_sz <= KMP_FREE_LIST_LIMIT) {
1798 *((
void **)ptr) = head;
1799 descr->size_allocated = q_sz;
1800 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1807 void *next = *((
void **)head);
1808 while (next != NULL) {
1811 ((kmp_mem_descr_t *)((
char *)next -
sizeof(kmp_mem_descr_t)))
1814 ((kmp_mem_descr_t *)((
char *)tail -
sizeof(kmp_mem_descr_t)))
1817 next = *((
void **)next);
1819 KMP_DEBUG_ASSERT(q_th != NULL);
1821 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync);
1824 *((
void **)tail) = old_ptr;
1826 while (!KMP_COMPARE_AND_STORE_PTR(
1827 &q_th->th.th_free_lists[index].th_free_list_sync, old_ptr, head)) {
1829 old_ptr = TCR_PTR(q_th->th.th_free_lists[index].th_free_list_sync);
1830 *((
void **)tail) = old_ptr;
1834 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1835 *((
void **)ptr) = NULL;
1836 descr->size_allocated = (size_t)1;
1843 KE_TRACE(25, (
"__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n",
1844 __kmp_gtid_from_thread(this_thr), size));
1845 __kmp_bget_dequeue(this_thr);
1846 brel(this_thr, descr->ptr_allocated);
1849 KE_TRACE(25, (
"<- __kmp_fast_free() returns\n"));
1855 void __kmp_initialize_fast_memory(kmp_info_t *this_thr) {
1856 KE_TRACE(10, (
"__kmp_initialize_fast_memory: Called from th %p\n", this_thr));
1858 memset(this_thr->th.th_free_lists, 0, NUM_LISTS *
sizeof(kmp_free_list_t));
1863 void __kmp_free_fast_memory(kmp_info_t *th) {
1866 thr_data_t *thr = get_thr_data(th);
1870 5, (
"__kmp_free_fast_memory: Called T#%d\n", __kmp_gtid_from_thread(th)));
1872 __kmp_bget_dequeue(th);
1875 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1876 bfhead_t *b = thr->freelist[bin].ql.flink;
1877 while (b != &thr->freelist[bin]) {
1878 if ((kmp_uintptr_t)b->bh.bb.bthr & 1) {
1886 while (lst != NULL) {
1888 KE_TRACE(10, (
"__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n",
1889 lst, next, th, __kmp_gtid_from_thread(th)));
1890 (*thr->relfcn)(lst);
1896 lst = (
void **)next;
1900 5, (
"__kmp_free_fast_memory: Freed T#%d\n", __kmp_gtid_from_thread(th)));
1903 #endif // USE_FAST_MEMORY