1 #define JEMALLOC_MUTEX_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
5 #include <dlfcn.h>
6 #endif
7
8 #ifndef _CRT_SPINCOUNT
9 #define _CRT_SPINCOUNT 4000
10 #endif
11
12 /******************************************************************************/
13 /* Data. */
14
15 #ifdef JEMALLOC_LAZY_LOCK
16 bool isthreaded = false;
17 #endif
18 #ifdef JEMALLOC_MUTEX_INIT_CB
19 static bool postpone_init = true;
20 static malloc_mutex_t *postponed_mutexes = NULL;
21 #endif
22
23 #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
24 static void pthread_create_once(void);
25 #endif
26
27 /******************************************************************************/
28 /*
29 * We intercept pthread_create() calls in order to toggle isthreaded if the
30 * process goes multi-threaded.
31 */
32
33 #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
34 static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
35 void *(*)(void *), void *__restrict);
36
37 static void
pthread_create_once(void)38 pthread_create_once(void)
39 {
40
41 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
42 if (pthread_create_fptr == NULL) {
43 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
44 "\"pthread_create\")\n");
45 abort();
46 }
47
48 isthreaded = true;
49 }
50
51 JEMALLOC_EXPORT int
pthread_create(pthread_t * __restrict thread,const pthread_attr_t * __restrict attr,void * (* start_routine)(void *),void * __restrict arg)52 pthread_create(pthread_t *__restrict thread,
53 const pthread_attr_t *__restrict attr, void *(*start_routine)(void *),
54 void *__restrict arg)
55 {
56 static pthread_once_t once_control = PTHREAD_ONCE_INIT;
57
58 pthread_once(&once_control, pthread_create_once);
59
60 return (pthread_create_fptr(thread, attr, start_routine, arg));
61 }
62 #endif
63
64 /******************************************************************************/
65
66 #ifdef JEMALLOC_MUTEX_INIT_CB
67 JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
68 void *(calloc_cb)(size_t, size_t));
69
70 #pragma weak _pthread_mutex_init_calloc_cb
71 int
_pthread_mutex_init_calloc_cb(pthread_mutex_t * mutex,void * (calloc_cb)(size_t,size_t))72 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
73 void *(calloc_cb)(size_t, size_t))
74 {
75
76 return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t)))
77 __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex,
78 calloc_cb));
79 }
80 #endif
81
82 bool
malloc_mutex_init(malloc_mutex_t * mutex)83 malloc_mutex_init(malloc_mutex_t *mutex)
84 {
85
86 #ifdef _WIN32
87 # if _WIN32_WINNT >= 0x0600
88 InitializeSRWLock(&mutex->lock);
89 # else
90 if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
91 _CRT_SPINCOUNT))
92 return (true);
93 # endif
94 #elif (defined(JEMALLOC_OSSPIN))
95 mutex->lock = 0;
96 #elif (defined(JEMALLOC_MUTEX_INIT_CB))
97 if (postpone_init) {
98 mutex->postponed_next = postponed_mutexes;
99 postponed_mutexes = mutex;
100 } else {
101 if (_pthread_mutex_init_calloc_cb(&mutex->lock,
102 bootstrap_calloc) != 0)
103 return (true);
104 }
105 #else
106 pthread_mutexattr_t attr;
107
108 if (pthread_mutexattr_init(&attr) != 0)
109 return (true);
110 pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
111 if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
112 pthread_mutexattr_destroy(&attr);
113 return (true);
114 }
115 pthread_mutexattr_destroy(&attr);
116 #endif
117 return (false);
118 }
119
120 void
malloc_mutex_prefork(malloc_mutex_t * mutex)121 malloc_mutex_prefork(malloc_mutex_t *mutex)
122 {
123
124 malloc_mutex_lock(mutex);
125 }
126
127 void
malloc_mutex_postfork_parent(malloc_mutex_t * mutex)128 malloc_mutex_postfork_parent(malloc_mutex_t *mutex)
129 {
130
131 malloc_mutex_unlock(mutex);
132 }
133
134 void
malloc_mutex_postfork_child(malloc_mutex_t * mutex)135 malloc_mutex_postfork_child(malloc_mutex_t *mutex)
136 {
137
138 #ifdef JEMALLOC_MUTEX_INIT_CB
139 malloc_mutex_unlock(mutex);
140 #else
141 if (malloc_mutex_init(mutex)) {
142 malloc_printf("<jemalloc>: Error re-initializing mutex in "
143 "child\n");
144 if (opt_abort)
145 abort();
146 }
147 #endif
148 }
149
150 bool
malloc_mutex_first_thread(void)151 malloc_mutex_first_thread(void)
152 {
153
154 #ifdef JEMALLOC_MUTEX_INIT_CB
155 postpone_init = false;
156 while (postponed_mutexes != NULL) {
157 if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock,
158 bootstrap_calloc) != 0)
159 return (true);
160 postponed_mutexes = postponed_mutexes->postponed_next;
161 }
162 #endif
163 return (false);
164 }
165
166 bool
mutex_boot(void)167 mutex_boot(void)
168 {
169
170 #ifndef JEMALLOC_MUTEX_INIT_CB
171 return (malloc_mutex_first_thread());
172 #else
173 return (false);
174 #endif
175 }
176