1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2004 Doug Rabson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * Define stubs for TLS internals so that programs and libraries can
31 * link. These functions will be replaced by functional versions at
32 * runtime from ld-elf.so.1.
33 */
34
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <elf.h>
40 #include <unistd.h>
41
42 #include "rtld.h"
43 #include "libc_private.h"
44
45 #define tls_assert(cond) ((cond) ? (void) 0 : \
46 (tls_msg(#cond ": assert failed: " __FILE__ ":" \
47 __XSTRING(__LINE__) "\n"), abort()))
48 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s))
49
50 /* Provided by jemalloc to avoid bootstrapping issues. */
51 void *__je_bootstrap_malloc(size_t size);
52 void *__je_bootstrap_calloc(size_t num, size_t size);
53 void __je_bootstrap_free(void *ptr);
54
55 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls);
56 __weak_reference(__libc_free_tls, _rtld_free_tls);
57
58 #ifdef __i386__
59
60 __weak_reference(___libc_tls_get_addr, ___tls_get_addr);
61 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *);
62
63 #endif
64
65 void * __libc_tls_get_addr(void *);
66 __weak_reference(__libc_tls_get_addr, __tls_get_addr);
67
68 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
69 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
70 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
71 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
72
73 #ifndef PIC
74
75 static size_t libc_tls_static_space;
76 static size_t libc_tls_init_size;
77 static size_t libc_tls_init_align;
78 static void *libc_tls_init;
79 #endif
80
81 void *
__libc_tls_get_addr(void * vti)82 __libc_tls_get_addr(void *vti)
83 {
84 uintptr_t *dtv;
85 tls_index *ti;
86
87 dtv = _tcb_get()->tcb_dtv;
88 ti = vti;
89 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) +
90 TLS_DTV_OFFSET);
91 }
92
93 #ifdef __i386__
94
95 /* GNU ABI */
96
97 __attribute__((__regparm__(1)))
98 void *
___libc_tls_get_addr(void * vti)99 ___libc_tls_get_addr(void *vti)
100 {
101 return (__libc_tls_get_addr(vti));
102 }
103
104 #endif
105
106 #ifndef PIC
107
108 static void *
libc_malloc_aligned(size_t size,size_t align)109 libc_malloc_aligned(size_t size, size_t align)
110 {
111 void *mem, *res;
112
113 if (align < sizeof(void *))
114 align = sizeof(void *);
115
116 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1);
117 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align);
118 *(void **)((uintptr_t)res - sizeof(void *)) = mem;
119 return (res);
120 }
121
122 static void
libc_free_aligned(void * ptr)123 libc_free_aligned(void *ptr)
124 {
125 void *mem;
126 uintptr_t x;
127
128 if (ptr == NULL)
129 return;
130
131 x = (uintptr_t)ptr;
132 x -= sizeof(void *);
133 mem = *(void **)x;
134 __je_bootstrap_free(mem);
135 }
136
137 #ifdef TLS_VARIANT_I
138
139 /*
140 * There are two versions of variant I of TLS
141 *
142 * - ARM and aarch64 uses original variant I as is described in [1] and [2],
143 * where TP points to start of TCB followed by aligned TLS segment.
144 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0]
145 * points to DTV vector and DTV values are real addresses (without bias).
146 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to
147 * TLS variables are computed by linker, so we cannot overalign TLS section.
148 *
149 * - MIPS, PowerPC and RISC-V use modified version of variant I,
150 * described in [3] where TP points (with bias) to TLS and TCB immediately
151 * precedes TLS without any alignment gap[4]. Only TLS should be aligned.
152 * The TCB[0] points to DTV vector and DTV values are biased by constant
153 * value (TLS_DTV_OFFSET) from real addresses[5].
154 *
155 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage
156 * www.akkadia.org/drepper/tls.pdf
157 *
158 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r)
159 * Architecture
160 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf
161 *
162 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification
163 * https://members.openpowerfoundation.org/document/dl/576
164 *
165 * [4] Its unclear if "without any alignment gap" is hard ABI requirement,
166 * but we must follow this rule due to suboptimal _tcb_set()
167 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but
168 * TCB as argument.
169 *
170 * [5] I'm not able to validate "values are biased" assertions.
171 */
172
173 /*
174 * Return pointer to allocated TLS block
175 */
176 static void *
get_tls_block_ptr(void * tcb,size_t tcbsize)177 get_tls_block_ptr(void *tcb, size_t tcbsize)
178 {
179 size_t extra_size, post_size, pre_size, tls_block_size;
180
181 /* Compute fragments sizes. */
182 extra_size = tcbsize - TLS_TCB_SIZE;
183 #if defined(__aarch64__) || defined(__arm__)
184 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE;
185 #else
186 post_size = 0;
187 #endif
188 tls_block_size = tcbsize + post_size;
189 pre_size = roundup2(tls_block_size, libc_tls_init_align) -
190 tls_block_size;
191
192 return ((char *)tcb - pre_size - extra_size);
193 }
194
195 /*
196 * Free Static TLS using the Variant I method. The tcbsize
197 * and tcbalign parameters must be the same as those used to allocate
198 * the block.
199 */
200 void
__libc_free_tls(void * tcb,size_t tcbsize,size_t tcbalign __unused)201 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused)
202 {
203 Elf_Addr *dtv;
204 Elf_Addr **tls;
205
206 tls = (Elf_Addr **)tcb;
207 dtv = tls[0];
208 __je_bootstrap_free(dtv);
209 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize));
210 }
211
212 /*
213 * Allocate Static TLS using the Variant I method.
214 *
215 * To handle all above requirements, we setup the following layout for
216 * TLS block:
217 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align))
218 *
219 * +----------+--------------+--------------+-----------+------------------+
220 * | pre gap | extended TCB | TCB | post gap | TLS segment |
221 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space |
222 * +----------+--------------+--------------+-----------+------------------+
223 *
224 * where:
225 * extra_size is tcbsize - TLS_TCB_SIZE
226 * post_size is used to adjust TCB to TLS alignment for first version of TLS
227 * layout and is always 0 for second version.
228 * pre_size is used to adjust TCB alignment for first version and to adjust
229 * TLS alignment for second version.
230 *
231 */
232 void *
__libc_allocate_tls(void * oldtcb,size_t tcbsize,size_t tcbalign)233 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign)
234 {
235 Elf_Addr *dtv, **tcb;
236 char *tls_block, *tls;
237 size_t extra_size, maxalign, post_size, pre_size, tls_block_size;
238
239 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE)
240 return (oldtcb);
241
242 tls_assert(tcbalign >= TLS_TCB_ALIGN);
243 maxalign = MAX(tcbalign, libc_tls_init_align);
244
245 /* Compute fragmets sizes. */
246 extra_size = tcbsize - TLS_TCB_SIZE;
247 #if defined(__aarch64__) || defined(__arm__)
248 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE;
249 #else
250 post_size = 0;
251 #endif
252 tls_block_size = tcbsize + post_size;
253 pre_size = roundup2(tls_block_size, libc_tls_init_align) -
254 tls_block_size;
255 tls_block_size += pre_size + libc_tls_static_space;
256
257 /* Allocate whole TLS block */
258 tls_block = libc_malloc_aligned(tls_block_size, maxalign);
259 if (tls_block == NULL) {
260 tls_msg("__libc_allocate_tls: Out of memory.\n");
261 abort();
262 }
263 memset(tls_block, 0, tls_block_size);
264 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size);
265 tls = (char *)tcb + TLS_TCB_SIZE + post_size;
266
267 if (oldtcb != NULL) {
268 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize),
269 tls_block_size);
270 libc_free_aligned(oldtcb);
271
272 /* Adjust the DTV. */
273 dtv = tcb[0];
274 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET);
275 } else {
276 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
277 if (dtv == NULL) {
278 tls_msg("__libc_allocate_tls: Out of memory.\n");
279 abort();
280 }
281 /* Build the DTV. */
282 tcb[0] = dtv;
283 dtv[0] = 1; /* Generation. */
284 dtv[1] = 1; /* Segments count. */
285 dtv[2] = (Elf_Addr)(tls + TLS_DTV_OFFSET);
286
287 if (libc_tls_init_size > 0)
288 memcpy(tls, libc_tls_init, libc_tls_init_size);
289 }
290
291 return (tcb);
292 }
293
294 #endif
295
296 #ifdef TLS_VARIANT_II
297
298 /*
299 * Free Static TLS using the Variant II method.
300 */
301 void
__libc_free_tls(void * tcb,size_t tcbsize __unused,size_t tcbalign)302 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign)
303 {
304 size_t size;
305 Elf_Addr* dtv;
306 Elf_Addr tlsstart, tlsend;
307
308 /*
309 * Figure out the size of the initial TLS block so that we can
310 * find stuff which ___tls_get_addr() allocated dynamically.
311 */
312 tcbalign = MAX(tcbalign, libc_tls_init_align);
313 size = roundup2(libc_tls_static_space, tcbalign);
314
315 dtv = ((Elf_Addr**)tcb)[1];
316 tlsend = (Elf_Addr) tcb;
317 tlsstart = tlsend - size;
318 libc_free_aligned((void*)tlsstart);
319 __je_bootstrap_free(dtv);
320 }
321
322 /*
323 * Allocate Static TLS using the Variant II method.
324 */
325 void *
__libc_allocate_tls(void * oldtls,size_t tcbsize,size_t tcbalign)326 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
327 {
328 size_t size;
329 char *tls;
330 Elf_Addr *dtv;
331 Elf_Addr segbase, oldsegbase;
332
333 tcbalign = MAX(tcbalign, libc_tls_init_align);
334 size = roundup2(libc_tls_static_space, tcbalign);
335
336 if (tcbsize < 2 * sizeof(Elf_Addr))
337 tcbsize = 2 * sizeof(Elf_Addr);
338 tls = libc_malloc_aligned(size + tcbsize, tcbalign);
339 if (tls == NULL) {
340 tls_msg("__libc_allocate_tls: Out of memory.\n");
341 abort();
342 }
343 memset(tls, 0, size + tcbsize);
344 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
345 if (dtv == NULL) {
346 tls_msg("__libc_allocate_tls: Out of memory.\n");
347 abort();
348 }
349
350 segbase = (Elf_Addr)(tls + size);
351 ((Elf_Addr*)segbase)[0] = segbase;
352 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv;
353
354 dtv[0] = 1;
355 dtv[1] = 1;
356 dtv[2] = segbase - libc_tls_static_space;
357
358 if (oldtls) {
359 /*
360 * Copy the static TLS block over whole.
361 */
362 oldsegbase = (Elf_Addr) oldtls;
363 memcpy((void *)(segbase - libc_tls_static_space),
364 (const void *)(oldsegbase - libc_tls_static_space),
365 libc_tls_static_space);
366
367 /*
368 * We assume that this block was the one we created with
369 * allocate_initial_tls().
370 */
371 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
372 } else {
373 memcpy((void *)(segbase - libc_tls_static_space),
374 libc_tls_init, libc_tls_init_size);
375 memset((void *)(segbase - libc_tls_static_space +
376 libc_tls_init_size), 0,
377 libc_tls_static_space - libc_tls_init_size);
378 }
379
380 return (void*) segbase;
381 }
382
383 #endif /* TLS_VARIANT_II */
384
385 #else
386
387 void *
__libc_allocate_tls(void * oldtls __unused,size_t tcbsize __unused,size_t tcbalign __unused)388 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused,
389 size_t tcbalign __unused)
390 {
391 return (0);
392 }
393
394 void
__libc_free_tls(void * tcb __unused,size_t tcbsize __unused,size_t tcbalign __unused)395 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused,
396 size_t tcbalign __unused)
397 {
398 }
399
400 #endif /* PIC */
401
402 void
_init_tls(void)403 _init_tls(void)
404 {
405 #ifndef PIC
406 Elf_Addr *sp;
407 Elf_Auxinfo *aux, *auxp;
408 Elf_Phdr *phdr;
409 size_t phent, phnum;
410 int i;
411 void *tls;
412
413 sp = (Elf_Addr *) environ;
414 while (*sp++ != 0)
415 ;
416 aux = (Elf_Auxinfo *) sp;
417 phdr = NULL;
418 phent = phnum = 0;
419 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) {
420 switch (auxp->a_type) {
421 case AT_PHDR:
422 phdr = auxp->a_un.a_ptr;
423 break;
424
425 case AT_PHENT:
426 phent = auxp->a_un.a_val;
427 break;
428
429 case AT_PHNUM:
430 phnum = auxp->a_un.a_val;
431 break;
432 }
433 }
434 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0)
435 return;
436
437 for (i = 0; (unsigned) i < phnum; i++) {
438 if (phdr[i].p_type == PT_TLS) {
439 libc_tls_static_space = roundup2(phdr[i].p_memsz,
440 phdr[i].p_align);
441 libc_tls_init_size = phdr[i].p_filesz;
442 libc_tls_init_align = phdr[i].p_align;
443 libc_tls_init = (void *)phdr[i].p_vaddr;
444 break;
445 }
446 }
447 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN);
448
449 _tcb_set(tls);
450 #endif
451 }
452