1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
23 * $FreeBSD$
24 */
25
26 /*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31 #ifdef illumos
32 #pragma ident "%Z%%M% %I% %E% SMI"
33 #endif
34
35 #include <sys/fasttrap_isa.h>
36 #include <sys/fasttrap_impl.h>
37 #include <sys/dtrace.h>
38 #include <sys/dtrace_impl.h>
39 #include <sys/cmn_err.h>
40 #ifdef illumos
41 #include <sys/regset.h>
42 #include <sys/privregs.h>
43 #include <sys/segments.h>
44 #include <sys/x86_archext.h>
45 #else
46 #include <cddl/dev/dtrace/dtrace_cddl.h>
47 #include <sys/types.h>
48 #include <sys/proc.h>
49 #include <sys/dtrace_bsd.h>
50 #include <cddl/dev/dtrace/x86/regset.h>
51 #include <machine/segments.h>
52 #include <machine/reg.h>
53 #include <machine/pcb.h>
54 #endif
55 #include <sys/sysmacros.h>
56 #ifdef illumos
57 #include <sys/trap.h>
58 #include <sys/archsystm.h>
59 #else
60 #include <sys/ptrace.h>
61
62 static int
uread(proc_t * p,void * kaddr,size_t len,uintptr_t uaddr)63 uread(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr)
64 {
65 ssize_t n;
66
67 PHOLD(p);
68 n = proc_readmem(curthread, p, uaddr, kaddr, len);
69 PRELE(p);
70 if (n != len)
71 return (ENOMEM);
72 return (0);
73 }
74
75 static int
uwrite(proc_t * p,void * kaddr,size_t len,uintptr_t uaddr)76 uwrite(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr)
77 {
78 ssize_t n;
79
80 PHOLD(p);
81 n = proc_writemem(curthread, p, uaddr, kaddr, len);
82 PRELE(p);
83 if (n != len)
84 return (ENOMEM);
85 return (0);
86 }
87
88 #endif /* illumos */
89 #ifdef __i386__
90 #define r_rax r_eax
91 #define r_rbx r_ebx
92 #define r_rip r_eip
93 #define r_rflags r_eflags
94 #define r_rsp r_esp
95 #define r_rbp r_ebp
96 #endif
97
98 /*
99 * Lossless User-Land Tracing on x86
100 * ---------------------------------
101 *
102 * The execution of most instructions is not dependent on the address; for
103 * these instructions it is sufficient to copy them into the user process's
104 * address space and execute them. To effectively single-step an instruction
105 * in user-land, we copy out the following sequence of instructions to scratch
106 * space in the user thread's ulwp_t structure.
107 *
108 * We then set the program counter (%eip or %rip) to point to this scratch
109 * space. Once execution resumes, the original instruction is executed and
110 * then control flow is redirected to what was originally the subsequent
111 * instruction. If the kernel attemps to deliver a signal while single-
112 * stepping, the signal is deferred and the program counter is moved into the
113 * second sequence of instructions. The second sequence ends in a trap into
114 * the kernel where the deferred signal is then properly handled and delivered.
115 *
116 * For instructions whose execute is position dependent, we perform simple
117 * emulation. These instructions are limited to control transfer
118 * instructions in 32-bit mode, but in 64-bit mode there's the added wrinkle
119 * of %rip-relative addressing that means that almost any instruction can be
120 * position dependent. For all the details on how we emulate generic
121 * instructions included %rip-relative instructions, see the code in
122 * fasttrap_pid_probe() below where we handle instructions of type
123 * FASTTRAP_T_COMMON (under the header: Generic Instruction Tracing).
124 */
125
126 #define FASTTRAP_MODRM_MOD(modrm) (((modrm) >> 6) & 0x3)
127 #define FASTTRAP_MODRM_REG(modrm) (((modrm) >> 3) & 0x7)
128 #define FASTTRAP_MODRM_RM(modrm) ((modrm) & 0x7)
129 #define FASTTRAP_MODRM(mod, reg, rm) (((mod) << 6) | ((reg) << 3) | (rm))
130
131 #define FASTTRAP_SIB_SCALE(sib) (((sib) >> 6) & 0x3)
132 #define FASTTRAP_SIB_INDEX(sib) (((sib) >> 3) & 0x7)
133 #define FASTTRAP_SIB_BASE(sib) ((sib) & 0x7)
134
135 #define FASTTRAP_REX_W(rex) (((rex) >> 3) & 1)
136 #define FASTTRAP_REX_R(rex) (((rex) >> 2) & 1)
137 #define FASTTRAP_REX_X(rex) (((rex) >> 1) & 1)
138 #define FASTTRAP_REX_B(rex) ((rex) & 1)
139 #define FASTTRAP_REX(w, r, x, b) \
140 (0x40 | ((w) << 3) | ((r) << 2) | ((x) << 1) | (b))
141
142 /*
143 * Single-byte op-codes.
144 */
145 #define FASTTRAP_PUSHL_EBP 0x55
146
147 #define FASTTRAP_JO 0x70
148 #define FASTTRAP_JNO 0x71
149 #define FASTTRAP_JB 0x72
150 #define FASTTRAP_JAE 0x73
151 #define FASTTRAP_JE 0x74
152 #define FASTTRAP_JNE 0x75
153 #define FASTTRAP_JBE 0x76
154 #define FASTTRAP_JA 0x77
155 #define FASTTRAP_JS 0x78
156 #define FASTTRAP_JNS 0x79
157 #define FASTTRAP_JP 0x7a
158 #define FASTTRAP_JNP 0x7b
159 #define FASTTRAP_JL 0x7c
160 #define FASTTRAP_JGE 0x7d
161 #define FASTTRAP_JLE 0x7e
162 #define FASTTRAP_JG 0x7f
163
164 #define FASTTRAP_NOP 0x90
165
166 #define FASTTRAP_MOV_EAX 0xb8
167 #define FASTTRAP_MOV_ECX 0xb9
168
169 #define FASTTRAP_RET16 0xc2
170 #define FASTTRAP_RET 0xc3
171
172 #define FASTTRAP_LOOPNZ 0xe0
173 #define FASTTRAP_LOOPZ 0xe1
174 #define FASTTRAP_LOOP 0xe2
175 #define FASTTRAP_JCXZ 0xe3
176
177 #define FASTTRAP_CALL 0xe8
178 #define FASTTRAP_JMP32 0xe9
179 #define FASTTRAP_JMP8 0xeb
180
181 #define FASTTRAP_INT3 0xcc
182 #define FASTTRAP_INT 0xcd
183
184 #define FASTTRAP_2_BYTE_OP 0x0f
185 #define FASTTRAP_GROUP5_OP 0xff
186
187 /*
188 * Two-byte op-codes (second byte only).
189 */
190 #define FASTTRAP_0F_JO 0x80
191 #define FASTTRAP_0F_JNO 0x81
192 #define FASTTRAP_0F_JB 0x82
193 #define FASTTRAP_0F_JAE 0x83
194 #define FASTTRAP_0F_JE 0x84
195 #define FASTTRAP_0F_JNE 0x85
196 #define FASTTRAP_0F_JBE 0x86
197 #define FASTTRAP_0F_JA 0x87
198 #define FASTTRAP_0F_JS 0x88
199 #define FASTTRAP_0F_JNS 0x89
200 #define FASTTRAP_0F_JP 0x8a
201 #define FASTTRAP_0F_JNP 0x8b
202 #define FASTTRAP_0F_JL 0x8c
203 #define FASTTRAP_0F_JGE 0x8d
204 #define FASTTRAP_0F_JLE 0x8e
205 #define FASTTRAP_0F_JG 0x8f
206
207 #define FASTTRAP_EFLAGS_OF 0x800
208 #define FASTTRAP_EFLAGS_DF 0x400
209 #define FASTTRAP_EFLAGS_SF 0x080
210 #define FASTTRAP_EFLAGS_ZF 0x040
211 #define FASTTRAP_EFLAGS_AF 0x010
212 #define FASTTRAP_EFLAGS_PF 0x004
213 #define FASTTRAP_EFLAGS_CF 0x001
214
215 /*
216 * Instruction prefixes.
217 */
218 #define FASTTRAP_PREFIX_OPERAND 0x66
219 #define FASTTRAP_PREFIX_ADDRESS 0x67
220 #define FASTTRAP_PREFIX_CS 0x2E
221 #define FASTTRAP_PREFIX_DS 0x3E
222 #define FASTTRAP_PREFIX_ES 0x26
223 #define FASTTRAP_PREFIX_FS 0x64
224 #define FASTTRAP_PREFIX_GS 0x65
225 #define FASTTRAP_PREFIX_SS 0x36
226 #define FASTTRAP_PREFIX_LOCK 0xF0
227 #define FASTTRAP_PREFIX_REP 0xF3
228 #define FASTTRAP_PREFIX_REPNE 0xF2
229
230 #define FASTTRAP_NOREG 0xff
231
232 /*
233 * Map between instruction register encodings and the kernel constants which
234 * correspond to indicies into struct regs.
235 */
236 #ifdef __amd64
237 static const uint8_t regmap[16] = {
238 REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI,
239 REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15,
240 };
241 #else
242 static const uint8_t regmap[8] = {
243 EAX, ECX, EDX, EBX, UESP, EBP, ESI, EDI
244 };
245 #endif
246
247 static ulong_t fasttrap_getreg(struct reg *, uint_t);
248
249 static uint64_t
fasttrap_anarg(struct reg * rp,int function_entry,int argno)250 fasttrap_anarg(struct reg *rp, int function_entry, int argno)
251 {
252 uint64_t value = 0;
253 int shift = function_entry ? 1 : 0;
254
255 #ifdef __amd64
256 if (curproc->p_model == DATAMODEL_LP64) {
257 uintptr_t *stack;
258
259 /*
260 * In 64-bit mode, the first six arguments are stored in
261 * registers.
262 */
263 if (argno < 6)
264 switch (argno) {
265 case 0:
266 return (rp->r_rdi);
267 case 1:
268 return (rp->r_rsi);
269 case 2:
270 return (rp->r_rdx);
271 case 3:
272 return (rp->r_rcx);
273 case 4:
274 return (rp->r_r8);
275 case 5:
276 return (rp->r_r9);
277 }
278
279 stack = (uintptr_t *)rp->r_rsp;
280 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
281 value = dtrace_fulword(&stack[argno - 6 + shift]);
282 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
283 } else {
284 #endif
285 #ifdef __i386
286 uint32_t *stack = (uint32_t *)rp->r_esp;
287 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
288 value = dtrace_fuword32(&stack[argno + shift]);
289 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
290 #endif
291 #ifdef __amd64
292 }
293 #endif
294
295 return (value);
296 }
297
298 /*ARGSUSED*/
299 int
fasttrap_tracepoint_init(proc_t * p,fasttrap_tracepoint_t * tp,uintptr_t pc,fasttrap_probe_type_t type)300 fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, uintptr_t pc,
301 fasttrap_probe_type_t type)
302 {
303 uint8_t instr[FASTTRAP_MAX_INSTR_SIZE + 10];
304 size_t len = FASTTRAP_MAX_INSTR_SIZE;
305 size_t first = MIN(len, PAGESIZE - (pc & PAGEOFFSET));
306 uint_t start = 0;
307 int rmindex, size;
308 uint8_t seg, rex = 0;
309
310 /*
311 * Read the instruction at the given address out of the process's
312 * address space. We don't have to worry about a debugger
313 * changing this instruction before we overwrite it with our trap
314 * instruction since P_PR_LOCK is set. Since instructions can span
315 * pages, we potentially read the instruction in two parts. If the
316 * second part fails, we just zero out that part of the instruction.
317 */
318 if (uread(p, &instr[0], first, pc) != 0)
319 return (-1);
320 if (len > first &&
321 uread(p, &instr[first], len - first, pc + first) != 0) {
322 bzero(&instr[first], len - first);
323 len = first;
324 }
325
326 /*
327 * If the disassembly fails, then we have a malformed instruction.
328 */
329 if ((size = dtrace_instr_size_isa(instr, p->p_model, &rmindex)) <= 0)
330 return (-1);
331
332 /*
333 * Make sure the disassembler isn't completely broken.
334 */
335 ASSERT(-1 <= rmindex && rmindex < size);
336
337 /*
338 * If the computed size is greater than the number of bytes read,
339 * then it was a malformed instruction possibly because it fell on a
340 * page boundary and the subsequent page was missing or because of
341 * some malicious user.
342 */
343 if (size > len)
344 return (-1);
345
346 tp->ftt_size = (uint8_t)size;
347 tp->ftt_segment = FASTTRAP_SEG_NONE;
348
349 /*
350 * Find the start of the instruction's opcode by processing any
351 * legacy prefixes.
352 */
353 for (;;) {
354 seg = 0;
355 switch (instr[start]) {
356 case FASTTRAP_PREFIX_SS:
357 seg++;
358 /*FALLTHRU*/
359 case FASTTRAP_PREFIX_GS:
360 seg++;
361 /*FALLTHRU*/
362 case FASTTRAP_PREFIX_FS:
363 seg++;
364 /*FALLTHRU*/
365 case FASTTRAP_PREFIX_ES:
366 seg++;
367 /*FALLTHRU*/
368 case FASTTRAP_PREFIX_DS:
369 seg++;
370 /*FALLTHRU*/
371 case FASTTRAP_PREFIX_CS:
372 seg++;
373 /*FALLTHRU*/
374 case FASTTRAP_PREFIX_OPERAND:
375 case FASTTRAP_PREFIX_ADDRESS:
376 case FASTTRAP_PREFIX_LOCK:
377 case FASTTRAP_PREFIX_REP:
378 case FASTTRAP_PREFIX_REPNE:
379 if (seg != 0) {
380 /*
381 * It's illegal for an instruction to specify
382 * two segment prefixes -- give up on this
383 * illegal instruction.
384 */
385 if (tp->ftt_segment != FASTTRAP_SEG_NONE)
386 return (-1);
387
388 tp->ftt_segment = seg;
389 }
390 start++;
391 continue;
392 }
393 break;
394 }
395
396 #ifdef __amd64
397 /*
398 * Identify the REX prefix on 64-bit processes.
399 */
400 if (p->p_model == DATAMODEL_LP64 && (instr[start] & 0xf0) == 0x40)
401 rex = instr[start++];
402 #endif
403
404 /*
405 * Now that we're pretty sure that the instruction is okay, copy the
406 * valid part to the tracepoint.
407 */
408 bcopy(instr, tp->ftt_instr, FASTTRAP_MAX_INSTR_SIZE);
409
410 tp->ftt_type = FASTTRAP_T_COMMON;
411 if (instr[start] == FASTTRAP_2_BYTE_OP) {
412 switch (instr[start + 1]) {
413 case FASTTRAP_0F_JO:
414 case FASTTRAP_0F_JNO:
415 case FASTTRAP_0F_JB:
416 case FASTTRAP_0F_JAE:
417 case FASTTRAP_0F_JE:
418 case FASTTRAP_0F_JNE:
419 case FASTTRAP_0F_JBE:
420 case FASTTRAP_0F_JA:
421 case FASTTRAP_0F_JS:
422 case FASTTRAP_0F_JNS:
423 case FASTTRAP_0F_JP:
424 case FASTTRAP_0F_JNP:
425 case FASTTRAP_0F_JL:
426 case FASTTRAP_0F_JGE:
427 case FASTTRAP_0F_JLE:
428 case FASTTRAP_0F_JG:
429 tp->ftt_type = FASTTRAP_T_JCC;
430 tp->ftt_code = (instr[start + 1] & 0x0f) | FASTTRAP_JO;
431 tp->ftt_dest = pc + tp->ftt_size +
432 /* LINTED - alignment */
433 *(int32_t *)&instr[start + 2];
434 break;
435 }
436 } else if (instr[start] == FASTTRAP_GROUP5_OP) {
437 uint_t mod = FASTTRAP_MODRM_MOD(instr[start + 1]);
438 uint_t reg = FASTTRAP_MODRM_REG(instr[start + 1]);
439 uint_t rm = FASTTRAP_MODRM_RM(instr[start + 1]);
440
441 if (reg == 2 || reg == 4) {
442 uint_t i, sz;
443
444 if (reg == 2)
445 tp->ftt_type = FASTTRAP_T_CALL;
446 else
447 tp->ftt_type = FASTTRAP_T_JMP;
448
449 if (mod == 3)
450 tp->ftt_code = 2;
451 else
452 tp->ftt_code = 1;
453
454 ASSERT(p->p_model == DATAMODEL_LP64 || rex == 0);
455
456 /*
457 * See AMD x86-64 Architecture Programmer's Manual
458 * Volume 3, Section 1.2.7, Table 1-12, and
459 * Appendix A.3.1, Table A-15.
460 */
461 if (mod != 3 && rm == 4) {
462 uint8_t sib = instr[start + 2];
463 uint_t index = FASTTRAP_SIB_INDEX(sib);
464 uint_t base = FASTTRAP_SIB_BASE(sib);
465
466 tp->ftt_scale = FASTTRAP_SIB_SCALE(sib);
467
468 tp->ftt_index = (index == 4) ?
469 FASTTRAP_NOREG :
470 regmap[index | (FASTTRAP_REX_X(rex) << 3)];
471 tp->ftt_base = (mod == 0 && base == 5) ?
472 FASTTRAP_NOREG :
473 regmap[base | (FASTTRAP_REX_B(rex) << 3)];
474
475 i = 3;
476 sz = mod == 1 ? 1 : 4;
477 } else {
478 /*
479 * In 64-bit mode, mod == 0 and r/m == 5
480 * denotes %rip-relative addressing; in 32-bit
481 * mode, the base register isn't used. In both
482 * modes, there is a 32-bit operand.
483 */
484 if (mod == 0 && rm == 5) {
485 #ifdef __amd64
486 if (p->p_model == DATAMODEL_LP64)
487 tp->ftt_base = REG_RIP;
488 else
489 #endif
490 tp->ftt_base = FASTTRAP_NOREG;
491 sz = 4;
492 } else {
493 uint8_t base = rm |
494 (FASTTRAP_REX_B(rex) << 3);
495
496 tp->ftt_base = regmap[base];
497 sz = mod == 1 ? 1 : mod == 2 ? 4 : 0;
498 }
499 tp->ftt_index = FASTTRAP_NOREG;
500 i = 2;
501 }
502
503 if (sz == 1) {
504 tp->ftt_dest = *(int8_t *)&instr[start + i];
505 } else if (sz == 4) {
506 /* LINTED - alignment */
507 tp->ftt_dest = *(int32_t *)&instr[start + i];
508 } else {
509 tp->ftt_dest = 0;
510 }
511 }
512 } else {
513 switch (instr[start]) {
514 case FASTTRAP_RET:
515 tp->ftt_type = FASTTRAP_T_RET;
516 break;
517
518 case FASTTRAP_RET16:
519 tp->ftt_type = FASTTRAP_T_RET16;
520 /* LINTED - alignment */
521 tp->ftt_dest = *(uint16_t *)&instr[start + 1];
522 break;
523
524 case FASTTRAP_JO:
525 case FASTTRAP_JNO:
526 case FASTTRAP_JB:
527 case FASTTRAP_JAE:
528 case FASTTRAP_JE:
529 case FASTTRAP_JNE:
530 case FASTTRAP_JBE:
531 case FASTTRAP_JA:
532 case FASTTRAP_JS:
533 case FASTTRAP_JNS:
534 case FASTTRAP_JP:
535 case FASTTRAP_JNP:
536 case FASTTRAP_JL:
537 case FASTTRAP_JGE:
538 case FASTTRAP_JLE:
539 case FASTTRAP_JG:
540 tp->ftt_type = FASTTRAP_T_JCC;
541 tp->ftt_code = instr[start];
542 tp->ftt_dest = pc + tp->ftt_size +
543 (int8_t)instr[start + 1];
544 break;
545
546 case FASTTRAP_LOOPNZ:
547 case FASTTRAP_LOOPZ:
548 case FASTTRAP_LOOP:
549 tp->ftt_type = FASTTRAP_T_LOOP;
550 tp->ftt_code = instr[start];
551 tp->ftt_dest = pc + tp->ftt_size +
552 (int8_t)instr[start + 1];
553 break;
554
555 case FASTTRAP_JCXZ:
556 tp->ftt_type = FASTTRAP_T_JCXZ;
557 tp->ftt_dest = pc + tp->ftt_size +
558 (int8_t)instr[start + 1];
559 break;
560
561 case FASTTRAP_CALL:
562 tp->ftt_type = FASTTRAP_T_CALL;
563 tp->ftt_dest = pc + tp->ftt_size +
564 /* LINTED - alignment */
565 *(int32_t *)&instr[start + 1];
566 tp->ftt_code = 0;
567 break;
568
569 case FASTTRAP_JMP32:
570 tp->ftt_type = FASTTRAP_T_JMP;
571 tp->ftt_dest = pc + tp->ftt_size +
572 /* LINTED - alignment */
573 *(int32_t *)&instr[start + 1];
574 break;
575 case FASTTRAP_JMP8:
576 tp->ftt_type = FASTTRAP_T_JMP;
577 tp->ftt_dest = pc + tp->ftt_size +
578 (int8_t)instr[start + 1];
579 break;
580
581 case FASTTRAP_PUSHL_EBP:
582 if (start == 0)
583 tp->ftt_type = FASTTRAP_T_PUSHL_EBP;
584 break;
585
586 case FASTTRAP_NOP:
587 #ifdef __amd64
588 ASSERT(p->p_model == DATAMODEL_LP64 || rex == 0);
589
590 /*
591 * On amd64 we have to be careful not to confuse a nop
592 * (actually xchgl %eax, %eax) with an instruction using
593 * the same opcode, but that does something different
594 * (e.g. xchgl %r8d, %eax or xcghq %r8, %rax).
595 */
596 if (FASTTRAP_REX_B(rex) == 0)
597 #endif
598 tp->ftt_type = FASTTRAP_T_NOP;
599 break;
600
601 case FASTTRAP_INT3:
602 /*
603 * The pid provider shares the int3 trap with debugger
604 * breakpoints so we can't instrument them.
605 */
606 ASSERT(instr[start] == FASTTRAP_INSTR);
607 return (-1);
608
609 case FASTTRAP_INT:
610 /*
611 * Interrupts seem like they could be traced with
612 * no negative implications, but it's possible that
613 * a thread could be redirected by the trap handling
614 * code which would eventually return to the
615 * instruction after the interrupt. If the interrupt
616 * were in our scratch space, the subsequent
617 * instruction might be overwritten before we return.
618 * Accordingly we refuse to instrument any interrupt.
619 */
620 return (-1);
621 }
622 }
623
624 #ifdef __amd64
625 if (p->p_model == DATAMODEL_LP64 && tp->ftt_type == FASTTRAP_T_COMMON) {
626 /*
627 * If the process is 64-bit and the instruction type is still
628 * FASTTRAP_T_COMMON -- meaning we're going to copy it out an
629 * execute it -- we need to watch for %rip-relative
630 * addressing mode. See the portion of fasttrap_pid_probe()
631 * below where we handle tracepoints with type
632 * FASTTRAP_T_COMMON for how we emulate instructions that
633 * employ %rip-relative addressing.
634 */
635 if (rmindex != -1) {
636 uint_t mod = FASTTRAP_MODRM_MOD(instr[rmindex]);
637 uint_t reg = FASTTRAP_MODRM_REG(instr[rmindex]);
638 uint_t rm = FASTTRAP_MODRM_RM(instr[rmindex]);
639
640 ASSERT(rmindex > start);
641
642 if (mod == 0 && rm == 5) {
643 /*
644 * We need to be sure to avoid other
645 * registers used by this instruction. While
646 * the reg field may determine the op code
647 * rather than denoting a register, assuming
648 * that it denotes a register is always safe.
649 * We leave the REX field intact and use
650 * whatever value's there for simplicity.
651 */
652 if (reg != 0) {
653 tp->ftt_ripmode = FASTTRAP_RIP_1 |
654 (FASTTRAP_RIP_X *
655 FASTTRAP_REX_B(rex));
656 rm = 0;
657 } else {
658 tp->ftt_ripmode = FASTTRAP_RIP_2 |
659 (FASTTRAP_RIP_X *
660 FASTTRAP_REX_B(rex));
661 rm = 1;
662 }
663
664 tp->ftt_modrm = tp->ftt_instr[rmindex];
665 tp->ftt_instr[rmindex] =
666 FASTTRAP_MODRM(2, reg, rm);
667 }
668 }
669 }
670 #endif
671
672 return (0);
673 }
674
675 int
fasttrap_tracepoint_install(proc_t * p,fasttrap_tracepoint_t * tp)676 fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp)
677 {
678 fasttrap_instr_t instr = FASTTRAP_INSTR;
679
680 if (uwrite(p, &instr, 1, tp->ftt_pc) != 0)
681 return (-1);
682
683 return (0);
684 }
685
686 int
fasttrap_tracepoint_remove(proc_t * p,fasttrap_tracepoint_t * tp)687 fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp)
688 {
689 uint8_t instr;
690
691 /*
692 * Distinguish between read or write failures and a changed
693 * instruction.
694 */
695 if (uread(p, &instr, 1, tp->ftt_pc) != 0)
696 return (0);
697 if (instr != FASTTRAP_INSTR)
698 return (0);
699 if (uwrite(p, &tp->ftt_instr[0], 1, tp->ftt_pc) != 0)
700 return (-1);
701
702 return (0);
703 }
704
705 #ifdef __amd64
706 static uintptr_t
fasttrap_fulword_noerr(const void * uaddr)707 fasttrap_fulword_noerr(const void *uaddr)
708 {
709 uintptr_t ret;
710
711 if ((ret = fasttrap_fulword(uaddr)) != -1)
712 return (ret);
713
714 return (0);
715 }
716 #endif
717
718 #ifdef __i386__
719 static uint32_t
fasttrap_fuword32_noerr(const void * uaddr)720 fasttrap_fuword32_noerr(const void *uaddr)
721 {
722 uint32_t ret;
723
724 if ((ret = fasttrap_fuword32(uaddr)) != -1)
725 return (ret);
726
727 return (0);
728 }
729 #endif
730
731 static void
fasttrap_return_common(struct reg * rp,uintptr_t pc,pid_t pid,uintptr_t new_pc)732 fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
733 uintptr_t new_pc)
734 {
735 fasttrap_tracepoint_t *tp;
736 fasttrap_bucket_t *bucket;
737 fasttrap_id_t *id;
738 #ifdef illumos
739 kmutex_t *pid_mtx;
740 #endif
741
742 #ifdef illumos
743 pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
744 mutex_enter(pid_mtx);
745 #endif
746 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
747
748 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
749 if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
750 tp->ftt_proc->ftpc_acount != 0)
751 break;
752 }
753
754 /*
755 * Don't sweat it if we can't find the tracepoint again; unlike
756 * when we're in fasttrap_pid_probe(), finding the tracepoint here
757 * is not essential to the correct execution of the process.
758 */
759 if (tp == NULL) {
760 #ifdef illumos
761 mutex_exit(pid_mtx);
762 #endif
763 return;
764 }
765
766 for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
767 /*
768 * If there's a branch that could act as a return site, we
769 * need to trace it, and check here if the program counter is
770 * external to the function.
771 */
772 if (tp->ftt_type != FASTTRAP_T_RET &&
773 tp->ftt_type != FASTTRAP_T_RET16 &&
774 new_pc - id->fti_probe->ftp_faddr <
775 id->fti_probe->ftp_fsize)
776 continue;
777
778 dtrace_probe(id->fti_probe->ftp_id,
779 pc - id->fti_probe->ftp_faddr,
780 rp->r_rax, rp->r_rbx, 0, 0);
781 }
782
783 #ifdef illumos
784 mutex_exit(pid_mtx);
785 #endif
786 }
787
788 static void
fasttrap_sigsegv(proc_t * p,kthread_t * t,uintptr_t addr)789 fasttrap_sigsegv(proc_t *p, kthread_t *t, uintptr_t addr)
790 {
791 #ifdef illumos
792 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
793
794 sqp->sq_info.si_signo = SIGSEGV;
795 sqp->sq_info.si_code = SEGV_MAPERR;
796 sqp->sq_info.si_addr = (caddr_t)addr;
797
798 mutex_enter(&p->p_lock);
799 sigaddqa(p, t, sqp);
800 mutex_exit(&p->p_lock);
801
802 if (t != NULL)
803 aston(t);
804 #else
805 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
806
807 ksiginfo_init(ksi);
808 ksi->ksi_signo = SIGSEGV;
809 ksi->ksi_code = SEGV_MAPERR;
810 ksi->ksi_addr = (caddr_t)addr;
811 (void) tdksignal(t, SIGSEGV, ksi);
812 #endif
813 }
814
815 #ifdef __amd64
816 static void
fasttrap_usdt_args64(fasttrap_probe_t * probe,struct reg * rp,int argc,uintptr_t * argv)817 fasttrap_usdt_args64(fasttrap_probe_t *probe, struct reg *rp, int argc,
818 uintptr_t *argv)
819 {
820 int i, x, cap = MIN(argc, probe->ftp_nargs);
821 uintptr_t *stack = (uintptr_t *)rp->r_rsp;
822
823 for (i = 0; i < cap; i++) {
824 x = probe->ftp_argmap[i];
825
826 if (x < 6)
827 argv[i] = (&rp->r_rdi)[x];
828 else
829 argv[i] = fasttrap_fulword_noerr(&stack[x]);
830 }
831
832 for (; i < argc; i++) {
833 argv[i] = 0;
834 }
835 }
836 #endif
837
838 #ifdef __i386__
839 static void
fasttrap_usdt_args32(fasttrap_probe_t * probe,struct reg * rp,int argc,uint32_t * argv)840 fasttrap_usdt_args32(fasttrap_probe_t *probe, struct reg *rp, int argc,
841 uint32_t *argv)
842 {
843 int i, x, cap = MIN(argc, probe->ftp_nargs);
844 uint32_t *stack = (uint32_t *)rp->r_rsp;
845
846 for (i = 0; i < cap; i++) {
847 x = probe->ftp_argmap[i];
848
849 argv[i] = fasttrap_fuword32_noerr(&stack[x]);
850 }
851
852 for (; i < argc; i++) {
853 argv[i] = 0;
854 }
855 }
856 #endif
857
858 static int
fasttrap_do_seg(fasttrap_tracepoint_t * tp,struct reg * rp,uintptr_t * addr)859 fasttrap_do_seg(fasttrap_tracepoint_t *tp, struct reg *rp, uintptr_t *addr)
860 {
861 proc_t *p = curproc;
862 #ifdef __i386__
863 struct segment_descriptor *desc;
864 #else
865 struct user_segment_descriptor *desc;
866 #endif
867 uint16_t sel = 0, ndx, type;
868 uintptr_t limit;
869
870 switch (tp->ftt_segment) {
871 case FASTTRAP_SEG_CS:
872 sel = rp->r_cs;
873 break;
874 case FASTTRAP_SEG_DS:
875 sel = rp->r_ds;
876 break;
877 case FASTTRAP_SEG_ES:
878 sel = rp->r_es;
879 break;
880 case FASTTRAP_SEG_FS:
881 sel = rp->r_fs;
882 break;
883 case FASTTRAP_SEG_GS:
884 sel = rp->r_gs;
885 break;
886 case FASTTRAP_SEG_SS:
887 sel = rp->r_ss;
888 break;
889 }
890
891 /*
892 * Make sure the given segment register specifies a user priority
893 * selector rather than a kernel selector.
894 */
895 if (ISPL(sel) != SEL_UPL)
896 return (-1);
897
898 ndx = IDXSEL(sel);
899
900 /*
901 * Check the bounds and grab the descriptor out of the specified
902 * descriptor table.
903 */
904 if (ISLDT(sel)) {
905 #ifdef __i386__
906 if (ndx > p->p_md.md_ldt->ldt_len)
907 return (-1);
908
909 desc = (struct segment_descriptor *)
910 p->p_md.md_ldt[ndx].ldt_base;
911 #else
912 if (ndx > max_ldt_segment)
913 return (-1);
914
915 desc = (struct user_segment_descriptor *)
916 p->p_md.md_ldt[ndx].ldt_base;
917 #endif
918
919 } else {
920 if (ndx >= NGDT)
921 return (-1);
922
923 #ifdef __i386__
924 desc = &gdt[ndx].sd;
925 #else
926 desc = &gdt[ndx];
927 #endif
928 }
929
930 /*
931 * The descriptor must have user privilege level and it must be
932 * present in memory.
933 */
934 if (desc->sd_dpl != SEL_UPL || desc->sd_p != 1)
935 return (-1);
936
937 type = desc->sd_type;
938
939 /*
940 * If the S bit in the type field is not set, this descriptor can
941 * only be used in system context.
942 */
943 if ((type & 0x10) != 0x10)
944 return (-1);
945
946 limit = USD_GETLIMIT(desc) * (desc->sd_gran ? PAGESIZE : 1);
947
948 if (tp->ftt_segment == FASTTRAP_SEG_CS) {
949 /*
950 * The code/data bit and readable bit must both be set.
951 */
952 if ((type & 0xa) != 0xa)
953 return (-1);
954
955 if (*addr > limit)
956 return (-1);
957 } else {
958 /*
959 * The code/data bit must be clear.
960 */
961 if ((type & 0x8) != 0)
962 return (-1);
963
964 /*
965 * If the expand-down bit is clear, we just check the limit as
966 * it would naturally be applied. Otherwise, we need to check
967 * that the address is the range [limit + 1 .. 0xffff] or
968 * [limit + 1 ... 0xffffffff] depending on if the default
969 * operand size bit is set.
970 */
971 if ((type & 0x4) == 0) {
972 if (*addr > limit)
973 return (-1);
974 } else if (desc->sd_def32) {
975 if (*addr < limit + 1 || 0xffff < *addr)
976 return (-1);
977 } else {
978 if (*addr < limit + 1 || 0xffffffff < *addr)
979 return (-1);
980 }
981 }
982
983 *addr += USD_GETBASE(desc);
984
985 return (0);
986 }
987
988 int
fasttrap_pid_probe(struct reg * rp)989 fasttrap_pid_probe(struct reg *rp)
990 {
991 proc_t *p = curproc;
992 #ifndef illumos
993 proc_t *pp;
994 #endif
995 uintptr_t pc = rp->r_rip - 1;
996 uintptr_t new_pc = 0;
997 fasttrap_bucket_t *bucket;
998 #ifdef illumos
999 kmutex_t *pid_mtx;
1000 #endif
1001 fasttrap_tracepoint_t *tp, tp_local;
1002 pid_t pid;
1003 dtrace_icookie_t cookie;
1004 uint_t is_enabled = 0;
1005
1006 /*
1007 * It's possible that a user (in a veritable orgy of bad planning)
1008 * could redirect this thread's flow of control before it reached the
1009 * return probe fasttrap. In this case we need to kill the process
1010 * since it's in a unrecoverable state.
1011 */
1012 if (curthread->t_dtrace_step) {
1013 ASSERT(curthread->t_dtrace_on);
1014 fasttrap_sigtrap(p, curthread, pc);
1015 return (0);
1016 }
1017
1018 /*
1019 * Clear all user tracing flags.
1020 */
1021 curthread->t_dtrace_ft = 0;
1022 curthread->t_dtrace_pc = 0;
1023 curthread->t_dtrace_npc = 0;
1024 curthread->t_dtrace_scrpc = 0;
1025 curthread->t_dtrace_astpc = 0;
1026 #ifdef __amd64
1027 curthread->t_dtrace_regv = 0;
1028 #endif
1029
1030 /*
1031 * Treat a child created by a call to vfork(2) as if it were its
1032 * parent. We know that there's only one thread of control in such a
1033 * process: this one.
1034 */
1035 #ifdef illumos
1036 while (p->p_flag & SVFORK) {
1037 p = p->p_parent;
1038 }
1039
1040 pid = p->p_pid;
1041 pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
1042 mutex_enter(pid_mtx);
1043 #else
1044 pp = p;
1045 sx_slock(&proctree_lock);
1046 while (pp->p_vmspace == pp->p_pptr->p_vmspace)
1047 pp = pp->p_pptr;
1048 pid = pp->p_pid;
1049 sx_sunlock(&proctree_lock);
1050 pp = NULL;
1051
1052 PROC_LOCK(p);
1053 _PHOLD(p);
1054 #endif
1055
1056 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
1057
1058 /*
1059 * Lookup the tracepoint that the process just hit.
1060 */
1061 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
1062 if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
1063 tp->ftt_proc->ftpc_acount != 0)
1064 break;
1065 }
1066
1067 /*
1068 * If we couldn't find a matching tracepoint, either a tracepoint has
1069 * been inserted without using the pid<pid> ioctl interface (see
1070 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
1071 */
1072 if (tp == NULL) {
1073 #ifdef illumos
1074 mutex_exit(pid_mtx);
1075 #else
1076 _PRELE(p);
1077 PROC_UNLOCK(p);
1078 #endif
1079 return (-1);
1080 }
1081
1082 /*
1083 * Set the program counter to the address of the traced instruction
1084 * so that it looks right in ustack() output.
1085 */
1086 rp->r_rip = pc;
1087
1088 if (tp->ftt_ids != NULL) {
1089 fasttrap_id_t *id;
1090
1091 #ifdef __amd64
1092 if (p->p_model == DATAMODEL_LP64) {
1093 for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
1094 fasttrap_probe_t *probe = id->fti_probe;
1095
1096 if (id->fti_ptype == DTFTP_ENTRY) {
1097 /*
1098 * We note that this was an entry
1099 * probe to help ustack() find the
1100 * first caller.
1101 */
1102 cookie = dtrace_interrupt_disable();
1103 DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
1104 dtrace_probe(probe->ftp_id, rp->r_rdi,
1105 rp->r_rsi, rp->r_rdx, rp->r_rcx,
1106 rp->r_r8);
1107 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
1108 dtrace_interrupt_enable(cookie);
1109 } else if (id->fti_ptype == DTFTP_IS_ENABLED) {
1110 /*
1111 * Note that in this case, we don't
1112 * call dtrace_probe() since it's only
1113 * an artificial probe meant to change
1114 * the flow of control so that it
1115 * encounters the true probe.
1116 */
1117 is_enabled = 1;
1118 } else if (probe->ftp_argmap == NULL) {
1119 dtrace_probe(probe->ftp_id, rp->r_rdi,
1120 rp->r_rsi, rp->r_rdx, rp->r_rcx,
1121 rp->r_r8);
1122 } else {
1123 uintptr_t t[5];
1124
1125 fasttrap_usdt_args64(probe, rp,
1126 sizeof (t) / sizeof (t[0]), t);
1127
1128 dtrace_probe(probe->ftp_id, t[0], t[1],
1129 t[2], t[3], t[4]);
1130 }
1131 }
1132 } else {
1133 #else /* __amd64 */
1134 uintptr_t s0, s1, s2, s3, s4, s5;
1135 uint32_t *stack = (uint32_t *)rp->r_esp;
1136
1137 /*
1138 * In 32-bit mode, all arguments are passed on the
1139 * stack. If this is a function entry probe, we need
1140 * to skip the first entry on the stack as it
1141 * represents the return address rather than a
1142 * parameter to the function.
1143 */
1144 s0 = fasttrap_fuword32_noerr(&stack[0]);
1145 s1 = fasttrap_fuword32_noerr(&stack[1]);
1146 s2 = fasttrap_fuword32_noerr(&stack[2]);
1147 s3 = fasttrap_fuword32_noerr(&stack[3]);
1148 s4 = fasttrap_fuword32_noerr(&stack[4]);
1149 s5 = fasttrap_fuword32_noerr(&stack[5]);
1150
1151 for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
1152 fasttrap_probe_t *probe = id->fti_probe;
1153
1154 if (id->fti_ptype == DTFTP_ENTRY) {
1155 /*
1156 * We note that this was an entry
1157 * probe to help ustack() find the
1158 * first caller.
1159 */
1160 cookie = dtrace_interrupt_disable();
1161 DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
1162 dtrace_probe(probe->ftp_id, s1, s2,
1163 s3, s4, s5);
1164 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
1165 dtrace_interrupt_enable(cookie);
1166 } else if (id->fti_ptype == DTFTP_IS_ENABLED) {
1167 /*
1168 * Note that in this case, we don't
1169 * call dtrace_probe() since it's only
1170 * an artificial probe meant to change
1171 * the flow of control so that it
1172 * encounters the true probe.
1173 */
1174 is_enabled = 1;
1175 } else if (probe->ftp_argmap == NULL) {
1176 dtrace_probe(probe->ftp_id, s0, s1,
1177 s2, s3, s4);
1178 } else {
1179 uint32_t t[5];
1180
1181 fasttrap_usdt_args32(probe, rp,
1182 sizeof (t) / sizeof (t[0]), t);
1183
1184 dtrace_probe(probe->ftp_id, t[0], t[1],
1185 t[2], t[3], t[4]);
1186 }
1187 }
1188 #endif /* __amd64 */
1189 #ifdef __amd64
1190 }
1191 #endif
1192 }
1193
1194 /*
1195 * We're about to do a bunch of work so we cache a local copy of
1196 * the tracepoint to emulate the instruction, and then find the
1197 * tracepoint again later if we need to light up any return probes.
1198 */
1199 tp_local = *tp;
1200 #ifdef illumos
1201 mutex_exit(pid_mtx);
1202 #else
1203 PROC_UNLOCK(p);
1204 #endif
1205 tp = &tp_local;
1206
1207 /*
1208 * Set the program counter to appear as though the traced instruction
1209 * had completely executed. This ensures that fasttrap_getreg() will
1210 * report the expected value for REG_RIP.
1211 */
1212 rp->r_rip = pc + tp->ftt_size;
1213
1214 /*
1215 * If there's an is-enabled probe connected to this tracepoint it
1216 * means that there was a 'xorl %eax, %eax' or 'xorq %rax, %rax'
1217 * instruction that was placed there by DTrace when the binary was
1218 * linked. As this probe is, in fact, enabled, we need to stuff 1
1219 * into %eax or %rax. Accordingly, we can bypass all the instruction
1220 * emulation logic since we know the inevitable result. It's possible
1221 * that a user could construct a scenario where the 'is-enabled'
1222 * probe was on some other instruction, but that would be a rather
1223 * exotic way to shoot oneself in the foot.
1224 */
1225 if (is_enabled) {
1226 rp->r_rax = 1;
1227 new_pc = rp->r_rip;
1228 goto done;
1229 }
1230
1231 /*
1232 * We emulate certain types of instructions to ensure correctness
1233 * (in the case of position dependent instructions) or optimize
1234 * common cases. The rest we have the thread execute back in user-
1235 * land.
1236 */
1237 switch (tp->ftt_type) {
1238 case FASTTRAP_T_RET:
1239 case FASTTRAP_T_RET16:
1240 {
1241 uintptr_t dst = 0;
1242 uintptr_t addr = 0;
1243 int ret = 0;
1244
1245 /*
1246 * We have to emulate _every_ facet of the behavior of a ret
1247 * instruction including what happens if the load from %esp
1248 * fails; in that case, we send a SIGSEGV.
1249 */
1250 #ifdef __amd64
1251 if (p->p_model == DATAMODEL_NATIVE) {
1252 ret = dst = fasttrap_fulword((void *)rp->r_rsp);
1253 addr = rp->r_rsp + sizeof (uintptr_t);
1254 } else {
1255 #endif
1256 #ifdef __i386__
1257 uint32_t dst32;
1258 ret = dst32 = fasttrap_fuword32((void *)rp->r_esp);
1259 dst = dst32;
1260 addr = rp->r_esp + sizeof (uint32_t);
1261 #endif
1262 #ifdef __amd64
1263 }
1264 #endif
1265
1266 if (ret == -1) {
1267 fasttrap_sigsegv(p, curthread, rp->r_rsp);
1268 new_pc = pc;
1269 break;
1270 }
1271
1272 if (tp->ftt_type == FASTTRAP_T_RET16)
1273 addr += tp->ftt_dest;
1274
1275 rp->r_rsp = addr;
1276 new_pc = dst;
1277 break;
1278 }
1279
1280 case FASTTRAP_T_JCC:
1281 {
1282 uint_t taken = 0;
1283
1284 switch (tp->ftt_code) {
1285 case FASTTRAP_JO:
1286 taken = (rp->r_rflags & FASTTRAP_EFLAGS_OF) != 0;
1287 break;
1288 case FASTTRAP_JNO:
1289 taken = (rp->r_rflags & FASTTRAP_EFLAGS_OF) == 0;
1290 break;
1291 case FASTTRAP_JB:
1292 taken = (rp->r_rflags & FASTTRAP_EFLAGS_CF) != 0;
1293 break;
1294 case FASTTRAP_JAE:
1295 taken = (rp->r_rflags & FASTTRAP_EFLAGS_CF) == 0;
1296 break;
1297 case FASTTRAP_JE:
1298 taken = (rp->r_rflags & FASTTRAP_EFLAGS_ZF) != 0;
1299 break;
1300 case FASTTRAP_JNE:
1301 taken = (rp->r_rflags & FASTTRAP_EFLAGS_ZF) == 0;
1302 break;
1303 case FASTTRAP_JBE:
1304 taken = (rp->r_rflags & FASTTRAP_EFLAGS_CF) != 0 ||
1305 (rp->r_rflags & FASTTRAP_EFLAGS_ZF) != 0;
1306 break;
1307 case FASTTRAP_JA:
1308 taken = (rp->r_rflags & FASTTRAP_EFLAGS_CF) == 0 &&
1309 (rp->r_rflags & FASTTRAP_EFLAGS_ZF) == 0;
1310 break;
1311 case FASTTRAP_JS:
1312 taken = (rp->r_rflags & FASTTRAP_EFLAGS_SF) != 0;
1313 break;
1314 case FASTTRAP_JNS:
1315 taken = (rp->r_rflags & FASTTRAP_EFLAGS_SF) == 0;
1316 break;
1317 case FASTTRAP_JP:
1318 taken = (rp->r_rflags & FASTTRAP_EFLAGS_PF) != 0;
1319 break;
1320 case FASTTRAP_JNP:
1321 taken = (rp->r_rflags & FASTTRAP_EFLAGS_PF) == 0;
1322 break;
1323 case FASTTRAP_JL:
1324 taken = ((rp->r_rflags & FASTTRAP_EFLAGS_SF) == 0) !=
1325 ((rp->r_rflags & FASTTRAP_EFLAGS_OF) == 0);
1326 break;
1327 case FASTTRAP_JGE:
1328 taken = ((rp->r_rflags & FASTTRAP_EFLAGS_SF) == 0) ==
1329 ((rp->r_rflags & FASTTRAP_EFLAGS_OF) == 0);
1330 break;
1331 case FASTTRAP_JLE:
1332 taken = (rp->r_rflags & FASTTRAP_EFLAGS_ZF) != 0 ||
1333 ((rp->r_rflags & FASTTRAP_EFLAGS_SF) == 0) !=
1334 ((rp->r_rflags & FASTTRAP_EFLAGS_OF) == 0);
1335 break;
1336 case FASTTRAP_JG:
1337 taken = (rp->r_rflags & FASTTRAP_EFLAGS_ZF) == 0 &&
1338 ((rp->r_rflags & FASTTRAP_EFLAGS_SF) == 0) ==
1339 ((rp->r_rflags & FASTTRAP_EFLAGS_OF) == 0);
1340 break;
1341
1342 }
1343
1344 if (taken)
1345 new_pc = tp->ftt_dest;
1346 else
1347 new_pc = pc + tp->ftt_size;
1348 break;
1349 }
1350
1351 case FASTTRAP_T_LOOP:
1352 {
1353 uint_t taken = 0;
1354 #ifdef __amd64
1355 greg_t cx = rp->r_rcx--;
1356 #else
1357 greg_t cx = rp->r_ecx--;
1358 #endif
1359
1360 switch (tp->ftt_code) {
1361 case FASTTRAP_LOOPNZ:
1362 taken = (rp->r_rflags & FASTTRAP_EFLAGS_ZF) == 0 &&
1363 cx != 0;
1364 break;
1365 case FASTTRAP_LOOPZ:
1366 taken = (rp->r_rflags & FASTTRAP_EFLAGS_ZF) != 0 &&
1367 cx != 0;
1368 break;
1369 case FASTTRAP_LOOP:
1370 taken = (cx != 0);
1371 break;
1372 }
1373
1374 if (taken)
1375 new_pc = tp->ftt_dest;
1376 else
1377 new_pc = pc + tp->ftt_size;
1378 break;
1379 }
1380
1381 case FASTTRAP_T_JCXZ:
1382 {
1383 #ifdef __amd64
1384 greg_t cx = rp->r_rcx;
1385 #else
1386 greg_t cx = rp->r_ecx;
1387 #endif
1388
1389 if (cx == 0)
1390 new_pc = tp->ftt_dest;
1391 else
1392 new_pc = pc + tp->ftt_size;
1393 break;
1394 }
1395
1396 case FASTTRAP_T_PUSHL_EBP:
1397 {
1398 int ret = 0;
1399
1400 #ifdef __amd64
1401 if (p->p_model == DATAMODEL_NATIVE) {
1402 rp->r_rsp -= sizeof (uintptr_t);
1403 ret = fasttrap_sulword((void *)rp->r_rsp, rp->r_rbp);
1404 } else {
1405 #endif
1406 #ifdef __i386__
1407 rp->r_rsp -= sizeof (uint32_t);
1408 ret = fasttrap_suword32((void *)rp->r_rsp, rp->r_rbp);
1409 #endif
1410 #ifdef __amd64
1411 }
1412 #endif
1413
1414 if (ret == -1) {
1415 fasttrap_sigsegv(p, curthread, rp->r_rsp);
1416 new_pc = pc;
1417 break;
1418 }
1419
1420 new_pc = pc + tp->ftt_size;
1421 break;
1422 }
1423
1424 case FASTTRAP_T_NOP:
1425 new_pc = pc + tp->ftt_size;
1426 break;
1427
1428 case FASTTRAP_T_JMP:
1429 case FASTTRAP_T_CALL:
1430 if (tp->ftt_code == 0) {
1431 new_pc = tp->ftt_dest;
1432 } else {
1433 uintptr_t value, addr = tp->ftt_dest;
1434
1435 if (tp->ftt_base != FASTTRAP_NOREG)
1436 addr += fasttrap_getreg(rp, tp->ftt_base);
1437 if (tp->ftt_index != FASTTRAP_NOREG)
1438 addr += fasttrap_getreg(rp, tp->ftt_index) <<
1439 tp->ftt_scale;
1440
1441 if (tp->ftt_code == 1) {
1442 /*
1443 * If there's a segment prefix for this
1444 * instruction, we'll need to check permissions
1445 * and bounds on the given selector, and adjust
1446 * the address accordingly.
1447 */
1448 if (tp->ftt_segment != FASTTRAP_SEG_NONE &&
1449 fasttrap_do_seg(tp, rp, &addr) != 0) {
1450 fasttrap_sigsegv(p, curthread, addr);
1451 new_pc = pc;
1452 break;
1453 }
1454
1455 #ifdef __amd64
1456 if (p->p_model == DATAMODEL_NATIVE) {
1457 #endif
1458 if ((value = fasttrap_fulword((void *)addr))
1459 == -1) {
1460 fasttrap_sigsegv(p, curthread,
1461 addr);
1462 new_pc = pc;
1463 break;
1464 }
1465 new_pc = value;
1466 #ifdef __amd64
1467 } else {
1468 uint32_t value32;
1469 addr = (uintptr_t)(uint32_t)addr;
1470 if ((value32 = fasttrap_fuword32((void *)addr))
1471 == -1) {
1472 fasttrap_sigsegv(p, curthread,
1473 addr);
1474 new_pc = pc;
1475 break;
1476 }
1477 new_pc = value32;
1478 }
1479 #endif
1480 } else {
1481 new_pc = addr;
1482 }
1483 }
1484
1485 /*
1486 * If this is a call instruction, we need to push the return
1487 * address onto the stack. If this fails, we send the process
1488 * a SIGSEGV and reset the pc to emulate what would happen if
1489 * this instruction weren't traced.
1490 */
1491 if (tp->ftt_type == FASTTRAP_T_CALL) {
1492 int ret = 0;
1493 uintptr_t addr = 0, pcps;
1494 #ifdef __amd64
1495 if (p->p_model == DATAMODEL_NATIVE) {
1496 addr = rp->r_rsp - sizeof (uintptr_t);
1497 pcps = pc + tp->ftt_size;
1498 ret = fasttrap_sulword((void *)addr, pcps);
1499 } else {
1500 #endif
1501 addr = rp->r_rsp - sizeof (uint32_t);
1502 pcps = (uint32_t)(pc + tp->ftt_size);
1503 ret = fasttrap_suword32((void *)addr, pcps);
1504 #ifdef __amd64
1505 }
1506 #endif
1507
1508 if (ret == -1) {
1509 fasttrap_sigsegv(p, curthread, addr);
1510 new_pc = pc;
1511 break;
1512 }
1513
1514 rp->r_rsp = addr;
1515 }
1516
1517 break;
1518
1519 case FASTTRAP_T_COMMON:
1520 {
1521 uintptr_t addr;
1522 #if defined(__amd64)
1523 uint8_t scratch[2 * FASTTRAP_MAX_INSTR_SIZE + 22];
1524 #else
1525 uint8_t scratch[2 * FASTTRAP_MAX_INSTR_SIZE + 7];
1526 #endif
1527 uint_t i = 0;
1528 #ifdef illumos
1529 klwp_t *lwp = ttolwp(curthread);
1530
1531 /*
1532 * Compute the address of the ulwp_t and step over the
1533 * ul_self pointer. The method used to store the user-land
1534 * thread pointer is very different on 32- and 64-bit
1535 * kernels.
1536 */
1537 #if defined(__amd64)
1538 if (p->p_model == DATAMODEL_LP64) {
1539 addr = lwp->lwp_pcb.pcb_fsbase;
1540 addr += sizeof (void *);
1541 } else {
1542 addr = lwp->lwp_pcb.pcb_gsbase;
1543 addr += sizeof (caddr32_t);
1544 }
1545 #else
1546 addr = USD_GETBASE(&lwp->lwp_pcb.pcb_gsdesc);
1547 addr += sizeof (void *);
1548 #endif
1549 #else /* !illumos */
1550 fasttrap_scrspace_t *scrspace;
1551 scrspace = fasttrap_scraddr(curthread, tp->ftt_proc);
1552 if (scrspace == NULL) {
1553 /*
1554 * We failed to allocate scratch space for this thread.
1555 * Try to write the original instruction back out and
1556 * reset the pc.
1557 */
1558 if (fasttrap_copyout(tp->ftt_instr, (void *)pc,
1559 tp->ftt_size))
1560 fasttrap_sigtrap(p, curthread, pc);
1561 new_pc = pc;
1562 break;
1563 }
1564 addr = scrspace->ftss_addr;
1565 #endif /* illumos */
1566
1567 /*
1568 * Generic Instruction Tracing
1569 * ---------------------------
1570 *
1571 * This is the layout of the scratch space in the user-land
1572 * thread structure for our generated instructions.
1573 *
1574 * 32-bit mode bytes
1575 * ------------------------ -----
1576 * a: <original instruction> <= 15
1577 * jmp <pc + tp->ftt_size> 5
1578 * b: <original instruction> <= 15
1579 * int T_DTRACE_RET 2
1580 * -----
1581 * <= 37
1582 *
1583 * 64-bit mode bytes
1584 * ------------------------ -----
1585 * a: <original instruction> <= 15
1586 * jmp 0(%rip) 6
1587 * <pc + tp->ftt_size> 8
1588 * b: <original instruction> <= 15
1589 * int T_DTRACE_RET 2
1590 * -----
1591 * <= 46
1592 *
1593 * The %pc is set to a, and curthread->t_dtrace_astpc is set
1594 * to b. If we encounter a signal on the way out of the
1595 * kernel, trap() will set %pc to curthread->t_dtrace_astpc
1596 * so that we execute the original instruction and re-enter
1597 * the kernel rather than redirecting to the next instruction.
1598 *
1599 * If there are return probes (so we know that we're going to
1600 * need to reenter the kernel after executing the original
1601 * instruction), the scratch space will just contain the
1602 * original instruction followed by an interrupt -- the same
1603 * data as at b.
1604 *
1605 * %rip-relative Addressing
1606 * ------------------------
1607 *
1608 * There's a further complication in 64-bit mode due to %rip-
1609 * relative addressing. While this is clearly a beneficial
1610 * architectural decision for position independent code, it's
1611 * hard not to see it as a personal attack against the pid
1612 * provider since before there was a relatively small set of
1613 * instructions to emulate; with %rip-relative addressing,
1614 * almost every instruction can potentially depend on the
1615 * address at which it's executed. Rather than emulating
1616 * the broad spectrum of instructions that can now be
1617 * position dependent, we emulate jumps and others as in
1618 * 32-bit mode, and take a different tack for instructions
1619 * using %rip-relative addressing.
1620 *
1621 * For every instruction that uses the ModRM byte, the
1622 * in-kernel disassembler reports its location. We use the
1623 * ModRM byte to identify that an instruction uses
1624 * %rip-relative addressing and to see what other registers
1625 * the instruction uses. To emulate those instructions,
1626 * we modify the instruction to be %rax-relative rather than
1627 * %rip-relative (or %rcx-relative if the instruction uses
1628 * %rax; or %r8- or %r9-relative if the REX.B is present so
1629 * we don't have to rewrite the REX prefix). We then load
1630 * the value that %rip would have been into the scratch
1631 * register and generate an instruction to reset the scratch
1632 * register back to its original value. The instruction
1633 * sequence looks like this:
1634 *
1635 * 64-mode %rip-relative bytes
1636 * ------------------------ -----
1637 * a: <modified instruction> <= 15
1638 * movq $<value>, %<scratch> 6
1639 * jmp 0(%rip) 6
1640 * <pc + tp->ftt_size> 8
1641 * b: <modified instruction> <= 15
1642 * int T_DTRACE_RET 2
1643 * -----
1644 * 52
1645 *
1646 * We set curthread->t_dtrace_regv so that upon receiving
1647 * a signal we can reset the value of the scratch register.
1648 */
1649
1650 ASSERT(tp->ftt_size < FASTTRAP_MAX_INSTR_SIZE);
1651
1652 curthread->t_dtrace_scrpc = addr;
1653 bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
1654 i += tp->ftt_size;
1655
1656 #ifdef __amd64
1657 if (tp->ftt_ripmode != 0) {
1658 greg_t *reg = NULL;
1659
1660 ASSERT(p->p_model == DATAMODEL_LP64);
1661 ASSERT(tp->ftt_ripmode &
1662 (FASTTRAP_RIP_1 | FASTTRAP_RIP_2));
1663
1664 /*
1665 * If this was a %rip-relative instruction, we change
1666 * it to be either a %rax- or %rcx-relative
1667 * instruction (depending on whether those registers
1668 * are used as another operand; or %r8- or %r9-
1669 * relative depending on the value of REX.B). We then
1670 * set that register and generate a movq instruction
1671 * to reset the value.
1672 */
1673 if (tp->ftt_ripmode & FASTTRAP_RIP_X)
1674 scratch[i++] = FASTTRAP_REX(1, 0, 0, 1);
1675 else
1676 scratch[i++] = FASTTRAP_REX(1, 0, 0, 0);
1677
1678 if (tp->ftt_ripmode & FASTTRAP_RIP_1)
1679 scratch[i++] = FASTTRAP_MOV_EAX;
1680 else
1681 scratch[i++] = FASTTRAP_MOV_ECX;
1682
1683 switch (tp->ftt_ripmode) {
1684 case FASTTRAP_RIP_1:
1685 reg = &rp->r_rax;
1686 curthread->t_dtrace_reg = REG_RAX;
1687 break;
1688 case FASTTRAP_RIP_2:
1689 reg = &rp->r_rcx;
1690 curthread->t_dtrace_reg = REG_RCX;
1691 break;
1692 case FASTTRAP_RIP_1 | FASTTRAP_RIP_X:
1693 reg = &rp->r_r8;
1694 curthread->t_dtrace_reg = REG_R8;
1695 break;
1696 case FASTTRAP_RIP_2 | FASTTRAP_RIP_X:
1697 reg = &rp->r_r9;
1698 curthread->t_dtrace_reg = REG_R9;
1699 break;
1700 }
1701
1702 /* LINTED - alignment */
1703 *(uint64_t *)&scratch[i] = *reg;
1704 curthread->t_dtrace_regv = *reg;
1705 *reg = pc + tp->ftt_size;
1706 i += sizeof (uint64_t);
1707 }
1708 #endif
1709
1710 /*
1711 * Generate the branch instruction to what would have
1712 * normally been the subsequent instruction. In 32-bit mode,
1713 * this is just a relative branch; in 64-bit mode this is a
1714 * %rip-relative branch that loads the 64-bit pc value
1715 * immediately after the jmp instruction.
1716 */
1717 #ifdef __amd64
1718 if (p->p_model == DATAMODEL_LP64) {
1719 scratch[i++] = FASTTRAP_GROUP5_OP;
1720 scratch[i++] = FASTTRAP_MODRM(0, 4, 5);
1721 /* LINTED - alignment */
1722 *(uint32_t *)&scratch[i] = 0;
1723 i += sizeof (uint32_t);
1724 /* LINTED - alignment */
1725 *(uint64_t *)&scratch[i] = pc + tp->ftt_size;
1726 i += sizeof (uint64_t);
1727 } else {
1728 #endif
1729 #ifdef __i386__
1730 /*
1731 * Set up the jmp to the next instruction; note that
1732 * the size of the traced instruction cancels out.
1733 */
1734 scratch[i++] = FASTTRAP_JMP32;
1735 /* LINTED - alignment */
1736 *(uint32_t *)&scratch[i] = pc - addr - 5;
1737 i += sizeof (uint32_t);
1738 #endif
1739 #ifdef __amd64
1740 }
1741 #endif
1742
1743 curthread->t_dtrace_astpc = addr + i;
1744 bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
1745 i += tp->ftt_size;
1746 scratch[i++] = FASTTRAP_INT;
1747 scratch[i++] = T_DTRACE_RET;
1748
1749 ASSERT(i <= sizeof (scratch));
1750
1751 #ifdef illumos
1752 if (fasttrap_copyout(scratch, (char *)addr, i)) {
1753 #else
1754 if (uwrite(p, scratch, i, addr)) {
1755 #endif
1756 fasttrap_sigtrap(p, curthread, pc);
1757 new_pc = pc;
1758 break;
1759 }
1760 if (tp->ftt_retids != NULL) {
1761 curthread->t_dtrace_step = 1;
1762 curthread->t_dtrace_ret = 1;
1763 new_pc = curthread->t_dtrace_astpc;
1764 } else {
1765 new_pc = curthread->t_dtrace_scrpc;
1766 }
1767
1768 curthread->t_dtrace_pc = pc;
1769 curthread->t_dtrace_npc = pc + tp->ftt_size;
1770 curthread->t_dtrace_on = 1;
1771 break;
1772 }
1773
1774 default:
1775 panic("fasttrap: mishandled an instruction");
1776 }
1777
1778 done:
1779 /*
1780 * If there were no return probes when we first found the tracepoint,
1781 * we should feel no obligation to honor any return probes that were
1782 * subsequently enabled -- they'll just have to wait until the next
1783 * time around.
1784 */
1785 if (tp->ftt_retids != NULL) {
1786 /*
1787 * We need to wait until the results of the instruction are
1788 * apparent before invoking any return probes. If this
1789 * instruction was emulated we can just call
1790 * fasttrap_return_common(); if it needs to be executed, we
1791 * need to wait until the user thread returns to the kernel.
1792 */
1793 if (tp->ftt_type != FASTTRAP_T_COMMON) {
1794 /*
1795 * Set the program counter to the address of the traced
1796 * instruction so that it looks right in ustack()
1797 * output. We had previously set it to the end of the
1798 * instruction to simplify %rip-relative addressing.
1799 */
1800 rp->r_rip = pc;
1801
1802 fasttrap_return_common(rp, pc, pid, new_pc);
1803 } else {
1804 ASSERT(curthread->t_dtrace_ret != 0);
1805 ASSERT(curthread->t_dtrace_pc == pc);
1806 ASSERT(curthread->t_dtrace_scrpc != 0);
1807 ASSERT(new_pc == curthread->t_dtrace_astpc);
1808 }
1809 }
1810
1811 rp->r_rip = new_pc;
1812
1813 #ifndef illumos
1814 PROC_LOCK(p);
1815 proc_write_regs(curthread, rp);
1816 _PRELE(p);
1817 PROC_UNLOCK(p);
1818 #endif
1819
1820 return (0);
1821 }
1822
1823 int
1824 fasttrap_return_probe(struct reg *rp)
1825 {
1826 proc_t *p = curproc;
1827 uintptr_t pc = curthread->t_dtrace_pc;
1828 uintptr_t npc = curthread->t_dtrace_npc;
1829
1830 curthread->t_dtrace_pc = 0;
1831 curthread->t_dtrace_npc = 0;
1832 curthread->t_dtrace_scrpc = 0;
1833 curthread->t_dtrace_astpc = 0;
1834
1835 #ifdef illumos
1836 /*
1837 * Treat a child created by a call to vfork(2) as if it were its
1838 * parent. We know that there's only one thread of control in such a
1839 * process: this one.
1840 */
1841 while (p->p_flag & SVFORK) {
1842 p = p->p_parent;
1843 }
1844 #endif
1845
1846 /*
1847 * We set rp->r_rip to the address of the traced instruction so
1848 * that it appears to dtrace_probe() that we're on the original
1849 * instruction, and so that the user can't easily detect our
1850 * complex web of lies. dtrace_return_probe() (our caller)
1851 * will correctly set %pc after we return.
1852 */
1853 rp->r_rip = pc;
1854
1855 fasttrap_return_common(rp, pc, p->p_pid, npc);
1856
1857 return (0);
1858 }
1859
1860 /*ARGSUSED*/
1861 uint64_t
1862 fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
1863 int aframes)
1864 {
1865 struct reg r;
1866
1867 fill_regs(curthread, &r);
1868
1869 return (fasttrap_anarg(&r, 1, argno));
1870 }
1871
1872 /*ARGSUSED*/
1873 uint64_t
1874 fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
1875 int aframes)
1876 {
1877 struct reg r;
1878
1879 fill_regs(curthread, &r);
1880
1881 return (fasttrap_anarg(&r, 0, argno));
1882 }
1883
1884 static ulong_t
1885 fasttrap_getreg(struct reg *rp, uint_t reg)
1886 {
1887 #ifdef __amd64
1888 switch (reg) {
1889 case REG_R15: return (rp->r_r15);
1890 case REG_R14: return (rp->r_r14);
1891 case REG_R13: return (rp->r_r13);
1892 case REG_R12: return (rp->r_r12);
1893 case REG_R11: return (rp->r_r11);
1894 case REG_R10: return (rp->r_r10);
1895 case REG_R9: return (rp->r_r9);
1896 case REG_R8: return (rp->r_r8);
1897 case REG_RDI: return (rp->r_rdi);
1898 case REG_RSI: return (rp->r_rsi);
1899 case REG_RBP: return (rp->r_rbp);
1900 case REG_RBX: return (rp->r_rbx);
1901 case REG_RDX: return (rp->r_rdx);
1902 case REG_RCX: return (rp->r_rcx);
1903 case REG_RAX: return (rp->r_rax);
1904 case REG_TRAPNO: return (rp->r_trapno);
1905 case REG_ERR: return (rp->r_err);
1906 case REG_RIP: return (rp->r_rip);
1907 case REG_CS: return (rp->r_cs);
1908 #ifdef illumos
1909 case REG_RFL: return (rp->r_rfl);
1910 #endif
1911 case REG_RSP: return (rp->r_rsp);
1912 case REG_SS: return (rp->r_ss);
1913 case REG_FS: return (rp->r_fs);
1914 case REG_GS: return (rp->r_gs);
1915 case REG_DS: return (rp->r_ds);
1916 case REG_ES: return (rp->r_es);
1917 case REG_FSBASE: return (rdmsr(MSR_FSBASE));
1918 case REG_GSBASE: return (rdmsr(MSR_GSBASE));
1919 }
1920
1921 panic("dtrace: illegal register constant");
1922 /*NOTREACHED*/
1923 #else
1924 #define _NGREG 19
1925 if (reg >= _NGREG)
1926 panic("dtrace: illegal register constant");
1927
1928 return (((greg_t *)&rp->r_gs)[reg]);
1929 #endif
1930 }
1931