1 /* $NetBSD: trap.c,v 1.199 2024/12/28 13:48:07 martin Exp $ */
2
3 /*
4 * Copyright (c) 1996-2002 Eduardo Horvath. All rights reserved.
5 * Copyright (c) 1996
6 * The President and Fellows of Harvard College. All rights reserved.
7 * Copyright (c) 1992, 1993
8 * The Regents of the University of California. All rights reserved.
9 *
10 * This software was developed by the Computer Systems Engineering group
11 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
12 * contributed to Berkeley.
13 *
14 * All advertising materials mentioning features or use of this software
15 * must display the following acknowledgement:
16 * This product includes software developed by the University of
17 * California, Lawrence Berkeley Laboratory.
18 * This product includes software developed by Harvard University.
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 * 1. Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * 2. Redistributions in binary form must reproduce the above copyright
26 * notice, this list of conditions and the following disclaimer in the
27 * documentation and/or other materials provided with the distribution.
28 * 3. All advertising materials mentioning features or use of this software
29 * must display the following acknowledgement:
30 * This product includes software developed by the University of
31 * California, Berkeley and its contributors.
32 * This product includes software developed by Harvard University.
33 * 4. Neither the name of the University nor the names of its contributors
34 * may be used to endorse or promote products derived from this software
35 * without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
38 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
39 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
40 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
41 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
42 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
43 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
44 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
45 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
46 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
47 * SUCH DAMAGE.
48 *
49 * @(#)trap.c 8.4 (Berkeley) 9/23/93
50 */
51
52 #include <sys/cdefs.h>
53 __KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.199 2024/12/28 13:48:07 martin Exp $");
54
55 #include "opt_ddb.h"
56 #include "opt_multiprocessor.h"
57 #include "opt_compat_netbsd32.h"
58
59 #include <sys/param.h>
60 #include <sys/systm.h>
61 #include <sys/pool.h>
62 #include <sys/proc.h>
63 #include <sys/ras.h>
64 #include <sys/kernel.h>
65 #include <sys/resource.h>
66 #include <sys/signal.h>
67 #include <sys/wait.h>
68 #include <sys/syscall.h>
69 #include <sys/syslog.h>
70 #include <sys/kauth.h>
71
72 #include <uvm/uvm_extern.h>
73
74 #include <machine/cpu.h>
75 #include <machine/ctlreg.h>
76 #include <machine/trap.h>
77 #include <machine/instr.h>
78 #include <machine/pcb.h>
79 #include <machine/pmap.h>
80 #include <machine/userret.h>
81
82 #ifdef DDB
83 #include <machine/db_machdep.h>
84 #else
85 #include <machine/frame.h>
86 #endif
87
88 #include <sparc64/sparc64/cache.h>
89
90 #include <sparc/fpu/fpu_extern.h>
91
92 #ifndef offsetof
93 #define offsetof(s, f) ((size_t)&((s *)0)->f)
94 #endif
95
96 #ifdef TRAPSTATS
97 /* trapstats */
98 int protfix = 0;
99 int udmiss = 0; /* Number of normal/nucleus data/text miss/protection faults */
100 int udhit = 0;
101 int udprot = 0;
102 int utmiss = 0;
103 int kdmiss = 0;
104 int kdhit = 0;
105 int kdprot = 0;
106 int ktmiss = 0;
107 int iveccnt = 0; /* number if normal/nucleus interrupt/interrupt vector faults */
108 int uintrcnt = 0;
109 int kiveccnt = 0;
110 int kintrcnt = 0;
111 int intristk = 0; /* interrupts when already on intrstack */
112 int intrpoll = 0; /* interrupts not using vector lists */
113 int wfill = 0;
114 int kwfill = 0;
115 int wspill = 0;
116 int wspillskip = 0;
117 int rftucnt = 0;
118 int rftuld = 0;
119 int rftudone = 0;
120 int rftkcnt[5] = { 0, 0, 0, 0, 0 };
121 #endif
122
123 #ifdef DEBUG
124 #define RW_64 0x1
125 #define RW_ERR 0x2
126 #define RW_FOLLOW 0x4
127 int rwindow_debug = RW_ERR;
128 #define TDB_ADDFLT 0x1
129 #define TDB_TXTFLT 0x2
130 #define TDB_TRAP 0x4
131 #define TDB_SYSCALL 0x8
132 #define TDB_FOLLOW 0x10
133 #define TDB_FRAME 0x20
134 #define TDB_NSAVED 0x40
135 #define TDB_TL 0x80
136 #define TDB_STOPSIG 0x100
137 #define TDB_STOPCALL 0x200
138 #define TDB_STOPCPIO 0x400
139 #define TDB_SYSTOP 0x800
140 int trapdebug = 0/*|TDB_SYSCALL|TDB_STOPSIG|TDB_STOPCPIO|TDB_ADDFLT|TDB_FOLLOW*/;
141 /* #define inline */
142 #endif
143
144 #ifdef DDB
145 #if 1
146 #define DEBUGGER(t,f) do { kdb_trap(t,f); } while (0)
147 #else
148 #define DEBUGGER(t,f) Debugger()
149 #endif
150 #else
151 #define DEBUGGER(t,f)
152 #define Debugger()
153 #endif
154
155 struct evcnt ecc_corrected =
156 EVCNT_INITIALIZER(EVCNT_TYPE_MISC,0,"ECC","corrected");
157 EVCNT_ATTACH_STATIC(ecc_corrected);
158
159 /*
160 * Initial FPU state is all registers == all 1s, everything else == all 0s.
161 * This makes every floating point register a signalling NaN, with sign bit
162 * set, no matter how it is interpreted. Appendix N of the Sparc V8 document
163 * seems to imply that we should do this, and it does make sense.
164 */
165 const struct fpstate64 initfpstate __aligned(SPARC64_BLOCK_SIZE) = {
166 .fs_regs =
167 { ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0,
168 ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0,
169 ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0,
170 ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0 },
171 .fs_qsize = 0
172 };
173
174 /*
175 * There are more than 100 trap types, but most are unused.
176 *
177 * Trap type 0 is taken over as an `Asynchronous System Trap'.
178 * This is left-over Vax emulation crap that should be fixed.
179 *
180 * Traps not supported on the spitfire are marked with `*',
181 * and additions are marked with `+'
182 */
183 static const char T[] = "*trap";
184 const char *trap_type[] = {
185 /* non-user vectors */
186 "ast", /* 0 */
187 "power on reset", /* 1 */
188 "watchdog reset", /* 2 */
189 "externally initiated reset",/*3 */
190 "software initiated reset",/* 4 */
191 "RED state exception", /* 5 */
192 T, T, /* 6..7 */
193 "instruction access exception", /* 8 */
194 "*instruction MMU miss",/* 9 */
195 "instruction access error",/* 0a */
196 T, T, T, T, T, /* 0b..0f */
197 "illegal instruction", /* 10 */
198 "privileged opcode", /* 11 */
199 "*unimplemented LDD", /* 12 */
200 "*unimplemented STD", /* 13 */
201 T, T, T, T, /* 14..17 */
202 T, T, T, T, T, T, T, T, /* 18..1f */
203 "fp disabled", /* 20 */
204 "fp exception ieee 754",/* 21 */
205 "fp exception other", /* 22 */
206 "tag overflow", /* 23 */
207 "clean window", /* 24 */
208 T, T, T, /* 25..27 -- trap continues */
209 "division by zero", /* 28 */
210 "*internal processor error",/* 29 */
211 T, T, T, T, T, T, /* 2a..2f */
212 "data access exception",/* 30 */
213 "*data access MMU miss",/* 31 */
214 "data access error", /* 32 */
215 "*data access protection",/* 33 */
216 "mem address not aligned", /* 34 */
217 "LDDF mem address not aligned",/* 35 */
218 "STDF mem address not aligned",/* 36 */
219 "privileged action", /* 37 */
220 "LDQF mem address not aligned",/* 38 */
221 "STQF mem address not aligned",/* 39 */
222 T, T, T, T, T, T, /* 3a..3f */
223 "*async data error", /* 40 */
224 "level 1 int", /* 41 */
225 "level 2 int", /* 42 */
226 "level 3 int", /* 43 */
227 "level 4 int", /* 44 */
228 "level 5 int", /* 45 */
229 "level 6 int", /* 46 */
230 "level 7 int", /* 47 */
231 "level 8 int", /* 48 */
232 "level 9 int", /* 49 */
233 "level 10 int", /* 4a */
234 "level 11 int", /* 4b */
235 "level 12 int", /* 4c */
236 "level 13 int", /* 4d */
237 "level 14 int", /* 4e */
238 "level 15 int", /* 4f */
239 T, T, T, T, T, T, T, T, /* 50..57 */
240 T, T, T, T, T, T, T, T, /* 58..5f */
241 "+interrupt vector", /* 60 */
242 "+PA_watchpoint", /* 61 */
243 "+VA_watchpoint", /* 62 */
244 "+corrected ECC error", /* 63 */
245 "+fast instruction access MMU miss",/* 64 */
246 T, T, T, /* 65..67 -- trap continues */
247 "+fast data access MMU miss",/* 68 */
248 T, T, T, /* 69..6b -- trap continues */
249 "+fast data access protection",/* 6c */
250 T, T, T, /* 6d..6f -- trap continues */
251 "+fast ECC error", /* 70 */
252 T, T, T, T, T, T, T, /* 71..77 */
253 T, T, T, T, T, T, T, T, /* 78..7f */
254 "spill 0 normal", /* 80 */
255 T, T, T, /* 81..83 -- trap continues */
256 "spill 1 normal", /* 84 */
257 T, T, T, /* 85..87 -- trap continues */
258 "spill 2 normal", /* 88 */
259 T, T, T, /* 89..8b -- trap continues */
260 "spill 3 normal", /* 8c */
261 T, T, T, /* 8d..8f -- trap continues */
262 "spill 4 normal", /* 90 */
263 T, T, T, /* 91..93 -- trap continues */
264 "spill 5 normal", /* 94 */
265 T, T, T, /* 95..97 -- trap continues */
266 "spill 6 normal", /* 98 */
267 T, T, T, /* 99..9b -- trap continues */
268 "spill 7 normal", /* 9c */
269 T, T, T, /* 9c..9f -- trap continues */
270 "spill 0 other", /* a0 */
271 T, T, T, /* a1..a3 -- trap continues */
272 "spill 1 other", /* a4 */
273 T, T, T, /* a5..a7 -- trap continues */
274 "spill 2 other", /* a8 */
275 T, T, T, /* a9..ab -- trap continues */
276 "spill 3 other", /* ac */
277 T, T, T, /* ad..af -- trap continues */
278 "spill 4 other", /* b0 */
279 T, T, T, /* b1..b3 -- trap continues */
280 "spill 5 other", /* b4 */
281 T, T, T, /* b5..b7 -- trap continues */
282 "spill 6 other", /* b8 */
283 T, T, T, /* b9..bb -- trap continues */
284 "spill 7 other", /* bc */
285 T, T, T, /* bc..bf -- trap continues */
286 "fill 0 normal", /* c0 */
287 T, T, T, /* c1..c3 -- trap continues */
288 "fill 1 normal", /* c4 */
289 T, T, T, /* c5..c7 -- trap continues */
290 "fill 2 normal", /* c8 */
291 T, T, T, /* c9..cb -- trap continues */
292 "fill 3 normal", /* cc */
293 T, T, T, /* cd..cf -- trap continues */
294 "fill 4 normal", /* d0 */
295 T, T, T, /* d1..d3 -- trap continues */
296 "fill 5 normal", /* d4 */
297 T, T, T, /* d5..d7 -- trap continues */
298 "fill 6 normal", /* d8 */
299 T, T, T, /* d9..db -- trap continues */
300 "fill 7 normal", /* dc */
301 T, T, T, /* dc..df -- trap continues */
302 "fill 0 other", /* e0 */
303 T, T, T, /* e1..e3 -- trap continues */
304 "fill 1 other", /* e4 */
305 T, T, T, /* e5..e7 -- trap continues */
306 "fill 2 other", /* e8 */
307 T, T, T, /* e9..eb -- trap continues */
308 "fill 3 other", /* ec */
309 T, T, T, /* ed..ef -- trap continues */
310 "fill 4 other", /* f0 */
311 T, T, T, /* f1..f3 -- trap continues */
312 "fill 5 other", /* f4 */
313 T, T, T, /* f5..f7 -- trap continues */
314 "fill 6 other", /* f8 */
315 T, T, T, /* f9..fb -- trap continues */
316 "fill 7 other", /* fc */
317 T, T, T, /* fc..ff -- trap continues */
318
319 /* user (software trap) vectors */
320 "syscall", /* 100 */
321 "breakpoint", /* 101 */
322 "zero divide", /* 102 */
323 "flush windows", /* 103 */
324 "clean windows", /* 104 */
325 "range check", /* 105 */
326 "fix align", /* 106 */
327 "integer overflow", /* 107 */
328 "svr4 syscall", /* 108 */
329 "4.4 syscall", /* 109 */
330 "kgdb exec", /* 10a */
331 T, T, T, T, T, /* 10b..10f */
332 T, T, T, T, T, T, T, T, /* 11a..117 */
333 T, T, T, T, T, T, T, T, /* 118..11f */
334 "svr4 getcc", /* 120 */
335 "svr4 setcc", /* 121 */
336 "svr4 getpsr", /* 122 */
337 "svr4 setpsr", /* 123 */
338 "svr4 gethrtime", /* 124 */
339 "svr4 gethrvtime", /* 125 */
340 T, /* 126 */
341 "svr4 gethrestime", /* 127 */
342 T, T, T, T, T, T, T, T, /* 128..12f */
343 T, T, /* 130..131 */
344 "get condition codes", /* 132 */
345 "set condition codes", /* 133 */
346 T, T, T, T, /* 134..137 */
347 T, T, T, T, T, T, T, T, /* 138..13f */
348 T, T, T, T, T, T, T, T, /* 140..147 */
349 T, T, T, T, T, T, T, T, /* 148..14f */
350 T, T, T, T, T, T, T, T, /* 150..157 */
351 T, T, T, T, T, T, T, T, /* 158..15f */
352 T, T, T, T, /* 160..163 */
353 "SVID syscall64", /* 164 */
354 "SPARC Intl syscall64", /* 165 */
355 "OS vendor spec syscall", /* 166 */
356 "HW OEM syscall", /* 167 */
357 "ret from deferred trap", /* 168 */
358 };
359
360 #define N_TRAP_TYPES (sizeof trap_type / sizeof *trap_type)
361
362 void trap(struct trapframe64 *, unsigned int, vaddr_t, long);
363 void data_access_fault(struct trapframe64 *, unsigned int, vaddr_t, vaddr_t,
364 vaddr_t, u_long);
365 void data_access_error(struct trapframe64 *, unsigned int, vaddr_t, u_long,
366 vaddr_t, u_long);
367 void text_access_fault(struct trapframe64 *tf, unsigned int type, vaddr_t pc,
368 u_long sfsr);
369 void text_access_error(struct trapframe64 *, unsigned int, vaddr_t, u_long,
370 vaddr_t, u_long);
371 void ecc_corrected_error(unsigned int type, vaddr_t pc);
372
373 #ifdef DEBUG
374 void print_trapframe(struct trapframe64 *);
375
376 void
print_trapframe(struct trapframe64 * tf)377 print_trapframe(struct trapframe64 *tf)
378 {
379
380 printf("Trapframe %p:\ttstate: %lx\tpc: %lx\tnpc: %lx\n",
381 tf, (u_long)tf->tf_tstate, (u_long)tf->tf_pc, (u_long)tf->tf_npc);
382 printf("fault: %p\ty: %x\t",
383 (void *)(u_long)tf->tf_fault, (int)tf->tf_y);
384 printf("pil: %d\toldpil: %d\ttt: %x\nGlobals:\n",
385 (int)tf->tf_pil, (int)tf->tf_oldpil, (int)tf->tf_tt);
386 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n",
387 (u_int)(tf->tf_global[0]>>32), (u_int)tf->tf_global[0],
388 (u_int)(tf->tf_global[1]>>32), (u_int)tf->tf_global[1],
389 (u_int)(tf->tf_global[2]>>32), (u_int)tf->tf_global[2],
390 (u_int)(tf->tf_global[3]>>32), (u_int)tf->tf_global[3]);
391 printf("%08x%08x %08x%08x %08x%08x %08x%08x\nOuts:\n",
392 (u_int)(tf->tf_global[4]>>32), (u_int)tf->tf_global[4],
393 (u_int)(tf->tf_global[5]>>32), (u_int)tf->tf_global[5],
394 (u_int)(tf->tf_global[6]>>32), (u_int)tf->tf_global[6],
395 (u_int)(tf->tf_global[7]>>32), (u_int)tf->tf_global[7]);
396 #ifdef DEBUG
397 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n",
398 (u_int)(tf->tf_out[0]>>32), (u_int)tf->tf_out[0],
399 (u_int)(tf->tf_out[1]>>32), (u_int)tf->tf_out[1],
400 (u_int)(tf->tf_out[2]>>32), (u_int)tf->tf_out[2],
401 (u_int)(tf->tf_out[3]>>32), (u_int)tf->tf_out[3]);
402 printf("%08x%08x %08x%08x %08x%08x %08x%08x\nLocals:\n",
403 (u_int)(tf->tf_out[4]>>32), (u_int)tf->tf_out[4],
404 (u_int)(tf->tf_out[5]>>32), (u_int)tf->tf_out[5],
405 (u_int)(tf->tf_out[6]>>32), (u_int)tf->tf_out[6],
406 (u_int)(tf->tf_out[7]>>32), (u_int)tf->tf_out[7]);
407 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n",
408 (u_int)(tf->tf_local[0]>>32), (u_int)tf->tf_local[0],
409 (u_int)(tf->tf_local[1]>>32), (u_int)tf->tf_local[1],
410 (u_int)(tf->tf_local[2]>>32), (u_int)tf->tf_local[2],
411 (u_int)(tf->tf_local[3]>>32), (u_int)tf->tf_local[3]);
412 printf("%08x%08x %08x%08x %08x%08x %08x%08x\nIns:\n",
413 (u_int)(tf->tf_local[4]>>32), (u_int)tf->tf_local[4],
414 (u_int)(tf->tf_local[5]>>32), (u_int)tf->tf_local[5],
415 (u_int)(tf->tf_local[6]>>32), (u_int)tf->tf_local[6],
416 (u_int)(tf->tf_local[7]>>32), (u_int)tf->tf_local[7]);
417 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n",
418 (u_int)(tf->tf_in[0]>>32), (u_int)tf->tf_in[0],
419 (u_int)(tf->tf_in[1]>>32), (u_int)tf->tf_in[1],
420 (u_int)(tf->tf_in[2]>>32), (u_int)tf->tf_in[2],
421 (u_int)(tf->tf_in[3]>>32), (u_int)tf->tf_in[3]);
422 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n",
423 (u_int)(tf->tf_in[4]>>32), (u_int)tf->tf_in[4],
424 (u_int)(tf->tf_in[5]>>32), (u_int)tf->tf_in[5],
425 (u_int)(tf->tf_in[6]>>32), (u_int)tf->tf_in[6],
426 (u_int)(tf->tf_in[7]>>32), (u_int)tf->tf_in[7]);
427 #endif
428
429 }
430 #endif
431
432 /*
433 * Called from locore.s trap handling, for non-MMU-related traps.
434 * (MMU-related traps go through data_access_fault, below.)
435 */
436 void
trap(struct trapframe64 * tf,unsigned int type,vaddr_t pc,long tstate)437 trap(struct trapframe64 *tf, unsigned int type, vaddr_t pc, long tstate)
438 {
439 struct lwp *l;
440 struct proc *p;
441 struct pcb *pcb;
442 int64_t n;
443 u_quad_t sticks;
444 int pstate = tstate >> TSTATE_PSTATE_SHIFT;
445 ksiginfo_t ksi;
446 int error;
447 int code, sig;
448 #ifdef MULTIPROCESSOR
449 int s;
450 #define disintr() s = intr_disable()
451 #define rstintr() intr_restore(s)
452 #else
453 #define disintr() /* nothing */
454 #define rstintr() /* nothing */
455 #endif
456
457 /* This steps the PC over the trap. */
458 #define ADVANCE (n = tf->tf_npc, tf->tf_pc = n, tf->tf_npc = n + 4)
459
460 #ifdef DEBUG
461 if (tf->tf_pc == tf->tf_npc) {
462 printf("trap: tpc %p == tnpc %p\n",
463 (void *)(u_long)tf->tf_pc, (void *)(u_long)tf->tf_npc);
464 Debugger();
465 }
466 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) ||
467 trapdebug & (TDB_FOLLOW | TDB_TRAP)) {
468 char sbuf[sizeof(PSTATE_BITS) + 64];
469
470 printf("trap: type 0x%x: pc=%lx &tf=%p\n",
471 type, pc, tf);
472 snprintb(sbuf, sizeof(sbuf), PSTATE_BITS, pstate);
473 printf(" npc=%lx pstate=%s %s\n",
474 (long)tf->tf_npc, sbuf,
475 type < N_TRAP_TYPES ? trap_type[type] :
476 ((type == T_AST) ? "ast" :
477 ((type == T_RWRET) ? "rwret" : T)));
478 }
479 if ((trapdebug & (TDB_FOLLOW | TDB_TRAP)) ||
480 ((trapdebug & TDB_TL) && gettl())) {
481 char sbuf[sizeof(PSTATE_BITS) + 64];
482
483 extern int trap_trace_dis;
484 trap_trace_dis = 1;
485 printf("trap: type 0x%x: lvl=%d pc=%lx &tf=%p",
486 type, gettl(), pc, tf);
487 snprintb(sbuf, sizeof(sbuf), PSTATE_BITS, pstate);
488 printf(" npc=%lx pstate=%s %s\n",
489 (long)tf->tf_npc, sbuf,
490 type < N_TRAP_TYPES ? trap_type[type] :
491 ((type == T_AST) ? "ast" :
492 ((type == T_RWRET) ? "rwret" : T)));
493 #ifdef DDB
494 kdb_trap(type, tf);
495 #endif
496 }
497 #endif
498
499 curcpu()->ci_data.cpu_ntrap++;
500
501 /*
502 * Generally, kernel traps cause a panic. Any exceptions are
503 * handled early here.
504 */
505 if (pstate & PSTATE_PRIV) {
506 #ifdef DDB
507 if (type == T_BREAKPOINT) {
508 write_all_windows();
509 if (kdb_trap(type, tf)) {
510 /* ADVANCE; */
511 return;
512 }
513 }
514 if (type == T_PA_WATCHPT || type == T_VA_WATCHPT) {
515 if (kdb_trap(type, tf)) {
516 /* DDB must turn off watchpoints or something */
517 return;
518 }
519 }
520 #endif
521 /*
522 * The kernel needs to use FPU registers for block
523 * load/store. If we trap in privileged code, save
524 * the FPU state if there is any and enable the FPU.
525 *
526 * We rely on the kernel code properly enabling the FPU
527 * in %fprs, otherwise we'll hang here trying to enable
528 * the FPU.
529 */
530 if (type == T_FPDISABLED) {
531 struct lwp *newfplwp;
532
533 /* New scheme */
534 if (CLKF_INTR((struct clockframe *)tf) || !curlwp) {
535 newfplwp = &lwp0;
536 } else {
537 newfplwp = curlwp;
538 /* force other cpus to give up this fpstate */
539 if (newfplwp->l_md.md_fpstate)
540 fpusave_lwp(newfplwp, true);
541 }
542 if (fplwp != newfplwp) {
543 disintr();
544 if (fplwp != NULL) {
545 /* someone else had it, maybe? */
546 KASSERT(fplwp->l_md.md_fpstate != NULL);
547 savefpstate(fplwp->l_md.md_fpstate);
548 fplwp = NULL;
549 }
550 rstintr();
551 /* If we have an allocated fpstate, load it */
552 if (newfplwp->l_md.md_fpstate != NULL) {
553 fplwp = newfplwp;
554 loadfpstate(fplwp->l_md.md_fpstate);
555 } else
556 fplwp = NULL;
557 }
558 /* Enable the FPU */
559 tf->tf_tstate |= TSTATE_PEF;
560 return;
561 } else if (type == T_ECCERR) {
562 ecc_corrected_error(type, pc);
563 return;
564 } else if (type == T_FAST_ECC_ERROR) {
565 /* Disable D$, clear error, enable D$, continue. */
566 membar_Sync();
567 sp_blast_dcache_disabled(dcache_size, dcache_line_size);
568 return;
569 }
570 goto dopanic;
571 }
572 l = curlwp;
573 p = l->l_proc;
574 sticks = p->p_sticks;
575 pcb = lwp_getpcb(l);
576 l->l_md.md_tf = tf; /* for ptrace/signals */
577
578 sig = 0;
579
580 switch (type) {
581
582 default:
583 if (type < 0x100) {
584 extern int trap_trace_dis;
585 dopanic:
586 trap_trace_dis = 1;
587
588 {
589 char sb[sizeof(PSTATE_BITS) + 64];
590
591 printf("trap type 0x%x: cpu %d, pc=%lx",
592 type, cpu_number(), pc);
593 snprintb(sb, sizeof(sb), PSTATE_BITS, pstate);
594 printf(" npc=%lx pstate=%s\n",
595 (long)tf->tf_npc, sb);
596 DEBUGGER(type, tf);
597 panic("%s", type < N_TRAP_TYPES ? trap_type[type] : T);
598 }
599 /* NOTREACHED */
600 }
601
602 /* the following message is gratuitous */
603 /* ... but leave it in until we find anything */
604 printf("%s[%d]: unimplemented software trap 0x%x\n",
605 p->p_comm, p->p_pid, type);
606
607 KSI_INIT_TRAP(&ksi);
608 sig = SIGILL;
609 ksi.ksi_trap = type;
610 ksi.ksi_code = ILL_ILLTRP;
611 ksi.ksi_addr = (void *)pc;
612 break;
613
614 case T_AST:
615 want_ast = 0;
616 if (l->l_pflag & LP_OWEUPC) {
617 l->l_pflag &= ~LP_OWEUPC;
618 ADDUPROF(l);
619 }
620 break;
621
622 case T_INST_EXCEPT:
623 case T_TEXTFAULT:
624 #ifdef DEBUG
625 /* This is not an MMU issue!!!! */
626 printf("trap: pid=%d.%d comm=%s textfault at %lx!! sending SIGILL due to trap %d: %s\n",
627 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm,
628 pc, type, type < N_TRAP_TYPES ? trap_type[type] : T);
629 #endif
630 /* FALLTHROUGH */
631 case T_ILLINST:
632 #if defined(DDB) && defined(DEBUG)
633 if (trapdebug & TDB_STOPSIG)
634 Debugger();
635 #endif
636 KSI_INIT_TRAP(&ksi);
637 sig = SIGILL;
638 ksi.ksi_trap = type;
639 ksi.ksi_code = ILL_ILLOPC;
640 ksi.ksi_addr = (void *)pc;
641 break;
642
643 case T_PRIVINST:
644 printf("trap: pid=%d.%d comm=%s privinst!! sending SIGILL due to trap %d: %s\n",
645 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm,
646 type, type < N_TRAP_TYPES ? trap_type[type] : T);
647 #if defined(DDB) && defined(DEBUG)
648 if (trapdebug & TDB_STOPSIG)
649 Debugger();
650 #endif
651 KSI_INIT_TRAP(&ksi);
652 sig = SIGILL;
653 ksi.ksi_trap = type;
654 ksi.ksi_code = ILL_PRVOPC;
655 ksi.ksi_addr = (void *)pc;
656 break;
657
658 case T_PRIVACT:
659 KSI_INIT_TRAP(&ksi);
660 sig = SIGILL;
661 ksi.ksi_trap = type;
662 ksi.ksi_code = ILL_PRVOPC;
663 ksi.ksi_addr = (void *)pc;
664 break;
665
666 case T_FPDISABLED: {
667 struct fpstate64 *fs = l->l_md.md_fpstate;
668
669 if (fs == NULL) {
670 /* NOTE: fpstate must be 64-byte aligned */
671 fs = pool_cache_get(fpstate_cache, PR_WAITOK);
672 *fs = initfpstate;
673 l->l_md.md_fpstate = fs;
674 }
675 /*
676 * We may have more FPEs stored up and/or ops queued.
677 * If they exist, handle them and get out. Otherwise,
678 * resolve the FPU state, turn it on, and try again.
679 *
680 * Ultras should never have a FPU queue.
681 */
682 if (fs->fs_qsize) {
683 printf("trap: Warning fs_qsize is %d\n",fs->fs_qsize);
684 fpu_cleanup(l, fs);
685 break;
686 }
687 if (fplwp != l) { /* we do not have it */
688 /* but maybe another CPU has it? */
689 fpusave_lwp(l, true);
690 disintr();
691 if (fplwp != NULL) { /* someone else had it */
692 KASSERT(fplwp->l_md.md_fpstate != NULL);
693 savefpstate(fplwp->l_md.md_fpstate);
694 }
695 loadfpstate(fs);
696 fplwp = l; /* now we do have it */
697 rstintr();
698 }
699 tf->tf_tstate |= TSTATE_PEF;
700 break;
701 }
702
703 case T_ALIGN:
704 case T_LDDF_ALIGN:
705 case T_STDF_ALIGN:
706 {
707 int64_t dsfsr = 0, dsfar = 0;
708 #ifdef DEBUG
709 int64_t isfsr = 0;
710 #endif
711 if (!CPU_ISSUN4V) {
712 dsfsr = ldxa(SFSR, ASI_DMMU);
713 if (dsfsr & SFSR_FV)
714 dsfar = ldxa(SFAR, ASI_DMMU);
715 } else {
716 paddr_t mmu_fsa_dfa = cpus->ci_mmufsa
717 + offsetof(struct mmufsa, dfa);
718 dsfar = ldxa(mmu_fsa_dfa, ASI_PHYS_CACHED);
719 }
720 #ifdef DEBUG
721 if (!CPU_ISSUN4V) {
722 isfsr = ldxa(SFSR, ASI_IMMU);
723 } else {
724 paddr_t mmu_fsa_ifa = cpus->ci_mmufsa
725 + offsetof(struct mmufsa, ifa);
726 isfsr = ldxa(mmu_fsa_ifa, ASI_PHYS_CACHED);
727 }
728 #endif
729 /*
730 * If we're busy doing copyin/copyout continue
731 */
732 pcb = lwp_getpcb(l);
733 if (pcb && pcb->pcb_onfault) {
734 tf->tf_pc = (vaddr_t)pcb->pcb_onfault;
735 tf->tf_npc = tf->tf_pc + 4;
736 break;
737 }
738
739 #ifdef DEBUG
740 #define fmt64(x) (u_int)((x)>>32), (u_int)((x))
741 if (!CPU_ISSUN4V) {
742 printf("Alignment error: pid=%d.%d comm=%s dsfsr=%08x:%08x "
743 "dsfar=%x:%x isfsr=%08x:%08x pc=%lx\n",
744 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, fmt64(dsfsr), fmt64(dsfar),
745 fmt64(isfsr), pc);
746 } else {
747
748 printf("Alignment error: pid=%d.%d comm=%s pc=%lx\n",
749 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, pc);
750 paddr_t mmufsa_dfa_addr = cpus->ci_mmufsa + offsetof(struct mmufsa, dfa);
751 paddr_t mmufsa_dfc_addr = cpus->ci_mmufsa + offsetof(struct mmufsa, dfc);
752 int64_t dfa = ldxa(mmufsa_dfa_addr, ASI_PHYS_CACHED);
753 printf("dfa = %016lx\n", dfa);
754 int64_t dfc = ldxa(mmufsa_dfc_addr, ASI_PHYS_CACHED);
755 printf("dfc = %016lx\n", dfc);
756 }
757 #endif
758
759 #if defined(DDB) && defined(DEBUG)
760 if (trapdebug & TDB_STOPSIG) {
761 write_all_windows();
762 kdb_trap(type, tf);
763 }
764 #endif
765 if ((l->l_proc->p_md.md_flags & MDP_FIXALIGN) != 0 &&
766 fixalign(l, tf) == 0) {
767 ADVANCE;
768 break;
769 }
770 KSI_INIT_TRAP(&ksi);
771 sig = SIGBUS;
772 ksi.ksi_trap = type;
773 ksi.ksi_code = BUS_ADRALN;
774 ksi.ksi_addr = (void*)(intptr_t)dsfar;
775 }
776 break;
777
778 case T_FP_IEEE_754:
779 case T_FP_OTHER:
780 /*
781 * Clean up after a floating point exception.
782 * fpu_cleanup can (and usually does) modify the
783 * state we save here, so we must `give up' the FPU
784 * chip context. (The software and hardware states
785 * will not match once fpu_cleanup does its job, so
786 * we must not save again later.)
787 */
788 if (l != fplwp)
789 panic("fpe without being the FP user");
790 disintr();
791 KASSERT(l->l_md.md_fpstate != NULL);
792 savefpstate(l->l_md.md_fpstate);
793 fplwp = NULL;
794 rstintr();
795 /* tf->tf_tstate &= ~TSTATE_PEF */ /* share_fpu will do this */
796 if (l->l_md.md_fpstate->fs_qsize == 0) {
797 error = copyin((void *)pc,
798 &l->l_md.md_fpstate->fs_queue[0].fq_instr,
799 sizeof(int));
800 if (error) {
801 sig = SIGBUS;
802 KSI_INIT_TRAP(&ksi);
803 ksi.ksi_trap = type;
804 ksi.ksi_code = BUS_OBJERR;
805 ksi.ksi_addr = (void *)pc;
806 break;
807 }
808 l->l_md.md_fpstate->fs_qsize = 1;
809 code = fpu_cleanup(l, l->l_md.md_fpstate);
810 ADVANCE;
811 } else
812 code = fpu_cleanup(l, l->l_md.md_fpstate);
813
814 if (code != 0) {
815 sig = SIGFPE;
816 KSI_INIT_TRAP(&ksi);
817 ksi.ksi_trap = type;
818 ksi.ksi_code = code;
819 ksi.ksi_addr = (void *)pc;
820 }
821 break;
822
823 case T_TAGOF:
824 KSI_INIT_TRAP(&ksi);
825 sig = SIGEMT;
826 ksi.ksi_trap = type;
827 ksi.ksi_code = SI_NOINFO;
828 ksi.ksi_addr = (void *)pc;
829 break;
830
831 case T_BREAKPOINT:
832 if (p->p_raslist == NULL ||
833 (ras_lookup(p, (void *)(intptr_t)tf->tf_pc) == (void *)-1)) {
834 sig = SIGTRAP;
835 KSI_INIT_TRAP(&ksi);
836 ksi.ksi_trap = type;
837 ksi.ksi_code = TRAP_BRKPT;
838 ksi.ksi_addr = (void *)pc;
839 }
840 break;
841
842 case T_IDIV0:
843 case T_DIV0:
844 ADVANCE;
845 sig = SIGFPE;
846 KSI_INIT_TRAP(&ksi);
847 ksi.ksi_trap = type;
848 ksi.ksi_code = FPE_INTDIV;
849 ksi.ksi_addr = (void *)pc;
850 break;
851
852 case T_CLEANWIN:
853 uprintf("T_CLEANWIN\n"); /* XXX Should not get this */
854 ADVANCE;
855 break;
856
857 case T_FLUSHWIN:
858 /* Software window flush for v8 software */
859 write_all_windows();
860 ADVANCE;
861 break;
862
863 case T_RANGECHECK:
864 printf("T_RANGECHECK\n"); /* XXX */
865 ADVANCE;
866 sig = SIGILL;
867 KSI_INIT_TRAP(&ksi);
868 ksi.ksi_trap = type;
869 ksi.ksi_code = ILL_ILLADR;
870 ksi.ksi_addr = (void *)pc;
871 break;
872
873 case T_FIXALIGN:
874 #ifdef DEBUG_ALIGN
875 uprintf("T_FIXALIGN\n");
876 #endif
877 /* User wants us to fix alignment faults */
878 l->l_proc->p_md.md_flags |= MDP_FIXALIGN;
879 ADVANCE;
880 break;
881
882 case T_INTOF:
883 uprintf("T_INTOF\n"); /* XXX */
884 ADVANCE;
885 sig = SIGFPE;
886 KSI_INIT_TRAP(&ksi);
887 ksi.ksi_trap = type;
888 ksi.ksi_code = FPE_INTOVF;
889 ksi.ksi_addr = (void *)pc;
890 break;
891 case T_ECCERR:
892 ecc_corrected_error(type, pc);
893 break;
894 }
895 if (sig != 0) {
896 ksi.ksi_signo = sig;
897 trapsignal(l, &ksi);
898 }
899 userret(l, pc, sticks);
900 share_fpu(l, tf);
901 #undef ADVANCE
902 #ifdef DEBUG
903 if (trapdebug & (TDB_FOLLOW | TDB_TRAP)) {
904 printf("trap: done\n");
905 /* if (type != T_BREAKPOINT) Debugger(); */
906 }
907 #endif
908 }
909
910 /*
911 * Save windows from PCB into user stack, and return 0. This is used on
912 * window overflow pseudo-traps (from locore.s, just before returning to
913 * user mode) and when ptrace or sendsig needs a consistent state.
914 * As a side effect, rwindow_save() always sets pcb_nsaved to 0.
915 *
916 * If the windows cannot be saved, pcb_nsaved is restored and we return -1.
917 *
918 * XXXXXX This cannot work properly. I need to re-examine this register
919 * window thing entirely.
920 */
921 int
rwindow_save(struct lwp * l)922 rwindow_save(struct lwp *l)
923 {
924 struct pcb *pcb = lwp_getpcb(l);
925 struct rwindow64 *rw = &pcb->pcb_rw[0];
926 uint64_t rwdest;
927 int i, j;
928
929 i = pcb->pcb_nsaved;
930 #ifdef DEBUG
931 if (rwindow_debug & RW_FOLLOW)
932 printf("rwindow_save(%p): nsaved %d\n", l, i);
933 #endif
934 if (i == 0)
935 return (0);
936 #ifdef DEBUG
937 if (rwindow_debug & RW_FOLLOW)
938 printf("%s[%d]: rwindow: pcb->stack:", l->l_proc->p_comm, l->l_proc->p_pid);
939 #endif
940
941 while (i > 0) {
942 rwdest = rw[i--].rw_in[6];
943 #ifdef DEBUG
944 if (rwindow_debug & RW_FOLLOW)
945 printf("window %d at %lx\n", i, (long)rwdest);
946 #endif
947 if (rwdest & 1) {
948 #ifdef DEBUG
949 if (rwindow_debug & RW_64) {
950 printf("rwindow_save: 64-bit tf to %p+BIAS "
951 "or %p\n",
952 (void *)(long)rwdest,
953 (void *)(long)(rwdest+BIAS));
954 Debugger();
955 }
956 #endif
957 rwdest += BIAS;
958 if (copyout((void *)&rw[i], (void *)(u_long)rwdest,
959 sizeof(*rw))) {
960 #ifdef DEBUG
961 if (rwindow_debug & (RW_ERR | RW_64))
962 printf("rwindow_save: 64-bit pcb copyout "
963 "to %p failed\n",
964 (void *)(long)rwdest);
965 #endif
966 return (-1);
967 }
968 #ifdef DEBUG
969 if (rwindow_debug & RW_64) {
970 printf("Finished copyout(%p, %p, %lx)\n",
971 (void *)&rw[i], (void *)(long)rwdest,
972 sizeof(*rw));
973 Debugger();
974 }
975 #endif
976 } else {
977 struct rwindow32 rwstack;
978
979 /* 32-bit window */
980 for (j = 0; j < 8; j++) {
981 rwstack.rw_local[j] = (int)rw[i].rw_local[j];
982 rwstack.rw_in[j] = (int)rw[i].rw_in[j];
983 }
984 /* Must truncate rwdest */
985 if (copyout(&rwstack, (void *)(u_long)(u_int)rwdest,
986 sizeof(rwstack))) {
987 #ifdef DEBUG
988 if (rwindow_debug & RW_ERR)
989 printf("rwindow_save: 32-bit pcb "
990 "copyout to %p (%p) failed\n",
991 (void *)(u_long)(u_int)rwdest,
992 (void *)(u_long)rwdest);
993 #endif
994 return (-1);
995 }
996 }
997 }
998 pcb->pcb_nsaved = 0;
999 #ifdef DEBUG
1000 if (rwindow_debug & RW_FOLLOW) {
1001 printf("\n");
1002 Debugger();
1003 }
1004 #endif
1005 return (0);
1006 }
1007
1008 /*
1009 * Kill user windows (before exec) by writing back to stack or pcb
1010 * and then erasing any pcb tracks. Otherwise we might try to write
1011 * the registers into the new process after the exec.
1012 */
1013 void
cpu_vmspace_exec(struct lwp * l,vaddr_t start,vaddr_t end)1014 cpu_vmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end)
1015 {
1016 struct pcb *pcb = lwp_getpcb(l);
1017
1018 write_user_windows();
1019 pcb->pcb_nsaved = 0;
1020 }
1021
1022 /*
1023 * This routine handles MMU generated faults. About half
1024 * of them could be recoverable through uvm_fault.
1025 */
1026 void
data_access_fault(struct trapframe64 * tf,unsigned int type,vaddr_t pc,vaddr_t addr,vaddr_t sfva,u_long sfsr)1027 data_access_fault(struct trapframe64 *tf, unsigned int type, vaddr_t pc,
1028 vaddr_t addr, vaddr_t sfva, u_long sfsr)
1029 {
1030 uint64_t tstate;
1031 struct lwp *l;
1032 struct proc *p;
1033 struct pcb *pcb;
1034 struct vmspace *vm;
1035 vaddr_t va;
1036 int rv;
1037 vm_prot_t access_type;
1038 vaddr_t onfault;
1039 u_quad_t sticks;
1040 ksiginfo_t ksi;
1041
1042 #ifdef DEBUG
1043 if (tf->tf_pc == tf->tf_npc) {
1044 printf("data_access_fault: tpc %lx == tnpc %lx\n",
1045 (long)tf->tf_pc, (long)tf->tf_npc);
1046 Debugger();
1047 }
1048 write_user_windows();
1049 if ((curpcb->pcb_nsaved > 8) ||
1050 (trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) ||
1051 (trapdebug & (TDB_ADDFLT | TDB_FOLLOW))) {
1052 printf("%ld: data_access_fault(%p, %x, %p, %p, %lx, %lx) "
1053 "nsaved=%d\n",
1054 (long)(curproc?curproc->p_pid:-1), tf, type,
1055 (void *)pc, (void *)addr,
1056 sfva, sfsr, (int)curpcb->pcb_nsaved);
1057 #ifdef DDB
1058 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved))
1059 Debugger();
1060 #endif
1061 }
1062 if (trapdebug & TDB_FRAME) {
1063 print_trapframe(tf);
1064 }
1065 if ((trapdebug & TDB_TL) && gettl()) {
1066 printf("%ld: data_access_fault(%p, %x, %p, %p, %lx, %lx) "
1067 "nsaved=%d\n",
1068 (long)(curproc?curproc->p_pid:-1), tf, type,
1069 (void*)addr, (void*)pc,
1070 sfva, sfsr, (int)curpcb->pcb_nsaved);
1071 Debugger();
1072 }
1073 if (trapdebug & TDB_STOPCALL) {
1074 Debugger();
1075 }
1076 #endif
1077
1078 curcpu()->ci_data.cpu_ntrap++;
1079 l = curlwp;
1080 p = l->l_proc;
1081 pcb = lwp_getpcb(l);
1082 onfault = (vaddr_t)pcb->pcb_onfault;
1083 sticks = p->p_sticks;
1084 tstate = tf->tf_tstate;
1085
1086 #ifdef _LP64
1087 /* deal with invalid VAs early */
1088 if (__predict_false(addr >= (1UL<<HOLESHIFT) && addr < (1UL<<63))) {
1089
1090 if (tstate & TSTATE_PRIV) {
1091 if (onfault) {
1092 tf->tf_pc = onfault;
1093 tf->tf_npc = onfault + 4;
1094 tf->tf_out[0] = EFAULT;
1095 return;
1096 }
1097 panic("fault type %u for invalid va %lx", type, addr);
1098 }
1099
1100 KSI_INIT_TRAP(&ksi);
1101 ksi.ksi_signo = SIGSEGV;
1102 ksi.ksi_code = SEGV_ACCERR;
1103 ksi.ksi_trap = type;
1104 ksi.ksi_addr = (void *)pc;
1105 trapsignal(l, &ksi);
1106 userret(l, pc, sticks);
1107 share_fpu(l, tf);
1108 return;
1109 }
1110 #endif
1111
1112 /* Find the faulting va to give to uvm_fault */
1113 va = trunc_page(addr);
1114
1115 /*
1116 * Now munch on protections.
1117 *
1118 * If it was a FAST_DATA_ACCESS_MMU_MISS we have no idea what the
1119 * access was since the SFSR is not set. But we should never get
1120 * here from there.
1121 */
1122 if (type == T_FDMMU_MISS || (sfsr & SFSR_FV) == 0) {
1123 /* Punt */
1124 access_type = VM_PROT_READ;
1125 } else {
1126 access_type = (sfsr & SFSR_W) ? VM_PROT_WRITE : VM_PROT_READ;
1127 }
1128 if (tstate & TSTATE_PRIV) {
1129 extern char Lfsbail[];
1130
1131 rv = EFAULT;
1132
1133 /*
1134 * If this was an access that we shouldn't try to page in,
1135 * resume at the fault handler without any action.
1136 */
1137 if (onfault == (vaddr_t)Lfsbail)
1138 goto kfault;
1139
1140 /*
1141 * During autoconfiguration, faults are never OK unless
1142 * pcb_onfault is set. Once running normally we must allow
1143 * exec() to cause copy-on-write faults to kernel addresses.
1144 */
1145 if (cold)
1146 goto kfault;
1147 if (!(addr & TLB_TAG_ACCESS_CTX)) {
1148 /* CTXT == NUCLEUS */
1149 curpcb->pcb_onfault = NULL;
1150 rv = uvm_fault(kernel_map, va, access_type);
1151 curpcb->pcb_onfault = (void *)onfault;
1152 #ifdef DEBUG
1153 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW))
1154 printf("cpu%d: data_access_fault: kernel "
1155 "uvm_fault(%p, %lx, %x) "
1156 "sez %x -- %s\n", cpu_number(),
1157 kernel_map, va, access_type, rv,
1158 rv ? "failure" : "success");
1159 #endif
1160 if (rv == 0)
1161 return;
1162 goto kfault;
1163 }
1164 } else {
1165 l->l_md.md_tf = tf;
1166 }
1167
1168 vm = p->p_vmspace;
1169 /* alas! must call the horrible vm code */
1170 pcb->pcb_onfault = NULL;
1171 rv = uvm_fault(&vm->vm_map, va, access_type);
1172 pcb->pcb_onfault = (void *)onfault;
1173
1174 #ifdef DEBUG
1175 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW))
1176 printf("cpu%d: data_access_fault: %s uvm_fault(%p, %lx, %x) "
1177 "sez %x -- %s\n", cpu_number(),
1178 &vm->vm_map == kernel_map ? "kernel!!!" : "user",
1179 &vm->vm_map, va, access_type, rv,
1180 rv ? "failure" : "success");
1181 #endif
1182
1183 /*
1184 * If this was a stack access we keep track of the maximum
1185 * accessed stack size. Also, if uvm_fault gets a protection
1186 * failure it is due to accessing the stack region outside
1187 * the current limit and we need to reflect that as an access
1188 * error.
1189 */
1190 if ((void *)va >= vm->vm_maxsaddr) {
1191 if (rv == 0)
1192 uvm_grow(p, va);
1193 else if (rv == EACCES)
1194 rv = EFAULT;
1195 }
1196 if (rv != 0) {
1197
1198 /*
1199 * Pagein failed. If doing copyin/out, return to onfault
1200 * address. Any other page fault in kernel, die; if user
1201 * fault, deliver SIGSEGV.
1202 */
1203 if (tstate & TSTATE_PRIV) {
1204 kfault:
1205 if (!onfault) {
1206 extern int trap_trace_dis;
1207
1208 /* Disable traptrace for printf */
1209 trap_trace_dis = 1;
1210 (void) splhigh();
1211 printf("cpu%d: data fault: pc=%lx rpc=%"PRIx64" addr=%lx\n",
1212 cpu_number(), pc, tf->tf_in[7], addr);
1213 DEBUGGER(type, tf);
1214 panic("kernel fault");
1215 /* NOTREACHED */
1216 }
1217 #ifdef DEBUG
1218 if (trapdebug &
1219 (TDB_ADDFLT | TDB_FOLLOW | TDB_STOPCPIO)) {
1220 printf("data_access_fault: copyin/out of %p "
1221 "fault -- recover\n", (void *)addr);
1222 DEBUGGER(type, tf);
1223 }
1224 #endif
1225 tf->tf_pc = onfault;
1226 tf->tf_npc = onfault + 4;
1227 tf->tf_out[0] = (rv == EACCES) ? EFAULT : rv;
1228 return;
1229 }
1230 #ifdef DEBUG
1231 if (trapdebug & (TDB_ADDFLT | TDB_STOPSIG)) {
1232 extern int trap_trace_dis;
1233 trap_trace_dis = 1;
1234 printf("cpu%d: data_access_fault at addr %p: "
1235 "sending SIGSEGV\n", cpu_number(), (void *)addr);
1236 printf("%ld: data_access_fault(%p, %x, %p, %p, "
1237 "%lx, %lx) nsaved=%d\n",
1238 (long)(curproc ? curproc->p_pid : -1), tf, type,
1239 (void *)addr, (void *)pc,
1240 sfva, sfsr, (int)curpcb->pcb_nsaved);
1241 Debugger();
1242 }
1243 #endif
1244 KSI_INIT_TRAP(&ksi);
1245 switch (rv) {
1246 case EINVAL:
1247 ksi.ksi_signo = SIGBUS;
1248 ksi.ksi_code = BUS_ADRERR;
1249 break;
1250 case EACCES:
1251 ksi.ksi_signo = SIGSEGV;
1252 ksi.ksi_code = SEGV_ACCERR;
1253 break;
1254 case ENOMEM:
1255 ksi.ksi_signo = SIGKILL;
1256 printf("UVM: pid %d (%s), uid %d killed: out of swap\n",
1257 p->p_pid, p->p_comm,
1258 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
1259 break;
1260 default:
1261 ksi.ksi_signo = SIGSEGV;
1262 ksi.ksi_code = SEGV_MAPERR;
1263 break;
1264 }
1265 ksi.ksi_trap = type;
1266 ksi.ksi_addr = (void *)sfva;
1267 trapsignal(l, &ksi);
1268 }
1269 if ((tstate & TSTATE_PRIV) == 0) {
1270 userret(l, pc, sticks);
1271 share_fpu(l, tf);
1272 }
1273 #ifdef DEBUG
1274 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW))
1275 printf("data_access_fault: done\n");
1276 if (trapdebug & TDB_FRAME) {
1277 print_trapframe(tf);
1278 }
1279 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) {
1280 extern void *return_from_trap(void);
1281
1282 if ((void *)(u_long)tf->tf_pc == (void *)return_from_trap) {
1283 printf("Returning from stack datafault\n");
1284 }
1285 }
1286 #endif
1287 }
1288
1289 /*
1290 * This routine handles deferred errors caused by the memory
1291 * or I/O bus subsystems. Most of these are fatal, and even
1292 * if they are not, recovery is painful. Also, the TPC and
1293 * TNPC values are probably not valid if we're not doing a
1294 * special PEEK/POKE code sequence.
1295 */
1296 void
data_access_error(struct trapframe64 * tf,unsigned int type,vaddr_t afva,u_long afsr,vaddr_t sfva,u_long sfsr)1297 data_access_error(struct trapframe64 *tf, unsigned int type, vaddr_t afva,
1298 u_long afsr, vaddr_t sfva, u_long sfsr)
1299 {
1300 u_long pc;
1301 uint64_t tstate;
1302 struct lwp *l;
1303 struct pcb *pcb;
1304 vaddr_t onfault;
1305 u_quad_t sticks;
1306 ksiginfo_t ksi;
1307
1308 #ifdef DEBUG
1309 if (tf->tf_pc == tf->tf_npc) {
1310 printf("data_access_error: tpc %lx == tnpc %lx\n",
1311 (long)tf->tf_pc, (long)tf->tf_npc);
1312 Debugger();
1313 }
1314 write_user_windows();
1315 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) ||
1316 trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) {
1317 char buf[768];
1318
1319 snprintb(buf, sizeof buf, SFSR_BITS, sfsr);
1320 printf("%d data_access_error(%lx, %lx, %lx, %p)=%lx @ %p %s\n",
1321 curproc?curproc->p_pid:-1,
1322 (long)type, (long)sfva, (long)afva, tf,
1323 (long)tf->tf_tstate,
1324 (void *)(u_long)tf->tf_pc, buf);
1325 }
1326 if (trapdebug & TDB_FRAME) {
1327 print_trapframe(tf);
1328 }
1329 if ((trapdebug & TDB_TL) && gettl()) {
1330 char buf[768];
1331 snprintb(buf, sizeof buf, SFSR_BITS, sfsr);
1332
1333 printf("%d tl %d data_access_error(%lx, %lx, %lx, %p)="
1334 "%lx @ %lx %s\n",
1335 curproc ? curproc->p_pid : -1, gettl(),
1336 (long)type, (long)sfva, (long)afva, tf,
1337 (long)tf->tf_tstate,
1338 (long)tf->tf_pc, buf);
1339 Debugger();
1340 }
1341 if (trapdebug & TDB_STOPCALL) {
1342 Debugger();
1343 }
1344 #endif
1345
1346 curcpu()->ci_data.cpu_ntrap++;
1347 pc = tf->tf_pc;
1348 tstate = tf->tf_tstate;
1349
1350 /*
1351 * Catch PCI config space reads.
1352 */
1353 if (curcpu()->ci_pci_probe) {
1354 #ifdef DIAGNOSTIC
1355 printf("data_access_error in pci_conf_read: pc=%lx addr=%lx\n",
1356 pc, afva);
1357 #endif
1358 curcpu()->ci_pci_fault = true;
1359 /*
1360 * The contents of TNPC are undefined, so make it points to
1361 * the next instruction.
1362 */
1363 tf->tf_npc = pc + 4;
1364 return;
1365 }
1366
1367 l = curlwp;
1368 pcb = lwp_getpcb(l);
1369 sticks = l->l_proc->p_sticks;
1370
1371 printf("data error type %x sfsr=%lx sfva=%lx afsr=%lx afva=%lx tf=%p\n",
1372 type, sfsr, sfva, afsr, afva, tf);
1373
1374 if (afsr == 0) {
1375 printf("data_access_error: no fault\n");
1376 goto out; /* No fault. Why were we called? */
1377 }
1378
1379 if (tstate & TSTATE_PRIV) {
1380 onfault = (vaddr_t)pcb->pcb_onfault;
1381 if (!onfault) {
1382 extern int trap_trace_dis;
1383 char buf[768];
1384
1385 trap_trace_dis = 1; /* Disable traptrace for printf */
1386 snprintb(buf, sizeof buf, SFSR_BITS, sfsr);
1387 (void) splhigh();
1388 printf("data fault: pc=%lx addr=%lx sfsr=%s\n",
1389 (u_long)pc, (long)sfva, buf);
1390 DEBUGGER(type, tf);
1391 panic("kernel fault");
1392 /* NOTREACHED */
1393 }
1394
1395 /*
1396 * If this was a privileged error but not a probe, we
1397 * cannot recover, so panic.
1398 */
1399 if (afsr & ASFR_PRIV) {
1400 char buf[128];
1401
1402 snprintb(buf, sizeof(buf), AFSR_BITS, afsr);
1403 panic("Privileged Async Fault: AFAR %p AFSR %lx\n%s",
1404 (void *)afva, afsr, buf);
1405 /* NOTREACHED */
1406 }
1407 #ifdef DEBUG
1408 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW | TDB_STOPCPIO)) {
1409 printf("data_access_error: kern fault -- "
1410 "skipping instr\n");
1411 if (trapdebug & TDB_STOPCPIO) {
1412 DEBUGGER(type, tf);
1413 }
1414 }
1415 #endif
1416 tf->tf_pc = onfault;
1417 tf->tf_npc = onfault + 4;
1418 tf->tf_out[0] = EFAULT;
1419 return;
1420 }
1421 #ifdef DEBUG
1422 if (trapdebug & (TDB_ADDFLT | TDB_STOPSIG)) {
1423 extern int trap_trace_dis;
1424
1425 trap_trace_dis = 1;
1426 printf("data_access_error at %p: sending SIGSEGV\n",
1427 (void *)(u_long)afva);
1428 Debugger();
1429 }
1430 #endif
1431 KSI_INIT_TRAP(&ksi);
1432 ksi.ksi_signo = SIGSEGV;
1433 ksi.ksi_code = SEGV_MAPERR;
1434 ksi.ksi_trap = type;
1435 ksi.ksi_addr = (void *)afva;
1436 trapsignal(l, &ksi);
1437 out:
1438 if ((tstate & TSTATE_PRIV) == 0) {
1439 userret(l, pc, sticks);
1440 share_fpu(l, tf);
1441 }
1442 #ifdef DEBUG
1443 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW))
1444 printf("data_access_error: done\n");
1445 if (trapdebug & TDB_FRAME) {
1446 print_trapframe(tf);
1447 }
1448 #endif
1449 }
1450
1451 /*
1452 * This routine handles MMU generated faults. About half
1453 * of them could be recoverable through uvm_fault.
1454 */
1455 void
text_access_fault(struct trapframe64 * tf,unsigned int type,vaddr_t pc,u_long sfsr)1456 text_access_fault(struct trapframe64 *tf, unsigned int type, vaddr_t pc,
1457 u_long sfsr)
1458 {
1459 uint64_t tstate;
1460 struct lwp *l;
1461 struct proc *p;
1462 struct vmspace *vm;
1463 vaddr_t va;
1464 int rv;
1465 vm_prot_t access_type;
1466 u_quad_t sticks;
1467 ksiginfo_t ksi;
1468
1469 #ifdef DEBUG
1470 if (tf->tf_pc == tf->tf_npc) {
1471 printf("text_access_fault: tpc %p == tnpc %p\n",
1472 (void *)(u_long)tf->tf_pc, (void *)(u_long)tf->tf_npc);
1473 Debugger();
1474 }
1475 write_user_windows();
1476 if (((trapdebug & TDB_NSAVED) && curpcb->pcb_nsaved) ||
1477 (trapdebug & (TDB_TXTFLT | TDB_FOLLOW)))
1478 printf("%d text_access_fault(%x, %lx, %p)\n",
1479 curproc?curproc->p_pid:-1, type, pc, tf);
1480 if (trapdebug & TDB_FRAME) {
1481 print_trapframe(tf);
1482 }
1483 if ((trapdebug & TDB_TL) && gettl()) {
1484 printf("%d tl %d text_access_fault(%x, %lx, %p)\n",
1485 curproc?curproc->p_pid:-1, gettl(), type, pc, tf);
1486 Debugger();
1487 }
1488 if (trapdebug & TDB_STOPCALL) {
1489 Debugger();
1490 }
1491 #endif
1492
1493 curcpu()->ci_data.cpu_ntrap++;
1494 l = curlwp;
1495 p = l->l_proc;
1496 sticks = p->p_sticks;
1497 tstate = tf->tf_tstate;
1498 va = trunc_page(pc);
1499
1500 /* Now munch on protections... */
1501
1502 access_type = VM_PROT_EXECUTE;
1503 if (tstate & TSTATE_PRIV) {
1504 extern int trap_trace_dis;
1505 trap_trace_dis = 1; /* Disable traptrace for printf */
1506 (void) splhigh();
1507 printf("text_access_fault: pc=%lx va=%lx\n", pc, va);
1508 DEBUGGER(type, tf);
1509 panic("kernel fault");
1510 /* NOTREACHED */
1511 } else
1512 l->l_md.md_tf = tf;
1513
1514 vm = p->p_vmspace;
1515 /* alas! must call the horrible vm code */
1516 KASSERT(curpcb->pcb_onfault == NULL);
1517 rv = uvm_fault(&vm->vm_map, va, access_type);
1518
1519 #ifdef DEBUG
1520 if (trapdebug & (TDB_TXTFLT | TDB_FOLLOW))
1521 printf("text_access_fault: uvm_fault(%p, %lx, %x) sez %x\n",
1522 &vm->vm_map, va, access_type, rv);
1523 #endif
1524 if (rv != 0) {
1525
1526 /*
1527 * Pagein failed, deliver SIGSEGV.
1528 */
1529 #ifdef DEBUG
1530 if (trapdebug & (TDB_TXTFLT | TDB_STOPSIG)) {
1531 extern int trap_trace_dis;
1532 trap_trace_dis = 1;
1533 printf("text_access_fault at %p: sending SIGSEGV\n",
1534 (void *)(u_long)va);
1535 Debugger();
1536 }
1537 #endif
1538 KSI_INIT_TRAP(&ksi);
1539 ksi.ksi_signo = SIGSEGV;
1540 ksi.ksi_code = (rv == EACCES ? SEGV_ACCERR : SEGV_MAPERR);
1541 ksi.ksi_trap = type;
1542 ksi.ksi_addr = (void *)pc;
1543 trapsignal(l, &ksi);
1544 }
1545 userret(l, pc, sticks);
1546 share_fpu(l, tf);
1547 #ifdef DEBUG
1548 if (trapdebug & (TDB_TXTFLT | TDB_FOLLOW)) {
1549 printf("text_access_fault: done\n");
1550 /* kdb_trap(T_BREAKPOINT, tf); */
1551 }
1552 if (trapdebug & TDB_FRAME) {
1553 print_trapframe(tf);
1554 }
1555 #endif
1556 }
1557
1558
1559 /*
1560 * This routine handles deferred errors caused by the memory
1561 * or I/O bus subsystems. Most of these are fatal, and even
1562 * if they are not, recovery is painful. Also, the TPC and
1563 * TNPC values are probably not valid if we're not doing a
1564 * special PEEK/POKE code sequence.
1565 */
1566 void
text_access_error(struct trapframe64 * tf,unsigned int type,vaddr_t pc,u_long sfsr,vaddr_t afva,u_long afsr)1567 text_access_error(struct trapframe64 *tf, unsigned int type, vaddr_t pc,
1568 u_long sfsr, vaddr_t afva, u_long afsr)
1569 {
1570 int64_t tstate;
1571 struct lwp *l;
1572 struct proc *p;
1573 struct vmspace *vm;
1574 vaddr_t va;
1575 int rv;
1576 vm_prot_t access_type;
1577 u_quad_t sticks;
1578 ksiginfo_t ksi;
1579 char buf[768];
1580
1581 #ifdef DEBUG
1582 if (tf->tf_pc == tf->tf_npc) {
1583 printf("text_access_error: tpc %p == tnpc %p\n",
1584 (void *)(u_long)tf->tf_pc, (void *)(u_long)tf->tf_npc);
1585 Debugger();
1586 }
1587 write_user_windows();
1588 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) ||
1589 trapdebug & (TDB_TXTFLT | TDB_FOLLOW)) {
1590 snprintb(buf, sizeof buf, SFSR_BITS, sfsr);
1591 printf("%ld text_access_error(%lx, %lx, %lx, %p)=%lx @ %lx %s\n",
1592 (long)(curproc?curproc->p_pid:-1),
1593 (long)type, pc, (long)afva, tf, (long)tf->tf_tstate,
1594 (long)tf->tf_pc, buf);
1595 }
1596 if (trapdebug & TDB_FRAME) {
1597 print_trapframe(tf);
1598 }
1599 if ((trapdebug & TDB_TL) && gettl()) {
1600 snprintb(buf, sizeof buf, SFSR_BITS, sfsr);
1601 printf("%d tl %d text_access_error(%lx, %lx, %lx, %p)=%lx @ %lx %s\n",
1602 curproc?curproc->p_pid:-1, gettl(),
1603 (long)type, (long)pc, (long)afva, tf,
1604 (long)tf->tf_tstate, (long)tf->tf_pc, buf);
1605 Debugger();
1606 }
1607 if (trapdebug & TDB_STOPCALL) {
1608 Debugger();
1609 }
1610 #endif
1611 curcpu()->ci_data.cpu_ntrap++;
1612 l = curlwp;
1613 p = l->l_proc;
1614 sticks = p->p_sticks;
1615
1616 tstate = tf->tf_tstate;
1617
1618 if ((afsr) != 0) {
1619 extern int trap_trace_dis;
1620
1621 trap_trace_dis++; /* Disable traptrace for printf */
1622 printf("text_access_error: memory error...\n");
1623 printf("text memory error type %d sfsr=%lx sfva=%lx afsr=%lx afva=%lx tf=%p\n",
1624 type, sfsr, pc, afsr, afva, tf);
1625 trap_trace_dis--; /* Reenable traptrace for printf */
1626
1627 if (tstate & TSTATE_PRIV)
1628 panic("text_access_error: kernel memory error");
1629
1630 /* User fault -- Berr */
1631 KSI_INIT_TRAP(&ksi);
1632 ksi.ksi_signo = SIGBUS;
1633 ksi.ksi_code = BUS_OBJERR;
1634 ksi.ksi_trap = type;
1635 ksi.ksi_addr = (void *)afva;
1636 trapsignal(l, &ksi);
1637 }
1638
1639 if ((sfsr & SFSR_FV) == 0 || (sfsr & SFSR_FT) == 0)
1640 goto out; /* No fault. Why were we called? */
1641
1642 va = trunc_page(pc);
1643
1644 /* Now munch on protections... */
1645
1646 access_type = VM_PROT_EXECUTE;
1647 if (tstate & TSTATE_PRIV) {
1648 extern int trap_trace_dis;
1649 trap_trace_dis = 1; /* Disable traptrace for printf */
1650 snprintb(buf, sizeof buf, SFSR_BITS, sfsr);
1651 (void) splhigh();
1652 printf("text error: pc=%lx sfsr=%s\n", pc, buf);
1653 DEBUGGER(type, tf);
1654 panic("kernel fault");
1655 /* NOTREACHED */
1656 } else
1657 l->l_md.md_tf = tf;
1658
1659 vm = p->p_vmspace;
1660 /* alas! must call the horrible vm code */
1661 KASSERT(curpcb->pcb_onfault == NULL);
1662 rv = uvm_fault(&vm->vm_map, va, access_type);
1663
1664 if (rv != 0) {
1665 /*
1666 * Pagein failed, deliver SIGSEGV.
1667 */
1668 #ifdef DEBUG
1669 if (trapdebug & (TDB_TXTFLT | TDB_STOPSIG)) {
1670 extern int trap_trace_dis;
1671 trap_trace_dis = 1;
1672 printf("text_access_error at %p: sending SIGSEGV\n",
1673 (void *)(u_long)va);
1674 Debugger();
1675 }
1676 #endif
1677 KSI_INIT_TRAP(&ksi);
1678 ksi.ksi_signo = SIGSEGV;
1679 ksi.ksi_code = (rv == EACCES ? SEGV_ACCERR : SEGV_MAPERR);
1680 ksi.ksi_trap = type;
1681 ksi.ksi_addr = (void *)pc;
1682 trapsignal(l, &ksi);
1683 }
1684 out:
1685 if ((tstate & TSTATE_PRIV) == 0) {
1686 userret(l, pc, sticks);
1687 share_fpu(l, tf);
1688 }
1689 #ifdef DEBUG
1690 if (trapdebug & (TDB_TXTFLT | TDB_FOLLOW))
1691 printf("text_access_error: done\n");
1692 if (trapdebug & TDB_FRAME) {
1693 print_trapframe(tf);
1694 }
1695 #endif
1696 }
1697
1698 /*
1699 * Handle an ECC corrected event.
1700 */
1701 void
ecc_corrected_error(unsigned int type,vaddr_t pc)1702 ecc_corrected_error(unsigned int type, vaddr_t pc)
1703 {
1704 uint64_t eeer, afar, afsr;
1705 char buf[128];
1706 int s;
1707
1708 /* Clear the error */
1709 eeer = ldxa(0, ASI_ERROR_EN_REG);
1710 s = intr_disable();
1711 stxa(0, ASI_ERROR_EN_REG,
1712 eeer & ~(P_EER_NCEEN | P_EER_CEEN));
1713 membar_Sync();
1714 intr_restore(s);
1715
1716 /* Flush the caches in order ensure no corrupt data got installed. */
1717 blast_dcache();
1718 blast_icache();
1719
1720 #if 0
1721 /* Ensure the caches are still turned on (should be). */
1722 cache_enable(PCPU_GET(impl));
1723 #endif
1724
1725 /* Grab the current AFSR/AFAR, and clear the error from the AFSR. */
1726 afar = ldxa(0, ASI_AFAR);
1727 afsr = ldxa(0, ASI_AFSR);
1728 s = intr_disable();
1729 stxa(0, ASI_AFSR, ldxa(0, ASI_AFSR));
1730 membar_Sync();
1731 intr_restore(s);
1732 ecc_corrected.ev_count++;
1733 snprintb(buf, sizeof(buf), AFSR_BITS, afsr);
1734 printf("corrected ECC error: pc %p afsr %"PRIx64" (%s) addr %"PRIx64"\n", (void *)pc, afsr, buf, afar);
1735
1736 /* Turn (non-)correctable error reporting back on. */
1737 s = intr_disable();
1738 stxa(0, ASI_ERROR_EN_REG, eeer);
1739 membar_Sync();
1740 intr_restore(s);
1741 }
1742