1 /* $NetBSD: vfp_init.c,v 1.78 2022/08/20 11:34:08 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2008 ARM Ltd
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the company may not be used to endorse or promote
16 * products derived from this software without specific prior written
17 * permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
25 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
28 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include "opt_cputypes.h"
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: vfp_init.c,v 1.78 2022/08/20 11:34:08 riastradh Exp $");
36
37 #include <sys/param.h>
38 #include <sys/types.h>
39 #include <sys/systm.h>
40 #include <sys/device.h>
41 #include <sys/kernel.h>
42 #include <sys/kthread.h>
43 #include <sys/proc.h>
44 #include <sys/cpu.h>
45
46 #include <arm/locore.h>
47 #include <arm/pcb.h>
48 #include <arm/undefined.h>
49 #include <arm/vfpreg.h>
50 #include <arm/mcontext.h>
51 #include <arm/fpu.h>
52
53 #include <uvm/uvm_extern.h> /* for pmap.h */
54
55 #include <crypto/aes/aes_impl.h>
56 #include <crypto/aes/arch/arm/aes_neon.h>
57 #include <crypto/chacha/arch/arm/chacha_neon.h>
58 #include <crypto/chacha/chacha_impl.h>
59
60 #ifdef FPU_VFP
61
62 #ifdef CPU_CORTEX
63 #define SETFPU __asm(".fpu\tvfpv4")
64 #else
65 #define SETFPU __asm(".fpu\tvfp")
66 #endif
67 SETFPU;
68
69 /* FLDMD <X>, {d0-d15} */
70 static inline void
load_vfpregs_lo(const uint64_t * p)71 load_vfpregs_lo(const uint64_t *p)
72 {
73 SETFPU;
74 __asm __volatile(".fpu vfp\n vldmia\t%0, {d0-d15}" :: "r" (p) : "memory");
75 }
76
77 /* FSTMD <X>, {d0-d15} */
78 static inline void
save_vfpregs_lo(uint64_t * p)79 save_vfpregs_lo(uint64_t *p)
80 {
81 SETFPU;
82 __asm __volatile(".fpu vfp\n vstmia\t%0, {d0-d15}" :: "r" (p) : "memory");
83 }
84
85 #ifdef CPU_CORTEX
86 /* FLDMD <X>, {d16-d31} */
87 static inline void
load_vfpregs_hi(const uint64_t * p)88 load_vfpregs_hi(const uint64_t *p)
89 {
90 SETFPU;
91 __asm __volatile(".fpu neon-vfpv4\n vldmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory");
92 }
93
94 /* FLDMD <X>, {d16-d31} */
95 static inline void
save_vfpregs_hi(uint64_t * p)96 save_vfpregs_hi(uint64_t *p)
97 {
98 SETFPU;
99 __asm __volatile(".fpu neon-vfpv4\nvstmia\t%0, {d16-d31}" :: "r" (&p[16]) : "memory");
100 }
101 #endif
102
103 static inline void
load_vfpregs(const struct vfpreg * fregs)104 load_vfpregs(const struct vfpreg *fregs)
105 {
106 load_vfpregs_lo(fregs->vfp_regs);
107 #ifdef CPU_CORTEX
108 #ifdef CPU_ARM11
109 switch (curcpu()->ci_vfp_id) {
110 case FPU_VFP_CORTEXA5:
111 case FPU_VFP_CORTEXA7:
112 case FPU_VFP_CORTEXA8:
113 case FPU_VFP_CORTEXA9:
114 case FPU_VFP_CORTEXA15:
115 case FPU_VFP_CORTEXA15_QEMU:
116 case FPU_VFP_CORTEXA53:
117 case FPU_VFP_CORTEXA57:
118 #endif
119 load_vfpregs_hi(fregs->vfp_regs);
120 #ifdef CPU_ARM11
121 break;
122 }
123 #endif
124 #endif
125 }
126
127 static inline void
save_vfpregs(struct vfpreg * fregs)128 save_vfpregs(struct vfpreg *fregs)
129 {
130 save_vfpregs_lo(fregs->vfp_regs);
131 #ifdef CPU_CORTEX
132 #ifdef CPU_ARM11
133 switch (curcpu()->ci_vfp_id) {
134 case FPU_VFP_CORTEXA5:
135 case FPU_VFP_CORTEXA7:
136 case FPU_VFP_CORTEXA8:
137 case FPU_VFP_CORTEXA9:
138 case FPU_VFP_CORTEXA15:
139 case FPU_VFP_CORTEXA15_QEMU:
140 case FPU_VFP_CORTEXA53:
141 case FPU_VFP_CORTEXA57:
142 #endif
143 save_vfpregs_hi(fregs->vfp_regs);
144 #ifdef CPU_ARM11
145 break;
146 }
147 #endif
148 #endif
149 }
150
151 /* The real handler for VFP bounces. */
152 static int vfp_handler(u_int, u_int, trapframe_t *, int);
153 #ifdef CPU_CORTEX
154 static int neon_handler(u_int, u_int, trapframe_t *, int);
155 #endif
156
157 static void vfp_state_load(lwp_t *, u_int);
158 static void vfp_state_save(lwp_t *);
159 static void vfp_state_release(lwp_t *);
160
161 const pcu_ops_t arm_vfp_ops = {
162 .pcu_id = PCU_FPU,
163 .pcu_state_save = vfp_state_save,
164 .pcu_state_load = vfp_state_load,
165 .pcu_state_release = vfp_state_release,
166 };
167
168 /* determine what bits can be changed */
169 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM;
170 /* default to run fast */
171 uint32_t vfp_fpscr_default = (VFP_FPSCR_DN | VFP_FPSCR_FZ | VFP_FPSCR_RN);
172
173 #else
174 /* determine what bits can be changed */
175 uint32_t vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM|VFP_FPSCR_RMODE;
176 #endif /* FPU_VFP */
177
178 static int
vfp_fpscr_handler(u_int address,u_int insn,trapframe_t * frame,int fault_code)179 vfp_fpscr_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code)
180 {
181 struct lwp * const l = curlwp;
182 const u_int regno = (insn >> 12) & 0xf;
183 /*
184 * Only match move to/from the FPSCR register and we
185 * can't be using the SP,LR,PC as a source.
186 */
187 if ((insn & 0xffef0fff) != 0xeee10a10 || regno > 12)
188 return 1;
189
190 struct pcb * const pcb = lwp_getpcb(l);
191
192 #ifdef FPU_VFP
193 /*
194 * If FPU is valid somewhere, let's just reenable VFP and
195 * retry the instruction (only safe thing to do since the
196 * pcb has a stale copy).
197 */
198 if (pcb->pcb_vfp.vfp_fpexc & VFP_FPEXC_EN)
199 return 1;
200
201 if (__predict_false(!vfp_used_p(l))) {
202 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default;
203 }
204 #endif
205
206 /*
207 * We now know the pcb has the saved copy.
208 */
209 register_t * const regp = &frame->tf_r0 + regno;
210 if (insn & 0x00100000) {
211 *regp = pcb->pcb_vfp.vfp_fpscr;
212 } else {
213 pcb->pcb_vfp.vfp_fpscr &= ~vfp_fpscr_changable;
214 pcb->pcb_vfp.vfp_fpscr |= *regp & vfp_fpscr_changable;
215 }
216
217 curcpu()->ci_vfp_evs[0].ev_count++;
218
219 frame->tf_pc += INSN_SIZE;
220 return 0;
221 }
222
223 #ifndef FPU_VFP
224 void
vfp_detect(struct cpu_info * ci)225 vfp_detect(struct cpu_info *ci)
226 {
227 ci->ci_vfp_id = 0;
228 return;
229 }
230 /*
231 * If we don't want VFP support, we still need to handle emulating VFP FPSCR
232 * instructions.
233 */
234 void
vfp_attach(struct cpu_info * ci)235 vfp_attach(struct cpu_info *ci)
236 {
237 if (CPU_IS_PRIMARY(ci)) {
238 replace_coproc_handler(VFP_COPROC, vfp_fpscr_handler);
239 }
240 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_TRAP, NULL,
241 ci->ci_cpuname, "vfp fpscr traps");
242 }
243
244 #else
245 void
vfp_detect(struct cpu_info * ci)246 vfp_detect(struct cpu_info *ci)
247 {
248
249 if (CPU_ID_ARM11_P(ci->ci_arm_cpuid)
250 || CPU_ID_MV88SV58XX_P(ci->ci_arm_cpuid)
251 || CPU_ID_CORTEX_P(ci->ci_arm_cpuid)) {
252 #if 0
253 const uint32_t nsacr = armreg_nsacr_read();
254 const uint32_t nsacr_vfp = __BITS(VFP_COPROC,VFP_COPROC2);
255 if ((nsacr & nsacr_vfp) != nsacr_vfp) {
256 ci->ci_fp_id = 0;
257 return;
258 }
259 #endif
260 const uint32_t cpacr_vfp = CPACR_CPn(VFP_COPROC);
261 const uint32_t cpacr_vfp2 = CPACR_CPn(VFP_COPROC2);
262
263 /*
264 * We first need to enable access to the coprocessors.
265 */
266 uint32_t cpacr = armreg_cpacr_read();
267 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp);
268 cpacr |= __SHIFTIN(CPACR_ALL, cpacr_vfp2);
269 armreg_cpacr_write(cpacr);
270
271 isb();
272
273 /*
274 * If we could enable them, then they exist.
275 */
276 cpacr = armreg_cpacr_read();
277 bool vfp_p = __SHIFTOUT(cpacr, cpacr_vfp2) == CPACR_ALL
278 && __SHIFTOUT(cpacr, cpacr_vfp) == CPACR_ALL;
279 if (!vfp_p) {
280 ci->ci_vfp_id = 0;
281 return;
282 }
283 }
284
285 /* borrow the ci_vfd_id field for VFP detection */
286 ci->ci_vfp_id = -1;
287
288 const uint32_t fpsid = armreg_fpsid_read();
289 if (ci->ci_vfp_id == 0) {
290 return;
291 }
292
293 ci->ci_vfp_id = fpsid;
294
295 ci->ci_mvfr[0] = armreg_mvfr0_read();
296 ci->ci_mvfr[1] = armreg_mvfr1_read();
297
298 }
299
300 void
vfp_attach(struct cpu_info * ci)301 vfp_attach(struct cpu_info *ci)
302 {
303 const char *model = NULL;
304
305 switch (ci->ci_vfp_id & ~ VFP_FPSID_REV_MSK) {
306 case FPU_VFP10_ARM10E:
307 model = "VFP10 R1";
308 break;
309 case FPU_VFP11_ARM11:
310 model = "VFP11";
311 break;
312 case FPU_VFP_MV88SV58XX:
313 model = "VFP3";
314 break;
315 case FPU_VFP_CORTEXA5:
316 case FPU_VFP_CORTEXA7:
317 case FPU_VFP_CORTEXA8:
318 case FPU_VFP_CORTEXA9:
319 case FPU_VFP_CORTEXA12:
320 case FPU_VFP_CORTEXA15:
321 case FPU_VFP_CORTEXA15_QEMU:
322 case FPU_VFP_CORTEXA17:
323 case FPU_VFP_CORTEXA53:
324 case FPU_VFP_CORTEXA57:
325 if (armreg_cpacr_read() & CPACR_V7_ASEDIS) {
326 model = "VFP 4.0+";
327 } else {
328 model = "NEON MPE (VFP 3.0+)";
329 cpu_neon_present = 1;
330 }
331 break;
332 default:
333 aprint_normal_dev(ci->ci_dev, "unrecognized VFP version %#x\n",
334 ci->ci_vfp_id);
335 if (CPU_IS_PRIMARY(ci))
336 replace_coproc_handler(VFP_COPROC, vfp_fpscr_handler);
337 vfp_fpscr_changable = VFP_FPSCR_CSUM|VFP_FPSCR_ESUM
338 |VFP_FPSCR_RMODE;
339 vfp_fpscr_default = 0;
340 return;
341 }
342
343 cpu_fpu_present = 1;
344
345 const uint32_t f0 = ci->ci_mvfr[0];
346 const uint32_t f1 = ci->ci_mvfr[1];
347 aprint_normal("vfp%d at %s: %s%s%s%s%s\n",
348 device_unit(ci->ci_dev),
349 device_xname(ci->ci_dev),
350 model,
351 ((f0 & ARM_MVFR0_ROUNDING_MASK) ? ", rounding" : ""),
352 ((f0 & ARM_MVFR0_EXCEPT_MASK) ? ", exceptions" : ""),
353 ((f1 & ARM_MVFR1_D_NAN_MASK) ? ", NaN propagation" : ""),
354 ((f1 & ARM_MVFR1_FTZ_MASK) ? ", denormals" : ""));
355
356 aprint_debug("vfp%d: mvfr: [0]=%#x [1]=%#x\n",
357 device_unit(ci->ci_dev), f0, f1);
358
359 if (CPU_IS_PRIMARY(ci)) {
360 cpu_media_and_vfp_features[0] = f0;
361 cpu_media_and_vfp_features[1] = f1;
362
363 if (f0 & ARM_MVFR0_ROUNDING_MASK) {
364 vfp_fpscr_changable |= VFP_FPSCR_RMODE;
365 }
366 if (f1 & ARM_MVFR0_EXCEPT_MASK) {
367 vfp_fpscr_changable |= VFP_FPSCR_ESUM;
368 }
369 // If hardware supports propagation of NaNs, select it.
370 if (f1 & ARM_MVFR1_D_NAN_MASK) {
371 vfp_fpscr_default &= ~VFP_FPSCR_DN;
372 vfp_fpscr_changable |= VFP_FPSCR_DN;
373 }
374 // If hardware supports denormalized numbers, use it.
375 if (f1 & ARM_MVFR1_FTZ_MASK) {
376 vfp_fpscr_default &= ~VFP_FPSCR_FZ;
377 vfp_fpscr_changable |= VFP_FPSCR_FZ;
378 }
379
380 replace_coproc_handler(VFP_COPROC, vfp_handler);
381 install_coproc_handler(VFP_COPROC2, vfp_handler);
382 #ifdef CPU_CORTEX
383 if (cpu_neon_present) {
384 install_coproc_handler(CORE_UNKNOWN_HANDLER,
385 neon_handler);
386 aes_md_init(&aes_neon_impl);
387 chacha_md_init(&chacha_neon_impl);
388 }
389 #endif
390 }
391
392 evcnt_attach_dynamic(&ci->ci_vfp_evs[0], EVCNT_TYPE_MISC, NULL,
393 ci->ci_cpuname, "vfp coproc use");
394 evcnt_attach_dynamic(&ci->ci_vfp_evs[1], EVCNT_TYPE_MISC, NULL,
395 ci->ci_cpuname, "vfp coproc re-use");
396 evcnt_attach_dynamic(&ci->ci_vfp_evs[2], EVCNT_TYPE_TRAP, NULL,
397 ci->ci_cpuname, "vfp coproc fault");
398 }
399
400 /* The real handler for VFP bounces. */
401 static int
vfp_handler(u_int address,u_int insn,trapframe_t * frame,int fault_code)402 vfp_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code)
403 {
404 struct cpu_info * const ci = curcpu();
405 uint32_t fpexc;
406
407 /* This shouldn't ever happen. */
408 if (fault_code != FAULT_USER &&
409 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM)
410 panic("VFP fault at %#x in non-user mode", frame->tf_pc);
411
412 if (ci->ci_vfp_id == 0) {
413 /* No VFP detected, just fault. */
414 return 1;
415 }
416
417 /*
418 * If we already own the FPU and it's enabled (and no exception), raise
419 * SIGILL. If there is an exception, raise SIGFPE.
420 */
421 if (curlwp->l_pcu_cpu[PCU_FPU] == ci) {
422 KASSERT(ci->ci_pcu_curlwp[PCU_FPU] == curlwp);
423
424 fpexc = armreg_fpexc_read();
425 if (fpexc & VFP_FPEXC_EN) {
426 if ((fpexc & VFP_FPEXC_EX) == 0) {
427 return 1; /* SIGILL */
428 } else {
429 goto fpe; /* SIGFPE; skip pcu_load(9) */
430 }
431 }
432 }
433
434 /*
435 * Make sure we own the FP.
436 */
437 pcu_load(&arm_vfp_ops);
438
439 fpexc = armreg_fpexc_read();
440 if (fpexc & VFP_FPEXC_EX) {
441 ksiginfo_t ksi;
442 KASSERT(fpexc & VFP_FPEXC_EN);
443
444 fpe:
445 curcpu()->ci_vfp_evs[2].ev_count++;
446
447 /*
448 * Need the clear the exception condition so any signal
449 * and future use can proceed.
450 */
451 armreg_fpexc_write(fpexc & ~(VFP_FPEXC_EX|VFP_FPEXC_FSUM));
452
453 pcu_save(&arm_vfp_ops, curlwp);
454
455 /*
456 * XXX Need to emulate bounce instructions here to get correct
457 * XXX exception codes, etc.
458 */
459 KSI_INIT_TRAP(&ksi);
460 ksi.ksi_signo = SIGFPE;
461 if (fpexc & VFP_FPEXC_IXF)
462 ksi.ksi_code = FPE_FLTRES;
463 else if (fpexc & VFP_FPEXC_UFF)
464 ksi.ksi_code = FPE_FLTUND;
465 else if (fpexc & VFP_FPEXC_OFF)
466 ksi.ksi_code = FPE_FLTOVF;
467 else if (fpexc & VFP_FPEXC_DZF)
468 ksi.ksi_code = FPE_FLTDIV;
469 else if (fpexc & VFP_FPEXC_IOF)
470 ksi.ksi_code = FPE_FLTINV;
471 ksi.ksi_addr = (uint32_t *)address;
472 ksi.ksi_trap = 0;
473 trapsignal(curlwp, &ksi);
474 return 0;
475 }
476
477 /* Need to restart the faulted instruction. */
478 // frame->tf_pc -= INSN_SIZE;
479 return 0;
480 }
481
482 #ifdef CPU_CORTEX
483 /* The real handler for NEON bounces. */
484 static int
neon_handler(u_int address,u_int insn,trapframe_t * frame,int fault_code)485 neon_handler(u_int address, u_int insn, trapframe_t *frame, int fault_code)
486 {
487 struct cpu_info * const ci = curcpu();
488
489 if (ci->ci_vfp_id == 0)
490 /* No VFP detected, just fault. */
491 return 1;
492
493 if ((insn & 0xfe000000) != 0xf2000000
494 && (insn & 0xfe000000) != 0xf4000000)
495 /* Not NEON instruction, just fault. */
496 return 1;
497
498 /* This shouldn't ever happen. */
499 if (fault_code != FAULT_USER &&
500 (curlwp->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) == LW_SYSTEM)
501 panic("NEON fault in non-user mode");
502
503 /* if we already own the FPU and it's enabled, raise SIGILL */
504 if (curcpu()->ci_pcu_curlwp[PCU_FPU] == curlwp
505 && (armreg_fpexc_read() & VFP_FPEXC_EN) != 0)
506 return 1;
507
508 pcu_load(&arm_vfp_ops);
509
510 /* Need to restart the faulted instruction. */
511 // frame->tf_pc -= INSN_SIZE;
512 return 0;
513 }
514 #endif
515
516 static void
vfp_state_load(lwp_t * l,u_int flags)517 vfp_state_load(lwp_t *l, u_int flags)
518 {
519 struct pcb * const pcb = lwp_getpcb(l);
520 struct vfpreg * const fregs = &pcb->pcb_vfp;
521
522 /*
523 * Instrument VFP usage -- if a process has not previously
524 * used the VFP, mark it as having used VFP for the first time,
525 * and count this event.
526 *
527 * If a process has used the VFP, count a "used VFP, and took
528 * a trap to use it again" event.
529 */
530 if (__predict_false((flags & PCU_VALID) == 0)) {
531 curcpu()->ci_vfp_evs[0].ev_count++;
532 pcb->pcb_vfp.vfp_fpscr = vfp_fpscr_default;
533 } else {
534 curcpu()->ci_vfp_evs[1].ev_count++;
535 }
536
537 KASSERT((armreg_fpexc_read() & VFP_FPEXC_EN) == 0);
538 /*
539 * If the VFP is already enabled we must be bouncing an instruction.
540 */
541 if (flags & PCU_REENABLE) {
542 uint32_t fpexc = armreg_fpexc_read();
543 armreg_fpexc_write(fpexc | VFP_FPEXC_EN);
544 fregs->vfp_fpexc |= VFP_FPEXC_EN;
545 return;
546 }
547 KASSERT((fregs->vfp_fpexc & VFP_FPEXC_EN) == 0);
548
549 /*
550 * Load and Enable the VFP (so that we can write the registers).
551 */
552 fregs->vfp_fpexc |= VFP_FPEXC_EN;
553 armreg_fpexc_write(fregs->vfp_fpexc);
554 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == NULL);
555 KASSERT(l->l_pcu_cpu[PCU_FPU] == NULL);
556
557 load_vfpregs(fregs);
558 armreg_fpscr_write(fregs->vfp_fpscr);
559
560 if (fregs->vfp_fpexc & VFP_FPEXC_EX) {
561 /* Need to restore the exception handling state. */
562 armreg_fpinst_write(fregs->vfp_fpinst);
563 if (fregs->vfp_fpexc & VFP_FPEXC_FP2V)
564 armreg_fpinst2_write(fregs->vfp_fpinst2);
565 }
566 }
567
568 void
vfp_state_save(lwp_t * l)569 vfp_state_save(lwp_t *l)
570 {
571 struct pcb * const pcb = lwp_getpcb(l);
572 struct vfpreg * const fregs = &pcb->pcb_vfp;
573 uint32_t fpexc = armreg_fpexc_read();
574
575 KASSERT(curcpu()->ci_pcu_curlwp[PCU_FPU] == l);
576 KASSERT(curcpu() == l->l_pcu_cpu[PCU_FPU]);
577 KASSERT(curlwp == l || curlwp->l_pcu_cpu[PCU_FPU] != curcpu());
578 /*
579 * Enable the VFP (so we can read the registers).
580 * Make sure the exception bit is cleared so that we can
581 * safely dump the registers.
582 */
583 armreg_fpexc_write((fpexc | VFP_FPEXC_EN) & ~VFP_FPEXC_EX);
584
585 fregs->vfp_fpexc = fpexc;
586 if (fpexc & VFP_FPEXC_EX) {
587 /* Need to save the exception handling state */
588 fregs->vfp_fpinst = armreg_fpinst_read();
589 if (fpexc & VFP_FPEXC_FP2V)
590 fregs->vfp_fpinst2 = armreg_fpinst2_read();
591 }
592 fregs->vfp_fpscr = armreg_fpscr_read();
593 save_vfpregs(fregs);
594
595 /* Disable the VFP. */
596 armreg_fpexc_write(fpexc & ~VFP_FPEXC_EN);
597 }
598
599 void
vfp_state_release(lwp_t * l)600 vfp_state_release(lwp_t *l)
601 {
602 struct pcb * const pcb = lwp_getpcb(l);
603
604 /*
605 * Now mark the VFP as disabled (and our state
606 * has been already saved or is being discarded).
607 */
608 pcb->pcb_vfp.vfp_fpexc &= ~VFP_FPEXC_EN;
609
610 /*
611 * Turn off the FPU so the next time a VFP instruction is issued
612 * an exception happens. We don't know if this LWP's state was
613 * loaded but if we turned off the FPU for some other LWP, when
614 * pcu_load invokes vfp_state_load it will see that VFP_FPEXC_EN
615 * is still set so it just restore fpexc and return since its
616 * contents are still sitting in the VFP.
617 */
618 armreg_fpexc_write(armreg_fpexc_read() & ~VFP_FPEXC_EN);
619 }
620
621 void
vfp_savecontext(lwp_t * l)622 vfp_savecontext(lwp_t *l)
623 {
624 pcu_save(&arm_vfp_ops, l);
625 }
626
627 void
vfp_discardcontext(lwp_t * l,bool used_p)628 vfp_discardcontext(lwp_t *l, bool used_p)
629 {
630 pcu_discard(&arm_vfp_ops, l, used_p);
631 }
632
633 bool
vfp_used_p(const lwp_t * l)634 vfp_used_p(const lwp_t *l)
635 {
636 return pcu_valid_p(&arm_vfp_ops, l);
637 }
638
639 void
vfp_getcontext(struct lwp * l,mcontext_t * mcp,int * flagsp)640 vfp_getcontext(struct lwp *l, mcontext_t *mcp, int *flagsp)
641 {
642 if (vfp_used_p(l)) {
643 const struct pcb * const pcb = lwp_getpcb(l);
644
645 pcu_save(&arm_vfp_ops, l);
646 mcp->__fpu.__vfpregs.__vfp_fpscr = pcb->pcb_vfp.vfp_fpscr;
647 memcpy(mcp->__fpu.__vfpregs.__vfp_fstmx, pcb->pcb_vfp.vfp_regs,
648 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx));
649 *flagsp |= _UC_FPU|_UC_ARM_VFP;
650 }
651 }
652
653 void
vfp_setcontext(struct lwp * l,const mcontext_t * mcp)654 vfp_setcontext(struct lwp *l, const mcontext_t *mcp)
655 {
656 struct pcb * const pcb = lwp_getpcb(l);
657
658 pcu_discard(&arm_vfp_ops, l, true);
659 pcb->pcb_vfp.vfp_fpscr = mcp->__fpu.__vfpregs.__vfp_fpscr;
660 memcpy(pcb->pcb_vfp.vfp_regs, mcp->__fpu.__vfpregs.__vfp_fstmx,
661 sizeof(mcp->__fpu.__vfpregs.__vfp_fstmx));
662 }
663
664 /*
665 * True if this is a system thread with its own private FPU state.
666 */
667 static inline bool
lwp_system_fpu_p(struct lwp * l)668 lwp_system_fpu_p(struct lwp *l)
669 {
670
671 return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) ==
672 (LW_SYSTEM|LW_SYSTEM_FPU);
673 }
674
675 static const struct vfpreg zero_vfpreg;
676
677 void
fpu_kern_enter(void)678 fpu_kern_enter(void)
679 {
680 struct cpu_info *ci;
681 uint32_t fpexc;
682 int s;
683
684 if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) {
685 KASSERT(!cpu_softintr_p());
686 return;
687 }
688
689 /*
690 * Block interrupts up to IPL_VM. We must block preemption
691 * since -- if this is a user thread -- there is nowhere to
692 * save the kernel fpu state, and if we want this to be usable
693 * in interrupts, we can't let interrupts interfere with the
694 * fpu state in use since there's nowhere for them to save it.
695 */
696 s = splvm();
697 ci = curcpu();
698 #if 0
699 /*
700 * Can't assert this because if the caller holds a spin lock at
701 * IPL_VM, and previously held and released a spin lock at
702 * higher IPL, the IPL remains raised above IPL_VM.
703 */
704 KASSERTMSG(ci->ci_cpl <= IPL_VM || cold, "cpl=%d", ci->ci_cpl);
705 #endif
706 KASSERT(ci->ci_kfpu_spl == -1);
707 ci->ci_kfpu_spl = s;
708
709 /* Save any fpu state on the current CPU. */
710 pcu_save_all_on_cpu();
711
712 /* Enable the fpu. */
713 fpexc = armreg_fpexc_read();
714 fpexc |= VFP_FPEXC_EN;
715 fpexc &= ~VFP_FPEXC_EX;
716 armreg_fpexc_write(fpexc);
717 }
718
719 void
fpu_kern_leave(void)720 fpu_kern_leave(void)
721 {
722 struct cpu_info *ci = curcpu();
723 int s;
724 uint32_t fpexc;
725
726 if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) {
727 KASSERT(!cpu_softintr_p());
728 return;
729 }
730
731 #if 0
732 /*
733 * Can't assert this because if the caller holds a spin lock at
734 * IPL_VM, and previously held and released a spin lock at
735 * higher IPL, the IPL remains raised above IPL_VM.
736 */
737 KASSERT(ci->ci_cpl == IPL_VM || cold);
738 #endif
739 KASSERT(ci->ci_kfpu_spl != -1);
740
741 /*
742 * Zero the fpu registers; otherwise we might leak secrets
743 * through Spectre-class attacks to userland, even if there are
744 * no bugs in fpu state management.
745 */
746 load_vfpregs(&zero_vfpreg);
747
748 /*
749 * Disable the fpu so that the kernel can't accidentally use
750 * it again.
751 */
752 fpexc = armreg_fpexc_read();
753 fpexc &= ~VFP_FPEXC_EN;
754 armreg_fpexc_write(fpexc);
755
756 /* Restore interrupts. */
757 s = ci->ci_kfpu_spl;
758 ci->ci_kfpu_spl = -1;
759 splx(s);
760 }
761
762 void
kthread_fpu_enter_md(void)763 kthread_fpu_enter_md(void)
764 {
765
766 pcu_load(&arm_vfp_ops);
767 }
768
769 void
kthread_fpu_exit_md(void)770 kthread_fpu_exit_md(void)
771 {
772
773 /* XXX Should vfp_state_release zero the registers itself? */
774 load_vfpregs(&zero_vfpreg);
775 vfp_discardcontext(curlwp, 0);
776 }
777
778 #endif /* FPU_VFP */
779