1 /* $NetBSD: altivec.c,v 1.35 2024/06/15 19:48:13 mrg Exp $ */
2
3 /*
4 * Copyright (C) 1996 Wolfgang Solfrank.
5 * Copyright (C) 1996 TooLs GmbH.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by TooLs GmbH.
19 * 4. The name of TooLs GmbH may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.35 2024/06/15 19:48:13 mrg Exp $");
36
37 #include <sys/param.h>
38 #include <sys/proc.h>
39 #include <sys/systm.h>
40 #include <sys/atomic.h>
41
42 #include <uvm/uvm_extern.h> /* for vcopypage/vzeropage */
43
44 #include <powerpc/pcb.h>
45 #include <powerpc/altivec.h>
46 #include <powerpc/spr.h>
47 #include <powerpc/oea/spr.h>
48 #include <powerpc/psl.h>
49
50 static void vec_state_load(lwp_t *, u_int);
51 static void vec_state_save(lwp_t *);
52 static void vec_state_release(lwp_t *);
53
54 const pcu_ops_t vec_ops = {
55 .pcu_id = PCU_VEC,
56 .pcu_state_load = vec_state_load,
57 .pcu_state_save = vec_state_save,
58 .pcu_state_release = vec_state_release,
59 };
60
61 bool
vec_used_p(lwp_t * l)62 vec_used_p(lwp_t *l)
63 {
64 return pcu_valid_p(&vec_ops, l);
65 }
66
67 void
vec_mark_used(lwp_t * l)68 vec_mark_used(lwp_t *l)
69 {
70 return pcu_discard(&vec_ops, l, true);
71 }
72
73 void
vec_state_load(lwp_t * l,u_int flags)74 vec_state_load(lwp_t *l, u_int flags)
75 {
76 struct pcb * const pcb = lwp_getpcb(l);
77
78 if ((flags & PCU_VALID) == 0) {
79 memset(&pcb->pcb_vr, 0, sizeof(pcb->pcb_vr));
80 vec_mark_used(l);
81 }
82
83 if ((flags & PCU_REENABLE) == 0) {
84 /*
85 * Enable AltiVec temporarily (and disable interrupts).
86 */
87 const register_t msr = mfmsr();
88 mtmsr((msr & ~PSL_EE) | PSL_VEC);
89 __asm volatile ("isync");
90
91 /*
92 * Load the vector unit from vreg which is best done in
93 * assembly.
94 */
95 vec_load_from_vreg(&pcb->pcb_vr);
96
97 /*
98 * Restore MSR (turn off AltiVec)
99 */
100 mtmsr(msr);
101 __asm volatile ("isync");
102 }
103
104 /*
105 * VRSAVE will be restored when trap frame returns
106 */
107 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave;
108
109 /*
110 * Mark vector registers as modified.
111 */
112 l->l_md.md_flags |= PSL_VEC;
113 l->l_md.md_utf->tf_srr1 |= PSL_VEC;
114 }
115
116 void
vec_state_save(lwp_t * l)117 vec_state_save(lwp_t *l)
118 {
119 struct pcb * const pcb = lwp_getpcb(l);
120
121 /*
122 * Turn on AltiVEC, turn off interrupts.
123 */
124 const register_t msr = mfmsr();
125 mtmsr((msr & ~PSL_EE) | PSL_VEC);
126 __asm volatile ("isync");
127
128 /*
129 * Grab contents of vector unit.
130 */
131 vec_unload_to_vreg(&pcb->pcb_vr);
132
133 /*
134 * Save VRSAVE
135 */
136 pcb->pcb_vr.vrsave = l->l_md.md_utf->tf_vrsave;
137
138 /*
139 * Note that we aren't using any CPU resources and stop any
140 * data streams.
141 */
142 __asm volatile ("dssall; sync");
143
144 /*
145 * Restore MSR (turn off AltiVec)
146 */
147 mtmsr(msr);
148 __asm volatile ("isync");
149 }
150
151 void
vec_state_release(lwp_t * l)152 vec_state_release(lwp_t *l)
153 {
154 __asm volatile("dssall;sync");
155 l->l_md.md_utf->tf_srr1 &= ~PSL_VEC;
156 l->l_md.md_flags &= ~PSL_VEC;
157 }
158
159 void
vec_restore_from_mcontext(struct lwp * l,const mcontext_t * mcp)160 vec_restore_from_mcontext(struct lwp *l, const mcontext_t *mcp)
161 {
162 struct pcb * const pcb = lwp_getpcb(l);
163
164 KASSERT(l == curlwp);
165
166 /* Nothing to do here. */
167 if (!vec_used_p(l))
168 return;
169
170 /* we don't need to save the state, just drop it */
171 pcu_discard(&vec_ops, l, true);
172
173 if (mcp != NULL) { /* XXX see compat_16_sys___sigreturn14() */
174 memcpy(pcb->pcb_vr.vreg, &mcp->__vrf.__vrs,
175 sizeof (pcb->pcb_vr.vreg));
176 pcb->pcb_vr.vscr = mcp->__vrf.__vscr;
177 pcb->pcb_vr.vrsave = mcp->__vrf.__vrsave;
178 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave;
179 }
180 }
181
182 bool
vec_save_to_mcontext(struct lwp * l,mcontext_t * mcp,unsigned int * flagp)183 vec_save_to_mcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flagp)
184 {
185 struct pcb * const pcb = lwp_getpcb(l);
186
187 KASSERT(l == curlwp);
188
189 /* Save AltiVec context, if any. */
190 if (!vec_used_p(l))
191 return false;
192
193 /*
194 * If we're the AltiVec owner, dump its context to the PCB first.
195 */
196 pcu_save(&vec_ops, l);
197
198 if (mcp != NULL) { /* XXX see sendsig_sigcontext() */
199 mcp->__gregs[_REG_MSR] |= PSL_VEC;
200 mcp->__vrf.__vscr = pcb->pcb_vr.vscr;
201 mcp->__vrf.__vrsave = l->l_md.md_utf->tf_vrsave;
202 memcpy(mcp->__vrf.__vrs, pcb->pcb_vr.vreg,
203 sizeof (mcp->__vrf.__vrs));
204 *flagp |= _UC_POWERPC_VEC;
205 }
206 return true;
207 }
208
209 #define ZERO_VEC 19
210
211 void
vzeropage(paddr_t pa)212 vzeropage(paddr_t pa)
213 {
214 const paddr_t ea = pa + PAGE_SIZE;
215 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16);
216 register_t omsr, msr;
217
218 __asm volatile("mfmsr %0" : "=r"(omsr) :);
219
220 /*
221 * Turn on AltiVec, turn off interrupts.
222 */
223 msr = (omsr & ~PSL_EE) | PSL_VEC;
224 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr));
225
226 /*
227 * Save the VEC register we are going to use before we disable
228 * relocation.
229 */
230 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC));
231 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC));
232
233 /*
234 * Zero the page using a single cache line.
235 */
236 __asm volatile(
237 " sync ;"
238 " mfmsr %[msr];"
239 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */
240 " mtmsr %[msr];" /* Turn off DMMU */
241 " isync;"
242 "1: stvx %[zv], %[pa], %[off0];"
243 " stvxl %[zv], %[pa], %[off16];"
244 " stvx %[zv], %[pa], %[off32];"
245 " stvxl %[zv], %[pa], %[off48];"
246 " addi %[pa], %[pa], 64;"
247 " cmplw %[pa], %[ea];"
248 " blt+ 1b;"
249 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */
250 " sync;"
251 " mtmsr %[msr];" /* Turn on DMMU */
252 " isync;"
253 :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea),
254 [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48),
255 [zv] "n"(ZERO_VEC));
256
257 /*
258 * Restore VEC register (now that we can access the stack again).
259 */
260 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC));
261
262 /*
263 * Restore old MSR (AltiVec OFF).
264 */
265 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr));
266 }
267
268 #define LO_VEC 16
269 #define HI_VEC 17
270
271 void
vcopypage(paddr_t dst,paddr_t src)272 vcopypage(paddr_t dst, paddr_t src)
273 {
274 const paddr_t edst = dst + PAGE_SIZE;
275 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16);
276 register_t omsr, msr;
277
278 __asm volatile("mfmsr %0" : "=r"(omsr) :);
279
280 /*
281 * Turn on AltiVec, turn off interrupts.
282 */
283 msr = (omsr & ~PSL_EE) | PSL_VEC;
284 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr));
285
286 /*
287 * Save the VEC registers we will be using before we disable
288 * relocation.
289 */
290 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC));
291 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC));
292
293 /*
294 * Copy the page using a single cache line, with DMMU
295 * disabled. On most PPCs, two vector registers occupy one
296 * cache line.
297 */
298 __asm volatile(
299 " sync ;"
300 " mfmsr %[msr];"
301 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */
302 " mtmsr %[msr];" /* Turn off DMMU */
303 " isync;"
304 "1: lvx %[lv], %[src], %[off0];"
305 " stvx %[lv], %[dst], %[off0];"
306 " lvxl %[hv], %[src], %[off16];"
307 " stvxl %[hv], %[dst], %[off16];"
308 " addi %[src], %[src], 32;"
309 " addi %[dst], %[dst], 32;"
310 " cmplw %[dst], %[edst];"
311 " blt+ 1b;"
312 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */
313 " sync;"
314 " mtmsr %[msr];" /* Turn on DMMU */
315 " isync;"
316 :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst),
317 [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16),
318 [lv] "n"(LO_VEC), [hv] "n"(HI_VEC));
319
320 /*
321 * Restore VEC registers (now that we can access the stack again).
322 */
323 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC));
324 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC));
325
326 /*
327 * Restore old MSR (AltiVec OFF).
328 */
329 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr));
330 }
331