1 /* $NetBSD: cpufunc.h,v 1.28 2024/12/30 19:13:48 jmcneill Exp $ */
2
3 /*
4 * Copyright (c) 2017 Ryo Shimizu
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #ifndef _AARCH64_CPUFUNC_H_
30 #define _AARCH64_CPUFUNC_H_
31
32 #ifdef _KERNEL
33
34 #include <arm/armreg.h>
35 #include <sys/device_if.h>
36
37 #include <sys/cpu.h>
38
39 #include <uvm/uvm_extern.h>
40 #include <uvm/pmap/pmap_devmap.h>
41
42 extern u_int aarch64_cache_vindexsize; /* cachesize/way (VIVT/VIPT) */
43 extern u_int aarch64_cache_prefer_mask;
44 extern u_int cputype; /* compat arm */
45
46 extern int aarch64_bti_enabled;
47 extern int aarch64_hafdbs_enabled;
48 extern int aarch64_pan_enabled;
49 extern int aarch64_pac_enabled;
50
51 void aarch64_hafdbs_init(int);
52 void aarch64_pan_init(int);
53 int aarch64_pac_init(int);
54
55 void aarch64_cpu_idle_wfi(void);
56
57 int set_cpufuncs(void);
58 int aarch64_setcpufuncs(struct cpu_info *);
59 void aarch64_getcacheinfo(struct cpu_info *);
60 void aarch64_parsecacheinfo(struct cpu_info *);
61 void aarch64_printcacheinfo(device_t, struct cpu_info *);
62
63 void aarch64_dcache_wbinv_all(void);
64 void aarch64_dcache_inv_all(void);
65 void aarch64_dcache_wb_all(void);
66 void aarch64_icache_inv_all(void);
67
68 /* cache op in cpufunc_asm_armv8.S */
69 void aarch64_nullop(void);
70 uint32_t aarch64_cpuid(void);
71 void aarch64_icache_sync_range(vaddr_t, vsize_t);
72 void aarch64_icache_inv_range(vaddr_t, vsize_t);
73 void aarch64_icache_barrier_range(vaddr_t, vsize_t);
74 void aarch64_idcache_wbinv_range(vaddr_t, vsize_t);
75 void aarch64_dcache_wbinv_range(vaddr_t, vsize_t);
76 void aarch64_dcache_inv_range(vaddr_t, vsize_t);
77 void aarch64_dcache_wb_range(vaddr_t, vsize_t);
78 void aarch64_icache_inv_all(void);
79 void aarch64_drain_writebuf(void);
80
81 /* tlb op in cpufunc_asm_armv8.S */
82 #define cpu_set_ttbr0(t) curcpu()->ci_cpufuncs.cf_set_ttbr0((t))
83 void aarch64_set_ttbr0(uint64_t);
84 void aarch64_set_ttbr0_thunderx(uint64_t);
85 void aarch64_tlbi_all(void); /* all ASID, all VA */
86 void aarch64_tlbi_by_asid(int); /* an ASID, all VA */
87 void aarch64_tlbi_by_va(vaddr_t); /* all ASID, a VA */
88 void aarch64_tlbi_by_va_ll(vaddr_t); /* all ASID, a VA, lastlevel */
89 void aarch64_tlbi_by_asid_va(int, vaddr_t); /* an ASID, a VA */
90 void aarch64_tlbi_by_asid_va_ll(int, vaddr_t); /* an ASID, a VA, lastlevel */
91
92 /* misc */
93 #define cpu_idnum() aarch64_cpuid()
94
95 /* cache op */
96 #define cpu_dcache_wbinv_all() aarch64_dcache_wbinv_all()
97 #define cpu_dcache_inv_all() aarch64_dcache_inv_all()
98 #define cpu_dcache_wb_all() aarch64_dcache_wb_all()
99 #define cpu_idcache_wbinv_all() \
100 (aarch64_dcache_wbinv_all(), aarch64_icache_inv_all())
101 #define cpu_icache_sync_all() \
102 (aarch64_dcache_wb_all(), aarch64_icache_inv_all())
103 #define cpu_icache_inv_all() aarch64_icache_inv_all()
104
105 #define cpu_dcache_wbinv_range(v,s) aarch64_dcache_wbinv_range((v),(s))
106 #define cpu_dcache_inv_range(v,s) aarch64_dcache_inv_range((v),(s))
107 #define cpu_dcache_wb_range(v,s) aarch64_dcache_wb_range((v),(s))
108 #define cpu_idcache_wbinv_range(v,s) aarch64_idcache_wbinv_range((v),(s))
109 #define cpu_icache_sync_range(v,s) \
110 curcpu()->ci_cpufuncs.cf_icache_sync_range((v),(s))
111
112 #define cpu_sdcache_wbinv_range(v,p,s) ((void)0)
113 #define cpu_sdcache_inv_range(v,p,s) ((void)0)
114 #define cpu_sdcache_wb_range(v,p,s) ((void)0)
115
116 /* others */
117 #define cpu_drain_writebuf() aarch64_drain_writebuf()
118
119 extern u_int arm_dcache_align;
120 extern u_int arm_dcache_align_mask;
121
122 static inline bool
cpu_gtmr_exists_p(void)123 cpu_gtmr_exists_p(void)
124 {
125
126 return true;
127 }
128
129 static inline u_int
cpu_clusterid(void)130 cpu_clusterid(void)
131 {
132
133 return __SHIFTOUT(reg_mpidr_el1_read(), MPIDR_AFF1);
134 }
135
136 static inline bool
cpu_earlydevice_va_p(void)137 cpu_earlydevice_va_p(void)
138 {
139 /* This function may be called before enabling MMU, or mapping KVA */
140 if ((reg_sctlr_el1_read() & SCTLR_M) == 0)
141 return false;
142
143 /* device mapping will be available after pmap_devmap_bootstrap() */
144 if (!pmap_devmap_bootstrapped_p())
145 return false;
146
147 return true;
148 }
149
150 #endif /* _KERNEL */
151
152 /* definitions of TAG and PAC in pointers */
153 #define AARCH64_ADDRTOP_TAG_BIT 55
154 #define AARCH64_ADDRTOP_TAG __BIT(55) /* ECR_EL1.TBI[01]=1 */
155 #define AARCH64_ADDRTOP_MSB __BIT(63) /* ECR_EL1.TBI[01]=0 */
156 #define AARCH64_ADDRESS_TAG_MASK __BITS(63,56) /* if TCR.TBI[01]=1 */
157 #define AARCH64_ADDRESS_PAC_MASK __BITS(54,48) /* depend on VIRT_BIT */
158 #define AARCH64_ADDRESS_TAGPAC_MASK \
159 (AARCH64_ADDRESS_TAG_MASK|AARCH64_ADDRESS_PAC_MASK)
160
161 #ifdef _KERNEL
162 /*
163 * Which is the address space of this VA?
164 * return the space considering TBI. (PAC is not yet)
165 *
166 * return value: AARCH64_ADDRSPACE_{LOWER,UPPER}{_OUTOFRANGE}?
167 */
168 #define AARCH64_ADDRSPACE_LOWER 0 /* -> TTBR0 */
169 #define AARCH64_ADDRSPACE_UPPER 1 /* -> TTBR1 */
170 #define AARCH64_ADDRSPACE_LOWER_OUTOFRANGE -1 /* certainly fault */
171 #define AARCH64_ADDRSPACE_UPPER_OUTOFRANGE -2 /* certainly fault */
172 static inline int
aarch64_addressspace(vaddr_t va)173 aarch64_addressspace(vaddr_t va)
174 {
175 uint64_t addrtop, tbi;
176
177 addrtop = va & AARCH64_ADDRTOP_TAG;
178 tbi = addrtop ? TCR_TBI1 : TCR_TBI0;
179 if (reg_tcr_el1_read() & tbi) {
180 if (addrtop == 0) {
181 /* lower address, and TBI0 enabled */
182 if ((va & AARCH64_ADDRESS_PAC_MASK) != 0)
183 return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE;
184 return AARCH64_ADDRSPACE_LOWER;
185 }
186 /* upper address, and TBI1 enabled */
187 if ((va & AARCH64_ADDRESS_PAC_MASK) != AARCH64_ADDRESS_PAC_MASK)
188 return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE;
189 return AARCH64_ADDRSPACE_UPPER;
190 }
191
192 addrtop = va & AARCH64_ADDRTOP_MSB;
193 if (addrtop == 0) {
194 /* lower address, and TBI0 disabled */
195 if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != 0)
196 return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE;
197 return AARCH64_ADDRSPACE_LOWER;
198 }
199 /* upper address, and TBI1 disabled */
200 if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != AARCH64_ADDRESS_TAGPAC_MASK)
201 return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE;
202 return AARCH64_ADDRSPACE_UPPER;
203 }
204
205 static inline vaddr_t
aarch64_untag_address(vaddr_t va)206 aarch64_untag_address(vaddr_t va)
207 {
208 uint64_t addrtop, tbi;
209
210 addrtop = va & AARCH64_ADDRTOP_TAG;
211 tbi = addrtop ? TCR_TBI1 : TCR_TBI0;
212 if (reg_tcr_el1_read() & tbi) {
213 if (addrtop == 0) {
214 /* lower address, and TBI0 enabled */
215 return va & ~AARCH64_ADDRESS_TAG_MASK;
216 }
217 /* upper address, and TBI1 enabled */
218 return va | AARCH64_ADDRESS_TAG_MASK;
219 }
220
221 /* TBI[01] is disabled, nothing to do */
222 return va;
223 }
224
225 #endif /* _KERNEL */
226
227 static __inline uint64_t
aarch64_strip_pac(uint64_t __val)228 aarch64_strip_pac(uint64_t __val)
229 {
230 if (__val & AARCH64_ADDRTOP_TAG)
231 return __val | AARCH64_ADDRESS_TAGPAC_MASK;
232 return __val & ~AARCH64_ADDRESS_TAGPAC_MASK;
233 }
234
235 #endif /* _AARCH64_CPUFUNC_H_ */
236