1 /*-
2 * Copyright (c) 2001 Jake Burkholder.
3 * Copyright (c) 2005, 2008 Marius Strobl <marius@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/pcpu.h>
35 #include <sys/proc.h>
36 #include <sys/sched.h>
37 #include <sys/smp.h>
38 #include <sys/sysctl.h>
39 #include <sys/timeet.h>
40 #include <sys/timetc.h>
41
42 #include <dev/ofw/openfirm.h>
43
44 #include <vm/vm.h>
45 #include <vm/pmap.h>
46
47 #include <machine/frame.h>
48 #include <machine/intr_machdep.h>
49 #include <machine/smp.h>
50 #include <machine/tick.h>
51 #include <machine/ver.h>
52
53 #define TICK_QUALITY_MP 10
54 #define TICK_QUALITY_UP 1000
55
56 static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
57
58 static int adjust_edges = 0;
59 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges,
60 0, "total number of times tick interrupts got more than 12.5% behind");
61
62 static int adjust_excess = 0;
63 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess,
64 0, "total number of ignored tick interrupts");
65
66 static int adjust_missed = 0;
67 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed,
68 0, "total number of missed tick interrupts");
69
70 static int adjust_ticks = 0;
71 SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks,
72 0, "total number of tick interrupts with adjustment");
73
74 u_int tick_et_use_stick = 0;
75 SYSCTL_INT(_machdep_tick, OID_AUTO, tick_et_use_stick, CTLFLAG_RD,
76 &tick_et_use_stick, 0, "tick event timer uses STICK instead of TICK");
77
78 typedef uint64_t rd_tick_t(void);
79 static rd_tick_t *rd_tick;
80 typedef void wr_tick_cmpr_t(uint64_t);
81 static wr_tick_cmpr_t *wr_tick_cmpr;
82
83 static struct timecounter stick_tc;
84 static struct eventtimer tick_et;
85 static struct timecounter tick_tc;
86
87 #ifdef SMP
88 static timecounter_get_t stick_get_timecount_mp;
89 #endif
90 static timecounter_get_t stick_get_timecount_up;
91 static rd_tick_t stick_rd;
92 static wr_tick_cmpr_t stick_wr_cmpr;
93 static int tick_et_start(struct eventtimer *et, sbintime_t first,
94 sbintime_t period);
95 static int tick_et_stop(struct eventtimer *et);
96 #ifdef SMP
97 static timecounter_get_t tick_get_timecount_mp;
98 #endif
99 static timecounter_get_t tick_get_timecount_up;
100 static void tick_intr(struct trapframe *tf);
101 static inline void tick_process(struct trapframe *tf);
102 static rd_tick_t tick_rd;
103 static wr_tick_cmpr_t tick_wr_cmpr;
104 static wr_tick_cmpr_t tick_wr_cmpr_bbwar;
105 static uint64_t tick_cputicks(void);
106
107 static uint64_t
stick_rd(void)108 stick_rd(void)
109 {
110
111 return (rdstick());
112 }
113
114 static void
stick_wr_cmpr(uint64_t tick)115 stick_wr_cmpr(uint64_t tick)
116 {
117
118 wrstickcmpr(tick, 0);
119 }
120
121 static uint64_t
tick_rd(void)122 tick_rd(void)
123 {
124
125 return (rd(tick));
126 }
127
128 static void
tick_wr_cmpr(uint64_t tick_cmpr)129 tick_wr_cmpr(uint64_t tick_cmpr)
130 {
131
132 wrtickcmpr(tick_cmpr, 0);
133 }
134
135 static void
tick_wr_cmpr_bbwar(uint64_t tick_cmpr)136 tick_wr_cmpr_bbwar(uint64_t tick_cmpr)
137 {
138
139 wrtickcmpr_bbwar(tick_cmpr, 0);
140 }
141
142 static uint64_t
tick_cputicks(void)143 tick_cputicks(void)
144 {
145
146 return (rd(tick));
147 }
148
149 void
cpu_initclocks(void)150 cpu_initclocks(void)
151 {
152 uint32_t clock, sclock;
153
154 clock = PCPU_GET(clock);
155 sclock = 0;
156 if (PCPU_GET(impl) == CPU_IMPL_SPARC64V ||
157 PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCIII) {
158 if (OF_getprop(OF_peer(0), "stick-frequency", &sclock,
159 sizeof(sclock)) == -1) {
160 panic("%s: could not determine STICK frequency",
161 __func__);
162 }
163 }
164 /*
165 * Given that the STICK timers typically are driven at rather low
166 * frequencies they shouldn't be used except when really necessary.
167 */
168 if (tick_et_use_stick != 0) {
169 rd_tick = stick_rd;
170 wr_tick_cmpr = stick_wr_cmpr;
171 /*
172 * We don't provide a CPU ticker as long as the frequency
173 * supplied isn't actually used per-CPU.
174 */
175 } else {
176 rd_tick = tick_rd;
177 if (PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCI &&
178 PCPU_GET(impl) < CPU_IMPL_ULTRASPARCIII)
179 wr_tick_cmpr = tick_wr_cmpr_bbwar;
180 else
181 wr_tick_cmpr = tick_wr_cmpr;
182 set_cputicker(tick_cputicks, clock, 0);
183 }
184 intr_setup(PIL_TICK, tick_intr, -1, NULL, NULL);
185
186 /*
187 * Initialize the (S)TICK-based timecounter(s).
188 * Note that we (try to) sync the (S)TICK timers of APs with the BSP
189 * during their startup but not afterwards. The resulting drift can
190 * cause problems when the time is calculated based on (S)TICK values
191 * read on different CPUs. Thus we always read the register on the
192 * BSP (if necessary via an IPI as sched_bind(9) isn't available in
193 * all circumstances) and use a low quality for the otherwise high
194 * quality (S)TICK timers in the MP case.
195 */
196 tick_tc.tc_get_timecount = tick_get_timecount_up;
197 tick_tc.tc_counter_mask = ~0u;
198 tick_tc.tc_frequency = clock;
199 tick_tc.tc_name = "tick";
200 tick_tc.tc_quality = TICK_QUALITY_UP;
201 #ifdef SMP
202 if (cpu_mp_probe()) {
203 tick_tc.tc_get_timecount = tick_get_timecount_mp;
204 tick_tc.tc_quality = TICK_QUALITY_MP;
205 }
206 #endif
207 tc_init(&tick_tc);
208 if (sclock != 0) {
209 stick_tc.tc_get_timecount = stick_get_timecount_up;
210 stick_tc.tc_counter_mask = ~0u;
211 stick_tc.tc_frequency = sclock;
212 stick_tc.tc_name = "stick";
213 stick_tc.tc_quality = TICK_QUALITY_UP;
214 #ifdef SMP
215 if (cpu_mp_probe()) {
216 stick_tc.tc_get_timecount = stick_get_timecount_mp;
217 stick_tc.tc_quality = TICK_QUALITY_MP;
218 }
219 #endif
220 tc_init(&stick_tc);
221 }
222 tick_et.et_name = tick_et_use_stick ? "stick" : "tick";
223 tick_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
224 ET_FLAGS_PERCPU;
225 tick_et.et_quality = 1000;
226 tick_et.et_frequency = tick_et_use_stick ? sclock : clock;
227 tick_et.et_min_period = 0x00010000LLU; /* To be safe. */
228 tick_et.et_max_period = (0xfffffffeLLU << 32) / tick_et.et_frequency;
229 tick_et.et_start = tick_et_start;
230 tick_et.et_stop = tick_et_stop;
231 tick_et.et_priv = NULL;
232 et_register(&tick_et);
233
234 cpu_initclocks_bsp();
235 }
236
237 static inline void
tick_process(struct trapframe * tf)238 tick_process(struct trapframe *tf)
239 {
240 struct trapframe *oldframe;
241 struct thread *td;
242
243 td = curthread;
244 td->td_intr_nesting_level++;
245 critical_enter();
246 if (tick_et.et_active) {
247 oldframe = td->td_intr_frame;
248 td->td_intr_frame = tf;
249 tick_et.et_event_cb(&tick_et, tick_et.et_arg);
250 td->td_intr_frame = oldframe;
251 }
252 td->td_intr_nesting_level--;
253 critical_exit();
254 }
255
256 static void
tick_intr(struct trapframe * tf)257 tick_intr(struct trapframe *tf)
258 {
259 u_long adj, ref, tick, tick_increment;
260 long delta;
261 register_t s;
262 int count;
263
264 tick_increment = PCPU_GET(tickincrement);
265 if (tick_increment != 0) {
266 /*
267 * NB: the sequence of reading the (S)TICK register,
268 * calculating the value of the next tick and writing it to
269 * the (S)TICK_COMPARE register must not be interrupted, not
270 * even by an IPI, otherwise a value that is in the past could
271 * be written in the worst case and thus causing the periodic
272 * timer to stop.
273 */
274 s = intr_disable();
275 adj = PCPU_GET(tickadj);
276 tick = rd_tick();
277 wr_tick_cmpr(tick + tick_increment - adj);
278 intr_restore(s);
279 ref = PCPU_GET(tickref);
280 delta = tick - ref;
281 count = 0;
282 while (delta >= tick_increment) {
283 tick_process(tf);
284 delta -= tick_increment;
285 ref += tick_increment;
286 if (adj != 0)
287 adjust_ticks++;
288 count++;
289 }
290 if (count > 0) {
291 adjust_missed += count - 1;
292 if (delta > (tick_increment >> 3)) {
293 if (adj == 0)
294 adjust_edges++;
295 adj = tick_increment >> 4;
296 } else
297 adj = 0;
298 } else {
299 adj = 0;
300 adjust_excess++;
301 }
302 PCPU_SET(tickref, ref);
303 PCPU_SET(tickadj, adj);
304 } else
305 tick_process(tf);
306 }
307
308 static u_int
stick_get_timecount_up(struct timecounter * tc)309 stick_get_timecount_up(struct timecounter *tc)
310 {
311
312 return ((u_int)rdstick());
313 }
314
315 static u_int
tick_get_timecount_up(struct timecounter * tc)316 tick_get_timecount_up(struct timecounter *tc)
317 {
318
319 return ((u_int)rd(tick));
320 }
321
322 #ifdef SMP
323 static u_int
stick_get_timecount_mp(struct timecounter * tc)324 stick_get_timecount_mp(struct timecounter *tc)
325 {
326 static u_long stick;
327
328 sched_pin();
329 if (curcpu == 0)
330 stick = rdstick();
331 else
332 ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
333 sched_unpin();
334 return (stick);
335 }
336
337 static u_int
tick_get_timecount_mp(struct timecounter * tc)338 tick_get_timecount_mp(struct timecounter *tc)
339 {
340 static u_long tick;
341
342 sched_pin();
343 if (curcpu == 0)
344 tick = rd(tick);
345 else
346 ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
347 sched_unpin();
348 return (tick);
349 }
350 #endif
351
352 static int
tick_et_start(struct eventtimer * et,sbintime_t first,sbintime_t period)353 tick_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
354 {
355 u_long base, div, fdiv;
356 register_t s;
357
358 if (period != 0)
359 div = (tick_et.et_frequency * period) >> 32;
360 else
361 div = 0;
362 if (first != 0)
363 fdiv = (tick_et.et_frequency * first) >> 32;
364 else
365 fdiv = div;
366 PCPU_SET(tickincrement, div);
367
368 /*
369 * Try to make the (S)TICK interrupts as synchronously as possible
370 * on all CPUs to avoid inaccuracies for migrating processes. Leave
371 * out one tick to make sure that it is not missed.
372 */
373 s = intr_disable();
374 base = rd_tick();
375 if (div != 0) {
376 PCPU_SET(tickadj, 0);
377 base = roundup(base, div);
378 }
379 PCPU_SET(tickref, base);
380 wr_tick_cmpr(base + fdiv);
381 intr_restore(s);
382 return (0);
383 }
384
385 static int
tick_et_stop(struct eventtimer * et)386 tick_et_stop(struct eventtimer *et)
387 {
388
389 PCPU_SET(tickincrement, 0);
390 tick_stop(PCPU_GET(impl));
391 return (0);
392 }
393
394 void
tick_clear(u_int cpu_impl)395 tick_clear(u_int cpu_impl)
396 {
397
398 if (cpu_impl == CPU_IMPL_SPARC64V ||
399 cpu_impl >= CPU_IMPL_ULTRASPARCIII)
400 wrstick(0, 0);
401 wrpr(tick, 0, 0);
402 }
403
404 void
tick_stop(u_int cpu_impl)405 tick_stop(u_int cpu_impl)
406 {
407
408 if (cpu_impl == CPU_IMPL_SPARC64V ||
409 cpu_impl >= CPU_IMPL_ULTRASPARCIII)
410 wrstickcmpr(1L << 63, 0);
411 wrtickcmpr(1L << 63, 0);
412 }
413