1 /******************************************************************************
2 * evtchn.c
3 *
4 * Driver for receiving and demuxing event-channel signals.
5 *
6 * Copyright (c) 2004-2005, K A Fraser
7 * Multi-process extensions Copyright (c) 2004, Steven Smith
8 * FreeBSD port Copyright (c) 2014, Roger Pau Monné
9 * Fetched from git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
10 * File: drivers/xen/evtchn.c
11 * Git commit: 0dc0064add422bc0ef5165ebe9ece3052bbd457d
12 *
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License version 2
15 * as published by the Free Software Foundation; or, when distributed
16 * separately from the Linux kernel or incorporated into other
17 * software packages, subject to the following license:
18 *
19 * Permission is hereby granted, free of charge, to any person obtaining a copy
20 * of this source file (the "Software"), to deal in the Software without
21 * restriction, including without limitation the rights to use, copy, modify,
22 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
23 * and to permit persons to whom the Software is furnished to do so, subject to
24 * the following conditions:
25 *
26 * The above copyright notice and this permission notice shall be included in
27 * all copies or substantial portions of the Software.
28 *
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
32 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
34 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
35 * IN THE SOFTWARE.
36 */
37
38 #include <sys/cdefs.h>
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/uio.h>
42 #include <sys/bus.h>
43 #include <sys/malloc.h>
44 #include <sys/kernel.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47 #include <sys/sx.h>
48 #include <sys/selinfo.h>
49 #include <sys/poll.h>
50 #include <sys/conf.h>
51 #include <sys/fcntl.h>
52 #include <sys/ioccom.h>
53 #include <sys/rman.h>
54 #include <sys/tree.h>
55 #include <sys/module.h>
56 #include <sys/filio.h>
57 #include <sys/vnode.h>
58
59 #include <machine/intr_machdep.h>
60 #include <machine/xen/synch_bitops.h>
61
62 #include <xen/xen-os.h>
63 #include <xen/evtchn.h>
64 #include <xen/xen_intr.h>
65
66 #include <xen/evtchn/evtchnvar.h>
67
68 MALLOC_DEFINE(M_EVTCHN, "evtchn_dev", "Xen event channel user-space device");
69
70 struct user_evtchn;
71
72 static int evtchn_cmp(struct user_evtchn *u1, struct user_evtchn *u2);
73
74 RB_HEAD(evtchn_tree, user_evtchn);
75
76 struct per_user_data {
77 struct mtx bind_mutex; /* serialize bind/unbind operations */
78 struct evtchn_tree evtchns;
79
80 /* Notification ring, accessed via /dev/xen/evtchn. */
81 #define EVTCHN_RING_SIZE (PAGE_SIZE / sizeof(evtchn_port_t))
82 #define EVTCHN_RING_MASK(_i) ((_i)&(EVTCHN_RING_SIZE-1))
83 evtchn_port_t *ring;
84 unsigned int ring_cons, ring_prod, ring_overflow;
85 struct sx ring_cons_mutex; /* protect against concurrent readers */
86 struct mtx ring_prod_mutex; /* product against concurrent interrupts */
87 struct selinfo ev_rsel;
88 };
89
90 struct user_evtchn {
91 RB_ENTRY(user_evtchn) node;
92 struct per_user_data *user;
93 evtchn_port_t port;
94 xen_intr_handle_t handle;
95 bool enabled;
96 };
97
98 RB_GENERATE_STATIC(evtchn_tree, user_evtchn, node, evtchn_cmp);
99
100 static device_t evtchn_dev;
101
102 static d_read_t evtchn_read;
103 static d_write_t evtchn_write;
104 static d_ioctl_t evtchn_ioctl;
105 static d_poll_t evtchn_poll;
106 static d_open_t evtchn_open;
107
108 static void evtchn_release(void *arg);
109
110 static struct cdevsw evtchn_devsw = {
111 .d_version = D_VERSION,
112 .d_open = evtchn_open,
113 .d_read = evtchn_read,
114 .d_write = evtchn_write,
115 .d_ioctl = evtchn_ioctl,
116 .d_poll = evtchn_poll,
117 .d_name = "evtchn",
118 };
119
120 /*------------------------- Red-black tree helpers ---------------------------*/
121 static int
evtchn_cmp(struct user_evtchn * u1,struct user_evtchn * u2)122 evtchn_cmp(struct user_evtchn *u1, struct user_evtchn *u2)
123 {
124
125 return (u1->port - u2->port);
126 }
127
128 static struct user_evtchn *
find_evtchn(struct per_user_data * u,evtchn_port_t port)129 find_evtchn(struct per_user_data *u, evtchn_port_t port)
130 {
131 struct user_evtchn tmp = {
132 .port = port,
133 };
134
135 return (RB_FIND(evtchn_tree, &u->evtchns, &tmp));
136 }
137
138 /*--------------------------- Interrupt handlers -----------------------------*/
139 static int
evtchn_filter(void * arg)140 evtchn_filter(void *arg)
141 {
142 struct user_evtchn *evtchn;
143
144 evtchn = arg;
145
146 if (!evtchn->enabled && bootverbose) {
147 device_printf(evtchn_dev,
148 "Received upcall for disabled event channel %d\n",
149 evtchn->port);
150 }
151
152 evtchn_mask_port(evtchn->port);
153 evtchn->enabled = false;
154
155 return (FILTER_SCHEDULE_THREAD);
156 }
157
158 static void
evtchn_interrupt(void * arg)159 evtchn_interrupt(void *arg)
160 {
161 struct user_evtchn *evtchn;
162 struct per_user_data *u;
163
164 evtchn = arg;
165 u = evtchn->user;
166
167 /*
168 * Protect against concurrent events using this handler
169 * on different CPUs.
170 */
171 mtx_lock(&u->ring_prod_mutex);
172 if ((u->ring_prod - u->ring_cons) < EVTCHN_RING_SIZE) {
173 u->ring[EVTCHN_RING_MASK(u->ring_prod)] = evtchn->port;
174 wmb(); /* Ensure ring contents visible */
175 if (u->ring_cons == u->ring_prod++) {
176 wakeup(u);
177 selwakeup(&u->ev_rsel);
178 }
179 } else
180 u->ring_overflow = 1;
181 mtx_unlock(&u->ring_prod_mutex);
182 }
183
184 /*------------------------- Character device methods -------------------------*/
185 static int
evtchn_open(struct cdev * dev,int flag,int otyp,struct thread * td)186 evtchn_open(struct cdev *dev, int flag, int otyp, struct thread *td)
187 {
188 struct per_user_data *u;
189 int error;
190
191 u = malloc(sizeof(*u), M_EVTCHN, M_WAITOK | M_ZERO);
192 u->ring = malloc(PAGE_SIZE, M_EVTCHN, M_WAITOK | M_ZERO);
193
194 /* Initialize locks */
195 mtx_init(&u->bind_mutex, "evtchn_bind_mutex", NULL, MTX_DEF);
196 sx_init(&u->ring_cons_mutex, "evtchn_ringc_sx");
197 mtx_init(&u->ring_prod_mutex, "evtchn_ringp_mutex", NULL, MTX_DEF);
198
199 /* Initialize red-black tree. */
200 RB_INIT(&u->evtchns);
201
202 /* Assign the allocated per_user_data to this open instance. */
203 error = devfs_set_cdevpriv(u, evtchn_release);
204 if (error != 0) {
205 mtx_destroy(&u->bind_mutex);
206 mtx_destroy(&u->ring_prod_mutex);
207 sx_destroy(&u->ring_cons_mutex);
208 free(u->ring, M_EVTCHN);
209 free(u, M_EVTCHN);
210 }
211
212 return (error);
213 }
214
215 static void
evtchn_release(void * arg)216 evtchn_release(void *arg)
217 {
218 struct per_user_data *u;
219 struct user_evtchn *evtchn, *tmp;
220
221 u = arg;
222
223 seldrain(&u->ev_rsel);
224
225 RB_FOREACH_SAFE(evtchn, evtchn_tree, &u->evtchns, tmp) {
226 xen_intr_unbind(&evtchn->handle);
227
228 RB_REMOVE(evtchn_tree, &u->evtchns, evtchn);
229 free(evtchn, M_EVTCHN);
230 }
231
232 mtx_destroy(&u->bind_mutex);
233 mtx_destroy(&u->ring_prod_mutex);
234 sx_destroy(&u->ring_cons_mutex);
235 free(u->ring, M_EVTCHN);
236 free(u, M_EVTCHN);
237 }
238
239 static int
evtchn_read(struct cdev * dev,struct uio * uio,int ioflag)240 evtchn_read(struct cdev *dev, struct uio *uio, int ioflag)
241 {
242 int error, count;
243 unsigned int c, p, bytes1 = 0, bytes2 = 0;
244 struct per_user_data *u;
245
246 error = devfs_get_cdevpriv((void **)&u);
247 if (error != 0)
248 return (EINVAL);
249
250 /* Whole number of ports. */
251 count = uio->uio_resid;
252 count &= ~(sizeof(evtchn_port_t)-1);
253
254 if (count == 0)
255 return (0);
256
257 if (count > PAGE_SIZE)
258 count = PAGE_SIZE;
259
260 sx_xlock(&u->ring_cons_mutex);
261 for (;;) {
262 if (u->ring_overflow) {
263 error = EFBIG;
264 goto unlock_out;
265 }
266
267 c = u->ring_cons;
268 p = u->ring_prod;
269 if (c != p)
270 break;
271
272 if (ioflag & IO_NDELAY) {
273 error = EWOULDBLOCK;
274 goto unlock_out;
275 }
276
277 error = sx_sleep(u, &u->ring_cons_mutex, PCATCH, "evtchw", 0);
278 if ((error != 0) && (error != EWOULDBLOCK))
279 goto unlock_out;
280 }
281
282 /* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */
283 if (((c ^ p) & EVTCHN_RING_SIZE) != 0) {
284 bytes1 = (EVTCHN_RING_SIZE - EVTCHN_RING_MASK(c)) *
285 sizeof(evtchn_port_t);
286 bytes2 = EVTCHN_RING_MASK(p) * sizeof(evtchn_port_t);
287 } else {
288 bytes1 = (p - c) * sizeof(evtchn_port_t);
289 bytes2 = 0;
290 }
291
292 /* Truncate chunks according to caller's maximum byte count. */
293 if (bytes1 > count) {
294 bytes1 = count;
295 bytes2 = 0;
296 } else if ((bytes1 + bytes2) > count) {
297 bytes2 = count - bytes1;
298 }
299
300 error = EFAULT;
301 rmb(); /* Ensure that we see the port before we copy it. */
302
303 if (uiomove(&u->ring[EVTCHN_RING_MASK(c)], bytes1, uio) ||
304 ((bytes2 != 0) && uiomove(&u->ring[0], bytes2, uio)))
305 goto unlock_out;
306
307 u->ring_cons += (bytes1 + bytes2) / sizeof(evtchn_port_t);
308 error = 0;
309
310 unlock_out:
311 sx_xunlock(&u->ring_cons_mutex);
312 return (error);
313 }
314
315 static int
evtchn_write(struct cdev * dev,struct uio * uio,int ioflag)316 evtchn_write(struct cdev *dev, struct uio *uio, int ioflag)
317 {
318 int error, i, count;
319 evtchn_port_t *kbuf;
320 struct per_user_data *u;
321
322 error = devfs_get_cdevpriv((void **)&u);
323 if (error != 0)
324 return (EINVAL);
325
326 kbuf = malloc(PAGE_SIZE, M_EVTCHN, M_WAITOK);
327
328 count = uio->uio_resid;
329 /* Whole number of ports. */
330 count &= ~(sizeof(evtchn_port_t)-1);
331
332 error = 0;
333 if (count == 0)
334 goto out;
335
336 if (count > PAGE_SIZE)
337 count = PAGE_SIZE;
338
339 error = uiomove(kbuf, count, uio);
340 if (error != 0)
341 goto out;
342
343 mtx_lock(&u->bind_mutex);
344
345 for (i = 0; i < (count/sizeof(evtchn_port_t)); i++) {
346 evtchn_port_t port = kbuf[i];
347 struct user_evtchn *evtchn;
348
349 evtchn = find_evtchn(u, port);
350 if (evtchn && !evtchn->enabled) {
351 evtchn->enabled = true;
352 evtchn_unmask_port(evtchn->port);
353 }
354 }
355
356 mtx_unlock(&u->bind_mutex);
357 error = 0;
358
359 out:
360 free(kbuf, M_EVTCHN);
361 return (error);
362 }
363
364 static inline int
evtchn_bind_user_port(struct per_user_data * u,struct user_evtchn * evtchn)365 evtchn_bind_user_port(struct per_user_data *u, struct user_evtchn *evtchn)
366 {
367 int error;
368
369 evtchn->port = xen_intr_port(evtchn->handle);
370 evtchn->user = u;
371 evtchn->enabled = true;
372 mtx_lock(&u->bind_mutex);
373 RB_INSERT(evtchn_tree, &u->evtchns, evtchn);
374 mtx_unlock(&u->bind_mutex);
375 error = xen_intr_add_handler(device_get_nameunit(evtchn_dev),
376 evtchn_filter, evtchn_interrupt, evtchn,
377 INTR_TYPE_MISC | INTR_MPSAFE, evtchn->handle);
378 if (error != 0) {
379 xen_intr_unbind(&evtchn->handle);
380 mtx_lock(&u->bind_mutex);
381 RB_REMOVE(evtchn_tree, &u->evtchns, evtchn);
382 mtx_unlock(&u->bind_mutex);
383 free(evtchn, M_EVTCHN);
384 }
385 return (error);
386 }
387
388 static int
evtchn_ioctl(struct cdev * dev,unsigned long cmd,caddr_t arg,int mode,struct thread * td __unused)389 evtchn_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg,
390 int mode, struct thread *td __unused)
391 {
392 struct per_user_data *u;
393 int error;
394
395 error = devfs_get_cdevpriv((void **)&u);
396 if (error != 0)
397 return (EINVAL);
398
399 switch (cmd) {
400 case IOCTL_EVTCHN_BIND_VIRQ: {
401 struct ioctl_evtchn_bind_virq *bind;
402 struct user_evtchn *evtchn;
403
404 evtchn = malloc(sizeof(*evtchn), M_EVTCHN, M_WAITOK | M_ZERO);
405
406 bind = (struct ioctl_evtchn_bind_virq *)arg;
407
408 error = xen_intr_bind_virq(evtchn_dev, bind->virq, 0,
409 NULL, NULL, NULL, 0, &evtchn->handle);
410 if (error != 0) {
411 free(evtchn, M_EVTCHN);
412 break;
413 }
414 error = evtchn_bind_user_port(u, evtchn);
415 if (error != 0)
416 break;
417 bind->port = evtchn->port;
418 break;
419 }
420
421 case IOCTL_EVTCHN_BIND_INTERDOMAIN: {
422 struct ioctl_evtchn_bind_interdomain *bind;
423 struct user_evtchn *evtchn;
424
425 evtchn = malloc(sizeof(*evtchn), M_EVTCHN, M_WAITOK | M_ZERO);
426
427 bind = (struct ioctl_evtchn_bind_interdomain *)arg;
428
429 error = xen_intr_bind_remote_port(evtchn_dev,
430 bind->remote_domain, bind->remote_port, NULL,
431 NULL, NULL, 0, &evtchn->handle);
432 if (error != 0) {
433 free(evtchn, M_EVTCHN);
434 break;
435 }
436 error = evtchn_bind_user_port(u, evtchn);
437 if (error != 0)
438 break;
439 bind->port = evtchn->port;
440 break;
441 }
442
443 case IOCTL_EVTCHN_BIND_UNBOUND_PORT: {
444 struct ioctl_evtchn_bind_unbound_port *bind;
445 struct user_evtchn *evtchn;
446
447 evtchn = malloc(sizeof(*evtchn), M_EVTCHN, M_WAITOK | M_ZERO);
448
449 bind = (struct ioctl_evtchn_bind_unbound_port *)arg;
450
451 error = xen_intr_alloc_and_bind_local_port(evtchn_dev,
452 bind->remote_domain, NULL, NULL, NULL, 0, &evtchn->handle);
453 if (error != 0) {
454 free(evtchn, M_EVTCHN);
455 break;
456 }
457 error = evtchn_bind_user_port(u, evtchn);
458 if (error != 0)
459 break;
460 bind->port = evtchn->port;
461 break;
462 }
463
464 case IOCTL_EVTCHN_UNBIND: {
465 struct ioctl_evtchn_unbind *unbind;
466 struct user_evtchn *evtchn;
467
468 unbind = (struct ioctl_evtchn_unbind *)arg;
469
470 mtx_lock(&u->bind_mutex);
471 evtchn = find_evtchn(u, unbind->port);
472 if (evtchn == NULL) {
473 error = ENOTCONN;
474 break;
475 }
476 RB_REMOVE(evtchn_tree, &u->evtchns, evtchn);
477 mtx_unlock(&u->bind_mutex);
478
479 xen_intr_unbind(&evtchn->handle);
480 free(evtchn, M_EVTCHN);
481 error = 0;
482 break;
483 }
484
485 case IOCTL_EVTCHN_NOTIFY: {
486 struct ioctl_evtchn_notify *notify;
487 struct user_evtchn *evtchn;
488
489 notify = (struct ioctl_evtchn_notify *)arg;
490
491 mtx_lock(&u->bind_mutex);
492 evtchn = find_evtchn(u, notify->port);
493 if (evtchn == NULL) {
494 error = ENOTCONN;
495 break;
496 }
497
498 xen_intr_signal(evtchn->handle);
499 mtx_unlock(&u->bind_mutex);
500 error = 0;
501 break;
502 }
503
504 case IOCTL_EVTCHN_RESET: {
505 /* Initialise the ring to empty. Clear errors. */
506 sx_xlock(&u->ring_cons_mutex);
507 mtx_lock(&u->ring_prod_mutex);
508 u->ring_cons = u->ring_prod = u->ring_overflow = 0;
509 mtx_unlock(&u->ring_prod_mutex);
510 sx_xunlock(&u->ring_cons_mutex);
511 error = 0;
512 break;
513 }
514
515 case FIONBIO:
516 case FIOASYNC:
517 /* Handled in an upper layer */
518 error = 0;
519 break;
520
521 default:
522 error = ENOTTY;
523 break;
524 }
525
526 return (error);
527 }
528
529 static int
evtchn_poll(struct cdev * dev,int events,struct thread * td)530 evtchn_poll(struct cdev *dev, int events, struct thread *td)
531 {
532 struct per_user_data *u;
533 int error, mask;
534
535 error = devfs_get_cdevpriv((void **)&u);
536 if (error != 0)
537 return (POLLERR);
538
539 /* we can always write */
540 mask = events & (POLLOUT | POLLWRNORM);
541
542 mtx_lock(&u->ring_prod_mutex);
543 if (events & (POLLIN | POLLRDNORM)) {
544 if (u->ring_cons != u->ring_prod) {
545 mask |= events & (POLLIN | POLLRDNORM);
546 } else {
547 /* Record that someone is waiting */
548 selrecord(td, &u->ev_rsel);
549 }
550 }
551 mtx_unlock(&u->ring_prod_mutex);
552
553 return (mask);
554 }
555
556 /*------------------ Private Device Attachment Functions --------------------*/
557 static void
evtchn_identify(driver_t * driver,device_t parent)558 evtchn_identify(driver_t *driver, device_t parent)
559 {
560
561 KASSERT((xen_domain()),
562 ("Trying to attach evtchn device on non Xen domain"));
563
564 evtchn_dev = BUS_ADD_CHILD(parent, 0, "evtchn", 0);
565 if (evtchn_dev == NULL)
566 panic("unable to attach evtchn user-space device");
567 }
568
569 static int
evtchn_probe(device_t dev)570 evtchn_probe(device_t dev)
571 {
572
573 device_set_desc(dev, "Xen event channel user-space device");
574 return (BUS_PROBE_NOWILDCARD);
575 }
576
577 static int
evtchn_attach(device_t dev)578 evtchn_attach(device_t dev)
579 {
580
581 make_dev_credf(MAKEDEV_ETERNAL, &evtchn_devsw, 0, NULL, UID_ROOT,
582 GID_WHEEL, 0600, "xen/evtchn");
583 return (0);
584 }
585
586 /*-------------------- Private Device Attachment Data -----------------------*/
587 static device_method_t evtchn_methods[] = {
588 DEVMETHOD(device_identify, evtchn_identify),
589 DEVMETHOD(device_probe, evtchn_probe),
590 DEVMETHOD(device_attach, evtchn_attach),
591
592 DEVMETHOD_END
593 };
594
595 static driver_t evtchn_driver = {
596 "evtchn",
597 evtchn_methods,
598 0,
599 };
600
601 devclass_t evtchn_devclass;
602
603 DRIVER_MODULE(evtchn, xenpv, evtchn_driver, evtchn_devclass, 0, 0);
604 MODULE_DEPEND(evtchn, xenpv, 1, 1, 1);
605