1 /*-
2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * This software was developed by SRI International and the University of
6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
7 * ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /*
32 * BERI virtio block backend driver
33 */
34
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/bus.h>
39 #include <sys/kernel.h>
40 #include <sys/module.h>
41 #include <sys/rman.h>
42 #include <sys/conf.h>
43 #include <sys/stat.h>
44 #include <sys/endian.h>
45 #include <sys/disk.h>
46 #include <sys/vnode.h>
47 #include <sys/fcntl.h>
48 #include <sys/kthread.h>
49 #include <sys/buf.h>
50 #include <sys/mdioctl.h>
51 #include <sys/namei.h>
52
53 #include <machine/bus.h>
54 #include <machine/fdt.h>
55 #include <machine/cpu.h>
56 #include <machine/intr.h>
57
58 #include <dev/fdt/fdt_common.h>
59 #include <dev/ofw/openfirm.h>
60 #include <dev/ofw/ofw_bus.h>
61 #include <dev/ofw/ofw_bus_subr.h>
62
63 #include <dev/beri/virtio/virtio.h>
64 #include <dev/beri/virtio/virtio_mmio_platform.h>
65 #include <dev/altera/pio/pio.h>
66 #include <dev/virtio/mmio/virtio_mmio.h>
67 #include <dev/virtio/block/virtio_blk.h>
68 #include <dev/virtio/virtio_ids.h>
69 #include <dev/virtio/virtio_config.h>
70 #include <dev/virtio/virtio_ring.h>
71
72 #include "pio_if.h"
73
74 #define DPRINTF(fmt, ...)
75
76 /* We use indirect descriptors */
77 #define NUM_DESCS 1
78 #define NUM_QUEUES 1
79
80 #define VTBLK_BLK_ID_BYTES 20
81 #define VTBLK_MAXSEGS 256
82
83 struct beri_vtblk_softc {
84 struct resource *res[1];
85 bus_space_tag_t bst;
86 bus_space_handle_t bsh;
87 struct cdev *cdev;
88 device_t dev;
89 int opened;
90 device_t pio_recv;
91 device_t pio_send;
92 struct vqueue_info vs_queues[NUM_QUEUES];
93 char ident[VTBLK_BLK_ID_BYTES];
94 struct ucred *cred;
95 struct vnode *vnode;
96 struct thread *vtblk_ktd;
97 struct sx sc_mtx;
98 int beri_mem_offset;
99 struct md_ioctl *mdio;
100 struct virtio_blk_config *cfg;
101 };
102
103 static struct resource_spec beri_spec[] = {
104 { SYS_RES_MEMORY, 0, RF_ACTIVE },
105 { -1, 0 }
106 };
107
108 static int
vtblk_rdwr(struct beri_vtblk_softc * sc,struct iovec * iov,int cnt,int offset,int operation,int iolen)109 vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov,
110 int cnt, int offset, int operation, int iolen)
111 {
112 struct vnode *vp;
113 struct mount *mp;
114 struct uio auio;
115 int error;
116
117 bzero(&auio, sizeof(auio));
118
119 vp = sc->vnode;
120
121 KASSERT(vp != NULL, ("file not opened"));
122
123 auio.uio_iov = iov;
124 auio.uio_iovcnt = cnt;
125 auio.uio_offset = offset;
126 auio.uio_segflg = UIO_SYSSPACE;
127 auio.uio_rw = operation;
128 auio.uio_resid = iolen;
129 auio.uio_td = curthread;
130
131 if (operation == 0) {
132 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
133 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred);
134 VOP_UNLOCK(vp);
135 } else {
136 (void) vn_start_write(vp, &mp, V_WAIT);
137 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
138 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred);
139 VOP_UNLOCK(vp);
140 vn_finished_write(mp);
141 }
142
143 return (error);
144 }
145
146 static void
vtblk_proc(struct beri_vtblk_softc * sc,struct vqueue_info * vq)147 vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq)
148 {
149 struct iovec iov[VTBLK_MAXSEGS + 2];
150 uint16_t flags[VTBLK_MAXSEGS + 2];
151 struct virtio_blk_outhdr *vbh;
152 struct iovec *tiov;
153 uint8_t *status;
154 off_t offset;
155 int iolen;
156 int type;
157 int i, n;
158 int err;
159
160 n = vq_getchain(sc->beri_mem_offset, vq, iov,
161 VTBLK_MAXSEGS + 2, flags);
162 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2,
163 ("wrong n value %d", n));
164
165 tiov = getcopy(iov, n);
166 vbh = iov[0].iov_base;
167
168 status = iov[n-1].iov_base;
169 KASSERT(iov[n-1].iov_len == 1,
170 ("iov_len == %d", iov[n-1].iov_len));
171
172 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER;
173 offset = be64toh(vbh->sector) * DEV_BSIZE;
174
175 iolen = 0;
176 for (i = 1; i < (n-1); i++) {
177 iolen += iov[i].iov_len;
178 }
179
180 switch (type) {
181 case VIRTIO_BLK_T_OUT:
182 case VIRTIO_BLK_T_IN:
183 err = vtblk_rdwr(sc, tiov + 1, i - 1,
184 offset, type, iolen);
185 break;
186 case VIRTIO_BLK_T_GET_ID:
187 /* Assume a single buffer */
188 strncpy(iov[1].iov_base, sc->ident,
189 MIN(iov[1].iov_len, sizeof(sc->ident)));
190 err = 0;
191 break;
192 case VIRTIO_BLK_T_FLUSH:
193 /* Possible? */
194 default:
195 err = -ENOSYS;
196 break;
197 }
198
199 if (err < 0) {
200 if (err == -ENOSYS) {
201 *status = VIRTIO_BLK_S_UNSUPP;
202 } else
203 *status = VIRTIO_BLK_S_IOERR;
204 } else
205 *status = VIRTIO_BLK_S_OK;
206
207 free(tiov, M_DEVBUF);
208 vq_relchain(vq, iov, n, 1);
209 }
210
211 static int
close_file(struct beri_vtblk_softc * sc,struct thread * td)212 close_file(struct beri_vtblk_softc *sc, struct thread *td)
213 {
214 int error;
215
216 if (sc->vnode != NULL) {
217 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY);
218 sc->vnode->v_vflag &= ~VV_MD;
219 VOP_UNLOCK(sc->vnode);
220 error = vn_close(sc->vnode, (FREAD|FWRITE),
221 sc->cred, td);
222 if (error != 0)
223 return (error);
224 sc->vnode = NULL;
225 }
226
227 if (sc->cred != NULL)
228 crfree(sc->cred);
229
230 return (0);
231 }
232
233 static int
open_file(struct beri_vtblk_softc * sc,struct thread * td)234 open_file(struct beri_vtblk_softc *sc, struct thread *td)
235 {
236 struct nameidata nd;
237 struct vattr vattr;
238 int error;
239 int flags;
240
241 flags = (FREAD | FWRITE);
242 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE,
243 sc->mdio->md_file, td);
244 error = vn_open(&nd, &flags, 0, NULL);
245 if (error != 0)
246 return (error);
247 NDFREE(&nd, NDF_ONLY_PNBUF);
248
249 if (nd.ni_vp->v_type != VREG) {
250 return (EINVAL);
251 }
252
253 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred);
254 if (error != 0)
255 return (error);
256
257 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) {
258 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY);
259 if (VN_IS_DOOMED(nd.ni_vp)) {
260 return (1);
261 }
262 }
263 nd.ni_vp->v_vflag |= VV_MD;
264 VOP_UNLOCK(nd.ni_vp);
265
266 sc->vnode = nd.ni_vp;
267 sc->cred = crhold(td->td_ucred);
268
269 return (0);
270 }
271
272 static int
vtblk_notify(struct beri_vtblk_softc * sc)273 vtblk_notify(struct beri_vtblk_softc *sc)
274 {
275 struct vqueue_info *vq;
276 int queue;
277 int reg;
278
279 vq = &sc->vs_queues[0];
280 if (!vq_ring_ready(vq))
281 return (0);
282
283 if (!sc->opened)
284 return (0);
285
286 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY);
287 queue = be16toh(reg);
288
289 KASSERT(queue == 0, ("we support single queue only"));
290
291 /* Process new descriptors */
292 vq = &sc->vs_queues[queue];
293 vq->vq_save_used = be16toh(vq->vq_used->idx);
294 while (vq_has_descs(vq))
295 vtblk_proc(sc, vq);
296
297 /* Interrupt the other side */
298 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) {
299 reg = htobe32(VIRTIO_MMIO_INT_VRING);
300 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg);
301 PIO_SET(sc->pio_send, Q_INTR, 1);
302 }
303
304 return (0);
305 }
306
307 static int
vq_init(struct beri_vtblk_softc * sc)308 vq_init(struct beri_vtblk_softc *sc)
309 {
310 struct vqueue_info *vq;
311 uint8_t *base;
312 int size;
313 int reg;
314 int pfn;
315
316 vq = &sc->vs_queues[0];
317 vq->vq_qsize = NUM_DESCS;
318
319 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN);
320 pfn = be32toh(reg);
321 vq->vq_pfn = pfn;
322
323 size = vring_size(vq->vq_qsize, VRING_ALIGN);
324 base = paddr_map(sc->beri_mem_offset,
325 (pfn << PAGE_SHIFT), size);
326
327 /* First pages are descriptors */
328 vq->vq_desc = (struct vring_desc *)base;
329 base += vq->vq_qsize * sizeof(struct vring_desc);
330
331 /* Then avail ring */
332 vq->vq_avail = (struct vring_avail *)base;
333 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t);
334
335 /* Then it's rounded up to the next page */
336 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN);
337
338 /* And the last pages are the used ring */
339 vq->vq_used = (struct vring_used *)base;
340
341 /* Mark queue as allocated, and start at 0 when we use it. */
342 vq->vq_flags = VQ_ALLOC;
343 vq->vq_last_avail = 0;
344
345 return (0);
346 }
347
348 static void
vtblk_thread(void * arg)349 vtblk_thread(void *arg)
350 {
351 struct beri_vtblk_softc *sc;
352 int err;
353
354 sc = arg;
355
356 sx_xlock(&sc->sc_mtx);
357 for (;;) {
358 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz);
359 vtblk_notify(sc);
360 }
361 sx_xunlock(&sc->sc_mtx);
362
363 kthread_exit();
364 }
365
366 static int
backend_info(struct beri_vtblk_softc * sc)367 backend_info(struct beri_vtblk_softc *sc)
368 {
369 struct virtio_blk_config *cfg;
370 uint32_t *s;
371 int reg;
372 int i;
373
374 /* Specify that we provide block device */
375 reg = htobe32(VIRTIO_ID_BLOCK);
376 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg);
377
378 /* Queue size */
379 reg = htobe32(NUM_DESCS);
380 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg);
381
382 /* Our features */
383 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC
384 | VIRTIO_BLK_F_BLK_SIZE
385 | VIRTIO_BLK_F_SEG_MAX);
386 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg);
387
388 cfg = sc->cfg;
389 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE);
390 cfg->size_max = 0; /* not negotiated */
391 cfg->seg_max = htobe32(VTBLK_MAXSEGS);
392 cfg->blk_size = htobe32(DEV_BSIZE);
393
394 s = (uint32_t *)cfg;
395
396 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) {
397 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s);
398 s+=1;
399 }
400
401 strncpy(sc->ident, "Virtio block backend", sizeof(sc->ident));
402
403 return (0);
404 }
405
406 static void
vtblk_intr(void * arg)407 vtblk_intr(void *arg)
408 {
409 struct beri_vtblk_softc *sc;
410 int pending;
411 int reg;
412
413 sc = arg;
414
415 reg = PIO_READ(sc->pio_recv);
416
417 /* Ack */
418 PIO_SET(sc->pio_recv, reg, 0);
419
420 pending = htobe32(reg);
421
422 if (pending & Q_PFN) {
423 vq_init(sc);
424 }
425
426 if (pending & Q_NOTIFY) {
427 wakeup(sc);
428 }
429 }
430
431 static int
beri_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)432 beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
433 int flags, struct thread *td)
434 {
435 struct beri_vtblk_softc *sc;
436 int err;
437
438 sc = dev->si_drv1;
439
440 switch (cmd) {
441 case MDIOCATTACH:
442 /* take file as argument */
443 if (sc->vnode != NULL) {
444 /* Already opened */
445 return (1);
446 }
447 sc->mdio = (struct md_ioctl *)addr;
448 backend_info(sc);
449 DPRINTF("opening file, td 0x%08x\n", (int)td);
450 err = open_file(sc, td);
451 if (err)
452 return (err);
453 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc);
454 sc->opened = 1;
455 break;
456 case MDIOCDETACH:
457 if (sc->vnode == NULL) {
458 /* File not opened */
459 return (1);
460 }
461 sc->opened = 0;
462 DPRINTF("closing file, td 0x%08x\n", (int)td);
463 err = close_file(sc, td);
464 if (err)
465 return (err);
466 PIO_TEARDOWN_IRQ(sc->pio_recv);
467 break;
468 default:
469 break;
470 }
471
472 return (0);
473 }
474
475 static struct cdevsw beri_cdevsw = {
476 .d_version = D_VERSION,
477 .d_ioctl = beri_ioctl,
478 .d_name = "virtio block backend",
479 };
480
481 static int
beri_vtblk_probe(device_t dev)482 beri_vtblk_probe(device_t dev)
483 {
484
485 if (!ofw_bus_status_okay(dev))
486 return (ENXIO);
487
488 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk"))
489 return (ENXIO);
490
491 device_set_desc(dev, "SRI-Cambridge BERI block");
492 return (BUS_PROBE_DEFAULT);
493 }
494
495 static int
beri_vtblk_attach(device_t dev)496 beri_vtblk_attach(device_t dev)
497 {
498 struct beri_vtblk_softc *sc;
499 int error;
500
501 sc = device_get_softc(dev);
502 sc->dev = dev;
503
504 if (bus_alloc_resources(dev, beri_spec, sc->res)) {
505 device_printf(dev, "could not allocate resources\n");
506 return (ENXIO);
507 }
508
509 /* Memory interface */
510 sc->bst = rman_get_bustag(sc->res[0]);
511 sc->bsh = rman_get_bushandle(sc->res[0]);
512
513 sc->cfg = malloc(sizeof(struct virtio_blk_config),
514 M_DEVBUF, M_NOWAIT|M_ZERO);
515
516 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev));
517
518 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd,
519 0, 0, "beri_virtio_block");
520 if (error) {
521 device_printf(dev, "cannot create kthread\n");
522 return (ENXIO);
523 }
524
525 if (setup_offset(dev, &sc->beri_mem_offset) != 0)
526 return (ENXIO);
527 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0)
528 return (ENXIO);
529 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0)
530 return (ENXIO);
531
532 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL,
533 S_IRWXU, "beri_vtblk");
534 if (sc->cdev == NULL) {
535 device_printf(dev, "Failed to create character device.\n");
536 return (ENXIO);
537 }
538
539 sc->cdev->si_drv1 = sc;
540 return (0);
541 }
542
543 static device_method_t beri_vtblk_methods[] = {
544 DEVMETHOD(device_probe, beri_vtblk_probe),
545 DEVMETHOD(device_attach, beri_vtblk_attach),
546 { 0, 0 }
547 };
548
549 static driver_t beri_vtblk_driver = {
550 "beri_vtblk",
551 beri_vtblk_methods,
552 sizeof(struct beri_vtblk_softc),
553 };
554
555 static devclass_t beri_vtblk_devclass;
556
557 DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver,
558 beri_vtblk_devclass, 0, 0);
559