1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /* Driver for VirtIO block devices. */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/bio.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/sglist.h>
39 #include <sys/sysctl.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/queue.h>
43
44 #include <geom/geom.h>
45 #include <geom/geom_disk.h>
46
47 #include <machine/bus.h>
48 #include <machine/resource.h>
49 #include <sys/bus.h>
50 #include <sys/rman.h>
51
52 #include <dev/virtio/virtio.h>
53 #include <dev/virtio/virtqueue.h>
54 #include <dev/virtio/block/virtio_blk.h>
55
56 #include "virtio_if.h"
57
58 struct vtblk_request {
59 struct virtio_blk_outhdr vbr_hdr;
60 struct bio *vbr_bp;
61 uint8_t vbr_ack;
62 TAILQ_ENTRY(vtblk_request) vbr_link;
63 };
64
65 enum vtblk_cache_mode {
66 VTBLK_CACHE_WRITETHROUGH,
67 VTBLK_CACHE_WRITEBACK,
68 VTBLK_CACHE_MAX
69 };
70
71 struct vtblk_softc {
72 device_t vtblk_dev;
73 struct mtx vtblk_mtx;
74 uint64_t vtblk_features;
75 uint32_t vtblk_flags;
76 #define VTBLK_FLAG_INDIRECT 0x0001
77 #define VTBLK_FLAG_DETACH 0x0002
78 #define VTBLK_FLAG_SUSPEND 0x0004
79 #define VTBLK_FLAG_BARRIER 0x0008
80 #define VTBLK_FLAG_WCE_CONFIG 0x0010
81
82 struct virtqueue *vtblk_vq;
83 struct sglist *vtblk_sglist;
84 struct disk *vtblk_disk;
85
86 struct bio_queue_head vtblk_bioq;
87 TAILQ_HEAD(, vtblk_request)
88 vtblk_req_free;
89 TAILQ_HEAD(, vtblk_request)
90 vtblk_req_ready;
91 struct vtblk_request *vtblk_req_ordered;
92
93 int vtblk_max_nsegs;
94 int vtblk_request_count;
95 enum vtblk_cache_mode vtblk_write_cache;
96
97 struct bio_queue vtblk_dump_queue;
98 struct vtblk_request vtblk_dump_request;
99 };
100
101 static struct virtio_feature_desc vtblk_feature_desc[] = {
102 { VIRTIO_BLK_F_BARRIER, "HostBarrier" },
103 { VIRTIO_BLK_F_SIZE_MAX, "MaxSegSize" },
104 { VIRTIO_BLK_F_SEG_MAX, "MaxNumSegs" },
105 { VIRTIO_BLK_F_GEOMETRY, "DiskGeometry" },
106 { VIRTIO_BLK_F_RO, "ReadOnly" },
107 { VIRTIO_BLK_F_BLK_SIZE, "BlockSize" },
108 { VIRTIO_BLK_F_SCSI, "SCSICmds" },
109 { VIRTIO_BLK_F_FLUSH, "FlushCmd" },
110 { VIRTIO_BLK_F_TOPOLOGY, "Topology" },
111 { VIRTIO_BLK_F_CONFIG_WCE, "ConfigWCE" },
112 { VIRTIO_BLK_F_MQ, "Multiqueue" },
113 { VIRTIO_BLK_F_DISCARD, "Discard" },
114 { VIRTIO_BLK_F_WRITE_ZEROES, "WriteZeros" },
115
116 { 0, NULL }
117 };
118
119 static int vtblk_modevent(module_t, int, void *);
120
121 static int vtblk_probe(device_t);
122 static int vtblk_attach(device_t);
123 static int vtblk_detach(device_t);
124 static int vtblk_suspend(device_t);
125 static int vtblk_resume(device_t);
126 static int vtblk_shutdown(device_t);
127 static int vtblk_attach_completed(device_t);
128 static int vtblk_config_change(device_t);
129
130 static int vtblk_open(struct disk *);
131 static int vtblk_close(struct disk *);
132 static int vtblk_ioctl(struct disk *, u_long, void *, int,
133 struct thread *);
134 static int vtblk_dump(void *, void *, off_t, size_t);
135 static void vtblk_strategy(struct bio *);
136
137 static int vtblk_negotiate_features(struct vtblk_softc *);
138 static int vtblk_setup_features(struct vtblk_softc *);
139 static int vtblk_maximum_segments(struct vtblk_softc *,
140 struct virtio_blk_config *);
141 static int vtblk_alloc_virtqueue(struct vtblk_softc *);
142 static void vtblk_resize_disk(struct vtblk_softc *, uint64_t);
143 static void vtblk_alloc_disk(struct vtblk_softc *,
144 struct virtio_blk_config *);
145 static void vtblk_create_disk(struct vtblk_softc *);
146
147 static int vtblk_request_prealloc(struct vtblk_softc *);
148 static void vtblk_request_free(struct vtblk_softc *);
149 static struct vtblk_request *
150 vtblk_request_dequeue(struct vtblk_softc *);
151 static void vtblk_request_enqueue(struct vtblk_softc *,
152 struct vtblk_request *);
153 static struct vtblk_request *
154 vtblk_request_next_ready(struct vtblk_softc *);
155 static void vtblk_request_requeue_ready(struct vtblk_softc *,
156 struct vtblk_request *);
157 static struct vtblk_request *
158 vtblk_request_next(struct vtblk_softc *);
159 static struct vtblk_request *
160 vtblk_request_bio(struct vtblk_softc *);
161 static int vtblk_request_execute(struct vtblk_softc *,
162 struct vtblk_request *);
163 static int vtblk_request_error(struct vtblk_request *);
164
165 static void vtblk_queue_completed(struct vtblk_softc *,
166 struct bio_queue *);
167 static void vtblk_done_completed(struct vtblk_softc *,
168 struct bio_queue *);
169 static void vtblk_drain_vq(struct vtblk_softc *);
170 static void vtblk_drain(struct vtblk_softc *);
171
172 static void vtblk_startio(struct vtblk_softc *);
173 static void vtblk_bio_done(struct vtblk_softc *, struct bio *, int);
174
175 static void vtblk_read_config(struct vtblk_softc *,
176 struct virtio_blk_config *);
177 static void vtblk_ident(struct vtblk_softc *);
178 static int vtblk_poll_request(struct vtblk_softc *,
179 struct vtblk_request *);
180 static int vtblk_quiesce(struct vtblk_softc *);
181 static void vtblk_vq_intr(void *);
182 static void vtblk_stop(struct vtblk_softc *);
183
184 static void vtblk_dump_quiesce(struct vtblk_softc *);
185 static int vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t);
186 static int vtblk_dump_flush(struct vtblk_softc *);
187 static void vtblk_dump_complete(struct vtblk_softc *);
188
189 static void vtblk_set_write_cache(struct vtblk_softc *, int);
190 static int vtblk_write_cache_enabled(struct vtblk_softc *sc,
191 struct virtio_blk_config *);
192 static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
193
194 static void vtblk_setup_sysctl(struct vtblk_softc *);
195 static int vtblk_tunable_int(struct vtblk_softc *, const char *, int);
196
197 #define vtblk_modern(_sc) (((_sc)->vtblk_features & VIRTIO_F_VERSION_1) != 0)
198 #define vtblk_htog16(_sc, _val) virtio_htog16(vtblk_modern(_sc), _val)
199 #define vtblk_htog32(_sc, _val) virtio_htog32(vtblk_modern(_sc), _val)
200 #define vtblk_htog64(_sc, _val) virtio_htog64(vtblk_modern(_sc), _val)
201 #define vtblk_gtoh16(_sc, _val) virtio_gtoh16(vtblk_modern(_sc), _val)
202 #define vtblk_gtoh32(_sc, _val) virtio_gtoh32(vtblk_modern(_sc), _val)
203 #define vtblk_gtoh64(_sc, _val) virtio_gtoh64(vtblk_modern(_sc), _val)
204
205 /* Tunables. */
206 static int vtblk_no_ident = 0;
207 TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
208 static int vtblk_writecache_mode = -1;
209 TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);
210
211 #define VTBLK_COMMON_FEATURES \
212 (VIRTIO_BLK_F_SIZE_MAX | \
213 VIRTIO_BLK_F_SEG_MAX | \
214 VIRTIO_BLK_F_GEOMETRY | \
215 VIRTIO_BLK_F_RO | \
216 VIRTIO_BLK_F_BLK_SIZE | \
217 VIRTIO_BLK_F_FLUSH | \
218 VIRTIO_BLK_F_TOPOLOGY | \
219 VIRTIO_BLK_F_CONFIG_WCE | \
220 VIRTIO_BLK_F_DISCARD | \
221 VIRTIO_RING_F_INDIRECT_DESC)
222
223 #define VTBLK_MODERN_FEATURES (VTBLK_COMMON_FEATURES)
224 #define VTBLK_LEGACY_FEATURES (VIRTIO_BLK_F_BARRIER | VTBLK_COMMON_FEATURES)
225
226 #define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx
227 #define VTBLK_LOCK_INIT(_sc, _name) \
228 mtx_init(VTBLK_MTX((_sc)), (_name), \
229 "VirtIO Block Lock", MTX_DEF)
230 #define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc)))
231 #define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc)))
232 #define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc)))
233 #define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
234 #define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
235 mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
236
237 #define VTBLK_DISK_NAME "vtbd"
238 #define VTBLK_QUIESCE_TIMEOUT (30 * hz)
239 #define VTBLK_BSIZE 512
240
241 /*
242 * Each block request uses at least two segments - one for the header
243 * and one for the status.
244 */
245 #define VTBLK_MIN_SEGMENTS 2
246
247 static device_method_t vtblk_methods[] = {
248 /* Device methods. */
249 DEVMETHOD(device_probe, vtblk_probe),
250 DEVMETHOD(device_attach, vtblk_attach),
251 DEVMETHOD(device_detach, vtblk_detach),
252 DEVMETHOD(device_suspend, vtblk_suspend),
253 DEVMETHOD(device_resume, vtblk_resume),
254 DEVMETHOD(device_shutdown, vtblk_shutdown),
255
256 /* VirtIO methods. */
257 DEVMETHOD(virtio_attach_completed, vtblk_attach_completed),
258 DEVMETHOD(virtio_config_change, vtblk_config_change),
259
260 DEVMETHOD_END
261 };
262
263 static driver_t vtblk_driver = {
264 "vtblk",
265 vtblk_methods,
266 sizeof(struct vtblk_softc)
267 };
268 static devclass_t vtblk_devclass;
269
270 VIRTIO_DRIVER_MODULE(virtio_blk, vtblk_driver, vtblk_devclass,
271 vtblk_modevent, 0);
272 MODULE_VERSION(virtio_blk, 1);
273 MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
274
275 VIRTIO_SIMPLE_PNPINFO(virtio_blk, VIRTIO_ID_BLOCK, "VirtIO Block Adapter");
276
277 static int
vtblk_modevent(module_t mod,int type,void * unused)278 vtblk_modevent(module_t mod, int type, void *unused)
279 {
280 int error;
281
282 error = 0;
283
284 switch (type) {
285 case MOD_LOAD:
286 case MOD_QUIESCE:
287 case MOD_UNLOAD:
288 case MOD_SHUTDOWN:
289 break;
290 default:
291 error = EOPNOTSUPP;
292 break;
293 }
294
295 return (error);
296 }
297
298 static int
vtblk_probe(device_t dev)299 vtblk_probe(device_t dev)
300 {
301 return (VIRTIO_SIMPLE_PROBE(dev, virtio_blk));
302 }
303
304 static int
vtblk_attach(device_t dev)305 vtblk_attach(device_t dev)
306 {
307 struct vtblk_softc *sc;
308 struct virtio_blk_config blkcfg;
309 int error;
310
311 sc = device_get_softc(dev);
312 sc->vtblk_dev = dev;
313 virtio_set_feature_desc(dev, vtblk_feature_desc);
314
315 VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
316 bioq_init(&sc->vtblk_bioq);
317 TAILQ_INIT(&sc->vtblk_dump_queue);
318 TAILQ_INIT(&sc->vtblk_req_free);
319 TAILQ_INIT(&sc->vtblk_req_ready);
320
321 vtblk_setup_sysctl(sc);
322
323 error = vtblk_setup_features(sc);
324 if (error) {
325 device_printf(dev, "cannot setup features\n");
326 goto fail;
327 }
328
329 vtblk_read_config(sc, &blkcfg);
330
331 /*
332 * With the current sglist(9) implementation, it is not easy
333 * for us to support a maximum segment size as adjacent
334 * segments are coalesced. For now, just make sure it's larger
335 * than the maximum supported transfer size.
336 */
337 if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
338 if (blkcfg.size_max < maxphys) {
339 error = ENOTSUP;
340 device_printf(dev, "host requires unsupported "
341 "maximum segment size feature\n");
342 goto fail;
343 }
344 }
345
346 sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
347 if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
348 error = EINVAL;
349 device_printf(dev, "fewer than minimum number of segments "
350 "allowed: %d\n", sc->vtblk_max_nsegs);
351 goto fail;
352 }
353
354 sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
355 if (sc->vtblk_sglist == NULL) {
356 error = ENOMEM;
357 device_printf(dev, "cannot allocate sglist\n");
358 goto fail;
359 }
360
361 error = vtblk_alloc_virtqueue(sc);
362 if (error) {
363 device_printf(dev, "cannot allocate virtqueue\n");
364 goto fail;
365 }
366
367 error = vtblk_request_prealloc(sc);
368 if (error) {
369 device_printf(dev, "cannot preallocate requests\n");
370 goto fail;
371 }
372
373 vtblk_alloc_disk(sc, &blkcfg);
374
375 error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY);
376 if (error) {
377 device_printf(dev, "cannot setup virtqueue interrupt\n");
378 goto fail;
379 }
380
381 virtqueue_enable_intr(sc->vtblk_vq);
382
383 fail:
384 if (error)
385 vtblk_detach(dev);
386
387 return (error);
388 }
389
390 static int
vtblk_detach(device_t dev)391 vtblk_detach(device_t dev)
392 {
393 struct vtblk_softc *sc;
394
395 sc = device_get_softc(dev);
396
397 VTBLK_LOCK(sc);
398 sc->vtblk_flags |= VTBLK_FLAG_DETACH;
399 if (device_is_attached(dev))
400 vtblk_stop(sc);
401 VTBLK_UNLOCK(sc);
402
403 vtblk_drain(sc);
404
405 if (sc->vtblk_disk != NULL) {
406 disk_destroy(sc->vtblk_disk);
407 sc->vtblk_disk = NULL;
408 }
409
410 if (sc->vtblk_sglist != NULL) {
411 sglist_free(sc->vtblk_sglist);
412 sc->vtblk_sglist = NULL;
413 }
414
415 VTBLK_LOCK_DESTROY(sc);
416
417 return (0);
418 }
419
420 static int
vtblk_suspend(device_t dev)421 vtblk_suspend(device_t dev)
422 {
423 struct vtblk_softc *sc;
424 int error;
425
426 sc = device_get_softc(dev);
427
428 VTBLK_LOCK(sc);
429 sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
430 /* XXX BMV: virtio_stop(), etc needed here? */
431 error = vtblk_quiesce(sc);
432 if (error)
433 sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
434 VTBLK_UNLOCK(sc);
435
436 return (error);
437 }
438
439 static int
vtblk_resume(device_t dev)440 vtblk_resume(device_t dev)
441 {
442 struct vtblk_softc *sc;
443
444 sc = device_get_softc(dev);
445
446 VTBLK_LOCK(sc);
447 /* XXX BMV: virtio_reinit(), etc needed here? */
448 sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
449 vtblk_startio(sc);
450 VTBLK_UNLOCK(sc);
451
452 return (0);
453 }
454
455 static int
vtblk_shutdown(device_t dev)456 vtblk_shutdown(device_t dev)
457 {
458
459 return (0);
460 }
461
462 static int
vtblk_attach_completed(device_t dev)463 vtblk_attach_completed(device_t dev)
464 {
465 struct vtblk_softc *sc;
466
467 sc = device_get_softc(dev);
468
469 /*
470 * Create disk after attach as VIRTIO_BLK_T_GET_ID can only be
471 * processed after the device acknowledged
472 * VIRTIO_CONFIG_STATUS_DRIVER_OK.
473 */
474 vtblk_create_disk(sc);
475 return (0);
476 }
477
478 static int
vtblk_config_change(device_t dev)479 vtblk_config_change(device_t dev)
480 {
481 struct vtblk_softc *sc;
482 struct virtio_blk_config blkcfg;
483 uint64_t capacity;
484
485 sc = device_get_softc(dev);
486
487 vtblk_read_config(sc, &blkcfg);
488
489 /* Capacity is always in 512-byte units. */
490 capacity = blkcfg.capacity * VTBLK_BSIZE;
491
492 if (sc->vtblk_disk->d_mediasize != capacity)
493 vtblk_resize_disk(sc, capacity);
494
495 return (0);
496 }
497
498 static int
vtblk_open(struct disk * dp)499 vtblk_open(struct disk *dp)
500 {
501 struct vtblk_softc *sc;
502
503 if ((sc = dp->d_drv1) == NULL)
504 return (ENXIO);
505
506 return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
507 }
508
509 static int
vtblk_close(struct disk * dp)510 vtblk_close(struct disk *dp)
511 {
512 struct vtblk_softc *sc;
513
514 if ((sc = dp->d_drv1) == NULL)
515 return (ENXIO);
516
517 return (0);
518 }
519
520 static int
vtblk_ioctl(struct disk * dp,u_long cmd,void * addr,int flag,struct thread * td)521 vtblk_ioctl(struct disk *dp, u_long cmd, void *addr, int flag,
522 struct thread *td)
523 {
524 struct vtblk_softc *sc;
525
526 if ((sc = dp->d_drv1) == NULL)
527 return (ENXIO);
528
529 return (ENOTTY);
530 }
531
532 static int
vtblk_dump(void * arg,void * virtual,off_t offset,size_t length)533 vtblk_dump(void *arg, void *virtual, off_t offset, size_t length)
534 {
535 struct disk *dp;
536 struct vtblk_softc *sc;
537 int error;
538
539 dp = arg;
540 error = 0;
541
542 if ((sc = dp->d_drv1) == NULL)
543 return (ENXIO);
544
545 VTBLK_LOCK(sc);
546
547 vtblk_dump_quiesce(sc);
548
549 if (length > 0)
550 error = vtblk_dump_write(sc, virtual, offset, length);
551 if (error || (virtual == NULL && offset == 0))
552 vtblk_dump_complete(sc);
553
554 VTBLK_UNLOCK(sc);
555
556 return (error);
557 }
558
559 static void
vtblk_strategy(struct bio * bp)560 vtblk_strategy(struct bio *bp)
561 {
562 struct vtblk_softc *sc;
563
564 if ((sc = bp->bio_disk->d_drv1) == NULL) {
565 vtblk_bio_done(NULL, bp, EINVAL);
566 return;
567 }
568
569 if ((bp->bio_cmd != BIO_READ) && (bp->bio_cmd != BIO_WRITE) &&
570 (bp->bio_cmd != BIO_FLUSH) && (bp->bio_cmd != BIO_DELETE)) {
571 vtblk_bio_done(sc, bp, EOPNOTSUPP);
572 return;
573 }
574
575 VTBLK_LOCK(sc);
576
577 if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
578 VTBLK_UNLOCK(sc);
579 vtblk_bio_done(sc, bp, ENXIO);
580 return;
581 }
582
583 bioq_insert_tail(&sc->vtblk_bioq, bp);
584 vtblk_startio(sc);
585
586 VTBLK_UNLOCK(sc);
587 }
588
589 static int
vtblk_negotiate_features(struct vtblk_softc * sc)590 vtblk_negotiate_features(struct vtblk_softc *sc)
591 {
592 device_t dev;
593 uint64_t features;
594
595 dev = sc->vtblk_dev;
596 features = virtio_bus_is_modern(dev) ? VTBLK_MODERN_FEATURES :
597 VTBLK_LEGACY_FEATURES;
598
599 sc->vtblk_features = virtio_negotiate_features(dev, features);
600 return (virtio_finalize_features(dev));
601 }
602
603 static int
vtblk_setup_features(struct vtblk_softc * sc)604 vtblk_setup_features(struct vtblk_softc *sc)
605 {
606 device_t dev;
607 int error;
608
609 dev = sc->vtblk_dev;
610
611 error = vtblk_negotiate_features(sc);
612 if (error)
613 return (error);
614
615 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
616 sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
617 if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
618 sc->vtblk_flags |= VTBLK_FLAG_WCE_CONFIG;
619
620 /* Legacy. */
621 if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
622 sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
623
624 return (0);
625 }
626
627 static int
vtblk_maximum_segments(struct vtblk_softc * sc,struct virtio_blk_config * blkcfg)628 vtblk_maximum_segments(struct vtblk_softc *sc,
629 struct virtio_blk_config *blkcfg)
630 {
631 device_t dev;
632 int nsegs;
633
634 dev = sc->vtblk_dev;
635 nsegs = VTBLK_MIN_SEGMENTS;
636
637 if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
638 nsegs += MIN(blkcfg->seg_max, maxphys / PAGE_SIZE + 1);
639 if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT)
640 nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);
641 } else
642 nsegs += 1;
643
644 return (nsegs);
645 }
646
647 static int
vtblk_alloc_virtqueue(struct vtblk_softc * sc)648 vtblk_alloc_virtqueue(struct vtblk_softc *sc)
649 {
650 device_t dev;
651 struct vq_alloc_info vq_info;
652
653 dev = sc->vtblk_dev;
654
655 VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs,
656 vtblk_vq_intr, sc, &sc->vtblk_vq,
657 "%s request", device_get_nameunit(dev));
658
659 return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
660 }
661
662 static void
vtblk_resize_disk(struct vtblk_softc * sc,uint64_t new_capacity)663 vtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity)
664 {
665 device_t dev;
666 struct disk *dp;
667 int error;
668
669 dev = sc->vtblk_dev;
670 dp = sc->vtblk_disk;
671
672 dp->d_mediasize = new_capacity;
673 if (bootverbose) {
674 device_printf(dev, "resized to %juMB (%ju %u byte sectors)\n",
675 (uintmax_t) dp->d_mediasize >> 20,
676 (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
677 dp->d_sectorsize);
678 }
679
680 error = disk_resize(dp, M_NOWAIT);
681 if (error) {
682 device_printf(dev,
683 "disk_resize(9) failed, error: %d\n", error);
684 }
685 }
686
687 static void
vtblk_alloc_disk(struct vtblk_softc * sc,struct virtio_blk_config * blkcfg)688 vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
689 {
690 device_t dev;
691 struct disk *dp;
692
693 dev = sc->vtblk_dev;
694
695 sc->vtblk_disk = dp = disk_alloc();
696 dp->d_open = vtblk_open;
697 dp->d_close = vtblk_close;
698 dp->d_ioctl = vtblk_ioctl;
699 dp->d_strategy = vtblk_strategy;
700 dp->d_name = VTBLK_DISK_NAME;
701 dp->d_unit = device_get_unit(dev);
702 dp->d_drv1 = sc;
703 dp->d_flags = DISKFLAG_UNMAPPED_BIO | DISKFLAG_DIRECT_COMPLETION;
704 dp->d_hba_vendor = virtio_get_vendor(dev);
705 dp->d_hba_device = virtio_get_device(dev);
706 dp->d_hba_subvendor = virtio_get_subvendor(dev);
707 dp->d_hba_subdevice = virtio_get_subdevice(dev);
708
709 if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
710 dp->d_flags |= DISKFLAG_WRITE_PROTECT;
711 else {
712 if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH))
713 dp->d_flags |= DISKFLAG_CANFLUSHCACHE;
714 dp->d_dump = vtblk_dump;
715 }
716
717 /* Capacity is always in 512-byte units. */
718 dp->d_mediasize = blkcfg->capacity * VTBLK_BSIZE;
719
720 if (virtio_with_feature(dev, VIRTIO_BLK_F_BLK_SIZE))
721 dp->d_sectorsize = blkcfg->blk_size;
722 else
723 dp->d_sectorsize = VTBLK_BSIZE;
724
725 /*
726 * The VirtIO maximum I/O size is given in terms of segments.
727 * However, FreeBSD limits I/O size by logical buffer size, not
728 * by physically contiguous pages. Therefore, we have to assume
729 * no pages are contiguous. This may impose an artificially low
730 * maximum I/O size. But in practice, since QEMU advertises 128
731 * segments, this gives us a maximum IO size of 125 * PAGE_SIZE,
732 * which is typically greater than maxphys. Eventually we should
733 * just advertise maxphys and split buffers that are too big.
734 *
735 * Note we must subtract one additional segment in case of non
736 * page aligned buffers.
737 */
738 dp->d_maxsize = (sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS - 1) *
739 PAGE_SIZE;
740 if (dp->d_maxsize < PAGE_SIZE)
741 dp->d_maxsize = PAGE_SIZE; /* XXX */
742
743 if (virtio_with_feature(dev, VIRTIO_BLK_F_GEOMETRY)) {
744 dp->d_fwsectors = blkcfg->geometry.sectors;
745 dp->d_fwheads = blkcfg->geometry.heads;
746 }
747
748 if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY) &&
749 blkcfg->topology.physical_block_exp > 0) {
750 dp->d_stripesize = dp->d_sectorsize *
751 (1 << blkcfg->topology.physical_block_exp);
752 dp->d_stripeoffset = (dp->d_stripesize -
753 blkcfg->topology.alignment_offset * dp->d_sectorsize) %
754 dp->d_stripesize;
755 }
756
757 if (virtio_with_feature(dev, VIRTIO_BLK_F_DISCARD)) {
758 dp->d_flags |= DISKFLAG_CANDELETE;
759 dp->d_delmaxsize = blkcfg->max_discard_sectors * VTBLK_BSIZE;
760 }
761
762 if (vtblk_write_cache_enabled(sc, blkcfg) != 0)
763 sc->vtblk_write_cache = VTBLK_CACHE_WRITEBACK;
764 else
765 sc->vtblk_write_cache = VTBLK_CACHE_WRITETHROUGH;
766 }
767
768 static void
vtblk_create_disk(struct vtblk_softc * sc)769 vtblk_create_disk(struct vtblk_softc *sc)
770 {
771 struct disk *dp;
772
773 dp = sc->vtblk_disk;
774
775 vtblk_ident(sc);
776
777 device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
778 (uintmax_t) dp->d_mediasize >> 20,
779 (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
780 dp->d_sectorsize);
781
782 disk_create(dp, DISK_VERSION);
783 }
784
785 static int
vtblk_request_prealloc(struct vtblk_softc * sc)786 vtblk_request_prealloc(struct vtblk_softc *sc)
787 {
788 struct vtblk_request *req;
789 int i, nreqs;
790
791 nreqs = virtqueue_size(sc->vtblk_vq);
792
793 /*
794 * Preallocate sufficient requests to keep the virtqueue full. Each
795 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
796 * the number allocated when indirect descriptors are not available.
797 */
798 if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
799 nreqs /= VTBLK_MIN_SEGMENTS;
800
801 for (i = 0; i < nreqs; i++) {
802 req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
803 if (req == NULL)
804 return (ENOMEM);
805
806 MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1);
807 MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1);
808
809 sc->vtblk_request_count++;
810 vtblk_request_enqueue(sc, req);
811 }
812
813 return (0);
814 }
815
816 static void
vtblk_request_free(struct vtblk_softc * sc)817 vtblk_request_free(struct vtblk_softc *sc)
818 {
819 struct vtblk_request *req;
820
821 MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready));
822
823 while ((req = vtblk_request_dequeue(sc)) != NULL) {
824 sc->vtblk_request_count--;
825 free(req, M_DEVBUF);
826 }
827
828 KASSERT(sc->vtblk_request_count == 0,
829 ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
830 }
831
832 static struct vtblk_request *
vtblk_request_dequeue(struct vtblk_softc * sc)833 vtblk_request_dequeue(struct vtblk_softc *sc)
834 {
835 struct vtblk_request *req;
836
837 req = TAILQ_FIRST(&sc->vtblk_req_free);
838 if (req != NULL) {
839 TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
840 bzero(req, sizeof(struct vtblk_request));
841 }
842
843 return (req);
844 }
845
846 static void
vtblk_request_enqueue(struct vtblk_softc * sc,struct vtblk_request * req)847 vtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req)
848 {
849
850 TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
851 }
852
853 static struct vtblk_request *
vtblk_request_next_ready(struct vtblk_softc * sc)854 vtblk_request_next_ready(struct vtblk_softc *sc)
855 {
856 struct vtblk_request *req;
857
858 req = TAILQ_FIRST(&sc->vtblk_req_ready);
859 if (req != NULL)
860 TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
861
862 return (req);
863 }
864
865 static void
vtblk_request_requeue_ready(struct vtblk_softc * sc,struct vtblk_request * req)866 vtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
867 {
868
869 /* NOTE: Currently, there will be at most one request in the queue. */
870 TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
871 }
872
873 static struct vtblk_request *
vtblk_request_next(struct vtblk_softc * sc)874 vtblk_request_next(struct vtblk_softc *sc)
875 {
876 struct vtblk_request *req;
877
878 req = vtblk_request_next_ready(sc);
879 if (req != NULL)
880 return (req);
881
882 return (vtblk_request_bio(sc));
883 }
884
885 static struct vtblk_request *
vtblk_request_bio(struct vtblk_softc * sc)886 vtblk_request_bio(struct vtblk_softc *sc)
887 {
888 struct bio_queue_head *bioq;
889 struct vtblk_request *req;
890 struct bio *bp;
891
892 bioq = &sc->vtblk_bioq;
893
894 if (bioq_first(bioq) == NULL)
895 return (NULL);
896
897 req = vtblk_request_dequeue(sc);
898 if (req == NULL)
899 return (NULL);
900
901 bp = bioq_takefirst(bioq);
902 req->vbr_bp = bp;
903 req->vbr_ack = -1;
904 req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
905
906 switch (bp->bio_cmd) {
907 case BIO_FLUSH:
908 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);
909 req->vbr_hdr.sector = 0;
910 break;
911 case BIO_READ:
912 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN);
913 req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
914 break;
915 case BIO_WRITE:
916 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);
917 req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
918 break;
919 case BIO_DELETE:
920 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_DISCARD);
921 req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
922 break;
923 default:
924 panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
925 }
926
927 if (bp->bio_flags & BIO_ORDERED)
928 req->vbr_hdr.type |= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER);
929
930 return (req);
931 }
932
933 static int
vtblk_request_execute(struct vtblk_softc * sc,struct vtblk_request * req)934 vtblk_request_execute(struct vtblk_softc *sc, struct vtblk_request *req)
935 {
936 struct virtqueue *vq;
937 struct sglist *sg;
938 struct bio *bp;
939 int ordered, readable, writable, error;
940
941 vq = sc->vtblk_vq;
942 sg = sc->vtblk_sglist;
943 bp = req->vbr_bp;
944 ordered = 0;
945 writable = 0;
946
947 /*
948 * Some hosts (such as bhyve) do not implement the barrier feature,
949 * so we emulate it in the driver by allowing the barrier request
950 * to be the only one in flight.
951 */
952 if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) {
953 if (sc->vtblk_req_ordered != NULL)
954 return (EBUSY);
955 if (bp->bio_flags & BIO_ORDERED) {
956 if (!virtqueue_empty(vq))
957 return (EBUSY);
958 ordered = 1;
959 req->vbr_hdr.type &= vtblk_gtoh32(sc,
960 ~VIRTIO_BLK_T_BARRIER);
961 }
962 }
963
964 sglist_reset(sg);
965 sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
966
967 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
968 error = sglist_append_bio(sg, bp);
969 if (error || sg->sg_nseg == sg->sg_maxseg) {
970 panic("%s: bio %p data buffer too big %d",
971 __func__, bp, error);
972 }
973
974 /* BIO_READ means the host writes into our buffer. */
975 if (bp->bio_cmd == BIO_READ)
976 writable = sg->sg_nseg - 1;
977 } else if (bp->bio_cmd == BIO_DELETE) {
978 struct virtio_blk_discard_write_zeroes *discard;
979
980 discard = malloc(sizeof(*discard), M_DEVBUF, M_NOWAIT | M_ZERO);
981 if (discard == NULL)
982 return (ENOMEM);
983
984 bp->bio_driver1 = discard;
985 discard->sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
986 discard->num_sectors = vtblk_gtoh32(sc, bp->bio_bcount / VTBLK_BSIZE);
987 error = sglist_append(sg, discard, sizeof(*discard));
988 if (error || sg->sg_nseg == sg->sg_maxseg) {
989 panic("%s: bio %p data buffer too big %d",
990 __func__, bp, error);
991 }
992 }
993
994 writable++;
995 sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
996 readable = sg->sg_nseg - writable;
997
998 error = virtqueue_enqueue(vq, req, sg, readable, writable);
999 if (error == 0 && ordered)
1000 sc->vtblk_req_ordered = req;
1001
1002 return (error);
1003 }
1004
1005 static int
vtblk_request_error(struct vtblk_request * req)1006 vtblk_request_error(struct vtblk_request *req)
1007 {
1008 int error;
1009
1010 switch (req->vbr_ack) {
1011 case VIRTIO_BLK_S_OK:
1012 error = 0;
1013 break;
1014 case VIRTIO_BLK_S_UNSUPP:
1015 error = ENOTSUP;
1016 break;
1017 default:
1018 error = EIO;
1019 break;
1020 }
1021
1022 return (error);
1023 }
1024
1025 static void
vtblk_queue_completed(struct vtblk_softc * sc,struct bio_queue * queue)1026 vtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue)
1027 {
1028 struct vtblk_request *req;
1029 struct bio *bp;
1030
1031 while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
1032 if (sc->vtblk_req_ordered != NULL) {
1033 MPASS(sc->vtblk_req_ordered == req);
1034 sc->vtblk_req_ordered = NULL;
1035 }
1036
1037 bp = req->vbr_bp;
1038 bp->bio_error = vtblk_request_error(req);
1039 TAILQ_INSERT_TAIL(queue, bp, bio_queue);
1040
1041 vtblk_request_enqueue(sc, req);
1042 }
1043 }
1044
1045 static void
vtblk_done_completed(struct vtblk_softc * sc,struct bio_queue * queue)1046 vtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue)
1047 {
1048 struct bio *bp, *tmp;
1049
1050 TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) {
1051 if (bp->bio_error != 0)
1052 disk_err(bp, "hard error", -1, 1);
1053 vtblk_bio_done(sc, bp, bp->bio_error);
1054 }
1055 }
1056
1057 static void
vtblk_drain_vq(struct vtblk_softc * sc)1058 vtblk_drain_vq(struct vtblk_softc *sc)
1059 {
1060 struct virtqueue *vq;
1061 struct vtblk_request *req;
1062 int last;
1063
1064 vq = sc->vtblk_vq;
1065 last = 0;
1066
1067 while ((req = virtqueue_drain(vq, &last)) != NULL) {
1068 vtblk_bio_done(sc, req->vbr_bp, ENXIO);
1069 vtblk_request_enqueue(sc, req);
1070 }
1071
1072 sc->vtblk_req_ordered = NULL;
1073 KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
1074 }
1075
1076 static void
vtblk_drain(struct vtblk_softc * sc)1077 vtblk_drain(struct vtblk_softc *sc)
1078 {
1079 struct bio_queue_head *bioq;
1080 struct vtblk_request *req;
1081 struct bio *bp;
1082
1083 bioq = &sc->vtblk_bioq;
1084
1085 if (sc->vtblk_vq != NULL) {
1086 struct bio_queue queue;
1087
1088 TAILQ_INIT(&queue);
1089 vtblk_queue_completed(sc, &queue);
1090 vtblk_done_completed(sc, &queue);
1091
1092 vtblk_drain_vq(sc);
1093 }
1094
1095 while ((req = vtblk_request_next_ready(sc)) != NULL) {
1096 vtblk_bio_done(sc, req->vbr_bp, ENXIO);
1097 vtblk_request_enqueue(sc, req);
1098 }
1099
1100 while (bioq_first(bioq) != NULL) {
1101 bp = bioq_takefirst(bioq);
1102 vtblk_bio_done(sc, bp, ENXIO);
1103 }
1104
1105 vtblk_request_free(sc);
1106 }
1107
1108 static void
vtblk_startio(struct vtblk_softc * sc)1109 vtblk_startio(struct vtblk_softc *sc)
1110 {
1111 struct virtqueue *vq;
1112 struct vtblk_request *req;
1113 int enq;
1114
1115 VTBLK_LOCK_ASSERT(sc);
1116 vq = sc->vtblk_vq;
1117 enq = 0;
1118
1119 if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
1120 return;
1121
1122 while (!virtqueue_full(vq)) {
1123 req = vtblk_request_next(sc);
1124 if (req == NULL)
1125 break;
1126
1127 if (vtblk_request_execute(sc, req) != 0) {
1128 vtblk_request_requeue_ready(sc, req);
1129 break;
1130 }
1131
1132 enq++;
1133 }
1134
1135 if (enq > 0)
1136 virtqueue_notify(vq);
1137 }
1138
1139 static void
vtblk_bio_done(struct vtblk_softc * sc,struct bio * bp,int error)1140 vtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error)
1141 {
1142
1143 /* Because of GEOM direct dispatch, we cannot hold any locks. */
1144 if (sc != NULL)
1145 VTBLK_LOCK_ASSERT_NOTOWNED(sc);
1146
1147 if (error) {
1148 bp->bio_resid = bp->bio_bcount;
1149 bp->bio_error = error;
1150 bp->bio_flags |= BIO_ERROR;
1151 }
1152
1153 if (bp->bio_driver1 != NULL) {
1154 free(bp->bio_driver1, M_DEVBUF);
1155 bp->bio_driver1 = NULL;
1156 }
1157
1158 biodone(bp);
1159 }
1160
1161 #define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg) \
1162 if (virtio_with_feature(_dev, _feature)) { \
1163 virtio_read_device_config(_dev, \
1164 offsetof(struct virtio_blk_config, _field), \
1165 &(_cfg)->_field, sizeof((_cfg)->_field)); \
1166 }
1167
1168 static void
vtblk_read_config(struct vtblk_softc * sc,struct virtio_blk_config * blkcfg)1169 vtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
1170 {
1171 device_t dev;
1172
1173 dev = sc->vtblk_dev;
1174
1175 bzero(blkcfg, sizeof(struct virtio_blk_config));
1176
1177 /* The capacity is always available. */
1178 virtio_read_device_config(dev, offsetof(struct virtio_blk_config,
1179 capacity), &blkcfg->capacity, sizeof(blkcfg->capacity));
1180
1181 /* Read the configuration if the feature was negotiated. */
1182 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg);
1183 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg);
1184 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1185 geometry.cylinders, blkcfg);
1186 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1187 geometry.heads, blkcfg);
1188 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1189 geometry.sectors, blkcfg);
1190 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg);
1191 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1192 topology.physical_block_exp, blkcfg);
1193 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1194 topology.alignment_offset, blkcfg);
1195 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1196 topology.min_io_size, blkcfg);
1197 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1198 topology.opt_io_size, blkcfg);
1199 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, wce, blkcfg);
1200 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_DISCARD, max_discard_sectors,
1201 blkcfg);
1202 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_DISCARD, max_discard_seg, blkcfg);
1203 VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_DISCARD, discard_sector_alignment,
1204 blkcfg);
1205 }
1206
1207 #undef VTBLK_GET_CONFIG
1208
1209 static void
vtblk_ident(struct vtblk_softc * sc)1210 vtblk_ident(struct vtblk_softc *sc)
1211 {
1212 struct bio buf;
1213 struct disk *dp;
1214 struct vtblk_request *req;
1215 int len, error;
1216
1217 dp = sc->vtblk_disk;
1218 len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE);
1219
1220 if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0)
1221 return;
1222
1223 req = vtblk_request_dequeue(sc);
1224 if (req == NULL)
1225 return;
1226
1227 req->vbr_ack = -1;
1228 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID);
1229 req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
1230 req->vbr_hdr.sector = 0;
1231
1232 req->vbr_bp = &buf;
1233 g_reset_bio(&buf);
1234
1235 buf.bio_cmd = BIO_READ;
1236 buf.bio_data = dp->d_ident;
1237 buf.bio_bcount = len;
1238
1239 VTBLK_LOCK(sc);
1240 error = vtblk_poll_request(sc, req);
1241 VTBLK_UNLOCK(sc);
1242
1243 vtblk_request_enqueue(sc, req);
1244
1245 if (error) {
1246 device_printf(sc->vtblk_dev,
1247 "error getting device identifier: %d\n", error);
1248 }
1249 }
1250
1251 static int
vtblk_poll_request(struct vtblk_softc * sc,struct vtblk_request * req)1252 vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
1253 {
1254 struct virtqueue *vq;
1255 int error;
1256
1257 vq = sc->vtblk_vq;
1258
1259 if (!virtqueue_empty(vq))
1260 return (EBUSY);
1261
1262 error = vtblk_request_execute(sc, req);
1263 if (error)
1264 return (error);
1265
1266 virtqueue_notify(vq);
1267 virtqueue_poll(vq, NULL);
1268
1269 error = vtblk_request_error(req);
1270 if (error && bootverbose) {
1271 device_printf(sc->vtblk_dev,
1272 "%s: IO error: %d\n", __func__, error);
1273 }
1274
1275 return (error);
1276 }
1277
1278 static int
vtblk_quiesce(struct vtblk_softc * sc)1279 vtblk_quiesce(struct vtblk_softc *sc)
1280 {
1281 int error;
1282
1283 VTBLK_LOCK_ASSERT(sc);
1284 error = 0;
1285
1286 while (!virtqueue_empty(sc->vtblk_vq)) {
1287 if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
1288 VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
1289 error = EBUSY;
1290 break;
1291 }
1292 }
1293
1294 return (error);
1295 }
1296
1297 static void
vtblk_vq_intr(void * xsc)1298 vtblk_vq_intr(void *xsc)
1299 {
1300 struct vtblk_softc *sc;
1301 struct virtqueue *vq;
1302 struct bio_queue queue;
1303
1304 sc = xsc;
1305 vq = sc->vtblk_vq;
1306 TAILQ_INIT(&queue);
1307
1308 VTBLK_LOCK(sc);
1309
1310 again:
1311 if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
1312 goto out;
1313
1314 vtblk_queue_completed(sc, &queue);
1315 vtblk_startio(sc);
1316
1317 if (virtqueue_enable_intr(vq) != 0) {
1318 virtqueue_disable_intr(vq);
1319 goto again;
1320 }
1321
1322 if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
1323 wakeup(&sc->vtblk_vq);
1324
1325 out:
1326 VTBLK_UNLOCK(sc);
1327 vtblk_done_completed(sc, &queue);
1328 }
1329
1330 static void
vtblk_stop(struct vtblk_softc * sc)1331 vtblk_stop(struct vtblk_softc *sc)
1332 {
1333
1334 virtqueue_disable_intr(sc->vtblk_vq);
1335 virtio_stop(sc->vtblk_dev);
1336 }
1337
1338 static void
vtblk_dump_quiesce(struct vtblk_softc * sc)1339 vtblk_dump_quiesce(struct vtblk_softc *sc)
1340 {
1341
1342 /*
1343 * Spin here until all the requests in-flight at the time of the
1344 * dump are completed and queued. The queued requests will be
1345 * biodone'd once the dump is finished.
1346 */
1347 while (!virtqueue_empty(sc->vtblk_vq))
1348 vtblk_queue_completed(sc, &sc->vtblk_dump_queue);
1349 }
1350
1351 static int
vtblk_dump_write(struct vtblk_softc * sc,void * virtual,off_t offset,size_t length)1352 vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset,
1353 size_t length)
1354 {
1355 struct bio buf;
1356 struct vtblk_request *req;
1357
1358 req = &sc->vtblk_dump_request;
1359 req->vbr_ack = -1;
1360 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);
1361 req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
1362 req->vbr_hdr.sector = vtblk_gtoh64(sc, offset / VTBLK_BSIZE);
1363
1364 req->vbr_bp = &buf;
1365 g_reset_bio(&buf);
1366
1367 buf.bio_cmd = BIO_WRITE;
1368 buf.bio_data = virtual;
1369 buf.bio_bcount = length;
1370
1371 return (vtblk_poll_request(sc, req));
1372 }
1373
1374 static int
vtblk_dump_flush(struct vtblk_softc * sc)1375 vtblk_dump_flush(struct vtblk_softc *sc)
1376 {
1377 struct bio buf;
1378 struct vtblk_request *req;
1379
1380 req = &sc->vtblk_dump_request;
1381 req->vbr_ack = -1;
1382 req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);
1383 req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
1384 req->vbr_hdr.sector = 0;
1385
1386 req->vbr_bp = &buf;
1387 g_reset_bio(&buf);
1388
1389 buf.bio_cmd = BIO_FLUSH;
1390
1391 return (vtblk_poll_request(sc, req));
1392 }
1393
1394 static void
vtblk_dump_complete(struct vtblk_softc * sc)1395 vtblk_dump_complete(struct vtblk_softc *sc)
1396 {
1397
1398 vtblk_dump_flush(sc);
1399
1400 VTBLK_UNLOCK(sc);
1401 vtblk_done_completed(sc, &sc->vtblk_dump_queue);
1402 VTBLK_LOCK(sc);
1403 }
1404
1405 static void
vtblk_set_write_cache(struct vtblk_softc * sc,int wc)1406 vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
1407 {
1408
1409 /* Set either writeback (1) or writethrough (0) mode. */
1410 virtio_write_dev_config_1(sc->vtblk_dev,
1411 offsetof(struct virtio_blk_config, wce), wc);
1412 }
1413
1414 static int
vtblk_write_cache_enabled(struct vtblk_softc * sc,struct virtio_blk_config * blkcfg)1415 vtblk_write_cache_enabled(struct vtblk_softc *sc,
1416 struct virtio_blk_config *blkcfg)
1417 {
1418 int wc;
1419
1420 if (sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) {
1421 wc = vtblk_tunable_int(sc, "writecache_mode",
1422 vtblk_writecache_mode);
1423 if (wc >= 0 && wc < VTBLK_CACHE_MAX)
1424 vtblk_set_write_cache(sc, wc);
1425 else
1426 wc = blkcfg->wce;
1427 } else
1428 wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_FLUSH);
1429
1430 return (wc);
1431 }
1432
1433 static int
vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)1434 vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
1435 {
1436 struct vtblk_softc *sc;
1437 int wc, error;
1438
1439 sc = oidp->oid_arg1;
1440 wc = sc->vtblk_write_cache;
1441
1442 error = sysctl_handle_int(oidp, &wc, 0, req);
1443 if (error || req->newptr == NULL)
1444 return (error);
1445 if ((sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) == 0)
1446 return (EPERM);
1447 if (wc < 0 || wc >= VTBLK_CACHE_MAX)
1448 return (EINVAL);
1449
1450 VTBLK_LOCK(sc);
1451 sc->vtblk_write_cache = wc;
1452 vtblk_set_write_cache(sc, sc->vtblk_write_cache);
1453 VTBLK_UNLOCK(sc);
1454
1455 return (0);
1456 }
1457
1458 static void
vtblk_setup_sysctl(struct vtblk_softc * sc)1459 vtblk_setup_sysctl(struct vtblk_softc *sc)
1460 {
1461 device_t dev;
1462 struct sysctl_ctx_list *ctx;
1463 struct sysctl_oid *tree;
1464 struct sysctl_oid_list *child;
1465
1466 dev = sc->vtblk_dev;
1467 ctx = device_get_sysctl_ctx(dev);
1468 tree = device_get_sysctl_tree(dev);
1469 child = SYSCTL_CHILDREN(tree);
1470
1471 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "writecache_mode",
1472 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
1473 vtblk_write_cache_sysctl, "I",
1474 "Write cache mode (writethrough (0) or writeback (1))");
1475 }
1476
1477 static int
vtblk_tunable_int(struct vtblk_softc * sc,const char * knob,int def)1478 vtblk_tunable_int(struct vtblk_softc *sc, const char *knob, int def)
1479 {
1480 char path[64];
1481
1482 snprintf(path, sizeof(path),
1483 "hw.vtblk.%d.%s", device_get_unit(sc->vtblk_dev), knob);
1484 TUNABLE_INT_FETCH(path, &def);
1485
1486 return (def);
1487 }
1488