[Midnightbsd-cvs] src [10042] trunk/sys/dev/virtio: sync with freebsd 10
laffer1 at midnightbsd.org
laffer1 at midnightbsd.org
Sun May 27 18:33:46 EDT 2018
Revision: 10042
http://svnweb.midnightbsd.org/src/?rev=10042
Author: laffer1
Date: 2018-05-27 18:33:45 -0400 (Sun, 27 May 2018)
Log Message:
-----------
sync with freebsd 10
Modified Paths:
--------------
trunk/sys/dev/virtio/balloon/virtio_balloon.c
trunk/sys/dev/virtio/balloon/virtio_balloon.h
trunk/sys/dev/virtio/block/virtio_blk.c
trunk/sys/dev/virtio/block/virtio_blk.h
trunk/sys/dev/virtio/network/if_vtnet.c
trunk/sys/dev/virtio/network/if_vtnetvar.h
trunk/sys/dev/virtio/network/virtio_net.h
trunk/sys/dev/virtio/pci/virtio_pci.c
trunk/sys/dev/virtio/pci/virtio_pci.h
trunk/sys/dev/virtio/scsi/virtio_scsi.c
trunk/sys/dev/virtio/scsi/virtio_scsi.h
trunk/sys/dev/virtio/scsi/virtio_scsivar.h
trunk/sys/dev/virtio/virtio.c
trunk/sys/dev/virtio/virtio.h
trunk/sys/dev/virtio/virtio_bus_if.m
trunk/sys/dev/virtio/virtio_if.m
trunk/sys/dev/virtio/virtio_ring.h
trunk/sys/dev/virtio/virtqueue.c
trunk/sys/dev/virtio/virtqueue.h
Added Paths:
-----------
trunk/sys/dev/virtio/console/
trunk/sys/dev/virtio/console/virtio_console.c
trunk/sys/dev/virtio/console/virtio_console.h
trunk/sys/dev/virtio/random/
trunk/sys/dev/virtio/random/virtio_random.c
trunk/sys/dev/virtio/virtio_config.h
trunk/sys/dev/virtio/virtio_ids.h
Property Changed:
----------------
trunk/sys/dev/virtio/virtio_bus_if.m
trunk/sys/dev/virtio/virtio_if.m
Modified: trunk/sys/dev/virtio/balloon/virtio_balloon.c
===================================================================
--- trunk/sys/dev/virtio/balloon/virtio_balloon.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/balloon/virtio_balloon.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -27,7 +28,7 @@
/* Driver for VirtIO memory balloon devices. */
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/balloon/virtio_balloon.c 292906 2015-12-30 08:15:43Z royger $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -334,7 +335,7 @@
KASSERT(m->queue == PQ_NONE,
("%s: allocated page %p on queue", __func__, m));
- TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
+ TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
}
if (i > 0)
@@ -362,8 +363,8 @@
sc->vtballoon_page_frames[i] =
VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
- TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
- TAILQ_INSERT_TAIL(&free_pages, m, pageq);
+ TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
+ TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
}
if (i > 0) {
@@ -371,7 +372,7 @@
vtballoon_send_page_frames(sc, vq, i);
while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
- TAILQ_REMOVE(&free_pages, m, pageq);
+ TAILQ_REMOVE(&free_pages, m, plinks.q);
vtballoon_free_page(sc, m);
}
}
@@ -438,8 +439,7 @@
{
vm_page_t m;
- m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
- VM_ALLOC_NOOBJ);
+ m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
if (m != NULL)
sc->vtballoon_current_npages++;
@@ -450,7 +450,6 @@
vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
{
- vm_page_unwire(m, 0);
vm_page_free(m);
sc->vtballoon_current_npages--;
}
Modified: trunk/sys/dev/virtio/balloon/virtio_balloon.h
===================================================================
--- trunk/sys/dev/virtio/balloon/virtio_balloon.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/balloon/virtio_balloon.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* This header is BSD licensed so anyone can use the definitions to implement
* compatible drivers/servers.
@@ -25,7 +26,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/balloon/virtio_balloon.h 238072 2012-07-03 15:15:41Z obrien $
*/
#ifndef _VIRTIO_BALLOON_H
Modified: trunk/sys/dev/virtio/block/virtio_blk.c
===================================================================
--- trunk/sys/dev/virtio/block/virtio_blk.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/block/virtio_blk.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -27,7 +28,7 @@
/* Driver for VirtIO block devices. */
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/block/virtio_blk.c 284344 2015-06-13 17:40:33Z bryanv $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -58,7 +59,6 @@
struct virtio_blk_outhdr vbr_hdr;
struct bio *vbr_bp;
uint8_t vbr_ack;
-
TAILQ_ENTRY(vtblk_request) vbr_link;
};
@@ -77,9 +77,8 @@
#define VTBLK_FLAG_READONLY 0x0002
#define VTBLK_FLAG_DETACH 0x0004
#define VTBLK_FLAG_SUSPEND 0x0008
-#define VTBLK_FLAG_DUMPING 0x0010
-#define VTBLK_FLAG_BARRIER 0x0020
-#define VTBLK_FLAG_WC_CONFIG 0x0040
+#define VTBLK_FLAG_BARRIER 0x0010
+#define VTBLK_FLAG_WC_CONFIG 0x0020
struct virtqueue *vtblk_vq;
struct sglist *vtblk_sglist;
@@ -96,6 +95,7 @@
int vtblk_request_count;
enum vtblk_cache_mode vtblk_write_cache;
+ struct bio_queue vtblk_dump_queue;
struct vtblk_request vtblk_dump_request;
};
@@ -132,53 +132,62 @@
static void vtblk_strategy(struct bio *);
static void vtblk_negotiate_features(struct vtblk_softc *);
+static void vtblk_setup_features(struct vtblk_softc *);
static int vtblk_maximum_segments(struct vtblk_softc *,
struct virtio_blk_config *);
static int vtblk_alloc_virtqueue(struct vtblk_softc *);
-static void vtblk_set_write_cache(struct vtblk_softc *, int);
-static int vtblk_write_cache_enabled(struct vtblk_softc *sc,
- struct virtio_blk_config *);
-static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
+static void vtblk_resize_disk(struct vtblk_softc *, uint64_t);
static void vtblk_alloc_disk(struct vtblk_softc *,
struct virtio_blk_config *);
static void vtblk_create_disk(struct vtblk_softc *);
-static int vtblk_quiesce(struct vtblk_softc *);
-static void vtblk_startio(struct vtblk_softc *);
-static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *);
-static int vtblk_execute_request(struct vtblk_softc *,
+static int vtblk_request_prealloc(struct vtblk_softc *);
+static void vtblk_request_free(struct vtblk_softc *);
+static struct vtblk_request *
+ vtblk_request_dequeue(struct vtblk_softc *);
+static void vtblk_request_enqueue(struct vtblk_softc *,
struct vtblk_request *);
+static struct vtblk_request *
+ vtblk_request_next_ready(struct vtblk_softc *);
+static void vtblk_request_requeue_ready(struct vtblk_softc *,
+ struct vtblk_request *);
+static struct vtblk_request *
+ vtblk_request_next(struct vtblk_softc *);
+static struct vtblk_request *
+ vtblk_request_bio(struct vtblk_softc *);
+static int vtblk_request_execute(struct vtblk_softc *,
+ struct vtblk_request *);
+static int vtblk_request_error(struct vtblk_request *);
-static void vtblk_vq_intr(void *);
+static void vtblk_queue_completed(struct vtblk_softc *,
+ struct bio_queue *);
+static void vtblk_done_completed(struct vtblk_softc *,
+ struct bio_queue *);
+static void vtblk_drain_vq(struct vtblk_softc *);
+static void vtblk_drain(struct vtblk_softc *);
-static void vtblk_stop(struct vtblk_softc *);
+static void vtblk_startio(struct vtblk_softc *);
+static void vtblk_bio_done(struct vtblk_softc *, struct bio *, int);
static void vtblk_read_config(struct vtblk_softc *,
struct virtio_blk_config *);
-static void vtblk_get_ident(struct vtblk_softc *);
-static void vtblk_prepare_dump(struct vtblk_softc *);
-static int vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
-static int vtblk_flush_dump(struct vtblk_softc *);
+static void vtblk_ident(struct vtblk_softc *);
static int vtblk_poll_request(struct vtblk_softc *,
struct vtblk_request *);
+static int vtblk_quiesce(struct vtblk_softc *);
+static void vtblk_vq_intr(void *);
+static void vtblk_stop(struct vtblk_softc *);
-static void vtblk_finish_completed(struct vtblk_softc *);
-static void vtblk_drain_vq(struct vtblk_softc *, int);
-static void vtblk_drain(struct vtblk_softc *);
+static void vtblk_dump_quiesce(struct vtblk_softc *);
+static int vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t);
+static int vtblk_dump_flush(struct vtblk_softc *);
+static void vtblk_dump_complete(struct vtblk_softc *);
-static int vtblk_alloc_requests(struct vtblk_softc *);
-static void vtblk_free_requests(struct vtblk_softc *);
-static struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *);
-static void vtblk_enqueue_request(struct vtblk_softc *,
- struct vtblk_request *);
+static void vtblk_set_write_cache(struct vtblk_softc *, int);
+static int vtblk_write_cache_enabled(struct vtblk_softc *sc,
+ struct virtio_blk_config *);
+static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
-static struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *);
-static void vtblk_enqueue_ready(struct vtblk_softc *,
- struct vtblk_request *);
-
-static int vtblk_request_error(struct vtblk_request *);
-static void vtblk_finish_bio(struct bio *, int);
-
static void vtblk_setup_sysctl(struct vtblk_softc *);
static int vtblk_tunable_int(struct vtblk_softc *, const char *, int);
@@ -197,6 +206,7 @@
VIRTIO_BLK_F_RO | \
VIRTIO_BLK_F_BLK_SIZE | \
VIRTIO_BLK_F_WCE | \
+ VIRTIO_BLK_F_TOPOLOGY | \
VIRTIO_BLK_F_CONFIG_WCE | \
VIRTIO_RING_F_INDIRECT_DESC)
@@ -287,30 +297,19 @@
struct virtio_blk_config blkcfg;
int error;
+ virtio_set_feature_desc(dev, vtblk_feature_desc);
+
sc = device_get_softc(dev);
sc->vtblk_dev = dev;
-
VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
-
bioq_init(&sc->vtblk_bioq);
+ TAILQ_INIT(&sc->vtblk_dump_queue);
TAILQ_INIT(&sc->vtblk_req_free);
TAILQ_INIT(&sc->vtblk_req_ready);
- virtio_set_feature_desc(dev, vtblk_feature_desc);
- vtblk_negotiate_features(sc);
-
- if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
- sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
- if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
- sc->vtblk_flags |= VTBLK_FLAG_READONLY;
- if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
- sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
- if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
- sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
-
vtblk_setup_sysctl(sc);
+ vtblk_setup_features(sc);
- /* Get local copy of config. */
vtblk_read_config(sc, &blkcfg);
/*
@@ -349,7 +348,7 @@
goto fail;
}
- error = vtblk_alloc_requests(sc);
+ error = vtblk_request_prealloc(sc);
if (error) {
device_printf(dev, "cannot preallocate requests\n");
goto fail;
@@ -449,7 +448,20 @@
static int
vtblk_config_change(device_t dev)
{
+ struct vtblk_softc *sc;
+ struct virtio_blk_config blkcfg;
+ uint64_t capacity;
+ sc = device_get_softc(dev);
+
+ vtblk_read_config(sc, &blkcfg);
+
+ /* Capacity is always in 512-byte units. */
+ capacity = blkcfg.capacity * 512;
+
+ if (sc->vtblk_disk->d_mediasize != capacity)
+ vtblk_resize_disk(sc, capacity);
+
return (0);
}
@@ -496,6 +508,7 @@
int error;
dp = arg;
+ error = 0;
if ((sc = dp->d_drv1) == NULL)
return (ENXIO);
@@ -502,19 +515,12 @@
VTBLK_LOCK(sc);
- if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
- vtblk_prepare_dump(sc);
- sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
- }
+ vtblk_dump_quiesce(sc);
if (length > 0)
- error = vtblk_write_dump(sc, virtual, offset, length);
- else if (virtual == NULL && offset == 0)
- error = vtblk_flush_dump(sc);
- else {
- error = EINVAL;
- sc->vtblk_flags &= ~VTBLK_FLAG_DUMPING;
- }
+ error = vtblk_dump_write(sc, virtual, offset, length);
+ if (error || (virtual == NULL && offset == 0))
+ vtblk_dump_complete(sc);
VTBLK_UNLOCK(sc);
@@ -527,7 +533,7 @@
struct vtblk_softc *sc;
if ((sc = bp->bio_disk->d_drv1) == NULL) {
- vtblk_finish_bio(bp, EINVAL);
+ vtblk_bio_done(NULL, bp, EINVAL);
return;
}
@@ -537,37 +543,21 @@
*/
if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
(bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
- vtblk_finish_bio(bp, EROFS);
+ vtblk_bio_done(sc, bp, EROFS);
return;
}
-#ifdef INVARIANTS
- /*
- * Prevent read/write buffers spanning too many segments from
- * getting into the queue. This should only trip if d_maxsize
- * was incorrectly set.
- */
- if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
- int nsegs, max_nsegs;
+ VTBLK_LOCK(sc);
- nsegs = sglist_count(bp->bio_data, bp->bio_bcount);
- max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS;
-
- KASSERT(nsegs <= max_nsegs,
- ("%s: bio %p spanned too many segments: %d, max: %d",
- __func__, bp, nsegs, max_nsegs));
+ if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
+ VTBLK_UNLOCK(sc);
+ vtblk_bio_done(sc, bp, ENXIO);
+ return;
}
-#endif
- VTBLK_LOCK(sc);
- if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
- vtblk_finish_bio(bp, ENXIO);
- else {
- bioq_disksort(&sc->vtblk_bioq, bp);
+ bioq_insert_tail(&sc->vtblk_bioq, bp);
+ vtblk_startio(sc);
- if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0)
- vtblk_startio(sc);
- }
VTBLK_UNLOCK(sc);
}
@@ -583,6 +573,25 @@
sc->vtblk_features = virtio_negotiate_features(dev, features);
}
+static void
+vtblk_setup_features(struct vtblk_softc *sc)
+{
+ device_t dev;
+
+ dev = sc->vtblk_dev;
+
+ vtblk_negotiate_features(sc);
+
+ if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
+ sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
+ if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
+ sc->vtblk_flags |= VTBLK_FLAG_READONLY;
+ if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
+ sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
+ if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
+ sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
+}
+
static int
vtblk_maximum_segments(struct vtblk_softc *sc,
struct virtio_blk_config *blkcfg)
@@ -619,58 +628,30 @@
}
static void
-vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
+vtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity)
{
+ device_t dev;
+ struct disk *dp;
+ int error;
- /* Set either writeback (1) or writethrough (0) mode. */
- virtio_write_dev_config_1(sc->vtblk_dev,
- offsetof(struct virtio_blk_config, writeback), wc);
-}
+ dev = sc->vtblk_dev;
+ dp = sc->vtblk_disk;
-static int
-vtblk_write_cache_enabled(struct vtblk_softc *sc,
- struct virtio_blk_config *blkcfg)
-{
- int wc;
+ dp->d_mediasize = new_capacity;
+ if (bootverbose) {
+ device_printf(dev, "resized to %juMB (%ju %u byte sectors)\n",
+ (uintmax_t) dp->d_mediasize >> 20,
+ (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
+ dp->d_sectorsize);
+ }
- if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
- wc = vtblk_tunable_int(sc, "writecache_mode",
- vtblk_writecache_mode);
- if (wc >= 0 && wc < VTBLK_CACHE_MAX)
- vtblk_set_write_cache(sc, wc);
- else
- wc = blkcfg->writeback;
- } else
- wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
-
- return (wc);
+ error = disk_resize(dp, M_NOWAIT);
+ if (error) {
+ device_printf(dev,
+ "disk_resize(9) failed, error: %d\n", error);
+ }
}
-static int
-vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
-{
- struct vtblk_softc *sc;
- int wc, error;
-
- sc = oidp->oid_arg1;
- wc = sc->vtblk_write_cache;
-
- error = sysctl_handle_int(oidp, &wc, 0, req);
- if (error || req->newptr == NULL)
- return (error);
- if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
- return (EPERM);
- if (wc < 0 || wc >= VTBLK_CACHE_MAX)
- return (EINVAL);
-
- VTBLK_LOCK(sc);
- sc->vtblk_write_cache = wc;
- vtblk_set_write_cache(sc, sc->vtblk_write_cache);
- VTBLK_UNLOCK(sc);
-
- return (0);
-}
-
static void
vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
{
@@ -687,7 +668,8 @@
dp->d_name = VTBLK_DISK_NAME;
dp->d_unit = device_get_unit(dev);
dp->d_drv1 = sc;
- dp->d_flags = DISKFLAG_CANFLUSHCACHE;
+ dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO |
+ DISKFLAG_DIRECT_COMPLETION;
dp->d_hba_vendor = virtio_get_vendor(dev);
dp->d_hba_device = virtio_get_device(dev);
dp->d_hba_subvendor = virtio_get_subvendor(dev);
@@ -727,7 +709,8 @@
dp->d_fwheads = blkcfg->geometry.heads;
}
- if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY)) {
+ if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY) &&
+ blkcfg->topology.physical_block_exp > 0) {
dp->d_stripesize = dp->d_sectorsize *
(1 << blkcfg->topology.physical_block_exp);
dp->d_stripeoffset = (dp->d_stripesize -
@@ -748,11 +731,7 @@
dp = sc->vtblk_disk;
- /*
- * Retrieving the identification string must be done after
- * the virtqueue interrupt is setup otherwise it will hang.
- */
- vtblk_get_ident(sc);
+ vtblk_ident(sc);
device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
(uintmax_t) dp->d_mediasize >> 20,
@@ -763,58 +742,108 @@
}
static int
-vtblk_quiesce(struct vtblk_softc *sc)
+vtblk_request_prealloc(struct vtblk_softc *sc)
{
- int error;
+ struct vtblk_request *req;
+ int i, nreqs;
- error = 0;
+ nreqs = virtqueue_size(sc->vtblk_vq);
- VTBLK_LOCK_ASSERT(sc);
+ /*
+ * Preallocate sufficient requests to keep the virtqueue full. Each
+ * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
+ * the number allocated when indirect descriptors are not available.
+ */
+ if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
+ nreqs /= VTBLK_MIN_SEGMENTS;
- while (!virtqueue_empty(sc->vtblk_vq)) {
- if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
- VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
- error = EBUSY;
- break;
- }
+ for (i = 0; i < nreqs; i++) {
+ req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
+ if (req == NULL)
+ return (ENOMEM);
+
+ MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1);
+ MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1);
+
+ sc->vtblk_request_count++;
+ vtblk_request_enqueue(sc, req);
}
- return (error);
+ return (0);
}
static void
-vtblk_startio(struct vtblk_softc *sc)
+vtblk_request_free(struct vtblk_softc *sc)
{
- struct virtqueue *vq;
struct vtblk_request *req;
- int enq;
- vq = sc->vtblk_vq;
- enq = 0;
+ MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready));
- VTBLK_LOCK_ASSERT(sc);
+ while ((req = vtblk_request_dequeue(sc)) != NULL) {
+ sc->vtblk_request_count--;
+ free(req, M_DEVBUF);
+ }
- while (!virtqueue_full(vq)) {
- if ((req = vtblk_dequeue_ready(sc)) == NULL)
- req = vtblk_bio_request(sc);
- if (req == NULL)
- break;
+ KASSERT(sc->vtblk_request_count == 0,
+ ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
+}
- if (vtblk_execute_request(sc, req) != 0) {
- vtblk_enqueue_ready(sc, req);
- break;
- }
+static struct vtblk_request *
+vtblk_request_dequeue(struct vtblk_softc *sc)
+{
+ struct vtblk_request *req;
- enq++;
+ req = TAILQ_FIRST(&sc->vtblk_req_free);
+ if (req != NULL) {
+ TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
+ bzero(req, sizeof(struct vtblk_request));
}
- if (enq > 0)
- virtqueue_notify(vq);
+ return (req);
}
+static void
+vtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+
+ TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
+}
+
static struct vtblk_request *
-vtblk_bio_request(struct vtblk_softc *sc)
+vtblk_request_next_ready(struct vtblk_softc *sc)
{
+ struct vtblk_request *req;
+
+ req = TAILQ_FIRST(&sc->vtblk_req_ready);
+ if (req != NULL)
+ TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
+
+ return (req);
+}
+
+static void
+vtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+
+ /* NOTE: Currently, there will be at most one request in the queue. */
+ TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
+}
+
+static struct vtblk_request *
+vtblk_request_next(struct vtblk_softc *sc)
+{
+ struct vtblk_request *req;
+
+ req = vtblk_request_next_ready(sc);
+ if (req != NULL)
+ return (req);
+
+ return (vtblk_request_bio(sc));
+}
+
+static struct vtblk_request *
+vtblk_request_bio(struct vtblk_softc *sc)
+{
struct bio_queue_head *bioq;
struct vtblk_request *req;
struct bio *bp;
@@ -824,7 +853,7 @@
if (bioq_first(bioq) == NULL)
return (NULL);
- req = vtblk_dequeue_request(sc);
+ req = vtblk_request_dequeue(sc);
if (req == NULL)
return (NULL);
@@ -849,11 +878,14 @@
panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
}
+ if (bp->bio_flags & BIO_ORDERED)
+ req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
+
return (req);
}
static int
-vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
+vtblk_request_execute(struct vtblk_softc *sc, struct vtblk_request *req)
{
struct virtqueue *vq;
struct sglist *sg;
@@ -866,26 +898,20 @@
ordered = 0;
writable = 0;
- VTBLK_LOCK_ASSERT(sc);
-
/*
- * Wait until the ordered request completes before
- * executing subsequent requests.
+ * Some hosts (such as bhyve) do not implement the barrier feature,
+ * so we emulate it in the driver by allowing the barrier request
+ * to be the only one in flight.
*/
- if (sc->vtblk_req_ordered != NULL)
- return (EBUSY);
-
- if (bp->bio_flags & BIO_ORDERED) {
- if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) {
- /*
- * This request will be executed once all
- * the in-flight requests are completed.
- */
+ if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) {
+ if (sc->vtblk_req_ordered != NULL)
+ return (EBUSY);
+ if (bp->bio_flags & BIO_ORDERED) {
if (!virtqueue_empty(vq))
return (EBUSY);
ordered = 1;
- } else
- req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
+ req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER;
+ }
}
sglist_reset(sg);
@@ -892,10 +918,11 @@
sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
- error = sglist_append(sg, bp->bio_data, bp->bio_bcount);
- if (error || sg->sg_nseg == sg->sg_maxseg)
- panic("%s: data buffer too big bio:%p error:%d",
+ error = sglist_append_bio(sg, bp);
+ if (error || sg->sg_nseg == sg->sg_maxseg) {
+ panic("%s: bio %p data buffer too big %d",
__func__, bp, error);
+ }
/* BIO_READ means the host writes into our buffer. */
if (bp->bio_cmd == BIO_READ)
@@ -913,46 +940,156 @@
return (error);
}
+static int
+vtblk_request_error(struct vtblk_request *req)
+{
+ int error;
+
+ switch (req->vbr_ack) {
+ case VIRTIO_BLK_S_OK:
+ error = 0;
+ break;
+ case VIRTIO_BLK_S_UNSUPP:
+ error = ENOTSUP;
+ break;
+ default:
+ error = EIO;
+ break;
+ }
+
+ return (error);
+}
+
static void
-vtblk_vq_intr(void *xsc)
+vtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue)
{
- struct vtblk_softc *sc;
+ struct vtblk_request *req;
+ struct bio *bp;
+
+ while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
+ if (sc->vtblk_req_ordered != NULL) {
+ MPASS(sc->vtblk_req_ordered == req);
+ sc->vtblk_req_ordered = NULL;
+ }
+
+ bp = req->vbr_bp;
+ bp->bio_error = vtblk_request_error(req);
+ TAILQ_INSERT_TAIL(queue, bp, bio_queue);
+
+ vtblk_request_enqueue(sc, req);
+ }
+}
+
+static void
+vtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue)
+{
+ struct bio *bp, *tmp;
+
+ TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) {
+ if (bp->bio_error != 0)
+ disk_err(bp, "hard error", -1, 1);
+ vtblk_bio_done(sc, bp, bp->bio_error);
+ }
+}
+
+static void
+vtblk_drain_vq(struct vtblk_softc *sc)
+{
struct virtqueue *vq;
+ struct vtblk_request *req;
+ int last;
- sc = xsc;
vq = sc->vtblk_vq;
+ last = 0;
-again:
- VTBLK_LOCK(sc);
- if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
- VTBLK_UNLOCK(sc);
- return;
+ while ((req = virtqueue_drain(vq, &last)) != NULL) {
+ vtblk_bio_done(sc, req->vbr_bp, ENXIO);
+ vtblk_request_enqueue(sc, req);
}
- vtblk_finish_completed(sc);
+ sc->vtblk_req_ordered = NULL;
+ KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
+}
- if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0)
- vtblk_startio(sc);
- else
- wakeup(&sc->vtblk_vq);
+static void
+vtblk_drain(struct vtblk_softc *sc)
+{
+ struct bio_queue queue;
+ struct bio_queue_head *bioq;
+ struct vtblk_request *req;
+ struct bio *bp;
- if (virtqueue_enable_intr(vq) != 0) {
- virtqueue_disable_intr(vq);
- VTBLK_UNLOCK(sc);
- goto again;
+ bioq = &sc->vtblk_bioq;
+ TAILQ_INIT(&queue);
+
+ if (sc->vtblk_vq != NULL) {
+ vtblk_queue_completed(sc, &queue);
+ vtblk_done_completed(sc, &queue);
+
+ vtblk_drain_vq(sc);
}
- VTBLK_UNLOCK(sc);
+ while ((req = vtblk_request_next_ready(sc)) != NULL) {
+ vtblk_bio_done(sc, req->vbr_bp, ENXIO);
+ vtblk_request_enqueue(sc, req);
+ }
+
+ while (bioq_first(bioq) != NULL) {
+ bp = bioq_takefirst(bioq);
+ vtblk_bio_done(sc, bp, ENXIO);
+ }
+
+ vtblk_request_free(sc);
}
static void
-vtblk_stop(struct vtblk_softc *sc)
+vtblk_startio(struct vtblk_softc *sc)
{
+ struct virtqueue *vq;
+ struct vtblk_request *req;
+ int enq;
- virtqueue_disable_intr(sc->vtblk_vq);
- virtio_stop(sc->vtblk_dev);
+ VTBLK_LOCK_ASSERT(sc);
+ vq = sc->vtblk_vq;
+ enq = 0;
+
+ if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
+ return;
+
+ while (!virtqueue_full(vq)) {
+ req = vtblk_request_next(sc);
+ if (req == NULL)
+ break;
+
+ if (vtblk_request_execute(sc, req) != 0) {
+ vtblk_request_requeue_ready(sc, req);
+ break;
+ }
+
+ enq++;
+ }
+
+ if (enq > 0)
+ virtqueue_notify(vq);
}
+static void
+vtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error)
+{
+
+ /* Because of GEOM direct dispatch, we cannot hold any locks. */
+ if (sc != NULL)
+ VTBLK_LOCK_ASSERT_NOTOWNED(sc);
+
+ if (error) {
+ bp->bio_resid = bp->bio_bcount;
+ bp->bio_error = error;
+ bp->bio_flags |= BIO_ERROR;
+ }
+
+ biodone(bp);
+}
+
#define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg) \
if (virtio_with_feature(_dev, _feature)) { \
virtio_read_device_config(_dev, \
@@ -985,7 +1122,7 @@
#undef VTBLK_GET_CONFIG
static void
-vtblk_get_ident(struct vtblk_softc *sc)
+vtblk_ident(struct vtblk_softc *sc)
{
struct bio buf;
struct disk *dp;
@@ -998,7 +1135,7 @@
if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0)
return;
- req = vtblk_dequeue_request(sc);
+ req = vtblk_request_dequeue(sc);
if (req == NULL)
return;
@@ -1018,7 +1155,7 @@
error = vtblk_poll_request(sc, req);
VTBLK_UNLOCK(sc);
- vtblk_enqueue_request(sc, req);
+ vtblk_request_enqueue(sc, req);
if (error) {
device_printf(sc->vtblk_dev,
@@ -1026,78 +1163,7 @@
}
}
-static void
-vtblk_prepare_dump(struct vtblk_softc *sc)
-{
- device_t dev;
- struct virtqueue *vq;
-
- dev = sc->vtblk_dev;
- vq = sc->vtblk_vq;
-
- vtblk_stop(sc);
-
- /*
- * Drain all requests caught in-flight in the virtqueue,
- * skipping biodone(). When dumping, only one request is
- * outstanding at a time, and we just poll the virtqueue
- * for the response.
- */
- vtblk_drain_vq(sc, 1);
-
- if (virtio_reinit(dev, sc->vtblk_features) != 0) {
- panic("%s: cannot reinit VirtIO block device during dump",
- device_get_nameunit(dev));
- }
-
- virtqueue_disable_intr(vq);
- virtio_reinit_complete(dev);
-}
-
static int
-vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset,
- size_t length)
-{
- struct bio buf;
- struct vtblk_request *req;
-
- req = &sc->vtblk_dump_request;
- req->vbr_ack = -1;
- req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
- req->vbr_hdr.ioprio = 1;
- req->vbr_hdr.sector = offset / 512;
-
- req->vbr_bp = &buf;
- bzero(&buf, sizeof(struct bio));
-
- buf.bio_cmd = BIO_WRITE;
- buf.bio_data = virtual;
- buf.bio_bcount = length;
-
- return (vtblk_poll_request(sc, req));
-}
-
-static int
-vtblk_flush_dump(struct vtblk_softc *sc)
-{
- struct bio buf;
- struct vtblk_request *req;
-
- req = &sc->vtblk_dump_request;
- req->vbr_ack = -1;
- req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
- req->vbr_hdr.ioprio = 1;
- req->vbr_hdr.sector = 0;
-
- req->vbr_bp = &buf;
- bzero(&buf, sizeof(struct bio));
-
- buf.bio_cmd = BIO_FLUSH;
-
- return (vtblk_poll_request(sc, req));
-}
-
-static int
vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
{
struct virtqueue *vq;
@@ -1108,7 +1174,7 @@
if (!virtqueue_empty(vq))
return (EBUSY);
- error = vtblk_execute_request(sc, req);
+ error = vtblk_request_execute(sc, req);
if (error)
return (error);
@@ -1124,212 +1190,184 @@
return (error);
}
-static void
-vtblk_finish_completed(struct vtblk_softc *sc)
+static int
+vtblk_quiesce(struct vtblk_softc *sc)
{
- struct vtblk_request *req;
- struct bio *bp;
int error;
- while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
- bp = req->vbr_bp;
+ VTBLK_LOCK_ASSERT(sc);
+ error = 0;
- if (sc->vtblk_req_ordered != NULL) {
- /* This should be the only outstanding request. */
- MPASS(sc->vtblk_req_ordered == req);
- sc->vtblk_req_ordered = NULL;
+ while (!virtqueue_empty(sc->vtblk_vq)) {
+ if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
+ VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
+ error = EBUSY;
+ break;
}
+ }
- error = vtblk_request_error(req);
- if (error)
- disk_err(bp, "hard error", -1, 1);
-
- vtblk_finish_bio(bp, error);
- vtblk_enqueue_request(sc, req);
- }
+ return (error);
}
static void
-vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
+vtblk_vq_intr(void *xsc)
{
+ struct vtblk_softc *sc;
struct virtqueue *vq;
- struct vtblk_request *req;
- int last;
+ struct bio_queue queue;
+ sc = xsc;
vq = sc->vtblk_vq;
- last = 0;
+ TAILQ_INIT(&queue);
- while ((req = virtqueue_drain(vq, &last)) != NULL) {
- if (!skip_done)
- vtblk_finish_bio(req->vbr_bp, ENXIO);
+ VTBLK_LOCK(sc);
- vtblk_enqueue_request(sc, req);
- }
+again:
+ if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
+ goto out;
- sc->vtblk_req_ordered = NULL;
- KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
-}
+ vtblk_queue_completed(sc, &queue);
+ vtblk_startio(sc);
-static void
-vtblk_drain(struct vtblk_softc *sc)
-{
- struct bio_queue_head *bioq;
- struct vtblk_request *req;
- struct bio *bp;
-
- bioq = &sc->vtblk_bioq;
-
- if (sc->vtblk_vq != NULL) {
- vtblk_finish_completed(sc);
- vtblk_drain_vq(sc, 0);
+ if (virtqueue_enable_intr(vq) != 0) {
+ virtqueue_disable_intr(vq);
+ goto again;
}
- while ((req = vtblk_dequeue_ready(sc)) != NULL) {
- vtblk_finish_bio(req->vbr_bp, ENXIO);
- vtblk_enqueue_request(sc, req);
- }
+ if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
+ wakeup(&sc->vtblk_vq);
- while (bioq_first(bioq) != NULL) {
- bp = bioq_takefirst(bioq);
- vtblk_finish_bio(bp, ENXIO);
- }
-
- vtblk_free_requests(sc);
+out:
+ VTBLK_UNLOCK(sc);
+ vtblk_done_completed(sc, &queue);
}
-#ifdef INVARIANTS
static void
-vtblk_request_invariants(struct vtblk_request *req)
+vtblk_stop(struct vtblk_softc *sc)
{
- int hdr_nsegs, ack_nsegs;
- hdr_nsegs = sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr));
- ack_nsegs = sglist_count(&req->vbr_ack, sizeof(req->vbr_ack));
-
- KASSERT(hdr_nsegs == 1, ("request header crossed page boundary"));
- KASSERT(ack_nsegs == 1, ("request ack crossed page boundary"));
+ virtqueue_disable_intr(sc->vtblk_vq);
+ virtio_stop(sc->vtblk_dev);
}
-#endif
-static int
-vtblk_alloc_requests(struct vtblk_softc *sc)
+static void
+vtblk_dump_quiesce(struct vtblk_softc *sc)
{
- struct vtblk_request *req;
- int i, nreqs;
- nreqs = virtqueue_size(sc->vtblk_vq);
-
/*
- * Preallocate sufficient requests to keep the virtqueue full. Each
- * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
- * the number allocated when indirect descriptors are not available.
+ * Spin here until all the requests in-flight at the time of the
+ * dump are completed and queued. The queued requests will be
+ * biodone'd once the dump is finished.
*/
- if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
- nreqs /= VTBLK_MIN_SEGMENTS;
-
- for (i = 0; i < nreqs; i++) {
- req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
- if (req == NULL)
- return (ENOMEM);
-
-#ifdef INVARIANTS
- vtblk_request_invariants(req);
-#endif
-
- sc->vtblk_request_count++;
- vtblk_enqueue_request(sc, req);
- }
-
- return (0);
+ while (!virtqueue_empty(sc->vtblk_vq))
+ vtblk_queue_completed(sc, &sc->vtblk_dump_queue);
}
-static void
-vtblk_free_requests(struct vtblk_softc *sc)
+static int
+vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset,
+ size_t length)
{
+ struct bio buf;
struct vtblk_request *req;
- KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready),
- ("%s: ready requests left on queue", __func__));
+ req = &sc->vtblk_dump_request;
+ req->vbr_ack = -1;
+ req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
+ req->vbr_hdr.ioprio = 1;
+ req->vbr_hdr.sector = offset / 512;
- while ((req = vtblk_dequeue_request(sc)) != NULL) {
- sc->vtblk_request_count--;
- free(req, M_DEVBUF);
- }
+ req->vbr_bp = &buf;
+ bzero(&buf, sizeof(struct bio));
- KASSERT(sc->vtblk_request_count == 0,
- ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
+ buf.bio_cmd = BIO_WRITE;
+ buf.bio_data = virtual;
+ buf.bio_bcount = length;
+
+ return (vtblk_poll_request(sc, req));
}
-static struct vtblk_request *
-vtblk_dequeue_request(struct vtblk_softc *sc)
+static int
+vtblk_dump_flush(struct vtblk_softc *sc)
{
+ struct bio buf;
struct vtblk_request *req;
- req = TAILQ_FIRST(&sc->vtblk_req_free);
- if (req != NULL)
- TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
+ req = &sc->vtblk_dump_request;
+ req->vbr_ack = -1;
+ req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
+ req->vbr_hdr.ioprio = 1;
+ req->vbr_hdr.sector = 0;
- return (req);
-}
+ req->vbr_bp = &buf;
+ bzero(&buf, sizeof(struct bio));
-static void
-vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req)
-{
+ buf.bio_cmd = BIO_FLUSH;
- bzero(req, sizeof(struct vtblk_request));
- TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
+ return (vtblk_poll_request(sc, req));
}
-static struct vtblk_request *
-vtblk_dequeue_ready(struct vtblk_softc *sc)
+static void
+vtblk_dump_complete(struct vtblk_softc *sc)
{
- struct vtblk_request *req;
- req = TAILQ_FIRST(&sc->vtblk_req_ready);
- if (req != NULL)
- TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
+ vtblk_dump_flush(sc);
- return (req);
+ VTBLK_UNLOCK(sc);
+ vtblk_done_completed(sc, &sc->vtblk_dump_queue);
+ VTBLK_LOCK(sc);
}
static void
-vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
+vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
{
- TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
+ /* Set either writeback (1) or writethrough (0) mode. */
+ virtio_write_dev_config_1(sc->vtblk_dev,
+ offsetof(struct virtio_blk_config, writeback), wc);
}
static int
-vtblk_request_error(struct vtblk_request *req)
+vtblk_write_cache_enabled(struct vtblk_softc *sc,
+ struct virtio_blk_config *blkcfg)
{
- int error;
+ int wc;
- switch (req->vbr_ack) {
- case VIRTIO_BLK_S_OK:
- error = 0;
- break;
- case VIRTIO_BLK_S_UNSUPP:
- error = ENOTSUP;
- break;
- default:
- error = EIO;
- break;
- }
+ if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
+ wc = vtblk_tunable_int(sc, "writecache_mode",
+ vtblk_writecache_mode);
+ if (wc >= 0 && wc < VTBLK_CACHE_MAX)
+ vtblk_set_write_cache(sc, wc);
+ else
+ wc = blkcfg->writeback;
+ } else
+ wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
- return (error);
+ return (wc);
}
-static void
-vtblk_finish_bio(struct bio *bp, int error)
+static int
+vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
{
+ struct vtblk_softc *sc;
+ int wc, error;
- if (error) {
- bp->bio_resid = bp->bio_bcount;
- bp->bio_error = error;
- bp->bio_flags |= BIO_ERROR;
- }
+ sc = oidp->oid_arg1;
+ wc = sc->vtblk_write_cache;
- biodone(bp);
+ error = sysctl_handle_int(oidp, &wc, 0, req);
+ if (error || req->newptr == NULL)
+ return (error);
+ if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
+ return (EPERM);
+ if (wc < 0 || wc >= VTBLK_CACHE_MAX)
+ return (EINVAL);
+
+ VTBLK_LOCK(sc);
+ sc->vtblk_write_cache = wc;
+ vtblk_set_write_cache(sc, sc->vtblk_write_cache);
+ VTBLK_UNLOCK(sc);
+
+ return (0);
}
static void
Modified: trunk/sys/dev/virtio/block/virtio_blk.h
===================================================================
--- trunk/sys/dev/virtio/block/virtio_blk.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/block/virtio_blk.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* This header is BSD licensed so anyone can use the definitions to implement
* compatible drivers/servers.
@@ -25,7 +26,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/block/virtio_blk.h 280243 2015-03-19 09:53:00Z mav $
*/
#ifndef _VIRTIO_BLK_H
@@ -67,7 +68,7 @@
uint8_t physical_block_exp;
uint8_t alignment_offset;
uint16_t min_io_size;
- uint16_t opt_io_size;
+ uint32_t opt_io_size;
} topology;
/* Writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
Added: trunk/sys/dev/virtio/console/virtio_console.c
===================================================================
--- trunk/sys/dev/virtio/console/virtio_console.c (rev 0)
+++ trunk/sys/dev/virtio/console/virtio_console.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -0,0 +1,1410 @@
+/* $MidnightBSD$ */
+/*-
+ * Copyright (c) 2014, Bryan Venteicher <bryanv at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Driver for VirtIO console devices. */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/console/virtio_console.c 275273 2014-11-29 22:48:40Z bryanv $");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/kdb.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/sglist.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+#include <sys/queue.h>
+
+#include <sys/conf.h>
+#include <sys/cons.h>
+#include <sys/tty.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+#include <dev/virtio/console/virtio_console.h>
+
+#include "virtio_if.h"
+
+#define VTCON_MAX_PORTS 32
+#define VTCON_TTY_PREFIX "V"
+#define VTCON_BULK_BUFSZ 128
+
+/*
+ * The buffer cannot cross more than one page boundary due to the
+ * size of the sglist segment array used.
+ */
+CTASSERT(VTCON_BULK_BUFSZ <= PAGE_SIZE);
+
+struct vtcon_softc;
+struct vtcon_softc_port;
+
+struct vtcon_port {
+ struct mtx vtcport_mtx;
+ struct vtcon_softc *vtcport_sc;
+ struct vtcon_softc_port *vtcport_scport;
+ struct tty *vtcport_tty;
+ struct virtqueue *vtcport_invq;
+ struct virtqueue *vtcport_outvq;
+ int vtcport_id;
+ int vtcport_flags;
+#define VTCON_PORT_FLAG_GONE 0x01
+#define VTCON_PORT_FLAG_CONSOLE 0x02
+
+#if defined(KDB)
+ int vtcport_alt_break_state;
+#endif
+};
+
+#define VTCON_PORT_LOCK(_port) mtx_lock(&(_port)->vtcport_mtx)
+#define VTCON_PORT_UNLOCK(_port) mtx_unlock(&(_port)->vtcport_mtx)
+
+struct vtcon_softc_port {
+ struct vtcon_softc *vcsp_sc;
+ struct vtcon_port *vcsp_port;
+ struct virtqueue *vcsp_invq;
+ struct virtqueue *vcsp_outvq;
+};
+
+struct vtcon_softc {
+ device_t vtcon_dev;
+ struct mtx vtcon_mtx;
+ uint64_t vtcon_features;
+ uint32_t vtcon_max_ports;
+ uint32_t vtcon_flags;
+#define VTCON_FLAG_DETACHED 0x01
+#define VTCON_FLAG_SIZE 0x02
+#define VTCON_FLAG_MULTIPORT 0x04
+
+ /*
+ * Ports can be added and removed during runtime, but we have
+ * to allocate all the virtqueues during attach. This array is
+ * indexed by the port ID.
+ */
+ struct vtcon_softc_port *vtcon_ports;
+
+ struct task vtcon_ctrl_task;
+ struct virtqueue *vtcon_ctrl_rxvq;
+ struct virtqueue *vtcon_ctrl_txvq;
+ struct mtx vtcon_ctrl_tx_mtx;
+};
+
+#define VTCON_LOCK(_sc) mtx_lock(&(_sc)->vtcon_mtx)
+#define VTCON_UNLOCK(_sc) mtx_unlock(&(_sc)->vtcon_mtx)
+#define VTCON_LOCK_ASSERT(_sc) \
+ mtx_assert(&(_sc)->vtcon_mtx, MA_OWNED)
+#define VTCON_LOCK_ASSERT_NOTOWNED(_sc) \
+ mtx_assert(&(_sc)->vtcon_mtx, MA_NOTOWNED)
+
+#define VTCON_CTRL_TX_LOCK(_sc) mtx_lock(&(_sc)->vtcon_ctrl_tx_mtx)
+#define VTCON_CTRL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->vtcon_ctrl_tx_mtx)
+
+#define VTCON_ASSERT_VALID_PORTID(_sc, _id) \
+ KASSERT((_id) >= 0 && (_id) < (_sc)->vtcon_max_ports, \
+ ("%s: port ID %d out of range", __func__, _id))
+
+#define VTCON_FEATURES VIRTIO_CONSOLE_F_MULTIPORT
+
+static struct virtio_feature_desc vtcon_feature_desc[] = {
+ { VIRTIO_CONSOLE_F_SIZE, "ConsoleSize" },
+ { VIRTIO_CONSOLE_F_MULTIPORT, "MultiplePorts" },
+ { VIRTIO_CONSOLE_F_EMERG_WRITE, "EmergencyWrite" },
+
+ { 0, NULL }
+};
+
+static int vtcon_modevent(module_t, int, void *);
+static void vtcon_drain_all(void);
+
+static int vtcon_probe(device_t);
+static int vtcon_attach(device_t);
+static int vtcon_detach(device_t);
+static int vtcon_config_change(device_t);
+
+static void vtcon_setup_features(struct vtcon_softc *);
+static void vtcon_negotiate_features(struct vtcon_softc *);
+static int vtcon_alloc_scports(struct vtcon_softc *);
+static int vtcon_alloc_virtqueues(struct vtcon_softc *);
+static void vtcon_read_config(struct vtcon_softc *,
+ struct virtio_console_config *);
+
+static void vtcon_determine_max_ports(struct vtcon_softc *,
+ struct virtio_console_config *);
+static void vtcon_destroy_ports(struct vtcon_softc *);
+static void vtcon_stop(struct vtcon_softc *);
+
+static int vtcon_ctrl_event_enqueue(struct vtcon_softc *,
+ struct virtio_console_control *);
+static int vtcon_ctrl_event_create(struct vtcon_softc *);
+static void vtcon_ctrl_event_requeue(struct vtcon_softc *,
+ struct virtio_console_control *);
+static int vtcon_ctrl_event_populate(struct vtcon_softc *);
+static void vtcon_ctrl_event_drain(struct vtcon_softc *);
+static int vtcon_ctrl_init(struct vtcon_softc *);
+static void vtcon_ctrl_deinit(struct vtcon_softc *);
+static void vtcon_ctrl_port_add_event(struct vtcon_softc *, int);
+static void vtcon_ctrl_port_remove_event(struct vtcon_softc *, int);
+static void vtcon_ctrl_port_console_event(struct vtcon_softc *, int);
+static void vtcon_ctrl_port_open_event(struct vtcon_softc *, int);
+static void vtcon_ctrl_process_event(struct vtcon_softc *,
+ struct virtio_console_control *);
+static void vtcon_ctrl_task_cb(void *, int);
+static void vtcon_ctrl_event_intr(void *);
+static void vtcon_ctrl_poll(struct vtcon_softc *,
+ struct virtio_console_control *control);
+static void vtcon_ctrl_send_control(struct vtcon_softc *, uint32_t,
+ uint16_t, uint16_t);
+
+static int vtcon_port_enqueue_buf(struct vtcon_port *, void *, size_t);
+static int vtcon_port_create_buf(struct vtcon_port *);
+static void vtcon_port_requeue_buf(struct vtcon_port *, void *);
+static int vtcon_port_populate(struct vtcon_port *);
+static void vtcon_port_destroy(struct vtcon_port *);
+static int vtcon_port_create(struct vtcon_softc *, int);
+static void vtcon_port_drain_bufs(struct virtqueue *);
+static void vtcon_port_drain(struct vtcon_port *);
+static void vtcon_port_teardown(struct vtcon_port *);
+static void vtcon_port_change_size(struct vtcon_port *, uint16_t,
+ uint16_t);
+static void vtcon_port_update_console_size(struct vtcon_softc *);
+static void vtcon_port_enable_intr(struct vtcon_port *);
+static void vtcon_port_disable_intr(struct vtcon_port *);
+static void vtcon_port_in(struct vtcon_port *);
+static void vtcon_port_intr(void *);
+static void vtcon_port_out(struct vtcon_port *, void *, int);
+static void vtcon_port_submit_event(struct vtcon_port *, uint16_t,
+ uint16_t);
+
+static int vtcon_tty_open(struct tty *);
+static void vtcon_tty_close(struct tty *);
+static void vtcon_tty_outwakeup(struct tty *);
+static void vtcon_tty_free(void *);
+
+static void vtcon_get_console_size(struct vtcon_softc *, uint16_t *,
+ uint16_t *);
+
+static void vtcon_enable_interrupts(struct vtcon_softc *);
+static void vtcon_disable_interrupts(struct vtcon_softc *);
+
+static int vtcon_pending_free;
+
+static struct ttydevsw vtcon_tty_class = {
+ .tsw_flags = 0,
+ .tsw_open = vtcon_tty_open,
+ .tsw_close = vtcon_tty_close,
+ .tsw_outwakeup = vtcon_tty_outwakeup,
+ .tsw_free = vtcon_tty_free,
+};
+
+static device_method_t vtcon_methods[] = {
+ /* Device methods. */
+ DEVMETHOD(device_probe, vtcon_probe),
+ DEVMETHOD(device_attach, vtcon_attach),
+ DEVMETHOD(device_detach, vtcon_detach),
+
+ /* VirtIO methods. */
+ DEVMETHOD(virtio_config_change, vtcon_config_change),
+
+ DEVMETHOD_END
+};
+
+static driver_t vtcon_driver = {
+ "vtcon",
+ vtcon_methods,
+ sizeof(struct vtcon_softc)
+};
+static devclass_t vtcon_devclass;
+
+DRIVER_MODULE(virtio_console, virtio_pci, vtcon_driver, vtcon_devclass,
+ vtcon_modevent, 0);
+MODULE_VERSION(virtio_console, 1);
+MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);
+
+static int
+vtcon_modevent(module_t mod, int type, void *unused)
+{
+ int error;
+
+ switch (type) {
+ case MOD_LOAD:
+ error = 0;
+ break;
+ case MOD_QUIESCE:
+ error = 0;
+ break;
+ case MOD_UNLOAD:
+ vtcon_drain_all();
+ error = 0;
+ break;
+ case MOD_SHUTDOWN:
+ error = 0;
+ break;
+ default:
+ error = EOPNOTSUPP;
+ break;
+ }
+
+ return (error);
+}
+
+static void
+vtcon_drain_all(void)
+{
+ int first;
+
+ for (first = 1; vtcon_pending_free != 0; first = 0) {
+ if (first != 0) {
+ printf("virtio_console: Waiting for all detached TTY "
+ "devices to have open fds closed.\n");
+ }
+ pause("vtcondra", hz);
+ }
+}
+
+static int
+vtcon_probe(device_t dev)
+{
+
+ if (virtio_get_device_type(dev) != VIRTIO_ID_CONSOLE)
+ return (ENXIO);
+
+ device_set_desc(dev, "VirtIO Console Adapter");
+
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtcon_attach(device_t dev)
+{
+ struct vtcon_softc *sc;
+ struct virtio_console_config concfg;
+ int error;
+
+ sc = device_get_softc(dev);
+ sc->vtcon_dev = dev;
+
+ mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);
+ mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);
+
+ virtio_set_feature_desc(dev, vtcon_feature_desc);
+ vtcon_setup_features(sc);
+
+ vtcon_read_config(sc, &concfg);
+ vtcon_determine_max_ports(sc, &concfg);
+
+ error = vtcon_alloc_scports(sc);
+ if (error) {
+ device_printf(dev, "cannot allocate softc port structures\n");
+ goto fail;
+ }
+
+ error = vtcon_alloc_virtqueues(sc);
+ if (error) {
+ device_printf(dev, "cannot allocate virtqueues\n");
+ goto fail;
+ }
+
+ if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
+ TASK_INIT(&sc->vtcon_ctrl_task, 0, vtcon_ctrl_task_cb, sc);
+ error = vtcon_ctrl_init(sc);
+ if (error)
+ goto fail;
+ } else {
+ error = vtcon_port_create(sc, 0);
+ if (error)
+ goto fail;
+ if (sc->vtcon_flags & VTCON_FLAG_SIZE)
+ vtcon_port_update_console_size(sc);
+ }
+
+ error = virtio_setup_intr(dev, INTR_TYPE_TTY);
+ if (error) {
+ device_printf(dev, "cannot setup virtqueue interrupts\n");
+ goto fail;
+ }
+
+ vtcon_enable_interrupts(sc);
+
+ vtcon_ctrl_send_control(sc, VIRTIO_CONSOLE_BAD_ID,
+ VIRTIO_CONSOLE_DEVICE_READY, 1);
+
+fail:
+ if (error)
+ vtcon_detach(dev);
+
+ return (error);
+}
+
+static int
+vtcon_detach(device_t dev)
+{
+ struct vtcon_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ VTCON_LOCK(sc);
+ sc->vtcon_flags |= VTCON_FLAG_DETACHED;
+ if (device_is_attached(dev))
+ vtcon_stop(sc);
+ VTCON_UNLOCK(sc);
+
+ if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
+ taskqueue_drain(taskqueue_thread, &sc->vtcon_ctrl_task);
+ vtcon_ctrl_deinit(sc);
+ }
+
+ vtcon_destroy_ports(sc);
+ mtx_destroy(&sc->vtcon_mtx);
+ mtx_destroy(&sc->vtcon_ctrl_tx_mtx);
+
+ return (0);
+}
+
+static int
+vtcon_config_change(device_t dev)
+{
+ struct vtcon_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ /*
+ * When the multiport feature is negotiated, all configuration
+ * changes are done through control virtqueue events.
+ */
+ if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0) {
+ if (sc->vtcon_flags & VTCON_FLAG_SIZE)
+ vtcon_port_update_console_size(sc);
+ }
+
+ return (0);
+}
+
+static void
+vtcon_negotiate_features(struct vtcon_softc *sc)
+{
+ device_t dev;
+ uint64_t features;
+
+ dev = sc->vtcon_dev;
+ features = VTCON_FEATURES;
+
+ sc->vtcon_features = virtio_negotiate_features(dev, features);
+}
+
+static void
+vtcon_setup_features(struct vtcon_softc *sc)
+{
+ device_t dev;
+
+ dev = sc->vtcon_dev;
+
+ vtcon_negotiate_features(sc);
+
+ if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))
+ sc->vtcon_flags |= VTCON_FLAG_SIZE;
+ if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))
+ sc->vtcon_flags |= VTCON_FLAG_MULTIPORT;
+}
+
+#define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg) \
+ if (virtio_with_feature(_dev, _feature)) { \
+ virtio_read_device_config(_dev, \
+ offsetof(struct virtio_console_config, _field), \
+ &(_cfg)->_field, sizeof((_cfg)->_field)); \
+ }
+
+static void
+vtcon_read_config(struct vtcon_softc *sc, struct virtio_console_config *concfg)
+{
+ device_t dev;
+
+ dev = sc->vtcon_dev;
+
+ bzero(concfg, sizeof(struct virtio_console_config));
+
+ VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, cols, concfg);
+ VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, rows, concfg);
+ VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_MULTIPORT, max_nr_ports, concfg);
+}
+
+#undef VTCON_GET_CONFIG
+
+static int
+vtcon_alloc_scports(struct vtcon_softc *sc)
+{
+ struct vtcon_softc_port *scport;
+ int max, i;
+
+ max = sc->vtcon_max_ports;
+
+ sc->vtcon_ports = malloc(sizeof(struct vtcon_softc_port) * max,
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (sc->vtcon_ports == NULL)
+ return (ENOMEM);
+
+ for (i = 0; i < max; i++) {
+ scport = &sc->vtcon_ports[i];
+ scport->vcsp_sc = sc;
+ }
+
+ return (0);
+}
+
+static int
+vtcon_alloc_virtqueues(struct vtcon_softc *sc)
+{
+ device_t dev;
+ struct vq_alloc_info *info;
+ struct vtcon_softc_port *scport;
+ int i, idx, portidx, nvqs, error;
+
+ dev = sc->vtcon_dev;
+
+ nvqs = sc->vtcon_max_ports * 2;
+ if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
+ nvqs += 2;
+
+ info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
+ if (info == NULL)
+ return (ENOMEM);
+
+ for (i = 0, idx = 0, portidx = 0; i < nvqs / 2; i++, idx += 2) {
+
+ if (i == 1) {
+ /* The control virtqueues are after the first port. */
+ VQ_ALLOC_INFO_INIT(&info[idx], 0,
+ vtcon_ctrl_event_intr, sc, &sc->vtcon_ctrl_rxvq,
+ "%s-control rx", device_get_nameunit(dev));
+ VQ_ALLOC_INFO_INIT(&info[idx+1], 0,
+ NULL, sc, &sc->vtcon_ctrl_txvq,
+ "%s-control tx", device_get_nameunit(dev));
+ continue;
+ }
+
+ scport = &sc->vtcon_ports[portidx];
+
+ VQ_ALLOC_INFO_INIT(&info[idx], 0, vtcon_port_intr,
+ scport, &scport->vcsp_invq, "%s-port%d in",
+ device_get_nameunit(dev), i);
+ VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL,
+ NULL, &scport->vcsp_outvq, "%s-port%d out",
+ device_get_nameunit(dev), i);
+
+ portidx++;
+ }
+
+ error = virtio_alloc_virtqueues(dev, 0, nvqs, info);
+ free(info, M_TEMP);
+
+ return (error);
+}
+
+static void
+vtcon_determine_max_ports(struct vtcon_softc *sc,
+ struct virtio_console_config *concfg)
+{
+
+ if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
+ sc->vtcon_max_ports =
+ min(concfg->max_nr_ports, VTCON_MAX_PORTS);
+ if (sc->vtcon_max_ports == 0)
+ sc->vtcon_max_ports = 1;
+ } else
+ sc->vtcon_max_ports = 1;
+}
+
+static void
+vtcon_destroy_ports(struct vtcon_softc *sc)
+{
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+ struct virtqueue *vq;
+ int i;
+
+ if (sc->vtcon_ports == NULL)
+ return;
+
+ VTCON_LOCK(sc);
+ for (i = 0; i < sc->vtcon_max_ports; i++) {
+ scport = &sc->vtcon_ports[i];
+
+ port = scport->vcsp_port;
+ if (port != NULL) {
+ scport->vcsp_port = NULL;
+ VTCON_PORT_LOCK(port);
+ VTCON_UNLOCK(sc);
+ vtcon_port_teardown(port);
+ VTCON_LOCK(sc);
+ }
+
+ vq = scport->vcsp_invq;
+ if (vq != NULL)
+ vtcon_port_drain_bufs(vq);
+ }
+ VTCON_UNLOCK(sc);
+
+ free(sc->vtcon_ports, M_DEVBUF);
+ sc->vtcon_ports = NULL;
+}
+
+static void
+vtcon_stop(struct vtcon_softc *sc)
+{
+
+ vtcon_disable_interrupts(sc);
+ virtio_stop(sc->vtcon_dev);
+}
+
+static int
+vtcon_ctrl_event_enqueue(struct vtcon_softc *sc,
+ struct virtio_console_control *control)
+{
+ struct sglist_seg segs[2];
+ struct sglist sg;
+ struct virtqueue *vq;
+ int error;
+
+ vq = sc->vtcon_ctrl_rxvq;
+
+ sglist_init(&sg, 2, segs);
+ error = sglist_append(&sg, control,
+ sizeof(struct virtio_console_control));
+ KASSERT(error == 0, ("%s: error %d adding control to sglist",
+ __func__, error));
+
+ return (virtqueue_enqueue(vq, control, &sg, 0, sg.sg_nseg));
+}
+
+static int
+vtcon_ctrl_event_create(struct vtcon_softc *sc)
+{
+ struct virtio_console_control *control;
+ int error;
+
+ control = malloc(sizeof(struct virtio_console_control), M_DEVBUF,
+ M_ZERO | M_NOWAIT);
+ if (control == NULL)
+ return (ENOMEM);
+
+ error = vtcon_ctrl_event_enqueue(sc, control);
+ if (error)
+ free(control, M_DEVBUF);
+
+ return (error);
+}
+
+static void
+vtcon_ctrl_event_requeue(struct vtcon_softc *sc,
+ struct virtio_console_control *control)
+{
+ int error;
+
+ bzero(control, sizeof(struct virtio_console_control));
+
+ error = vtcon_ctrl_event_enqueue(sc, control);
+ KASSERT(error == 0,
+ ("%s: cannot requeue control buffer %d", __func__, error));
+}
+
+static int
+vtcon_ctrl_event_populate(struct vtcon_softc *sc)
+{
+ struct virtqueue *vq;
+ int nbufs, error;
+
+ vq = sc->vtcon_ctrl_rxvq;
+ error = ENOSPC;
+
+ for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
+ error = vtcon_ctrl_event_create(sc);
+ if (error)
+ break;
+ }
+
+ if (nbufs > 0) {
+ virtqueue_notify(vq);
+ error = 0;
+ }
+
+ return (error);
+}
+
+static void
+vtcon_ctrl_event_drain(struct vtcon_softc *sc)
+{
+ struct virtio_console_control *control;
+ struct virtqueue *vq;
+ int last;
+
+ vq = sc->vtcon_ctrl_rxvq;
+ last = 0;
+
+ if (vq == NULL)
+ return;
+
+ VTCON_LOCK(sc);
+ while ((control = virtqueue_drain(vq, &last)) != NULL)
+ free(control, M_DEVBUF);
+ VTCON_UNLOCK(sc);
+}
+
+static int
+vtcon_ctrl_init(struct vtcon_softc *sc)
+{
+ int error;
+
+ error = vtcon_ctrl_event_populate(sc);
+
+ return (error);
+}
+
+static void
+vtcon_ctrl_deinit(struct vtcon_softc *sc)
+{
+
+ vtcon_ctrl_event_drain(sc);
+}
+
+static void
+vtcon_ctrl_port_add_event(struct vtcon_softc *sc, int id)
+{
+ device_t dev;
+ int error;
+
+ dev = sc->vtcon_dev;
+
+ /* This single thread only way for ports to be created. */
+ if (sc->vtcon_ports[id].vcsp_port != NULL) {
+ device_printf(dev, "%s: adding port %d, but already exists\n",
+ __func__, id);
+ return;
+ }
+
+ error = vtcon_port_create(sc, id);
+ if (error) {
+ device_printf(dev, "%s: cannot create port %d: %d\n",
+ __func__, id, error);
+ vtcon_ctrl_send_control(sc, id, VIRTIO_CONSOLE_PORT_READY, 0);
+ return;
+ }
+}
+
+static void
+vtcon_ctrl_port_remove_event(struct vtcon_softc *sc, int id)
+{
+ device_t dev;
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+
+ dev = sc->vtcon_dev;
+ scport = &sc->vtcon_ports[id];
+
+ VTCON_LOCK(sc);
+ port = scport->vcsp_port;
+ if (port == NULL) {
+ VTCON_UNLOCK(sc);
+ device_printf(dev, "%s: remove port %d, but does not exist\n",
+ __func__, id);
+ return;
+ }
+
+ scport->vcsp_port = NULL;
+ VTCON_PORT_LOCK(port);
+ VTCON_UNLOCK(sc);
+ vtcon_port_teardown(port);
+}
+
+static void
+vtcon_ctrl_port_console_event(struct vtcon_softc *sc, int id)
+{
+ device_t dev;
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+
+ dev = sc->vtcon_dev;
+ scport = &sc->vtcon_ports[id];
+
+ VTCON_LOCK(sc);
+ port = scport->vcsp_port;
+ if (port == NULL) {
+ VTCON_UNLOCK(sc);
+ device_printf(dev, "%s: console port %d, but does not exist\n",
+ __func__, id);
+ return;
+ }
+
+ VTCON_PORT_LOCK(port);
+ VTCON_UNLOCK(sc);
+ port->vtcport_flags |= VTCON_PORT_FLAG_CONSOLE;
+ vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
+ VTCON_PORT_UNLOCK(port);
+}
+
+static void
+vtcon_ctrl_port_open_event(struct vtcon_softc *sc, int id)
+{
+ device_t dev;
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+
+ dev = sc->vtcon_dev;
+ scport = &sc->vtcon_ports[id];
+
+ VTCON_LOCK(sc);
+ port = scport->vcsp_port;
+ if (port == NULL) {
+ VTCON_UNLOCK(sc);
+ device_printf(dev, "%s: open port %d, but does not exist\n",
+ __func__, id);
+ return;
+ }
+
+ VTCON_PORT_LOCK(port);
+ VTCON_UNLOCK(sc);
+ vtcon_port_enable_intr(port);
+ VTCON_PORT_UNLOCK(port);
+}
+
+static void
+vtcon_ctrl_process_event(struct vtcon_softc *sc,
+ struct virtio_console_control *control)
+{
+ device_t dev;
+ int id;
+
+ dev = sc->vtcon_dev;
+ id = control->id;
+
+ if (id < 0 || id >= sc->vtcon_max_ports) {
+ device_printf(dev, "%s: invalid port ID %d\n", __func__, id);
+ return;
+ }
+
+ switch (control->event) {
+ case VIRTIO_CONSOLE_PORT_ADD:
+ vtcon_ctrl_port_add_event(sc, id);
+ break;
+
+ case VIRTIO_CONSOLE_PORT_REMOVE:
+ vtcon_ctrl_port_remove_event(sc, id);
+ break;
+
+ case VIRTIO_CONSOLE_CONSOLE_PORT:
+ vtcon_ctrl_port_console_event(sc, id);
+ break;
+
+ case VIRTIO_CONSOLE_RESIZE:
+ break;
+
+ case VIRTIO_CONSOLE_PORT_OPEN:
+ vtcon_ctrl_port_open_event(sc, id);
+ break;
+
+ case VIRTIO_CONSOLE_PORT_NAME:
+ break;
+ }
+}
+
+static void
+vtcon_ctrl_task_cb(void *xsc, int pending)
+{
+ struct vtcon_softc *sc;
+ struct virtqueue *vq;
+ struct virtio_console_control *control;
+ int detached;
+
+ sc = xsc;
+ vq = sc->vtcon_ctrl_rxvq;
+
+ VTCON_LOCK(sc);
+
+ while ((detached = (sc->vtcon_flags & VTCON_FLAG_DETACHED)) == 0) {
+ control = virtqueue_dequeue(vq, NULL);
+ if (control == NULL)
+ break;
+
+ VTCON_UNLOCK(sc);
+ vtcon_ctrl_process_event(sc, control);
+ VTCON_LOCK(sc);
+ vtcon_ctrl_event_requeue(sc, control);
+ }
+
+ if (!detached) {
+ virtqueue_notify(vq);
+ if (virtqueue_enable_intr(vq) != 0)
+ taskqueue_enqueue(taskqueue_thread,
+ &sc->vtcon_ctrl_task);
+ }
+
+ VTCON_UNLOCK(sc);
+}
+
+static void
+vtcon_ctrl_event_intr(void *xsc)
+{
+ struct vtcon_softc *sc;
+
+ sc = xsc;
+
+ /*
+ * Only some events require us to potentially block, but it
+ * easier to just defer all event handling to the taskqueue.
+ */
+ taskqueue_enqueue(taskqueue_thread, &sc->vtcon_ctrl_task);
+}
+
+static void
+vtcon_ctrl_poll(struct vtcon_softc *sc,
+ struct virtio_console_control *control)
+{
+ struct sglist_seg segs[2];
+ struct sglist sg;
+ struct virtqueue *vq;
+ int error;
+
+ vq = sc->vtcon_ctrl_txvq;
+
+ sglist_init(&sg, 2, segs);
+ error = sglist_append(&sg, control,
+ sizeof(struct virtio_console_control));
+ KASSERT(error == 0, ("%s: error %d adding control to sglist",
+ __func__, error));
+
+ /*
+ * We cannot use the softc lock to serialize access to this
+ * virtqueue since this is called from the tty layer with the
+ * port lock held. Acquiring the softc would violate our lock
+ * ordering.
+ */
+ VTCON_CTRL_TX_LOCK(sc);
+ KASSERT(virtqueue_empty(vq),
+ ("%s: virtqueue is not emtpy", __func__));
+ error = virtqueue_enqueue(vq, control, &sg, sg.sg_nseg, 0);
+ if (error == 0) {
+ virtqueue_notify(vq);
+ virtqueue_poll(vq, NULL);
+ }
+ VTCON_CTRL_TX_UNLOCK(sc);
+}
+
+static void
+vtcon_ctrl_send_control(struct vtcon_softc *sc, uint32_t portid,
+ uint16_t event, uint16_t value)
+{
+ struct virtio_console_control control;
+
+ if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)
+ return;
+
+ control.id = portid;
+ control.event = event;
+ control.value = value;
+
+ vtcon_ctrl_poll(sc, &control);
+}
+
+static int
+vtcon_port_enqueue_buf(struct vtcon_port *port, void *buf, size_t len)
+{
+ struct sglist_seg segs[2];
+ struct sglist sg;
+ struct virtqueue *vq;
+ int error;
+
+ vq = port->vtcport_invq;
+
+ sglist_init(&sg, 2, segs);
+ error = sglist_append(&sg, buf, len);
+ KASSERT(error == 0,
+ ("%s: error %d adding buffer to sglist", __func__, error));
+
+ error = virtqueue_enqueue(vq, buf, &sg, 0, sg.sg_nseg);
+
+ return (error);
+}
+
+static int
+vtcon_port_create_buf(struct vtcon_port *port)
+{
+ void *buf;
+ int error;
+
+ buf = malloc(VTCON_BULK_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
+ if (buf == NULL)
+ return (ENOMEM);
+
+ error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
+ if (error)
+ free(buf, M_DEVBUF);
+
+ return (error);
+}
+
+static void
+vtcon_port_requeue_buf(struct vtcon_port *port, void *buf)
+{
+ int error;
+
+ error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
+ KASSERT(error == 0,
+ ("%s: cannot requeue input buffer %d", __func__, error));
+}
+
+static int
+vtcon_port_populate(struct vtcon_port *port)
+{
+ struct virtqueue *vq;
+ int nbufs, error;
+
+ vq = port->vtcport_invq;
+ error = ENOSPC;
+
+ for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
+ error = vtcon_port_create_buf(port);
+ if (error)
+ break;
+ }
+
+ if (nbufs > 0) {
+ virtqueue_notify(vq);
+ error = 0;
+ }
+
+ return (error);
+}
+
+static void
+vtcon_port_destroy(struct vtcon_port *port)
+{
+
+ port->vtcport_sc = NULL;
+ port->vtcport_scport = NULL;
+ port->vtcport_invq = NULL;
+ port->vtcport_outvq = NULL;
+ port->vtcport_id = -1;
+ mtx_destroy(&port->vtcport_mtx);
+ free(port, M_DEVBUF);
+}
+
+static int
+vtcon_port_init_vqs(struct vtcon_port *port)
+{
+ struct vtcon_softc_port *scport;
+ int error;
+
+ scport = port->vtcport_scport;
+
+ port->vtcport_invq = scport->vcsp_invq;
+ port->vtcport_outvq = scport->vcsp_outvq;
+
+ /*
+ * Free any data left over from when this virtqueue was in use by a
+ * prior port. We have not yet notified the host that the port is
+ * ready, so assume nothing in the virtqueue can be for us.
+ */
+ vtcon_port_drain(port);
+
+ KASSERT(virtqueue_empty(port->vtcport_invq),
+ ("%s: in virtqueue is not empty", __func__));
+ KASSERT(virtqueue_empty(port->vtcport_outvq),
+ ("%s: out virtqueue is not empty", __func__));
+
+ error = vtcon_port_populate(port);
+ if (error)
+ return (error);
+
+ return (0);
+}
+
+static int
+vtcon_port_create(struct vtcon_softc *sc, int id)
+{
+ device_t dev;
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+ int error;
+
+ dev = sc->vtcon_dev;
+ scport = &sc->vtcon_ports[id];
+
+ VTCON_ASSERT_VALID_PORTID(sc, id);
+ MPASS(scport->vcsp_port == NULL);
+
+ port = malloc(sizeof(struct vtcon_port), M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (port == NULL)
+ return (ENOMEM);
+
+ port->vtcport_sc = sc;
+ port->vtcport_scport = scport;
+ port->vtcport_id = id;
+ mtx_init(&port->vtcport_mtx, "vtcpmtx", NULL, MTX_DEF);
+ port->vtcport_tty = tty_alloc_mutex(&vtcon_tty_class, port,
+ &port->vtcport_mtx);
+
+ error = vtcon_port_init_vqs(port);
+ if (error) {
+ VTCON_PORT_LOCK(port);
+ vtcon_port_teardown(port);
+ return (error);
+ }
+
+ VTCON_LOCK(sc);
+ VTCON_PORT_LOCK(port);
+ scport->vcsp_port = port;
+ vtcon_port_enable_intr(port);
+ vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_READY, 1);
+ VTCON_PORT_UNLOCK(port);
+ VTCON_UNLOCK(sc);
+
+ tty_makedev(port->vtcport_tty, NULL, "%s%r.%r", VTCON_TTY_PREFIX,
+ device_get_unit(dev), id);
+
+ return (0);
+}
+
+static void
+vtcon_port_drain_bufs(struct virtqueue *vq)
+{
+ void *buf;
+ int last;
+
+ last = 0;
+
+ while ((buf = virtqueue_drain(vq, &last)) != NULL)
+ free(buf, M_DEVBUF);
+}
+
+static void
+vtcon_port_drain(struct vtcon_port *port)
+{
+
+ vtcon_port_drain_bufs(port->vtcport_invq);
+}
+
+static void
+vtcon_port_teardown(struct vtcon_port *port)
+{
+ struct tty *tp;
+
+ tp = port->vtcport_tty;
+
+ port->vtcport_flags |= VTCON_PORT_FLAG_GONE;
+
+ if (tp != NULL) {
+ atomic_add_int(&vtcon_pending_free, 1);
+ tty_rel_gone(tp);
+ } else
+ vtcon_port_destroy(port);
+}
+
+static void
+vtcon_port_change_size(struct vtcon_port *port, uint16_t cols, uint16_t rows)
+{
+ struct tty *tp;
+ struct winsize sz;
+
+ tp = port->vtcport_tty;
+
+ if (tp == NULL)
+ return;
+
+ bzero(&sz, sizeof(struct winsize));
+ sz.ws_col = cols;
+ sz.ws_row = rows;
+
+ tty_set_winsize(tp, &sz);
+}
+
+static void
+vtcon_port_update_console_size(struct vtcon_softc *sc)
+{
+ struct vtcon_port *port;
+ struct vtcon_softc_port *scport;
+ uint16_t cols, rows;
+
+ vtcon_get_console_size(sc, &cols, &rows);
+
+ /*
+ * For now, assume the first (only) port is the console. Note
+ * QEMU does not implement this feature yet.
+ */
+ scport = &sc->vtcon_ports[0];
+
+ VTCON_LOCK(sc);
+ port = scport->vcsp_port;
+
+ if (port != NULL) {
+ VTCON_PORT_LOCK(port);
+ VTCON_UNLOCK(sc);
+ vtcon_port_change_size(port, cols, rows);
+ VTCON_PORT_UNLOCK(port);
+ } else
+ VTCON_UNLOCK(sc);
+}
+
+static void
+vtcon_port_enable_intr(struct vtcon_port *port)
+{
+
+ /*
+ * NOTE: The out virtqueue is always polled, so its interupt
+ * kept disabled.
+ */
+ virtqueue_enable_intr(port->vtcport_invq);
+}
+
+static void
+vtcon_port_disable_intr(struct vtcon_port *port)
+{
+
+ if (port->vtcport_invq != NULL)
+ virtqueue_disable_intr(port->vtcport_invq);
+ if (port->vtcport_outvq != NULL)
+ virtqueue_disable_intr(port->vtcport_outvq);
+}
+
+static void
+vtcon_port_in(struct vtcon_port *port)
+{
+ struct virtqueue *vq;
+ struct tty *tp;
+ char *buf;
+ uint32_t len;
+ int i, deq;
+
+ tp = port->vtcport_tty;
+ vq = port->vtcport_invq;
+
+again:
+ deq = 0;
+
+ while ((buf = virtqueue_dequeue(vq, &len)) != NULL) {
+ for (i = 0; i < len; i++) {
+#if defined(KDB)
+ if (port->vtcport_flags & VTCON_PORT_FLAG_CONSOLE)
+ kdb_alt_break(buf[i],
+ &port->vtcport_alt_break_state);
+#endif
+ ttydisc_rint(tp, buf[i], 0);
+ }
+ vtcon_port_requeue_buf(port, buf);
+ deq++;
+ }
+ ttydisc_rint_done(tp);
+
+ if (deq > 0)
+ virtqueue_notify(vq);
+
+ if (virtqueue_enable_intr(vq) != 0)
+ goto again;
+}
+
+static void
+vtcon_port_intr(void *scportx)
+{
+ struct vtcon_softc_port *scport;
+ struct vtcon_softc *sc;
+ struct vtcon_port *port;
+
+ scport = scportx;
+ sc = scport->vcsp_sc;
+
+ VTCON_LOCK(sc);
+ port = scport->vcsp_port;
+ if (port == NULL) {
+ VTCON_UNLOCK(sc);
+ return;
+ }
+ VTCON_PORT_LOCK(port);
+ VTCON_UNLOCK(sc);
+ if ((port->vtcport_flags & VTCON_PORT_FLAG_GONE) == 0)
+ vtcon_port_in(port);
+ VTCON_PORT_UNLOCK(port);
+}
+
+static void
+vtcon_port_out(struct vtcon_port *port, void *buf, int bufsize)
+{
+ struct sglist_seg segs[2];
+ struct sglist sg;
+ struct virtqueue *vq;
+ int error;
+
+ vq = port->vtcport_outvq;
+ KASSERT(virtqueue_empty(vq),
+ ("%s: port %p out virtqueue not emtpy", __func__, port));
+
+ sglist_init(&sg, 2, segs);
+ error = sglist_append(&sg, buf, bufsize);
+ KASSERT(error == 0, ("%s: error %d adding buffer to sglist",
+ __func__, error));
+
+ error = virtqueue_enqueue(vq, buf, &sg, sg.sg_nseg, 0);
+ if (error == 0) {
+ virtqueue_notify(vq);
+ virtqueue_poll(vq, NULL);
+ }
+}
+
+static void
+vtcon_port_submit_event(struct vtcon_port *port, uint16_t event,
+ uint16_t value)
+{
+ struct vtcon_softc *sc;
+
+ sc = port->vtcport_sc;
+
+ vtcon_ctrl_send_control(sc, port->vtcport_id, event, value);
+}
+
+static int
+vtcon_tty_open(struct tty *tp)
+{
+ struct vtcon_port *port;
+
+ port = tty_softc(tp);
+
+ if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
+ return (ENXIO);
+
+ vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
+
+ return (0);
+}
+
+static void
+vtcon_tty_close(struct tty *tp)
+{
+ struct vtcon_port *port;
+
+ port = tty_softc(tp);
+
+ if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
+ return;
+
+ vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
+}
+
+static void
+vtcon_tty_outwakeup(struct tty *tp)
+{
+ struct vtcon_port *port;
+ char buf[VTCON_BULK_BUFSZ];
+ int len;
+
+ port = tty_softc(tp);
+
+ if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
+ return;
+
+ while ((len = ttydisc_getc(tp, buf, sizeof(buf))) != 0)
+ vtcon_port_out(port, buf, len);
+}
+
+static void
+vtcon_tty_free(void *xport)
+{
+ struct vtcon_port *port;
+
+ port = xport;
+
+ vtcon_port_destroy(port);
+ atomic_subtract_int(&vtcon_pending_free, 1);
+}
+
+static void
+vtcon_get_console_size(struct vtcon_softc *sc, uint16_t *cols, uint16_t *rows)
+{
+ struct virtio_console_config concfg;
+
+ KASSERT(sc->vtcon_flags & VTCON_FLAG_SIZE,
+ ("%s: size feature not negotiated", __func__));
+
+ vtcon_read_config(sc, &concfg);
+
+ *cols = concfg.cols;
+ *rows = concfg.rows;
+}
+
+static void
+vtcon_enable_interrupts(struct vtcon_softc *sc)
+{
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+ int i;
+
+ VTCON_LOCK(sc);
+
+ if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
+ virtqueue_enable_intr(sc->vtcon_ctrl_rxvq);
+
+ for (i = 0; i < sc->vtcon_max_ports; i++) {
+ scport = &sc->vtcon_ports[i];
+
+ port = scport->vcsp_port;
+ if (port == NULL)
+ continue;
+
+ VTCON_PORT_LOCK(port);
+ vtcon_port_enable_intr(port);
+ VTCON_PORT_UNLOCK(port);
+ }
+
+ VTCON_UNLOCK(sc);
+}
+
+static void
+vtcon_disable_interrupts(struct vtcon_softc *sc)
+{
+ struct vtcon_softc_port *scport;
+ struct vtcon_port *port;
+ int i;
+
+ VTCON_LOCK_ASSERT(sc);
+
+ if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
+ virtqueue_disable_intr(sc->vtcon_ctrl_rxvq);
+
+ for (i = 0; i < sc->vtcon_max_ports; i++) {
+ scport = &sc->vtcon_ports[i];
+
+ port = scport->vcsp_port;
+ if (port == NULL)
+ continue;
+
+ VTCON_PORT_LOCK(port);
+ vtcon_port_disable_intr(port);
+ VTCON_PORT_UNLOCK(port);
+ }
+}
Property changes on: trunk/sys/dev/virtio/console/virtio_console.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Added: trunk/sys/dev/virtio/console/virtio_console.h
===================================================================
--- trunk/sys/dev/virtio/console/virtio_console.h (rev 0)
+++ trunk/sys/dev/virtio/console/virtio_console.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -0,0 +1,76 @@
+/* $MidnightBSD$ */
+/*-
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Copyright (C) Red Hat, Inc., 2009, 2010, 2011
+ * Copyright (C) Amit Shah <amit.shah at redhat.com>, 2009, 2010, 2011
+ *
+ * $FreeBSD: stable/10/sys/dev/virtio/console/virtio_console.h 273515 2014-10-23 04:47:32Z bryanv $
+ */
+
+#ifndef _VIRTIO_CONSOLE_H
+#define _VIRTIO_CONSOLE_H
+
+/* Feature bits */
+#define VIRTIO_CONSOLE_F_SIZE 0x01 /* Console size */
+#define VIRTIO_CONSOLE_F_MULTIPORT 0x02 /* Multiple ports */
+#define VIRTIO_CONSOLE_F_EMERG_WRITE 0x04 /* Emergency write */
+
+#define VIRTIO_CONSOLE_BAD_ID (~(uint32_t)0)
+
+struct virtio_console_config {
+ /* colums of the screens */
+ uint16_t cols;
+ /* rows of the screens */
+ uint16_t rows;
+ /* max. number of ports this device can hold */
+ uint32_t max_nr_ports;
+ /* emergency write register */
+ uint32_t emerg_wr;
+} __packed;
+
+/*
+ * A message that's passed between the Host and the Guest for a
+ * particular port.
+ */
+struct virtio_console_control {
+ uint32_t id; /* Port number */
+ uint16_t event; /* The kind of control event (see below) */
+ uint16_t value; /* Extra information for the key */
+};
+
+/* Some events for control messages */
+#define VIRTIO_CONSOLE_DEVICE_READY 0
+#define VIRTIO_CONSOLE_PORT_ADD 1
+#define VIRTIO_CONSOLE_PORT_REMOVE 2
+#define VIRTIO_CONSOLE_PORT_READY 3
+#define VIRTIO_CONSOLE_CONSOLE_PORT 4
+#define VIRTIO_CONSOLE_RESIZE 5
+#define VIRTIO_CONSOLE_PORT_OPEN 6
+#define VIRTIO_CONSOLE_PORT_NAME 7
+
+#endif /* _VIRTIO_CONSOLE_H */
Property changes on: trunk/sys/dev/virtio/console/virtio_console.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Modified: trunk/sys/dev/virtio/network/if_vtnet.c
===================================================================
--- trunk/sys/dev/virtio/network/if_vtnet.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/network/if_vtnet.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -27,12 +28,8 @@
/* Driver for VirtIO network devices. */
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/network/if_vtnet.c 304081 2016-08-14 15:27:59Z smh $");
-#ifdef HAVE_KERNEL_OPTION_HEADERS
-#include "opt_device_polling.h"
-#endif
-
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
@@ -46,6 +43,9 @@
#include <sys/sglist.h>
#include <sys/lock.h>
#include <sys/mutex.h>
+#include <sys/taskqueue.h>
+#include <sys/smp.h>
+#include <machine/smp.h>
#include <vm/uma.h>
@@ -63,6 +63,7 @@
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
+#include <netinet6/ip6_var.h>
#include <netinet/udp.h>
#include <netinet/tcp.h>
#include <netinet/sctp.h>
@@ -79,6 +80,9 @@
#include "virtio_if.h"
+#include "opt_inet.h"
+#include "opt_inet6.h"
+
static int vtnet_modevent(module_t, int, void *);
static int vtnet_probe(device_t);
@@ -87,101 +91,173 @@
static int vtnet_suspend(device_t);
static int vtnet_resume(device_t);
static int vtnet_shutdown(device_t);
+static int vtnet_attach_completed(device_t);
static int vtnet_config_change(device_t);
static void vtnet_negotiate_features(struct vtnet_softc *);
+static void vtnet_setup_features(struct vtnet_softc *);
+static int vtnet_init_rxq(struct vtnet_softc *, int);
+static int vtnet_init_txq(struct vtnet_softc *, int);
+static int vtnet_alloc_rxtx_queues(struct vtnet_softc *);
+static void vtnet_free_rxtx_queues(struct vtnet_softc *);
+static int vtnet_alloc_rx_filters(struct vtnet_softc *);
+static void vtnet_free_rx_filters(struct vtnet_softc *);
static int vtnet_alloc_virtqueues(struct vtnet_softc *);
-static void vtnet_get_hwaddr(struct vtnet_softc *);
-static void vtnet_set_hwaddr(struct vtnet_softc *);
-static int vtnet_is_link_up(struct vtnet_softc *);
-static void vtnet_update_link_status(struct vtnet_softc *);
-static void vtnet_watchdog(struct vtnet_softc *);
+static int vtnet_setup_interface(struct vtnet_softc *);
static int vtnet_change_mtu(struct vtnet_softc *, int);
static int vtnet_ioctl(struct ifnet *, u_long, caddr_t);
-static int vtnet_init_rx_vq(struct vtnet_softc *);
-static void vtnet_free_rx_mbufs(struct vtnet_softc *);
-static void vtnet_free_tx_mbufs(struct vtnet_softc *);
-static void vtnet_free_ctrl_vq(struct vtnet_softc *);
-
-#ifdef DEVICE_POLLING
-static poll_handler_t vtnet_poll;
-#endif
-
-static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int,
- struct mbuf **);
-static int vtnet_replace_rxbuf(struct vtnet_softc *,
+static int vtnet_rxq_populate(struct vtnet_rxq *);
+static void vtnet_rxq_free_mbufs(struct vtnet_rxq *);
+static struct mbuf *
+ vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **);
+static int vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *,
struct mbuf *, int);
-static int vtnet_newbuf(struct vtnet_softc *);
-static void vtnet_discard_merged_rxbuf(struct vtnet_softc *, int);
-static void vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *);
-static int vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *);
-static void vtnet_vlan_tag_remove(struct mbuf *);
-static int vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
+static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int);
+static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *);
+static int vtnet_rxq_new_buf(struct vtnet_rxq *);
+static int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *,
+ struct virtio_net_hdr *);
+static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int);
+static void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *);
+static int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int);
+static void vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *,
struct virtio_net_hdr *);
-static int vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
-static int vtnet_rxeof(struct vtnet_softc *, int, int *);
+static int vtnet_rxq_eof(struct vtnet_rxq *);
static void vtnet_rx_vq_intr(void *);
+static void vtnet_rxq_tq_intr(void *, int);
-static void vtnet_txeof(struct vtnet_softc *);
-static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
+static int vtnet_txq_below_threshold(struct vtnet_txq *);
+static int vtnet_txq_notify(struct vtnet_txq *);
+static void vtnet_txq_free_mbufs(struct vtnet_txq *);
+static int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *,
+ int *, int *, int *);
+static int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int,
+ int, struct virtio_net_hdr *);
+static struct mbuf *
+ vtnet_txq_offload(struct vtnet_txq *, struct mbuf *,
struct virtio_net_hdr *);
-static int vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **,
+static int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **,
struct vtnet_tx_header *);
-static int vtnet_encap(struct vtnet_softc *, struct mbuf **);
-static void vtnet_start_locked(struct ifnet *);
+static int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **);
+#ifdef VTNET_LEGACY_TX
+static void vtnet_start_locked(struct vtnet_txq *, struct ifnet *);
static void vtnet_start(struct ifnet *);
-static void vtnet_tick(void *);
+#else
+static int vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *);
+static int vtnet_txq_mq_start(struct ifnet *, struct mbuf *);
+static void vtnet_txq_tq_deferred(void *, int);
+#endif
+static void vtnet_txq_start(struct vtnet_txq *);
+static void vtnet_txq_tq_intr(void *, int);
+static int vtnet_txq_eof(struct vtnet_txq *);
static void vtnet_tx_vq_intr(void *);
+static void vtnet_tx_start_all(struct vtnet_softc *);
+#ifndef VTNET_LEGACY_TX
+static void vtnet_qflush(struct ifnet *);
+#endif
+
+static int vtnet_watchdog(struct vtnet_txq *);
+static void vtnet_rxq_accum_stats(struct vtnet_rxq *,
+ struct vtnet_rxq_stats *);
+static void vtnet_txq_accum_stats(struct vtnet_txq *,
+ struct vtnet_txq_stats *);
+static void vtnet_accumulate_stats(struct vtnet_softc *);
+static void vtnet_tick(void *);
+
+static void vtnet_start_taskqueues(struct vtnet_softc *);
+static void vtnet_free_taskqueues(struct vtnet_softc *);
+static void vtnet_drain_taskqueues(struct vtnet_softc *);
+
+static void vtnet_drain_rxtx_queues(struct vtnet_softc *);
+static void vtnet_stop_rendezvous(struct vtnet_softc *);
static void vtnet_stop(struct vtnet_softc *);
+static int vtnet_virtio_reinit(struct vtnet_softc *);
+static void vtnet_init_rx_filters(struct vtnet_softc *);
+static int vtnet_init_rx_queues(struct vtnet_softc *);
+static int vtnet_init_tx_queues(struct vtnet_softc *);
+static int vtnet_init_rxtx_queues(struct vtnet_softc *);
+static void vtnet_set_active_vq_pairs(struct vtnet_softc *);
static int vtnet_reinit(struct vtnet_softc *);
static void vtnet_init_locked(struct vtnet_softc *);
static void vtnet_init(void *);
+static void vtnet_free_ctrl_vq(struct vtnet_softc *);
static void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
struct sglist *, int, int);
-
-static void vtnet_rx_filter(struct vtnet_softc *sc);
+static int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
+static int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t);
static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
static int vtnet_set_promisc(struct vtnet_softc *, int);
static int vtnet_set_allmulti(struct vtnet_softc *, int);
+static void vtnet_attach_disable_promisc(struct vtnet_softc *);
+static void vtnet_rx_filter(struct vtnet_softc *);
static void vtnet_rx_filter_mac(struct vtnet_softc *);
-
static int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);
static void vtnet_rx_filter_vlan(struct vtnet_softc *);
-static void vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t);
+static void vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t);
static void vtnet_register_vlan(void *, struct ifnet *, uint16_t);
static void vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);
+static int vtnet_is_link_up(struct vtnet_softc *);
+static void vtnet_update_link_status(struct vtnet_softc *);
static int vtnet_ifmedia_upd(struct ifnet *);
static void vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
+static void vtnet_get_hwaddr(struct vtnet_softc *);
+static void vtnet_set_hwaddr(struct vtnet_softc *);
+static void vtnet_vlan_tag_remove(struct mbuf *);
+static void vtnet_set_rx_process_limit(struct vtnet_softc *);
+static void vtnet_set_tx_intr_threshold(struct vtnet_softc *);
-static void vtnet_add_statistics(struct vtnet_softc *);
+static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *,
+ struct sysctl_oid_list *, struct vtnet_rxq *);
+static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *,
+ struct sysctl_oid_list *, struct vtnet_txq *);
+static void vtnet_setup_queue_sysctl(struct vtnet_softc *);
+static void vtnet_setup_sysctl(struct vtnet_softc *);
-static int vtnet_enable_rx_intr(struct vtnet_softc *);
-static int vtnet_enable_tx_intr(struct vtnet_softc *);
-static void vtnet_disable_rx_intr(struct vtnet_softc *);
-static void vtnet_disable_tx_intr(struct vtnet_softc *);
+static int vtnet_rxq_enable_intr(struct vtnet_rxq *);
+static void vtnet_rxq_disable_intr(struct vtnet_rxq *);
+static int vtnet_txq_enable_intr(struct vtnet_txq *);
+static void vtnet_txq_disable_intr(struct vtnet_txq *);
+static void vtnet_enable_rx_interrupts(struct vtnet_softc *);
+static void vtnet_enable_tx_interrupts(struct vtnet_softc *);
+static void vtnet_enable_interrupts(struct vtnet_softc *);
+static void vtnet_disable_rx_interrupts(struct vtnet_softc *);
+static void vtnet_disable_tx_interrupts(struct vtnet_softc *);
+static void vtnet_disable_interrupts(struct vtnet_softc *);
+static int vtnet_tunable_int(struct vtnet_softc *, const char *, int);
+
/* Tunables. */
+static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD, 0, "VNET driver parameters");
static int vtnet_csum_disable = 0;
TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
+SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN,
+ &vtnet_csum_disable, 0, "Disables receive and send checksum offload");
static int vtnet_tso_disable = 0;
TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
+SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN, &vtnet_tso_disable,
+ 0, "Disables TCP Segmentation Offload");
static int vtnet_lro_disable = 0;
TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
+SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN, &vtnet_lro_disable,
+ 0, "Disables TCP Large Receive Offload");
+static int vtnet_mq_disable = 0;
+TUNABLE_INT("hw.vtnet.mq_disable", &vtnet_mq_disable);
+SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN, &vtnet_mq_disable,
+ 0, "Disables Multi Queue support");
+static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS;
+TUNABLE_INT("hw.vtnet.mq_max_pairs", &vtnet_mq_max_pairs);
+SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN,
+ &vtnet_mq_max_pairs, 0, "Sets the maximum number of Multi Queue pairs");
+static int vtnet_rx_process_limit = 512;
+TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit);
+SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
+ &vtnet_rx_process_limit, 0,
+ "Limits the number RX segments processed in a single pass");
-/*
- * Reducing the number of transmit completed interrupts can
- * improve performance. To do so, the define below keeps the
- * Tx vq interrupt disabled and adds calls to vtnet_txeof()
- * in the start and watchdog paths. The price to pay for this
- * is the m_free'ing of transmitted mbufs may be delayed until
- * the watchdog fires.
- */
-#define VTNET_TX_INTR_MODERATION
-
static uma_zone_t vtnet_tx_header_zone;
static struct virtio_feature_desc vtnet_feature_desc[] = {
@@ -203,6 +279,9 @@
{ VIRTIO_NET_F_CTRL_RX, "RxMode" },
{ VIRTIO_NET_F_CTRL_VLAN, "VLanFilter" },
{ VIRTIO_NET_F_CTRL_RX_EXTRA, "RxModeExtra" },
+ { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" },
+ { VIRTIO_NET_F_MQ, "Multiqueue" },
+ { VIRTIO_NET_F_CTRL_MAC_ADDR, "SetMacAddress" },
{ 0, NULL }
};
@@ -209,19 +288,24 @@
static device_method_t vtnet_methods[] = {
/* Device methods. */
- DEVMETHOD(device_probe, vtnet_probe),
- DEVMETHOD(device_attach, vtnet_attach),
- DEVMETHOD(device_detach, vtnet_detach),
- DEVMETHOD(device_suspend, vtnet_suspend),
- DEVMETHOD(device_resume, vtnet_resume),
- DEVMETHOD(device_shutdown, vtnet_shutdown),
+ DEVMETHOD(device_probe, vtnet_probe),
+ DEVMETHOD(device_attach, vtnet_attach),
+ DEVMETHOD(device_detach, vtnet_detach),
+ DEVMETHOD(device_suspend, vtnet_suspend),
+ DEVMETHOD(device_resume, vtnet_resume),
+ DEVMETHOD(device_shutdown, vtnet_shutdown),
/* VirtIO methods. */
- DEVMETHOD(virtio_config_change, vtnet_config_change),
+ DEVMETHOD(virtio_attach_completed, vtnet_attach_completed),
+ DEVMETHOD(virtio_config_change, vtnet_config_change),
DEVMETHOD_END
};
+#ifdef DEV_NETMAP
+#include <dev/netmap/if_vtnet_netmap.h>
+#endif /* DEV_NETMAP */
+
static driver_t vtnet_driver = {
"vtnet",
vtnet_methods,
@@ -282,57 +366,32 @@
vtnet_attach(device_t dev)
{
struct vtnet_softc *sc;
- struct ifnet *ifp;
- int tx_size, error;
+ int error;
sc = device_get_softc(dev);
sc->vtnet_dev = dev;
- VTNET_LOCK_INIT(sc);
- callout_init_mtx(&sc->vtnet_tick_ch, VTNET_MTX(sc), 0);
-
- ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
- vtnet_ifmedia_sts);
- ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
- ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
-
- vtnet_add_statistics(sc);
-
+ /* Register our feature descriptions. */
virtio_set_feature_desc(dev, vtnet_feature_desc);
- vtnet_negotiate_features(sc);
- if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
- sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
- sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
- } else
- sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
+ VTNET_CORE_LOCK_INIT(sc);
+ callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0);
- sc->vtnet_rx_mbuf_size = MCLBYTES;
- sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
+ vtnet_setup_sysctl(sc);
+ vtnet_setup_features(sc);
- if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
- sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
+ error = vtnet_alloc_rx_filters(sc);
+ if (error) {
+ device_printf(dev, "cannot allocate Rx filters\n");
+ goto fail;
+ }
- if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) {
- sc->vtnet_mac_filter = malloc(
- sizeof(struct vtnet_mac_filter), M_DEVBUF,
- M_NOWAIT | M_ZERO);
- if (sc->vtnet_mac_filter == NULL) {
- device_printf(dev,
- "cannot allocate mac filter table\n");
- error = ENOMEM;
- goto fail;
- }
-
- sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
- }
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
- sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
+ error = vtnet_alloc_rxtx_queues(sc);
+ if (error) {
+ device_printf(dev, "cannot allocate queues\n");
+ goto fail;
}
- vtnet_get_hwaddr(sc);
-
error = vtnet_alloc_virtqueues(sc);
if (error) {
device_printf(dev, "cannot allocate virtqueues\n");
@@ -339,112 +398,26 @@
goto fail;
}
- ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
- if (ifp == NULL) {
- device_printf(dev, "cannot allocate ifnet structure\n");
- error = ENOSPC;
+ error = vtnet_setup_interface(sc);
+ if (error) {
+ device_printf(dev, "cannot setup interface\n");
goto fail;
}
- ifp->if_softc = sc;
- if_initname(ifp, device_get_name(dev), device_get_unit(dev));
- ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
- ifp->if_init = vtnet_init;
- ifp->if_start = vtnet_start;
- ifp->if_ioctl = vtnet_ioctl;
-
- sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq);
- sc->vtnet_rx_process_limit = sc->vtnet_rx_size;
-
- tx_size = virtqueue_size(sc->vtnet_tx_vq);
- sc->vtnet_tx_size = tx_size;
- IFQ_SET_MAXLEN(&ifp->if_snd, tx_size - 1);
- ifp->if_snd.ifq_drv_maxlen = tx_size - 1;
- IFQ_SET_READY(&ifp->if_snd);
-
- ether_ifattach(ifp, sc->vtnet_hwaddr);
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
- ifp->if_capabilities |= IFCAP_LINKSTATE;
-
- /* Tell the upper layer(s) we support long frames. */
- ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
- ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
- ifp->if_capabilities |= IFCAP_TXCSUM;
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
- ifp->if_capabilities |= IFCAP_TSO4;
- if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
- ifp->if_capabilities |= IFCAP_TSO6;
- if (ifp->if_capabilities & IFCAP_TSO)
- ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
- sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
- }
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
- ifp->if_capabilities |= IFCAP_RXCSUM;
-
- if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
- virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
- ifp->if_capabilities |= IFCAP_LRO;
- }
-
- if (ifp->if_capabilities & IFCAP_HWCSUM) {
- /*
- * VirtIO does not support VLAN tagging, but we can fake
- * it by inserting and removing the 802.1Q header during
- * transmit and receive. We are then able to do checksum
- * offloading of VLAN frames.
- */
- ifp->if_capabilities |=
- IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
- }
-
- ifp->if_capenable = ifp->if_capabilities;
-
- /*
- * Capabilities after here are not enabled by default.
- */
-
- if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
- ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
-
- sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
- vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
- sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
- vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
- }
-
-#ifdef DEVICE_POLLING
- ifp->if_capabilities |= IFCAP_POLLING;
-#endif
-
error = virtio_setup_intr(dev, INTR_TYPE_NET);
if (error) {
device_printf(dev, "cannot setup virtqueue interrupts\n");
- ether_ifdetach(ifp);
+ /* BMV: This will crash if during boot! */
+ ether_ifdetach(sc->vtnet_ifp);
goto fail;
}
- /*
- * Device defaults to promiscuous mode for backwards
- * compatibility. Turn it off if possible.
- */
- if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
- VTNET_LOCK(sc);
- if (vtnet_set_promisc(sc, 0) != 0) {
- ifp->if_flags |= IFF_PROMISC;
- device_printf(dev,
- "cannot disable promiscuous mode\n");
- }
- VTNET_UNLOCK(sc);
- } else
- ifp->if_flags |= IFF_PROMISC;
+#ifdef DEV_NETMAP
+ vtnet_netmap_attach(sc);
+#endif /* DEV_NETMAP */
+ vtnet_start_taskqueues(sc);
+
fail:
if (error)
vtnet_detach(dev);
@@ -461,24 +434,23 @@
sc = device_get_softc(dev);
ifp = sc->vtnet_ifp;
- KASSERT(mtx_initialized(VTNET_MTX(sc)),
- ("vtnet mutex not initialized"));
-
-#ifdef DEVICE_POLLING
- if (ifp != NULL && ifp->if_capenable & IFCAP_POLLING)
- ether_poll_deregister(ifp);
-#endif
-
if (device_is_attached(dev)) {
- VTNET_LOCK(sc);
+ VTNET_CORE_LOCK(sc);
vtnet_stop(sc);
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
callout_drain(&sc->vtnet_tick_ch);
+ vtnet_drain_taskqueues(sc);
ether_ifdetach(ifp);
}
+#ifdef DEV_NETMAP
+ netmap_detach(ifp);
+#endif /* DEV_NETMAP */
+
+ vtnet_free_taskqueues(sc);
+
if (sc->vtnet_vlan_attach != NULL) {
EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
sc->vtnet_vlan_attach = NULL;
@@ -488,10 +460,7 @@
sc->vtnet_vlan_detach = NULL;
}
- if (sc->vtnet_mac_filter != NULL) {
- free(sc->vtnet_mac_filter, M_DEVBUF);
- sc->vtnet_mac_filter = NULL;
- }
+ ifmedia_removeall(&sc->vtnet_media);
if (ifp != NULL) {
if_free(ifp);
@@ -498,15 +467,13 @@
sc->vtnet_ifp = NULL;
}
- if (sc->vtnet_rx_vq != NULL)
- vtnet_free_rx_mbufs(sc);
- if (sc->vtnet_tx_vq != NULL)
- vtnet_free_tx_mbufs(sc);
+ vtnet_free_rxtx_queues(sc);
+ vtnet_free_rx_filters(sc);
+
if (sc->vtnet_ctrl_vq != NULL)
vtnet_free_ctrl_vq(sc);
- ifmedia_removeall(&sc->vtnet_media);
- VTNET_LOCK_DESTROY(sc);
+ VTNET_CORE_LOCK_DESTROY(sc);
return (0);
}
@@ -518,10 +485,10 @@
sc = device_get_softc(dev);
- VTNET_LOCK(sc);
+ VTNET_CORE_LOCK(sc);
vtnet_stop(sc);
sc->vtnet_flags |= VTNET_FLAG_SUSPENDED;
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
return (0);
}
@@ -535,11 +502,11 @@
sc = device_get_softc(dev);
ifp = sc->vtnet_ifp;
- VTNET_LOCK(sc);
+ VTNET_CORE_LOCK(sc);
if (ifp->if_flags & IFF_UP)
vtnet_init_locked(sc);
sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED;
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
return (0);
}
@@ -556,6 +523,15 @@
}
static int
+vtnet_attach_completed(device_t dev)
+{
+
+ vtnet_attach_disable_promisc(device_get_softc(dev));
+
+ return (0);
+}
+
+static int
vtnet_config_change(device_t dev)
{
struct vtnet_softc *sc;
@@ -562,9 +538,11 @@
sc = device_get_softc(dev);
- VTNET_LOCK(sc);
+ VTNET_CORE_LOCK(sc);
vtnet_update_link_status(sc);
- VTNET_UNLOCK(sc);
+ if (sc->vtnet_link_active != 0)
+ vtnet_tx_start_all(sc);
+ VTNET_CORE_UNLOCK(sc);
return (0);
}
@@ -578,188 +556,509 @@
dev = sc->vtnet_dev;
mask = 0;
- if (vtnet_csum_disable)
- mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
-
/*
- * TSO and LRO are only available when their corresponding
- * checksum offload feature is also negotiated.
+ * TSO and LRO are only available when their corresponding checksum
+ * offload feature is also negotiated.
*/
-
- if (vtnet_csum_disable || vtnet_tso_disable)
- mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 |
- VIRTIO_NET_F_HOST_ECN;
-
- if (vtnet_csum_disable || vtnet_lro_disable)
+ if (vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable)) {
+ mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
+ mask |= VTNET_TSO_FEATURES | VTNET_LRO_FEATURES;
+ }
+ if (vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable))
+ mask |= VTNET_TSO_FEATURES;
+ if (vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable))
mask |= VTNET_LRO_FEATURES;
+#ifndef VTNET_LEGACY_TX
+ if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable))
+ mask |= VIRTIO_NET_F_MQ;
+#else
+ mask |= VIRTIO_NET_F_MQ;
+#endif
features = VTNET_FEATURES & ~mask;
-#ifdef VTNET_TX_INTR_MODERATION
- features |= VIRTIO_F_NOTIFY_ON_EMPTY;
-#endif
sc->vtnet_features = virtio_negotiate_features(dev, features);
- if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0 &&
- virtio_with_feature(dev, VTNET_LRO_FEATURES)) {
+ if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
+ virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
/*
* LRO without mergeable buffers requires special care. This
* is not ideal because every receive buffer must be large
* enough to hold the maximum TCP packet, the Ethernet header,
- * and the vtnet_rx_header. This requires up to 34 descriptors
- * when using MCLBYTES clusters. If we do not have indirect
- * descriptors, LRO is disabled since the virtqueue will not
- * be able to contain very many receive buffers.
+ * and the header. This requires up to 34 descriptors with
+ * MCLBYTES clusters. If we do not have indirect descriptors,
+ * LRO is disabled since the virtqueue will not contain very
+ * many receive buffers.
*/
- if (virtio_with_feature(dev,
- VIRTIO_RING_F_INDIRECT_DESC) == 0) {
+ if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
device_printf(dev,
- "LRO disabled due to lack of both mergeable "
- "buffers and indirect descriptors\n");
+ "LRO disabled due to both mergeable buffers and "
+ "indirect descriptors not negotiated\n");
- sc->vtnet_features = virtio_negotiate_features(dev,
- features & ~VTNET_LRO_FEATURES);
+ features &= ~VTNET_LRO_FEATURES;
+ sc->vtnet_features =
+ virtio_negotiate_features(dev, features);
} else
sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
}
}
-static int
-vtnet_alloc_virtqueues(struct vtnet_softc *sc)
+static void
+vtnet_setup_features(struct vtnet_softc *sc)
{
device_t dev;
- struct vq_alloc_info vq_info[3];
- int nvqs, rxsegs;
dev = sc->vtnet_dev;
- nvqs = 2;
- /*
- * Indirect descriptors are not needed for the Rx
- * virtqueue when mergeable buffers are negotiated.
- * The header is placed inline with the data, not
- * in a separate descriptor, and mbuf clusters are
- * always physically contiguous.
- */
- if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
- rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ?
- VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS;
+ vtnet_negotiate_features(sc);
+
+ if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
+ sc->vtnet_flags |= VTNET_FLAG_INDIRECT;
+ if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX))
+ sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX;
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
+ /* This feature should always be negotiated. */
+ sc->vtnet_flags |= VTNET_FLAG_MAC;
+ }
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
+ sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
+ sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
} else
- rxsegs = 0;
+ sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
- VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs,
- vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq,
- "%s receive", device_get_nameunit(dev));
+ if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
+ sc->vtnet_rx_nsegs = VTNET_MRG_RX_SEGS;
+ else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
+ sc->vtnet_rx_nsegs = VTNET_MAX_RX_SEGS;
+ else
+ sc->vtnet_rx_nsegs = VTNET_MIN_RX_SEGS;
- VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS,
- vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq,
- "%s transmit", device_get_nameunit(dev));
+ if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) ||
+ virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) ||
+ virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
+ sc->vtnet_tx_nsegs = VTNET_MAX_TX_SEGS;
+ else
+ sc->vtnet_tx_nsegs = VTNET_MIN_TX_SEGS;
- if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
- nvqs++;
+ if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
+ sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;
- VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL,
- &sc->vtnet_ctrl_vq, "%s control",
- device_get_nameunit(dev));
+ if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX))
+ sc->vtnet_flags |= VTNET_FLAG_CTRL_RX;
+ if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN))
+ sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
+ if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR))
+ sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;
}
- return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
+ if (virtio_with_feature(dev, VIRTIO_NET_F_MQ) &&
+ sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
+ sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev,
+ offsetof(struct virtio_net_config, max_virtqueue_pairs));
+ } else
+ sc->vtnet_max_vq_pairs = 1;
+
+ if (sc->vtnet_max_vq_pairs > 1) {
+ /*
+ * Limit the maximum number of queue pairs to the lower of
+ * the number of CPUs and the configured maximum.
+ * The actual number of queues that get used may be less.
+ */
+ int max;
+
+ max = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs);
+ if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN) {
+ if (max > mp_ncpus)
+ max = mp_ncpus;
+ if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
+ max = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX;
+ if (max > 1) {
+ sc->vtnet_requested_vq_pairs = max;
+ sc->vtnet_flags |= VTNET_FLAG_MULTIQ;
+ }
+ }
+ }
}
+static int
+vtnet_init_rxq(struct vtnet_softc *sc, int id)
+{
+ struct vtnet_rxq *rxq;
+
+ rxq = &sc->vtnet_rxqs[id];
+
+ snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d",
+ device_get_nameunit(sc->vtnet_dev), id);
+ mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF);
+
+ rxq->vtnrx_sc = sc;
+ rxq->vtnrx_id = id;
+
+ rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT);
+ if (rxq->vtnrx_sg == NULL)
+ return (ENOMEM);
+
+ TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);
+ rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT,
+ taskqueue_thread_enqueue, &rxq->vtnrx_tq);
+
+ return (rxq->vtnrx_tq == NULL ? ENOMEM : 0);
+}
+
+static int
+vtnet_init_txq(struct vtnet_softc *sc, int id)
+{
+ struct vtnet_txq *txq;
+
+ txq = &sc->vtnet_txqs[id];
+
+ snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d",
+ device_get_nameunit(sc->vtnet_dev), id);
+ mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF);
+
+ txq->vtntx_sc = sc;
+ txq->vtntx_id = id;
+
+ txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT);
+ if (txq->vtntx_sg == NULL)
+ return (ENOMEM);
+
+#ifndef VTNET_LEGACY_TX
+ txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF,
+ M_NOWAIT, &txq->vtntx_mtx);
+ if (txq->vtntx_br == NULL)
+ return (ENOMEM);
+
+ TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq);
+#endif
+ TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq);
+ txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT,
+ taskqueue_thread_enqueue, &txq->vtntx_tq);
+ if (txq->vtntx_tq == NULL)
+ return (ENOMEM);
+
+ return (0);
+}
+
+static int
+vtnet_alloc_rxtx_queues(struct vtnet_softc *sc)
+{
+ int i, npairs, error;
+
+ npairs = sc->vtnet_max_vq_pairs;
+
+ sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF,
+ M_NOWAIT | M_ZERO);
+ sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF,
+ M_NOWAIT | M_ZERO);
+ if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL)
+ return (ENOMEM);
+
+ for (i = 0; i < npairs; i++) {
+ error = vtnet_init_rxq(sc, i);
+ if (error)
+ return (error);
+ error = vtnet_init_txq(sc, i);
+ if (error)
+ return (error);
+ }
+
+ vtnet_setup_queue_sysctl(sc);
+
+ return (0);
+}
+
static void
-vtnet_get_hwaddr(struct vtnet_softc *sc)
+vtnet_destroy_rxq(struct vtnet_rxq *rxq)
{
- device_t dev;
- dev = sc->vtnet_dev;
+ rxq->vtnrx_sc = NULL;
+ rxq->vtnrx_id = -1;
- if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) {
- virtio_read_device_config(dev,
- offsetof(struct virtio_net_config, mac),
- sc->vtnet_hwaddr, ETHER_ADDR_LEN);
- } else {
- /* Generate random locally administered unicast address. */
- sc->vtnet_hwaddr[0] = 0xB2;
- arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
+ if (rxq->vtnrx_sg != NULL) {
+ sglist_free(rxq->vtnrx_sg);
+ rxq->vtnrx_sg = NULL;
+ }
- vtnet_set_hwaddr(sc);
+ if (mtx_initialized(&rxq->vtnrx_mtx) != 0)
+ mtx_destroy(&rxq->vtnrx_mtx);
+}
+
+static void
+vtnet_destroy_txq(struct vtnet_txq *txq)
+{
+
+ txq->vtntx_sc = NULL;
+ txq->vtntx_id = -1;
+
+ if (txq->vtntx_sg != NULL) {
+ sglist_free(txq->vtntx_sg);
+ txq->vtntx_sg = NULL;
}
+
+#ifndef VTNET_LEGACY_TX
+ if (txq->vtntx_br != NULL) {
+ buf_ring_free(txq->vtntx_br, M_DEVBUF);
+ txq->vtntx_br = NULL;
+ }
+#endif
+
+ if (mtx_initialized(&txq->vtntx_mtx) != 0)
+ mtx_destroy(&txq->vtntx_mtx);
}
static void
-vtnet_set_hwaddr(struct vtnet_softc *sc)
+vtnet_free_rxtx_queues(struct vtnet_softc *sc)
{
- device_t dev;
+ int i;
- dev = sc->vtnet_dev;
+ if (sc->vtnet_rxqs != NULL) {
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
+ vtnet_destroy_rxq(&sc->vtnet_rxqs[i]);
+ free(sc->vtnet_rxqs, M_DEVBUF);
+ sc->vtnet_rxqs = NULL;
+ }
- virtio_write_device_config(dev,
- offsetof(struct virtio_net_config, mac),
- sc->vtnet_hwaddr, ETHER_ADDR_LEN);
+ if (sc->vtnet_txqs != NULL) {
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
+ vtnet_destroy_txq(&sc->vtnet_txqs[i]);
+ free(sc->vtnet_txqs, M_DEVBUF);
+ sc->vtnet_txqs = NULL;
+ }
}
static int
-vtnet_is_link_up(struct vtnet_softc *sc)
+vtnet_alloc_rx_filters(struct vtnet_softc *sc)
{
+
+ if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
+ sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (sc->vtnet_mac_filter == NULL)
+ return (ENOMEM);
+ }
+
+ if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
+ sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) *
+ VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (sc->vtnet_vlan_filter == NULL)
+ return (ENOMEM);
+ }
+
+ return (0);
+}
+
+static void
+vtnet_free_rx_filters(struct vtnet_softc *sc)
+{
+
+ if (sc->vtnet_mac_filter != NULL) {
+ free(sc->vtnet_mac_filter, M_DEVBUF);
+ sc->vtnet_mac_filter = NULL;
+ }
+
+ if (sc->vtnet_vlan_filter != NULL) {
+ free(sc->vtnet_vlan_filter, M_DEVBUF);
+ sc->vtnet_vlan_filter = NULL;
+ }
+}
+
+static int
+vtnet_alloc_virtqueues(struct vtnet_softc *sc)
+{
device_t dev;
- struct ifnet *ifp;
- uint16_t status;
+ struct vq_alloc_info *info;
+ struct vtnet_rxq *rxq;
+ struct vtnet_txq *txq;
+ int i, idx, flags, nvqs, error;
dev = sc->vtnet_dev;
- ifp = sc->vtnet_ifp;
+ flags = 0;
- VTNET_LOCK_ASSERT(sc);
+ nvqs = sc->vtnet_max_vq_pairs * 2;
+ if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
+ nvqs++;
- if ((ifp->if_capenable & IFCAP_LINKSTATE) == 0)
- return (1);
+ info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
+ if (info == NULL)
+ return (ENOMEM);
- status = virtio_read_dev_config_2(dev,
- offsetof(struct virtio_net_config, status));
+ for (i = 0, idx = 0; i < sc->vtnet_max_vq_pairs; i++, idx+=2) {
+ rxq = &sc->vtnet_rxqs[i];
+ VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs,
+ vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq,
+ "%s-%d rx", device_get_nameunit(dev), rxq->vtnrx_id);
- return ((status & VIRTIO_NET_S_LINK_UP) != 0);
+ txq = &sc->vtnet_txqs[i];
+ VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs,
+ vtnet_tx_vq_intr, txq, &txq->vtntx_vq,
+ "%s-%d tx", device_get_nameunit(dev), txq->vtntx_id);
+ }
+
+ if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
+ VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL,
+ &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev));
+ }
+
+ /*
+ * Enable interrupt binding if this is multiqueue. This only matters
+ * when per-vq MSIX is available.
+ */
+ if (sc->vtnet_flags & VTNET_FLAG_MULTIQ)
+ flags |= 0;
+
+ error = virtio_alloc_virtqueues(dev, flags, nvqs, info);
+ free(info, M_TEMP);
+
+ return (error);
}
-static void
-vtnet_update_link_status(struct vtnet_softc *sc)
+static int
+vtnet_setup_interface(struct vtnet_softc *sc)
{
+ device_t dev;
struct ifnet *ifp;
- int link;
- ifp = sc->vtnet_ifp;
+ dev = sc->vtnet_dev;
- link = vtnet_is_link_up(sc);
+ ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
+ if (ifp == NULL) {
+ device_printf(dev, "cannot allocate ifnet structure\n");
+ return (ENOSPC);
+ }
- if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) {
- sc->vtnet_flags |= VTNET_FLAG_LINK;
- if_link_state_change(ifp, LINK_STATE_UP);
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- vtnet_start_locked(ifp);
- } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) {
- sc->vtnet_flags &= ~VTNET_FLAG_LINK;
- if_link_state_change(ifp, LINK_STATE_DOWN);
+ if_initname(ifp, device_get_name(dev), device_get_unit(dev));
+ if_initbaudrate(ifp, IF_Gbps(10)); /* Approx. */
+ ifp->if_softc = sc;
+ ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+ ifp->if_init = vtnet_init;
+ ifp->if_ioctl = vtnet_ioctl;
+
+#ifndef VTNET_LEGACY_TX
+ ifp->if_transmit = vtnet_txq_mq_start;
+ ifp->if_qflush = vtnet_qflush;
+#else
+ struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq;
+ ifp->if_start = vtnet_start;
+ IFQ_SET_MAXLEN(&ifp->if_snd, virtqueue_size(vq) - 1);
+ ifp->if_snd.ifq_drv_maxlen = virtqueue_size(vq) - 1;
+ IFQ_SET_READY(&ifp->if_snd);
+#endif
+
+ ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
+ vtnet_ifmedia_sts);
+ ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
+ ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);
+
+ /* Read (or generate) the MAC address for the adapter. */
+ vtnet_get_hwaddr(sc);
+
+ ether_ifattach(ifp, sc->vtnet_hwaddr);
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
+ ifp->if_capabilities |= IFCAP_LINKSTATE;
+
+ /* Tell the upper layer(s) we support long frames. */
+ ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
+ ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
+ ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6;
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_GSO)) {
+ ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
+ sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
+ } else {
+ if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
+ ifp->if_capabilities |= IFCAP_TSO4;
+ if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
+ ifp->if_capabilities |= IFCAP_TSO6;
+ if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
+ sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
+ }
+
+ if (ifp->if_capabilities & IFCAP_TSO)
+ ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
}
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
+ ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6;
+
+ if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
+ virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
+ ifp->if_capabilities |= IFCAP_LRO;
+ }
+
+ if (ifp->if_capabilities & IFCAP_HWCSUM) {
+ /*
+ * VirtIO does not support VLAN tagging, but we can fake
+ * it by inserting and removing the 802.1Q header during
+ * transmit and receive. We are then able to do checksum
+ * offloading of VLAN frames.
+ */
+ ifp->if_capabilities |=
+ IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
+ }
+
+ ifp->if_capenable = ifp->if_capabilities;
+
+ /*
+ * Capabilities after here are not enabled by default.
+ */
+
+ if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
+ ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
+
+ sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
+ vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
+ sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
+ vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
+ }
+
+ vtnet_set_rx_process_limit(sc);
+ vtnet_set_tx_intr_threshold(sc);
+
+ return (0);
}
-static void
-vtnet_watchdog(struct vtnet_softc *sc)
+static int
+vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
{
struct ifnet *ifp;
+ int frame_size, clsize;
ifp = sc->vtnet_ifp;
-#ifdef VTNET_TX_INTR_MODERATION
- vtnet_txeof(sc);
-#endif
+ if (new_mtu < ETHERMIN || new_mtu > VTNET_MAX_MTU)
+ return (EINVAL);
- if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer)
- return;
+ frame_size = sc->vtnet_hdr_size + sizeof(struct ether_vlan_header) +
+ new_mtu;
- if_printf(ifp, "watchdog timeout -- resetting\n");
-#ifdef VTNET_DEBUG
- virtqueue_dump(sc->vtnet_tx_vq);
-#endif
- ifp->if_oerrors++;
- ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
- vtnet_init_locked(sc);
+ /*
+ * Based on the new MTU (and hence frame size) determine which
+ * cluster size is most appropriate for the receive queues.
+ */
+ if (frame_size <= MCLBYTES) {
+ clsize = MCLBYTES;
+ } else if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
+ /* Avoid going past 9K jumbos. */
+ if (frame_size > MJUM9BYTES)
+ return (EINVAL);
+ clsize = MJUM9BYTES;
+ } else
+ clsize = MJUMPAGESIZE;
+
+ ifp->if_mtu = new_mtu;
+ sc->vtnet_rx_new_clsize = clsize;
+
+ if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+ ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+ vtnet_init_locked(sc);
+ }
+
+ return (0);
}
static int
@@ -771,22 +1070,19 @@
sc = ifp->if_softc;
ifr = (struct ifreq *) data;
- reinit = 0;
error = 0;
switch (cmd) {
case SIOCSIFMTU:
- if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU)
- error = EINVAL;
- else if (ifp->if_mtu != ifr->ifr_mtu) {
- VTNET_LOCK(sc);
+ if (ifp->if_mtu != ifr->ifr_mtu) {
+ VTNET_CORE_LOCK(sc);
error = vtnet_change_mtu(sc, ifr->ifr_mtu);
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
}
break;
case SIOCSIFFLAGS:
- VTNET_LOCK(sc);
+ VTNET_CORE_LOCK(sc);
if ((ifp->if_flags & IFF_UP) == 0) {
if (ifp->if_drv_flags & IFF_DRV_RUNNING)
vtnet_stop(sc);
@@ -795,8 +1091,12 @@
(IFF_PROMISC | IFF_ALLMULTI)) {
if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
vtnet_rx_filter(sc);
- else
- error = ENOTSUP;
+ else {
+ ifp->if_flags |= IFF_PROMISC;
+ if ((ifp->if_flags ^ sc->vtnet_if_flags)
+ & IFF_ALLMULTI)
+ error = ENOTSUP;
+ }
}
} else
vtnet_init_locked(sc);
@@ -803,16 +1103,17 @@
if (error == 0)
sc->vtnet_if_flags = ifp->if_flags;
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
- VTNET_LOCK(sc);
- if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) &&
- (ifp->if_drv_flags & IFF_DRV_RUNNING))
+ if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
+ break;
+ VTNET_CORE_LOCK(sc);
+ if (ifp->if_drv_flags & IFF_DRV_RUNNING)
vtnet_rx_filter_mac(sc);
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
break;
case SIOCSIFMEDIA:
@@ -821,68 +1122,36 @@
break;
case SIOCSIFCAP:
+ VTNET_CORE_LOCK(sc);
mask = ifr->ifr_reqcap ^ ifp->if_capenable;
-#ifdef DEVICE_POLLING
- if (mask & IFCAP_POLLING) {
- if (ifr->ifr_reqcap & IFCAP_POLLING) {
- error = ether_poll_register(vtnet_poll, ifp);
- if (error)
- break;
-
- VTNET_LOCK(sc);
- vtnet_disable_rx_intr(sc);
- vtnet_disable_tx_intr(sc);
- ifp->if_capenable |= IFCAP_POLLING;
- VTNET_UNLOCK(sc);
- } else {
- error = ether_poll_deregister(ifp);
-
- /* Enable interrupts even in error case. */
- VTNET_LOCK(sc);
- vtnet_enable_tx_intr(sc);
- vtnet_enable_rx_intr(sc);
- ifp->if_capenable &= ~IFCAP_POLLING;
- VTNET_UNLOCK(sc);
- }
- }
-#endif
- VTNET_LOCK(sc);
-
- if (mask & IFCAP_TXCSUM) {
+ if (mask & IFCAP_TXCSUM)
ifp->if_capenable ^= IFCAP_TXCSUM;
- if (ifp->if_capenable & IFCAP_TXCSUM)
- ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
- else
- ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD;
- }
-
- if (mask & IFCAP_TSO4) {
+ if (mask & IFCAP_TXCSUM_IPV6)
+ ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
+ if (mask & IFCAP_TSO4)
ifp->if_capenable ^= IFCAP_TSO4;
- if (ifp->if_capenable & IFCAP_TSO4)
- ifp->if_hwassist |= CSUM_TSO;
- else
- ifp->if_hwassist &= ~CSUM_TSO;
- }
+ if (mask & IFCAP_TSO6)
+ ifp->if_capenable ^= IFCAP_TSO6;
- if (mask & IFCAP_RXCSUM) {
- ifp->if_capenable ^= IFCAP_RXCSUM;
+ if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
+ IFCAP_VLAN_HWFILTER)) {
+ /* These Rx features require us to renegotiate. */
reinit = 1;
- }
- if (mask & IFCAP_LRO) {
- ifp->if_capenable ^= IFCAP_LRO;
- reinit = 1;
- }
+ if (mask & IFCAP_RXCSUM)
+ ifp->if_capenable ^= IFCAP_RXCSUM;
+ if (mask & IFCAP_RXCSUM_IPV6)
+ ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
+ if (mask & IFCAP_LRO)
+ ifp->if_capenable ^= IFCAP_LRO;
+ if (mask & IFCAP_VLAN_HWFILTER)
+ ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
+ } else
+ reinit = 0;
- if (mask & IFCAP_VLAN_HWFILTER) {
- ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
- reinit = 1;
- }
-
if (mask & IFCAP_VLAN_HWTSO)
ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
-
if (mask & IFCAP_VLAN_HWTAGGING)
ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
@@ -890,9 +1159,10 @@
ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
vtnet_init_locked(sc);
}
+
+ VTNET_CORE_UNLOCK(sc);
VLAN_CAPABILITIES(ifp);
- VTNET_UNLOCK(sc);
break;
default:
@@ -900,80 +1170,32 @@
break;
}
- VTNET_LOCK_ASSERT_NOTOWNED(sc);
+ VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc);
return (error);
}
static int
-vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
+vtnet_rxq_populate(struct vtnet_rxq *rxq)
{
- struct ifnet *ifp;
- int new_frame_size, clsize;
-
- ifp = sc->vtnet_ifp;
-
- if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
- new_frame_size = sizeof(struct vtnet_rx_header) +
- sizeof(struct ether_vlan_header) + new_mtu;
-
- if (new_frame_size > MJUM9BYTES)
- return (EINVAL);
-
- if (new_frame_size <= MCLBYTES)
- clsize = MCLBYTES;
- else
- clsize = MJUM9BYTES;
- } else {
- new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) +
- sizeof(struct ether_vlan_header) + new_mtu;
-
- if (new_frame_size <= MCLBYTES)
- clsize = MCLBYTES;
- else
- clsize = MJUMPAGESIZE;
- }
-
- sc->vtnet_rx_mbuf_size = clsize;
- sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc);
- KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS,
- ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count));
-
- ifp->if_mtu = new_mtu;
-
- if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
- ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
- vtnet_init_locked(sc);
- }
-
- return (0);
-}
-
-static int
-vtnet_init_rx_vq(struct vtnet_softc *sc)
-{
struct virtqueue *vq;
int nbufs, error;
- vq = sc->vtnet_rx_vq;
- nbufs = 0;
+ vq = rxq->vtnrx_vq;
error = ENOSPC;
- while (!virtqueue_full(vq)) {
- if ((error = vtnet_newbuf(sc)) != 0)
+ for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
+ error = vtnet_rxq_new_buf(rxq);
+ if (error)
break;
- nbufs++;
}
if (nbufs > 0) {
virtqueue_notify(vq);
-
/*
* EMSGSIZE signifies the virtqueue did not have enough
* entries available to hold the last mbuf. This is not
- * an error. We should not get ENOSPC since we check if
- * the virtqueue is full before attempting to add a
- * buffer.
+ * an error.
*/
if (error == EMSGSIZE)
error = 0;
@@ -983,87 +1205,33 @@
}
static void
-vtnet_free_rx_mbufs(struct vtnet_softc *sc)
+vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq)
{
struct virtqueue *vq;
struct mbuf *m;
int last;
- vq = sc->vtnet_rx_vq;
+ vq = rxq->vtnrx_vq;
last = 0;
while ((m = virtqueue_drain(vq, &last)) != NULL)
m_freem(m);
- KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq"));
+ KASSERT(virtqueue_empty(vq),
+ ("%s: mbufs remaining in rx queue %p", __func__, rxq));
}
-static void
-vtnet_free_tx_mbufs(struct vtnet_softc *sc)
-{
- struct virtqueue *vq;
- struct vtnet_tx_header *txhdr;
- int last;
-
- vq = sc->vtnet_tx_vq;
- last = 0;
-
- while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
- m_freem(txhdr->vth_mbuf);
- uma_zfree(vtnet_tx_header_zone, txhdr);
- }
-
- KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq"));
-}
-
-static void
-vtnet_free_ctrl_vq(struct vtnet_softc *sc)
-{
-
- /*
- * The control virtqueue is only polled, therefore
- * it should already be empty.
- */
- KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq),
- ("Ctrl Vq not empty"));
-}
-
-#ifdef DEVICE_POLLING
-static int
-vtnet_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
-{
- struct vtnet_softc *sc;
- int rx_done;
-
- sc = ifp->if_softc;
- rx_done = 0;
-
- VTNET_LOCK(sc);
- if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
- if (cmd == POLL_AND_CHECK_STATUS)
- vtnet_update_link_status(sc);
-
- if (virtqueue_nused(sc->vtnet_rx_vq) > 0)
- vtnet_rxeof(sc, count, &rx_done);
-
- vtnet_txeof(sc);
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- vtnet_start_locked(ifp);
- }
- VTNET_UNLOCK(sc);
-
- return (rx_done);
-}
-#endif /* DEVICE_POLLING */
-
static struct mbuf *
-vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
+vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
{
struct mbuf *m_head, *m_tail, *m;
int i, clsize;
- clsize = sc->vtnet_rx_mbuf_size;
+ clsize = sc->vtnet_rx_clsize;
+ KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
+ ("%s: chained mbuf %d request without LRO_NOMRG", __func__, nbufs));
+
m_head = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, clsize);
if (m_head == NULL)
goto fail;
@@ -1071,19 +1239,15 @@
m_head->m_len = clsize;
m_tail = m_head;
- if (nbufs > 1) {
- KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
- ("chained Rx mbuf requested without LRO_NOMRG"));
+ /* Allocate the rest of the chain. */
+ for (i = 1; i < nbufs; i++) {
+ m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize);
+ if (m == NULL)
+ goto fail;
- for (i = 1; i < nbufs; i++) {
- m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize);
- if (m == NULL)
- goto fail;
-
- m->m_len = clsize;
- m_tail->m_next = m;
- m_tail = m;
- }
+ m->m_len = clsize;
+ m_tail->m_next = m;
+ m_tail = m;
}
if (m_tailp != NULL)
@@ -1098,34 +1262,38 @@
return (NULL);
}
+/*
+ * Slow path for when LRO without mergeable buffers is negotiated.
+ */
static int
-vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0)
+vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *rxq, struct mbuf *m0,
+ int len0)
{
+ struct vtnet_softc *sc;
struct mbuf *m, *m_prev;
struct mbuf *m_new, *m_tail;
int len, clsize, nreplace, error;
- m = m0;
+ sc = rxq->vtnrx_sc;
+ clsize = sc->vtnet_rx_clsize;
+
m_prev = NULL;
- len = len0;
-
m_tail = NULL;
- clsize = sc->vtnet_rx_mbuf_size;
nreplace = 0;
- KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ||
- m->m_next == NULL, ("chained Rx mbuf without LRO_NOMRG"));
+ m = m0;
+ len = len0;
/*
- * Since LRO_NOMRG mbuf chains are so large, we want to avoid
- * allocating an entire chain for each received frame. When
- * the received frame's length is less than that of the chain,
- * the unused mbufs are reassigned to the new chain.
+ * Since these mbuf chains are so large, we avoid allocating an
+ * entire replacement chain if possible. When the received frame
+ * did not consume the entire chain, the unused mbufs are moved
+ * to the replacement chain.
*/
while (len > 0) {
/*
- * Something is seriously wrong if we received
- * a frame larger than the mbuf chain. Drop it.
+ * Something is seriously wrong if we received a frame
+ * larger than the chain. Drop it.
*/
if (m == NULL) {
sc->vtnet_stats.rx_frame_too_large++;
@@ -1132,9 +1300,10 @@
return (EMSGSIZE);
}
+ /* We always allocate the same cluster size. */
KASSERT(m->m_len == clsize,
- ("mbuf length not expected cluster size: %d",
- m->m_len));
+ ("%s: mbuf size %d is not the cluster size %d",
+ __func__, m->m_len, clsize));
m->m_len = MIN(m->m_len, len);
len -= m->m_len;
@@ -1144,12 +1313,11 @@
nreplace++;
}
- KASSERT(m_prev != NULL, ("m_prev == NULL"));
- KASSERT(nreplace <= sc->vtnet_rx_mbuf_count,
- ("too many replacement mbufs: %d/%d", nreplace,
- sc->vtnet_rx_mbuf_count));
+ KASSERT(nreplace <= sc->vtnet_rx_nmbufs,
+ ("%s: too many replacement mbufs %d max %d", __func__, nreplace,
+ sc->vtnet_rx_nmbufs));
- m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail);
+ m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail);
if (m_new == NULL) {
m_prev->m_len = clsize;
return (ENOBUFS);
@@ -1156,8 +1324,8 @@
}
/*
- * Move unused mbufs, if any, from the original chain
- * onto the end of the new chain.
+ * Move any unused mbufs from the received chain onto the end
+ * of the new chain.
*/
if (m_prev->m_next != NULL) {
m_tail->m_next = m_prev->m_next;
@@ -1164,7 +1332,7 @@
m_prev->m_next = NULL;
}
- error = vtnet_enqueue_rxbuf(sc, m_new);
+ error = vtnet_rxq_enqueue_buf(rxq, m_new);
if (error) {
/*
* BAD! We could not enqueue the replacement mbuf chain. We
@@ -1189,343 +1357,321 @@
}
static int
-vtnet_newbuf(struct vtnet_softc *sc)
+vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len)
{
- struct mbuf *m;
+ struct vtnet_softc *sc;
+ struct mbuf *m_new;
int error;
- m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL);
- if (m == NULL)
- return (ENOBUFS);
+ sc = rxq->vtnrx_sc;
- error = vtnet_enqueue_rxbuf(sc, m);
- if (error)
- m_freem(m);
+ KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL,
+ ("%s: chained mbuf without LRO_NOMRG", __func__));
- return (error);
-}
+ if (m->m_next == NULL) {
+ /* Fast-path for the common case of just one mbuf. */
+ if (m->m_len < len)
+ return (EINVAL);
-static void
-vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs)
-{
- struct virtqueue *vq;
- struct mbuf *m;
+ m_new = vtnet_rx_alloc_buf(sc, 1, NULL);
+ if (m_new == NULL)
+ return (ENOBUFS);
- vq = sc->vtnet_rx_vq;
+ error = vtnet_rxq_enqueue_buf(rxq, m_new);
+ if (error) {
+ /*
+ * The new mbuf is suppose to be an identical
+ * copy of the one just dequeued so this is an
+ * unexpected error.
+ */
+ m_freem(m_new);
+ sc->vtnet_stats.rx_enq_replacement_failed++;
+ } else
+ m->m_len = len;
+ } else
+ error = vtnet_rxq_replace_lro_nomgr_buf(rxq, m, len);
- while (--nbufs > 0) {
- if ((m = virtqueue_dequeue(vq, NULL)) == NULL)
- break;
- vtnet_discard_rxbuf(sc, m);
- }
+ return (error);
}
-static void
-vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
-{
- int error;
-
- /*
- * Requeue the discarded mbuf. This should always be
- * successful since it was just dequeued.
- */
- error = vtnet_enqueue_rxbuf(sc, m);
- KASSERT(error == 0, ("cannot requeue discarded mbuf"));
-}
-
static int
-vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m)
+vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)
{
- struct sglist sg;
- struct sglist_seg segs[VTNET_MAX_RX_SEGS];
+ struct vtnet_softc *sc;
+ struct sglist *sg;
struct vtnet_rx_header *rxhdr;
- struct virtio_net_hdr *hdr;
uint8_t *mdata;
int offset, error;
- VTNET_LOCK_ASSERT(sc);
- KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ||
- m->m_next == NULL, ("chained Rx mbuf without LRO_NOMRG"));
+ sc = rxq->vtnrx_sc;
+ sg = rxq->vtnrx_sg;
+ mdata = mtod(m, uint8_t *);
- sglist_init(&sg, VTNET_MAX_RX_SEGS, segs);
+ VTNET_RXQ_LOCK_ASSERT(rxq);
+ KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL,
+ ("%s: chained mbuf without LRO_NOMRG", __func__));
+ KASSERT(m->m_len == sc->vtnet_rx_clsize,
+ ("%s: unexpected cluster size %d/%d", __func__, m->m_len,
+ sc->vtnet_rx_clsize));
- mdata = mtod(m, uint8_t *);
- offset = 0;
-
+ sglist_reset(sg);
if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
+ MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr));
rxhdr = (struct vtnet_rx_header *) mdata;
- hdr = &rxhdr->vrh_hdr;
- offset += sizeof(struct vtnet_rx_header);
+ sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size);
+ offset = sizeof(struct vtnet_rx_header);
+ } else
+ offset = 0;
- error = sglist_append(&sg, hdr, sc->vtnet_hdr_size);
- KASSERT(error == 0, ("cannot add header to sglist"));
+ sglist_append(sg, mdata + offset, m->m_len - offset);
+ if (m->m_next != NULL) {
+ error = sglist_append_mbuf(sg, m->m_next);
+ MPASS(error == 0);
}
- error = sglist_append(&sg, mdata + offset, m->m_len - offset);
- if (error)
- return (error);
+ error = virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg);
- if (m->m_next != NULL) {
- error = sglist_append_mbuf(&sg, m->m_next);
- if (error)
- return (error);
- }
-
- return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg));
+ return (error);
}
-static void
-vtnet_vlan_tag_remove(struct mbuf *m)
+static int
+vtnet_rxq_new_buf(struct vtnet_rxq *rxq)
{
- struct ether_vlan_header *evl;
+ struct vtnet_softc *sc;
+ struct mbuf *m;
+ int error;
- evl = mtod(m, struct ether_vlan_header *);
+ sc = rxq->vtnrx_sc;
- m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag);
- m->m_flags |= M_VLANTAG;
+ m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL);
+ if (m == NULL)
+ return (ENOBUFS);
- /* Strip the 802.1Q header. */
- bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN,
- ETHER_HDR_LEN - ETHER_TYPE_LEN);
- m_adj(m, ETHER_VLAN_ENCAP_LEN);
+ error = vtnet_rxq_enqueue_buf(rxq, m);
+ if (error)
+ m_freem(m);
+
+ return (error);
}
-#ifdef notyet
+/*
+ * Use the checksum offset in the VirtIO header to set the
+ * correct CSUM_* flags.
+ */
static int
-vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
- struct virtio_net_hdr *hdr)
+vtnet_rxq_csum_by_offset(struct vtnet_rxq *rxq, struct mbuf *m,
+ uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr)
{
- struct ether_header *eh;
- struct ether_vlan_header *evh;
- struct ip *ip;
- struct ip6_hdr *ip6;
- struct udphdr *udp;
- int ip_offset, csum_start, csum_offset, hlen;
- uint16_t eth_type;
- uint8_t ip_proto;
+ struct vtnet_softc *sc;
+#if defined(INET) || defined(INET6)
+ int offset = hdr->csum_start + hdr->csum_offset;
+#endif
- /*
- * Convert the VirtIO checksum interface to FreeBSD's interface.
- * The host only provides us with the offset at which to start
- * checksumming, and the offset from that to place the completed
- * checksum. While this maps well with how Linux does checksums,
- * for FreeBSD, we must parse the received packet in order to set
- * the appropriate CSUM_* flags.
- */
+ sc = rxq->vtnrx_sc;
- /*
- * Every mbuf added to the receive virtqueue is always at least
- * MCLBYTES big, so assume something is amiss if the first mbuf
- * does not contain both the Ethernet and protocol headers.
- */
- ip_offset = sizeof(struct ether_header);
- if (m->m_len < ip_offset)
- return (1);
-
- eh = mtod(m, struct ether_header *);
- eth_type = ntohs(eh->ether_type);
- if (eth_type == ETHERTYPE_VLAN) {
- ip_offset = sizeof(struct ether_vlan_header);
- if (m->m_len < ip_offset)
- return (1);
- evh = mtod(m, struct ether_vlan_header *);
- eth_type = ntohs(evh->evl_proto);
- }
-
+ /* Only do a basic sanity check on the offset. */
switch (eth_type) {
+#if defined(INET)
case ETHERTYPE_IP:
- if (m->m_len < ip_offset + sizeof(struct ip))
+ if (__predict_false(offset < ip_start + sizeof(struct ip)))
return (1);
-
- ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
- /* Sanity check the IP header. */
- if (ip->ip_v != IPVERSION)
- return (1);
- hlen = ip->ip_hl << 2;
- if (hlen < sizeof(struct ip))
- return (1);
- if (ntohs(ip->ip_len) < hlen)
- return (1);
- if (ntohs(ip->ip_len) != (m->m_pkthdr.len - ip_offset))
- return (1);
-
- ip_proto = ip->ip_p;
- csum_start = ip_offset + hlen;
break;
-
+#endif
+#if defined(INET6)
case ETHERTYPE_IPV6:
- if (m->m_len < ip_offset + sizeof(struct ip6_hdr))
+ if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr)))
return (1);
-
- /*
- * XXX FreeBSD does not handle any IPv6 checksum offloading
- * at the moment.
- */
-
- ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
- /* XXX Assume no extension headers are present. */
- ip_proto = ip6->ip6_nxt;
- csum_start = ip_offset + sizeof(struct ip6_hdr);
break;
-
+#endif
default:
sc->vtnet_stats.rx_csum_bad_ethtype++;
return (1);
}
- /* Assume checksum begins right after the IP header. */
- if (hdr->csum_start != csum_start) {
- sc->vtnet_stats.rx_csum_bad_start++;
- return (1);
- }
-
- switch (ip_proto) {
- case IPPROTO_TCP:
- csum_offset = offsetof(struct tcphdr, th_sum);
+ /*
+ * Use the offset to determine the appropriate CSUM_* flags. This is
+ * a bit dirty, but we can get by with it since the checksum offsets
+ * happen to be different. We assume the host host does not do IPv4
+ * header checksum offloading.
+ */
+ switch (hdr->csum_offset) {
+ case offsetof(struct udphdr, uh_sum):
+ case offsetof(struct tcphdr, th_sum):
+ m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+ m->m_pkthdr.csum_data = 0xFFFF;
break;
-
- case IPPROTO_UDP:
- csum_offset = offsetof(struct udphdr, uh_sum);
+ case offsetof(struct sctphdr, checksum):
+ m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
break;
-
- case IPPROTO_SCTP:
- csum_offset = offsetof(struct sctphdr, checksum);
- break;
-
default:
- sc->vtnet_stats.rx_csum_bad_ipproto++;
- return (1);
- }
-
- if (hdr->csum_offset != csum_offset) {
sc->vtnet_stats.rx_csum_bad_offset++;
return (1);
}
- /*
- * The IP header checksum is almost certainly valid but I'm
- * uncertain if that is guaranteed.
- *
- * m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
- */
+ return (0);
+}
- switch (ip_proto) {
- case IPPROTO_UDP:
- if (m->m_len < csum_start + sizeof(struct udphdr))
- return (1);
+static int
+vtnet_rxq_csum_by_parse(struct vtnet_rxq *rxq, struct mbuf *m,
+ uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr)
+{
+ struct vtnet_softc *sc;
+ int offset, proto;
- udp = (struct udphdr *)(mtod(m, uint8_t *) + csum_start);
- if (udp->uh_sum == 0)
- return (0);
+ sc = rxq->vtnrx_sc;
- /* FALLTHROUGH */
+ switch (eth_type) {
+#if defined(INET)
+ case ETHERTYPE_IP: {
+ struct ip *ip;
+ if (__predict_false(m->m_len < ip_start + sizeof(struct ip)))
+ return (1);
+ ip = (struct ip *)(m->m_data + ip_start);
+ proto = ip->ip_p;
+ offset = ip_start + (ip->ip_hl << 2);
+ break;
+ }
+#endif
+#if defined(INET6)
+ case ETHERTYPE_IPV6:
+ if (__predict_false(m->m_len < ip_start +
+ sizeof(struct ip6_hdr)))
+ return (1);
+ offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto);
+ if (__predict_false(offset < 0))
+ return (1);
+ break;
+#endif
+ default:
+ sc->vtnet_stats.rx_csum_bad_ethtype++;
+ return (1);
+ }
+ switch (proto) {
case IPPROTO_TCP:
+ if (__predict_false(m->m_len < offset + sizeof(struct tcphdr)))
+ return (1);
m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
m->m_pkthdr.csum_data = 0xFFFF;
break;
-
+ case IPPROTO_UDP:
+ if (__predict_false(m->m_len < offset + sizeof(struct udphdr)))
+ return (1);
+ m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+ m->m_pkthdr.csum_data = 0xFFFF;
+ break;
case IPPROTO_SCTP:
+ if (__predict_false(m->m_len < offset + sizeof(struct sctphdr)))
+ return (1);
m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
break;
+ default:
+ /*
+ * For the remaining protocols, FreeBSD does not support
+ * checksum offloading, so the checksum will be recomputed.
+ */
+#if 0
+ if_printf(sc->vtnet_ifp, "cksum offload of unsupported "
+ "protocol eth_type=%#x proto=%d csum_start=%d "
+ "csum_offset=%d\n", __func__, eth_type, proto,
+ hdr->csum_start, hdr->csum_offset);
+#endif
+ break;
}
- sc->vtnet_stats.rx_csum_offloaded++;
-
return (0);
}
-#endif
/*
- * Alternative method of doing receive checksum offloading. Rather
- * than parsing the received frame down to the IP header, use the
- * csum_offset to determine which CSUM_* flags are appropriate. We
- * can get by with doing this only because the checksum offsets are
- * unique for the things we care about.
+ * Set the appropriate CSUM_* flags. Unfortunately, the information
+ * provided is not directly useful to us. The VirtIO header gives the
+ * offset of the checksum, which is all Linux needs, but this is not
+ * how FreeBSD does things. We are forced to peek inside the packet
+ * a bit.
+ *
+ * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
+ * could accept the offsets and let the stack figure it out.
*/
static int
-vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m,
+vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,
struct virtio_net_hdr *hdr)
{
struct ether_header *eh;
struct ether_vlan_header *evh;
- struct udphdr *udp;
- int csum_len;
uint16_t eth_type;
+ int offset, error;
- csum_len = hdr->csum_start + hdr->csum_offset;
-
- if (csum_len < sizeof(struct ether_header) + sizeof(struct ip))
- return (1);
- if (m->m_len < csum_len)
- return (1);
-
eh = mtod(m, struct ether_header *);
eth_type = ntohs(eh->ether_type);
if (eth_type == ETHERTYPE_VLAN) {
+ /* BMV: We should handle nested VLAN tags too. */
evh = mtod(m, struct ether_vlan_header *);
eth_type = ntohs(evh->evl_proto);
- }
+ offset = sizeof(struct ether_vlan_header);
+ } else
+ offset = sizeof(struct ether_header);
- if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) {
- sc->vtnet_stats.rx_csum_bad_ethtype++;
- return (1);
- }
+ if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
+ error = vtnet_rxq_csum_by_offset(rxq, m, eth_type, offset, hdr);
+ else
+ error = vtnet_rxq_csum_by_parse(rxq, m, eth_type, offset, hdr);
- /* Use the offset to determine the appropriate CSUM_* flags. */
- switch (hdr->csum_offset) {
- case offsetof(struct udphdr, uh_sum):
- if (m->m_len < hdr->csum_start + sizeof(struct udphdr))
- return (1);
- udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start);
- if (udp->uh_sum == 0)
- return (0);
+ return (error);
+}
- /* FALLTHROUGH */
+static void
+vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs)
+{
+ struct mbuf *m;
- case offsetof(struct tcphdr, th_sum):
- m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
- m->m_pkthdr.csum_data = 0xFFFF;
- break;
-
- case offsetof(struct sctphdr, checksum):
- m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
- break;
-
- default:
- sc->vtnet_stats.rx_csum_bad_offset++;
- return (1);
+ while (--nbufs > 0) {
+ m = virtqueue_dequeue(rxq->vtnrx_vq, NULL);
+ if (m == NULL)
+ break;
+ vtnet_rxq_discard_buf(rxq, m);
}
+}
- sc->vtnet_stats.rx_csum_offloaded++;
+static void
+vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m)
+{
+ int error;
- return (0);
+ /*
+ * Requeue the discarded mbuf. This should always be successful
+ * since it was just dequeued.
+ */
+ error = vtnet_rxq_enqueue_buf(rxq, m);
+ KASSERT(error == 0,
+ ("%s: cannot requeue discarded mbuf %d", __func__, error));
}
static int
-vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs)
+vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
{
+ struct vtnet_softc *sc;
struct ifnet *ifp;
struct virtqueue *vq;
struct mbuf *m, *m_tail;
int len;
+ sc = rxq->vtnrx_sc;
+ vq = rxq->vtnrx_vq;
ifp = sc->vtnet_ifp;
- vq = sc->vtnet_rx_vq;
m_tail = m_head;
while (--nbufs > 0) {
m = virtqueue_dequeue(vq, &len);
if (m == NULL) {
- ifp->if_ierrors++;
+ rxq->vtnrx_stats.vrxs_ierrors++;
goto fail;
}
- if (vtnet_newbuf(sc) != 0) {
- ifp->if_iqdrops++;
- vtnet_discard_rxbuf(sc, m);
+ if (vtnet_rxq_new_buf(rxq) != 0) {
+ rxq->vtnrx_stats.vrxs_iqdrops++;
+ vtnet_rxq_discard_buf(rxq, m);
if (nbufs > 1)
- vtnet_discard_merged_rxbuf(sc, nbufs);
+ vtnet_rxq_discard_merged_bufs(rxq, nbufs);
goto fail;
}
@@ -1549,27 +1695,81 @@
return (1);
}
+static void
+vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
+ struct virtio_net_hdr *hdr)
+{
+ struct vtnet_softc *sc;
+ struct ifnet *ifp;
+ struct ether_header *eh;
+
+ sc = rxq->vtnrx_sc;
+ ifp = sc->vtnet_ifp;
+
+ if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
+ eh = mtod(m, struct ether_header *);
+ if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
+ vtnet_vlan_tag_remove(m);
+ /*
+ * With the 802.1Q header removed, update the
+ * checksum starting location accordingly.
+ */
+ if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
+ hdr->csum_start -= ETHER_VLAN_ENCAP_LEN;
+ }
+ }
+
+ m->m_pkthdr.flowid = rxq->vtnrx_id;
+ M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
+
+ /*
+ * BMV: FreeBSD does not have the UNNECESSARY and PARTIAL checksum
+ * distinction that Linux does. Need to reevaluate if performing
+ * offloading for the NEEDS_CSUM case is really appropriate.
+ */
+ if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM |
+ VIRTIO_NET_HDR_F_DATA_VALID)) {
+ if (vtnet_rxq_csum(rxq, m, hdr) == 0)
+ rxq->vtnrx_stats.vrxs_csum++;
+ else
+ rxq->vtnrx_stats.vrxs_csum_failed++;
+ }
+
+ rxq->vtnrx_stats.vrxs_ipackets++;
+ rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;
+
+ VTNET_RXQ_UNLOCK(rxq);
+ (*ifp->if_input)(ifp, m);
+ VTNET_RXQ_LOCK(rxq);
+}
+
static int
-vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
+vtnet_rxq_eof(struct vtnet_rxq *rxq)
{
- struct virtio_net_hdr lhdr;
+ struct virtio_net_hdr lhdr, *hdr;
+ struct vtnet_softc *sc;
struct ifnet *ifp;
struct virtqueue *vq;
struct mbuf *m;
- struct ether_header *eh;
- struct virtio_net_hdr *hdr;
struct virtio_net_hdr_mrg_rxbuf *mhdr;
- int len, deq, nbufs, adjsz, rx_npkts;
+ int len, deq, nbufs, adjsz, count;
+ sc = rxq->vtnrx_sc;
+ vq = rxq->vtnrx_vq;
ifp = sc->vtnet_ifp;
- vq = sc->vtnet_rx_vq;
hdr = &lhdr;
deq = 0;
- rx_npkts = 0;
+ count = sc->vtnet_rx_process_limit;
- VTNET_LOCK_ASSERT(sc);
+ VTNET_RXQ_LOCK_ASSERT(rxq);
- while (--count >= 0) {
+#ifdef DEV_NETMAP
+ if (netmap_rx_irq(ifp, 0, &deq)) {
+ return (FALSE);
+ }
+#endif /* DEV_NETMAP */
+
+ while (count-- > 0) {
m = virtqueue_dequeue(vq, &len);
if (m == NULL)
break;
@@ -1576,8 +1776,8 @@
deq++;
if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
- ifp->if_ierrors++;
- vtnet_discard_rxbuf(sc, m);
+ rxq->vtnrx_stats.vrxs_ierrors++;
+ vtnet_rxq_discard_buf(rxq, m);
continue;
}
@@ -1585,8 +1785,8 @@
nbufs = 1;
adjsz = sizeof(struct vtnet_rx_header);
/*
- * Account for our pad between the header and
- * the actual start of the frame.
+ * Account for our pad inserted between the header
+ * and the actual start of the frame.
*/
len += VTNET_RX_HEADER_PAD;
} else {
@@ -1595,11 +1795,11 @@
adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
}
- if (vtnet_replace_rxbuf(sc, m, len) != 0) {
- ifp->if_iqdrops++;
- vtnet_discard_rxbuf(sc, m);
+ if (vtnet_rxq_replace_buf(rxq, m, len) != 0) {
+ rxq->vtnrx_stats.vrxs_iqdrops++;
+ vtnet_rxq_discard_buf(rxq, m);
if (nbufs > 1)
- vtnet_discard_merged_rxbuf(sc, nbufs);
+ vtnet_rxq_discard_merged_bufs(rxq, nbufs);
continue;
}
@@ -1608,51 +1808,26 @@
m->m_pkthdr.csum_flags = 0;
if (nbufs > 1) {
- if (vtnet_rxeof_merged(sc, m, nbufs) != 0)
+ /* Dequeue the rest of chain. */
+ if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0)
continue;
}
- ifp->if_ipackets++;
-
/*
* Save copy of header before we strip it. For both mergeable
- * and non-mergeable, the VirtIO header is placed first in the
- * mbuf's data. We no longer need num_buffers, so always use a
- * virtio_net_hdr.
+ * and non-mergeable, the header is at the beginning of the
+ * mbuf data. We no longer need num_buffers, so always use a
+ * regular header.
+ *
+ * BMV: Is this memcpy() expensive? We know the mbuf data is
+ * still valid even after the m_adj().
*/
memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
m_adj(m, adjsz);
- if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
- eh = mtod(m, struct ether_header *);
- if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
- vtnet_vlan_tag_remove(m);
+ vtnet_rxq_input(rxq, m, hdr);
- /*
- * With the 802.1Q header removed, update the
- * checksum starting location accordingly.
- */
- if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
- hdr->csum_start -=
- ETHER_VLAN_ENCAP_LEN;
- }
- }
-
- if (ifp->if_capenable & IFCAP_RXCSUM &&
- hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
- if (vtnet_rx_csum(sc, m, hdr) != 0)
- sc->vtnet_stats.rx_csum_failed++;
- }
-
- VTNET_UNLOCK(sc);
- rx_npkts++;
- (*ifp->if_input)(ifp, m);
- VTNET_LOCK(sc);
-
- /*
- * The interface may have been stopped while we were
- * passing the packet up the network stack.
- */
+ /* Must recheck after dropping the Rx lock. */
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
break;
}
@@ -1660,232 +1835,335 @@
if (deq > 0)
virtqueue_notify(vq);
- if (rx_npktsp != NULL)
- *rx_npktsp = rx_npkts;
-
return (count > 0 ? 0 : EAGAIN);
}
static void
-vtnet_rx_vq_intr(void *xsc)
+vtnet_rx_vq_intr(void *xrxq)
{
struct vtnet_softc *sc;
+ struct vtnet_rxq *rxq;
struct ifnet *ifp;
- int more;
+ int tries, more;
- sc = xsc;
+ rxq = xrxq;
+ sc = rxq->vtnrx_sc;
ifp = sc->vtnet_ifp;
+ tries = 0;
+ if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) {
+ /*
+ * Ignore this interrupt. Either this is a spurious interrupt
+ * or multiqueue without per-VQ MSIX so every queue needs to
+ * be polled (a brain dead configuration we could try harder
+ * to avoid).
+ */
+ vtnet_rxq_disable_intr(rxq);
+ return;
+ }
+
+ VTNET_RXQ_LOCK(rxq);
+
again:
- VTNET_LOCK(sc);
-
-#ifdef DEVICE_POLLING
- if (ifp->if_capenable & IFCAP_POLLING) {
- VTNET_UNLOCK(sc);
+ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+ VTNET_RXQ_UNLOCK(rxq);
return;
}
-#endif
+ more = vtnet_rxq_eof(rxq);
+ if (more || vtnet_rxq_enable_intr(rxq) != 0) {
+ if (!more)
+ vtnet_rxq_disable_intr(rxq);
+ /*
+ * This is an occasional condition or race (when !more),
+ * so retry a few times before scheduling the taskqueue.
+ */
+ if (tries++ < VTNET_INTR_DISABLE_RETRIES)
+ goto again;
+
+ VTNET_RXQ_UNLOCK(rxq);
+ rxq->vtnrx_stats.vrxs_rescheduled++;
+ taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
+ } else
+ VTNET_RXQ_UNLOCK(rxq);
+}
+
+static void
+vtnet_rxq_tq_intr(void *xrxq, int pending)
+{
+ struct vtnet_softc *sc;
+ struct vtnet_rxq *rxq;
+ struct ifnet *ifp;
+ int more;
+
+ rxq = xrxq;
+ sc = rxq->vtnrx_sc;
+ ifp = sc->vtnet_ifp;
+
+ VTNET_RXQ_LOCK(rxq);
+
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
- vtnet_enable_rx_intr(sc);
- VTNET_UNLOCK(sc);
+ VTNET_RXQ_UNLOCK(rxq);
return;
}
- more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
- if (more || vtnet_enable_rx_intr(sc) != 0) {
+ more = vtnet_rxq_eof(rxq);
+ if (more || vtnet_rxq_enable_intr(rxq) != 0) {
if (!more)
- vtnet_disable_rx_intr(sc);
- sc->vtnet_stats.rx_task_rescheduled++;
- VTNET_UNLOCK(sc);
- goto again;
+ vtnet_rxq_disable_intr(rxq);
+ rxq->vtnrx_stats.vrxs_rescheduled++;
+ taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
}
- VTNET_UNLOCK(sc);
+ VTNET_RXQ_UNLOCK(rxq);
}
+static int
+vtnet_txq_below_threshold(struct vtnet_txq *txq)
+{
+ struct vtnet_softc *sc;
+ struct virtqueue *vq;
+
+ sc = txq->vtntx_sc;
+ vq = txq->vtntx_vq;
+
+ return (virtqueue_nfree(vq) <= sc->vtnet_tx_intr_thresh);
+}
+
+static int
+vtnet_txq_notify(struct vtnet_txq *txq)
+{
+ struct virtqueue *vq;
+
+ vq = txq->vtntx_vq;
+
+ txq->vtntx_watchdog = VTNET_TX_TIMEOUT;
+ virtqueue_notify(vq);
+
+ if (vtnet_txq_enable_intr(txq) == 0)
+ return (0);
+
+ /*
+ * Drain frames that were completed since last checked. If this
+ * causes the queue to go above the threshold, the caller should
+ * continue transmitting.
+ */
+ if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) {
+ virtqueue_disable_intr(vq);
+ return (1);
+ }
+
+ return (0);
+}
+
static void
-vtnet_txeof(struct vtnet_softc *sc)
+vtnet_txq_free_mbufs(struct vtnet_txq *txq)
{
struct virtqueue *vq;
- struct ifnet *ifp;
struct vtnet_tx_header *txhdr;
- int deq;
+ int last;
- vq = sc->vtnet_tx_vq;
- ifp = sc->vtnet_ifp;
- deq = 0;
+ vq = txq->vtntx_vq;
+ last = 0;
- VTNET_LOCK_ASSERT(sc);
-
- while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
- deq++;
- ifp->if_opackets++;
+ while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
m_freem(txhdr->vth_mbuf);
uma_zfree(vtnet_tx_header_zone, txhdr);
}
- if (deq > 0) {
- ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
- if (virtqueue_empty(vq))
- sc->vtnet_watchdog_timer = 0;
- }
+ KASSERT(virtqueue_empty(vq),
+ ("%s: mbufs remaining in tx queue %p", __func__, txq));
}
-static struct mbuf *
-vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
- struct virtio_net_hdr *hdr)
+/*
+ * BMV: Much of this can go away once we finally have offsets in
+ * the mbuf packet header. Bug andre at .
+ */
+static int
+vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m,
+ int *etype, int *proto, int *start)
{
- struct ifnet *ifp;
- struct ether_header *eh;
+ struct vtnet_softc *sc;
struct ether_vlan_header *evh;
- struct ip *ip;
- struct ip6_hdr *ip6;
- struct tcphdr *tcp;
- int ip_offset;
- uint16_t eth_type, csum_start;
- uint8_t ip_proto, gso_type;
+ int offset;
- ifp = sc->vtnet_ifp;
+ sc = txq->vtntx_sc;
- ip_offset = sizeof(struct ether_header);
- if (m->m_len < ip_offset) {
- if ((m = m_pullup(m, ip_offset)) == NULL)
- return (NULL);
+ evh = mtod(m, struct ether_vlan_header *);
+ if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
+ /* BMV: We should handle nested VLAN tags too. */
+ *etype = ntohs(evh->evl_proto);
+ offset = sizeof(struct ether_vlan_header);
+ } else {
+ *etype = ntohs(evh->evl_encap_proto);
+ offset = sizeof(struct ether_header);
}
- eh = mtod(m, struct ether_header *);
- eth_type = ntohs(eh->ether_type);
- if (eth_type == ETHERTYPE_VLAN) {
- ip_offset = sizeof(struct ether_vlan_header);
- if (m->m_len < ip_offset) {
- if ((m = m_pullup(m, ip_offset)) == NULL)
- return (NULL);
- }
- evh = mtod(m, struct ether_vlan_header *);
- eth_type = ntohs(evh->evl_proto);
+ switch (*etype) {
+#if defined(INET)
+ case ETHERTYPE_IP: {
+ struct ip *ip, iphdr;
+ if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
+ m_copydata(m, offset, sizeof(struct ip),
+ (caddr_t) &iphdr);
+ ip = &iphdr;
+ } else
+ ip = (struct ip *)(m->m_data + offset);
+ *proto = ip->ip_p;
+ *start = offset + (ip->ip_hl << 2);
+ break;
}
+#endif
+#if defined(INET6)
+ case ETHERTYPE_IPV6:
+ *proto = -1;
+ *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
+ /* Assert the network stack sent us a valid packet. */
+ KASSERT(*start > offset,
+ ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
+ *start, offset, *proto));
+ break;
+#endif
+ default:
+ sc->vtnet_stats.tx_csum_bad_ethtype++;
+ return (EINVAL);
+ }
- switch (eth_type) {
- case ETHERTYPE_IP:
- if (m->m_len < ip_offset + sizeof(struct ip)) {
- m = m_pullup(m, ip_offset + sizeof(struct ip));
- if (m == NULL)
- return (NULL);
- }
+ return (0);
+}
- ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset);
- ip_proto = ip->ip_p;
- csum_start = ip_offset + (ip->ip_hl << 2);
- gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
- break;
+static int
+vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type,
+ int offset, struct virtio_net_hdr *hdr)
+{
+ static struct timeval lastecn;
+ static int curecn;
+ struct vtnet_softc *sc;
+ struct tcphdr *tcp, tcphdr;
- case ETHERTYPE_IPV6:
- if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) {
- m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr));
- if (m == NULL)
- return (NULL);
- }
+ sc = txq->vtntx_sc;
- ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset);
+ if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
+ m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
+ tcp = &tcphdr;
+ } else
+ tcp = (struct tcphdr *)(m->m_data + offset);
+
+ hdr->hdr_len = offset + (tcp->th_off << 2);
+ hdr->gso_size = m->m_pkthdr.tso_segsz;
+ hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
+ VIRTIO_NET_HDR_GSO_TCPV6;
+
+ if (tcp->th_flags & TH_CWR) {
/*
- * XXX Assume no extension headers are present. Presently,
- * this will always be true in the case of TSO, and FreeBSD
- * does not perform checksum offloading of IPv6 yet.
+ * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD,
+ * ECN support is not on a per-interface basis, but globally via
+ * the net.inet.tcp.ecn.enable sysctl knob. The default is off.
*/
- ip_proto = ip6->ip6_nxt;
- csum_start = ip_offset + sizeof(struct ip6_hdr);
- gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
- break;
-
- default:
- return (m);
+ if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
+ if (ppsratecheck(&lastecn, &curecn, 1))
+ if_printf(sc->vtnet_ifp,
+ "TSO with ECN not negotiated with host\n");
+ return (ENOTSUP);
+ }
+ hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
}
- if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) {
+ txq->vtntx_stats.vtxs_tso++;
+
+ return (0);
+}
+
+static struct mbuf *
+vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m,
+ struct virtio_net_hdr *hdr)
+{
+ struct vtnet_softc *sc;
+ int flags, etype, csum_start, proto, error;
+
+ sc = txq->vtntx_sc;
+ flags = m->m_pkthdr.csum_flags;
+
+ error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start);
+ if (error)
+ goto drop;
+
+ if ((etype == ETHERTYPE_IP && flags & VTNET_CSUM_OFFLOAD) ||
+ (etype == ETHERTYPE_IPV6 && flags & VTNET_CSUM_OFFLOAD_IPV6)) {
+ /*
+ * We could compare the IP protocol vs the CSUM_ flag too,
+ * but that really should not be necessary.
+ */
hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = csum_start;
hdr->csum_offset = m->m_pkthdr.csum_data;
-
- sc->vtnet_stats.tx_csum_offloaded++;
+ txq->vtntx_stats.vtxs_csum++;
}
- if (m->m_pkthdr.csum_flags & CSUM_TSO) {
- if (ip_proto != IPPROTO_TCP)
- return (m);
-
- if (m->m_len < csum_start + sizeof(struct tcphdr)) {
- m = m_pullup(m, csum_start + sizeof(struct tcphdr));
- if (m == NULL)
- return (NULL);
+ if (flags & CSUM_TSO) {
+ if (__predict_false(proto != IPPROTO_TCP)) {
+ /* Likely failed to correctly parse the mbuf. */
+ sc->vtnet_stats.tx_tso_not_tcp++;
+ goto drop;
}
- tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start);
- hdr->gso_type = gso_type;
- hdr->hdr_len = csum_start + (tcp->th_off << 2);
- hdr->gso_size = m->m_pkthdr.tso_segsz;
+ KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM,
+ ("%s: mbuf %p TSO without checksum offload %#x",
+ __func__, m, flags));
- if (tcp->th_flags & TH_CWR) {
- /*
- * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN.
- * ECN support is only configurable globally with the
- * net.inet.tcp.ecn.enable sysctl knob.
- */
- if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
- if_printf(ifp, "TSO with ECN not supported "
- "by host\n");
- m_freem(m);
- return (NULL);
- }
-
- hdr->flags |= VIRTIO_NET_HDR_GSO_ECN;
- }
-
- sc->vtnet_stats.tx_tso_offloaded++;
+ error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr);
+ if (error)
+ goto drop;
}
return (m);
+
+drop:
+ m_freem(m);
+ return (NULL);
}
static int
-vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
+vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
struct vtnet_tx_header *txhdr)
{
- struct sglist sg;
- struct sglist_seg segs[VTNET_MAX_TX_SEGS];
+ struct vtnet_softc *sc;
struct virtqueue *vq;
+ struct sglist *sg;
struct mbuf *m;
- int collapsed, error;
+ int error;
- vq = sc->vtnet_tx_vq;
+ sc = txq->vtntx_sc;
+ vq = txq->vtntx_vq;
+ sg = txq->vtntx_sg;
m = *m_head;
- collapsed = 0;
- sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
- error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
- KASSERT(error == 0 && sg.sg_nseg == 1,
- ("%s: cannot add header to sglist error %d", __func__, error));
+ sglist_reset(sg);
+ error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
+ KASSERT(error == 0 && sg->sg_nseg == 1,
+ ("%s: error %d adding header to sglist", __func__, error));
-again:
- error = sglist_append_mbuf(&sg, m);
+ error = sglist_append_mbuf(sg, m);
if (error) {
- if (collapsed)
- goto fail;
-
- m = m_collapse(m, M_NOWAIT, VTNET_MAX_TX_SEGS - 1);
+ m = m_defrag(m, M_NOWAIT);
if (m == NULL)
goto fail;
*m_head = m;
- collapsed = 1;
- goto again;
+ sc->vtnet_stats.tx_defragged++;
+
+ error = sglist_append_mbuf(sg, m);
+ if (error)
+ goto fail;
}
txhdr->vth_mbuf = m;
+ error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0);
- return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0));
+ return (error);
fail:
+ sc->vtnet_stats.tx_defrag_failed++;
m_freem(*m_head);
*m_head = NULL;
@@ -1893,7 +2171,7 @@
}
static int
-vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
+vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head)
{
struct vtnet_tx_header *txhdr;
struct virtio_net_hdr *hdr;
@@ -1905,16 +2183,15 @@
txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO);
if (txhdr == NULL) {
+ m_freem(m);
*m_head = NULL;
- m_freem(m);
return (ENOMEM);
}
/*
- * Always use the non-mergeable header to simplify things. When
- * the mergeable feature is negotiated, the num_buffers field
- * must be set to zero. We use vtnet_hdr_size later to enqueue
- * the correct header size to the host.
+ * Always use the non-mergeable header, regardless if the feature
+ * was negotiated. For transmit, num_buffers is always zero. The
+ * vtnet_hdr_size is used to enqueue the correct header size.
*/
hdr = &txhdr->vth_uhdr.hdr;
@@ -1927,8 +2204,8 @@
m->m_flags &= ~M_VLANTAG;
}
- if (m->m_pkthdr.csum_flags != 0) {
- m = vtnet_tx_offload(sc, m, hdr);
+ if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) {
+ m = vtnet_txq_offload(txq, m, hdr);
if ((*m_head = m) == NULL) {
error = ENOBUFS;
goto fail;
@@ -1935,135 +2212,606 @@
}
}
- error = vtnet_enqueue_txbuf(sc, m_head, txhdr);
+ error = vtnet_txq_enqueue_buf(txq, m_head, txhdr);
+ if (error == 0)
+ return (0);
+
fail:
- if (error)
- uma_zfree(vtnet_tx_header_zone, txhdr);
+ uma_zfree(vtnet_tx_header_zone, txhdr);
return (error);
}
+#ifdef VTNET_LEGACY_TX
+
static void
+vtnet_start_locked(struct vtnet_txq *txq, struct ifnet *ifp)
+{
+ struct vtnet_softc *sc;
+ struct virtqueue *vq;
+ struct mbuf *m0;
+ int tries, enq;
+
+ sc = txq->vtntx_sc;
+ vq = txq->vtntx_vq;
+ tries = 0;
+
+ VTNET_TXQ_LOCK_ASSERT(txq);
+
+ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
+ sc->vtnet_link_active == 0)
+ return;
+
+ vtnet_txq_eof(txq);
+
+again:
+ enq = 0;
+
+ while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
+ if (virtqueue_full(vq))
+ break;
+
+ IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
+ if (m0 == NULL)
+ break;
+
+ if (vtnet_txq_encap(txq, &m0) != 0) {
+ if (m0 != NULL)
+ IFQ_DRV_PREPEND(&ifp->if_snd, m0);
+ break;
+ }
+
+ enq++;
+ ETHER_BPF_MTAP(ifp, m0);
+ }
+
+ if (enq > 0 && vtnet_txq_notify(txq) != 0) {
+ if (tries++ < VTNET_NOTIFY_RETRIES)
+ goto again;
+
+ txq->vtntx_stats.vtxs_rescheduled++;
+ taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
+ }
+}
+
+static void
vtnet_start(struct ifnet *ifp)
{
struct vtnet_softc *sc;
+ struct vtnet_txq *txq;
sc = ifp->if_softc;
+ txq = &sc->vtnet_txqs[0];
- VTNET_LOCK(sc);
- vtnet_start_locked(ifp);
- VTNET_UNLOCK(sc);
+ VTNET_TXQ_LOCK(txq);
+ vtnet_start_locked(txq, ifp);
+ VTNET_TXQ_UNLOCK(txq);
}
-static void
-vtnet_start_locked(struct ifnet *ifp)
+#else /* !VTNET_LEGACY_TX */
+
+static int
+vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m)
{
struct vtnet_softc *sc;
struct virtqueue *vq;
- struct mbuf *m0;
- int enq;
+ struct buf_ring *br;
+ struct ifnet *ifp;
+ int enq, tries, error;
- sc = ifp->if_softc;
- vq = sc->vtnet_tx_vq;
- enq = 0;
+ sc = txq->vtntx_sc;
+ vq = txq->vtntx_vq;
+ br = txq->vtntx_br;
+ ifp = sc->vtnet_ifp;
+ tries = 0;
+ error = 0;
- VTNET_LOCK_ASSERT(sc);
+ VTNET_TXQ_LOCK_ASSERT(txq);
- if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
- IFF_DRV_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0))
- return;
+ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
+ sc->vtnet_link_active == 0) {
+ if (m != NULL)
+ error = drbr_enqueue(ifp, br, m);
+ return (error);
+ }
-#ifdef VTNET_TX_INTR_MODERATION
- if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2)
- vtnet_txeof(sc);
-#endif
+ if (m != NULL) {
+ error = drbr_enqueue(ifp, br, m);
+ if (error)
+ return (error);
+ }
- while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
+ vtnet_txq_eof(txq);
+
+again:
+ enq = 0;
+
+ while ((m = drbr_peek(ifp, br)) != NULL) {
if (virtqueue_full(vq)) {
- ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+ drbr_putback(ifp, br, m);
break;
}
- IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
- if (m0 == NULL)
+ if (vtnet_txq_encap(txq, &m) != 0) {
+ if (m != NULL)
+ drbr_putback(ifp, br, m);
+ else
+ drbr_advance(ifp, br);
break;
-
- if (vtnet_encap(sc, &m0) != 0) {
- if (m0 == NULL)
- break;
- IFQ_DRV_PREPEND(&ifp->if_snd, m0);
- ifp->if_drv_flags |= IFF_DRV_OACTIVE;
- break;
}
+ drbr_advance(ifp, br);
enq++;
- ETHER_BPF_MTAP(ifp, m0);
+ ETHER_BPF_MTAP(ifp, m);
}
- if (enq > 0) {
- virtqueue_notify(vq);
- sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT;
+ if (enq > 0 && vtnet_txq_notify(txq) != 0) {
+ if (tries++ < VTNET_NOTIFY_RETRIES)
+ goto again;
+
+ txq->vtntx_stats.vtxs_rescheduled++;
+ taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
}
+
+ return (0);
}
+static int
+vtnet_txq_mq_start(struct ifnet *ifp, struct mbuf *m)
+{
+ struct vtnet_softc *sc;
+ struct vtnet_txq *txq;
+ int i, npairs, error;
+
+ sc = ifp->if_softc;
+ npairs = sc->vtnet_act_vq_pairs;
+
+ /* check if flowid is set */
+ if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
+ i = m->m_pkthdr.flowid % npairs;
+ else
+ i = curcpu % npairs;
+
+ txq = &sc->vtnet_txqs[i];
+
+ if (VTNET_TXQ_TRYLOCK(txq) != 0) {
+ error = vtnet_txq_mq_start_locked(txq, m);
+ VTNET_TXQ_UNLOCK(txq);
+ } else {
+ error = drbr_enqueue(ifp, txq->vtntx_br, m);
+ taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask);
+ }
+
+ return (error);
+}
+
static void
-vtnet_tick(void *xsc)
+vtnet_txq_tq_deferred(void *xtxq, int pending)
{
struct vtnet_softc *sc;
+ struct vtnet_txq *txq;
- sc = xsc;
+ txq = xtxq;
+ sc = txq->vtntx_sc;
- VTNET_LOCK_ASSERT(sc);
-#ifdef VTNET_DEBUG
- virtqueue_dump(sc->vtnet_rx_vq);
- virtqueue_dump(sc->vtnet_tx_vq);
+ VTNET_TXQ_LOCK(txq);
+ if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br))
+ vtnet_txq_mq_start_locked(txq, NULL);
+ VTNET_TXQ_UNLOCK(txq);
+}
+
+#endif /* VTNET_LEGACY_TX */
+
+static void
+vtnet_txq_start(struct vtnet_txq *txq)
+{
+ struct vtnet_softc *sc;
+ struct ifnet *ifp;
+
+ sc = txq->vtntx_sc;
+ ifp = sc->vtnet_ifp;
+
+#ifdef VTNET_LEGACY_TX
+ if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+ vtnet_start_locked(txq, ifp);
+#else
+ if (!drbr_empty(ifp, txq->vtntx_br))
+ vtnet_txq_mq_start_locked(txq, NULL);
#endif
-
- vtnet_watchdog(sc);
- callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
}
static void
-vtnet_tx_vq_intr(void *xsc)
+vtnet_txq_tq_intr(void *xtxq, int pending)
{
struct vtnet_softc *sc;
+ struct vtnet_txq *txq;
struct ifnet *ifp;
- sc = xsc;
+ txq = xtxq;
+ sc = txq->vtntx_sc;
ifp = sc->vtnet_ifp;
-again:
- VTNET_LOCK(sc);
+ VTNET_TXQ_LOCK(txq);
-#ifdef DEVICE_POLLING
- if (ifp->if_capenable & IFCAP_POLLING) {
- VTNET_UNLOCK(sc);
+ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+ VTNET_TXQ_UNLOCK(txq);
return;
}
-#endif
+ vtnet_txq_eof(txq);
+ vtnet_txq_start(txq);
+
+ VTNET_TXQ_UNLOCK(txq);
+}
+
+static int
+vtnet_txq_eof(struct vtnet_txq *txq)
+{
+ struct virtqueue *vq;
+ struct vtnet_tx_header *txhdr;
+ struct mbuf *m;
+ int deq;
+
+ vq = txq->vtntx_vq;
+ deq = 0;
+ VTNET_TXQ_LOCK_ASSERT(txq);
+
+#ifdef DEV_NETMAP
+ if (netmap_tx_irq(txq->vtntx_sc->vtnet_ifp, txq->vtntx_id)) {
+ virtqueue_disable_intr(vq); // XXX luigi
+ return 0; // XXX or 1 ?
+ }
+#endif /* DEV_NETMAP */
+
+ while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
+ m = txhdr->vth_mbuf;
+ deq++;
+
+ txq->vtntx_stats.vtxs_opackets++;
+ txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len;
+ if (m->m_flags & M_MCAST)
+ txq->vtntx_stats.vtxs_omcasts++;
+
+ m_freem(m);
+ uma_zfree(vtnet_tx_header_zone, txhdr);
+ }
+
+ if (virtqueue_empty(vq))
+ txq->vtntx_watchdog = 0;
+
+ return (deq);
+}
+
+static void
+vtnet_tx_vq_intr(void *xtxq)
+{
+ struct vtnet_softc *sc;
+ struct vtnet_txq *txq;
+ struct ifnet *ifp;
+
+ txq = xtxq;
+ sc = txq->vtntx_sc;
+ ifp = sc->vtnet_ifp;
+
+ if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) {
+ /*
+ * Ignore this interrupt. Either this is a spurious interrupt
+ * or multiqueue without per-VQ MSIX so every queue needs to
+ * be polled (a brain dead configuration we could try harder
+ * to avoid).
+ */
+ vtnet_txq_disable_intr(txq);
+ return;
+ }
+
+ VTNET_TXQ_LOCK(txq);
+
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
- vtnet_enable_tx_intr(sc);
- VTNET_UNLOCK(sc);
+ VTNET_TXQ_UNLOCK(txq);
return;
}
- vtnet_txeof(sc);
+ vtnet_txq_eof(txq);
+ vtnet_txq_start(txq);
- if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
- vtnet_start_locked(ifp);
+ VTNET_TXQ_UNLOCK(txq);
+}
- if (vtnet_enable_tx_intr(sc) != 0) {
- vtnet_disable_tx_intr(sc);
- sc->vtnet_stats.tx_task_rescheduled++;
- VTNET_UNLOCK(sc);
- goto again;
+static void
+vtnet_tx_start_all(struct vtnet_softc *sc)
+{
+ struct vtnet_txq *txq;
+ int i;
+
+ VTNET_CORE_LOCK_ASSERT(sc);
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
+ txq = &sc->vtnet_txqs[i];
+
+ VTNET_TXQ_LOCK(txq);
+ vtnet_txq_start(txq);
+ VTNET_TXQ_UNLOCK(txq);
}
+}
- VTNET_UNLOCK(sc);
+#ifndef VTNET_LEGACY_TX
+static void
+vtnet_qflush(struct ifnet *ifp)
+{
+ struct vtnet_softc *sc;
+ struct vtnet_txq *txq;
+ struct mbuf *m;
+ int i;
+
+ sc = ifp->if_softc;
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
+ txq = &sc->vtnet_txqs[i];
+
+ VTNET_TXQ_LOCK(txq);
+ while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL)
+ m_freem(m);
+ VTNET_TXQ_UNLOCK(txq);
+ }
+
+ if_qflush(ifp);
}
+#endif
+static int
+vtnet_watchdog(struct vtnet_txq *txq)
+{
+ struct ifnet *ifp;
+
+ ifp = txq->vtntx_sc->vtnet_ifp;
+
+ VTNET_TXQ_LOCK(txq);
+ if (txq->vtntx_watchdog == 1) {
+ /*
+ * Only drain completed frames if the watchdog is about to
+ * expire. If any frames were drained, there may be enough
+ * free descriptors now available to transmit queued frames.
+ * In that case, the timer will immediately be decremented
+ * below, but the timeout is generous enough that should not
+ * be a problem.
+ */
+ if (vtnet_txq_eof(txq) != 0)
+ vtnet_txq_start(txq);
+ }
+
+ if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) {
+ VTNET_TXQ_UNLOCK(txq);
+ return (0);
+ }
+ VTNET_TXQ_UNLOCK(txq);
+
+ if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id);
+ return (1);
+}
+
static void
+vtnet_rxq_accum_stats(struct vtnet_rxq *rxq, struct vtnet_rxq_stats *accum)
+{
+ struct vtnet_rxq_stats *st;
+
+ st = &rxq->vtnrx_stats;
+
+ accum->vrxs_ipackets += st->vrxs_ipackets;
+ accum->vrxs_ibytes += st->vrxs_ibytes;
+ accum->vrxs_iqdrops += st->vrxs_iqdrops;
+ accum->vrxs_csum += st->vrxs_csum;
+ accum->vrxs_csum_failed += st->vrxs_csum_failed;
+ accum->vrxs_rescheduled += st->vrxs_rescheduled;
+}
+
+static void
+vtnet_txq_accum_stats(struct vtnet_txq *txq, struct vtnet_txq_stats *accum)
+{
+ struct vtnet_txq_stats *st;
+
+ st = &txq->vtntx_stats;
+
+ accum->vtxs_opackets += st->vtxs_opackets;
+ accum->vtxs_obytes += st->vtxs_obytes;
+ accum->vtxs_csum += st->vtxs_csum;
+ accum->vtxs_tso += st->vtxs_tso;
+ accum->vtxs_rescheduled += st->vtxs_rescheduled;
+}
+
+static void
+vtnet_accumulate_stats(struct vtnet_softc *sc)
+{
+ struct ifnet *ifp;
+ struct vtnet_statistics *st;
+ struct vtnet_rxq_stats rxaccum;
+ struct vtnet_txq_stats txaccum;
+ int i;
+
+ ifp = sc->vtnet_ifp;
+ st = &sc->vtnet_stats;
+ bzero(&rxaccum, sizeof(struct vtnet_rxq_stats));
+ bzero(&txaccum, sizeof(struct vtnet_txq_stats));
+
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
+ vtnet_rxq_accum_stats(&sc->vtnet_rxqs[i], &rxaccum);
+ vtnet_txq_accum_stats(&sc->vtnet_txqs[i], &txaccum);
+ }
+
+ st->rx_csum_offloaded = rxaccum.vrxs_csum;
+ st->rx_csum_failed = rxaccum.vrxs_csum_failed;
+ st->rx_task_rescheduled = rxaccum.vrxs_rescheduled;
+ st->tx_csum_offloaded = txaccum.vtxs_csum;
+ st->tx_tso_offloaded = txaccum.vtxs_tso;
+ st->tx_task_rescheduled = txaccum.vtxs_rescheduled;
+
+ /*
+ * With the exception of if_ierrors, these ifnet statistics are
+ * only updated in the driver, so just set them to our accumulated
+ * values. if_ierrors is updated in ether_input() for malformed
+ * frames that we should have already discarded.
+ */
+ ifp->if_ipackets = rxaccum.vrxs_ipackets;
+ ifp->if_iqdrops = rxaccum.vrxs_iqdrops;
+ ifp->if_ierrors = rxaccum.vrxs_ierrors;
+ ifp->if_opackets = txaccum.vtxs_opackets;
+#ifndef VTNET_LEGACY_TX
+ ifp->if_obytes = txaccum.vtxs_obytes;
+ ifp->if_omcasts = txaccum.vtxs_omcasts;
+#endif
+}
+
+static void
+vtnet_tick(void *xsc)
+{
+ struct vtnet_softc *sc;
+ struct ifnet *ifp;
+ int i, timedout;
+
+ sc = xsc;
+ ifp = sc->vtnet_ifp;
+ timedout = 0;
+
+ VTNET_CORE_LOCK_ASSERT(sc);
+ vtnet_accumulate_stats(sc);
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
+ timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]);
+
+ if (timedout != 0) {
+ ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+ vtnet_init_locked(sc);
+ } else
+ callout_schedule(&sc->vtnet_tick_ch, hz);
+}
+
+static void
+vtnet_start_taskqueues(struct vtnet_softc *sc)
+{
+ device_t dev;
+ struct vtnet_rxq *rxq;
+ struct vtnet_txq *txq;
+ int i, error;
+
+ dev = sc->vtnet_dev;
+
+ /*
+ * Errors here are very difficult to recover from - we cannot
+ * easily fail because, if this is during boot, we will hang
+ * when freeing any successfully started taskqueues because
+ * the scheduler isn't up yet.
+ *
+ * Most drivers just ignore the return value - it only fails
+ * with ENOMEM so an error is not likely.
+ */
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
+ rxq = &sc->vtnet_rxqs[i];
+ error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET,
+ "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id);
+ if (error) {
+ device_printf(dev, "failed to start rx taskq %d\n",
+ rxq->vtnrx_id);
+ }
+
+ txq = &sc->vtnet_txqs[i];
+ error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET,
+ "%s txq %d", device_get_nameunit(dev), txq->vtntx_id);
+ if (error) {
+ device_printf(dev, "failed to start tx taskq %d\n",
+ txq->vtntx_id);
+ }
+ }
+}
+
+static void
+vtnet_free_taskqueues(struct vtnet_softc *sc)
+{
+ struct vtnet_rxq *rxq;
+ struct vtnet_txq *txq;
+ int i;
+
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
+ rxq = &sc->vtnet_rxqs[i];
+ if (rxq->vtnrx_tq != NULL) {
+ taskqueue_free(rxq->vtnrx_tq);
+ rxq->vtnrx_vq = NULL;
+ }
+
+ txq = &sc->vtnet_txqs[i];
+ if (txq->vtntx_tq != NULL) {
+ taskqueue_free(txq->vtntx_tq);
+ txq->vtntx_tq = NULL;
+ }
+ }
+}
+
+static void
+vtnet_drain_taskqueues(struct vtnet_softc *sc)
+{
+ struct vtnet_rxq *rxq;
+ struct vtnet_txq *txq;
+ int i;
+
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
+ rxq = &sc->vtnet_rxqs[i];
+ if (rxq->vtnrx_tq != NULL)
+ taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
+
+ txq = &sc->vtnet_txqs[i];
+ if (txq->vtntx_tq != NULL) {
+ taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask);
+#ifndef VTNET_LEGACY_TX
+ taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask);
+#endif
+ }
+ }
+}
+
+static void
+vtnet_drain_rxtx_queues(struct vtnet_softc *sc)
+{
+ struct vtnet_rxq *rxq;
+ struct vtnet_txq *txq;
+ int i;
+
+#ifdef DEV_NETMAP
+ if (nm_native_on(NA(sc->vtnet_ifp)))
+ return;
+#endif /* DEV_NETMAP */
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
+ rxq = &sc->vtnet_rxqs[i];
+ vtnet_rxq_free_mbufs(rxq);
+
+ txq = &sc->vtnet_txqs[i];
+ vtnet_txq_free_mbufs(txq);
+ }
+}
+
+static void
+vtnet_stop_rendezvous(struct vtnet_softc *sc)
+{
+ struct vtnet_rxq *rxq;
+ struct vtnet_txq *txq;
+ int i;
+
+ /*
+ * Lock and unlock the per-queue mutex so we known the stop
+ * state is visible. Doing only the active queues should be
+ * sufficient, but it does not cost much extra to do all the
+ * queues. Note we hold the core mutex here too.
+ */
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
+ rxq = &sc->vtnet_rxqs[i];
+ VTNET_RXQ_LOCK(rxq);
+ VTNET_RXQ_UNLOCK(rxq);
+
+ txq = &sc->vtnet_txqs[i];
+ VTNET_TXQ_LOCK(txq);
+ VTNET_TXQ_UNLOCK(txq);
+ }
+}
+
+static void
vtnet_stop(struct vtnet_softc *sc)
{
device_t dev;
@@ -2072,38 +2820,47 @@
dev = sc->vtnet_dev;
ifp = sc->vtnet_ifp;
- VTNET_LOCK_ASSERT(sc);
+ VTNET_CORE_LOCK_ASSERT(sc);
- sc->vtnet_watchdog_timer = 0;
+ ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+ sc->vtnet_link_active = 0;
callout_stop(&sc->vtnet_tick_ch);
- ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
- vtnet_disable_rx_intr(sc);
- vtnet_disable_tx_intr(sc);
+ /* Only advisory. */
+ vtnet_disable_interrupts(sc);
/*
- * Stop the host VirtIO adapter. Note this will reset the host
- * adapter's state back to the pre-initialized state, so in
- * order to make the device usable again, we must drive it
- * through virtio_reinit() and virtio_reinit_complete().
+ * Stop the host adapter. This resets it to the pre-initialized
+ * state. It will not generate any interrupts until after it is
+ * reinitialized.
*/
virtio_stop(dev);
+ vtnet_stop_rendezvous(sc);
- sc->vtnet_flags &= ~VTNET_FLAG_LINK;
-
- vtnet_free_rx_mbufs(sc);
- vtnet_free_tx_mbufs(sc);
+ /* Free any mbufs left in the virtqueues. */
+ vtnet_drain_rxtx_queues(sc);
}
static int
-vtnet_reinit(struct vtnet_softc *sc)
+vtnet_virtio_reinit(struct vtnet_softc *sc)
{
+ device_t dev;
struct ifnet *ifp;
uint64_t features;
+ int mask, error;
+ dev = sc->vtnet_dev;
ifp = sc->vtnet_ifp;
features = sc->vtnet_features;
+ mask = 0;
+#if defined(INET)
+ mask |= IFCAP_RXCSUM;
+#endif
+#if defined (INET6)
+ mask |= IFCAP_RXCSUM_IPV6;
+#endif
+
/*
* Re-negotiate with the host, removing any disabled receive
* features. Transmit features are disabled only on our side
@@ -2110,8 +2867,13 @@
* via if_capenable and if_hwassist.
*/
- if (ifp->if_capabilities & IFCAP_RXCSUM) {
- if ((ifp->if_capenable & IFCAP_RXCSUM) == 0)
+ if (ifp->if_capabilities & mask) {
+ /*
+ * We require both IPv4 and IPv6 offloading to be enabled
+ * in order to negotiated it: VirtIO does not distinguish
+ * between the two.
+ */
+ if ((ifp->if_capenable & mask) != mask)
features &= ~VIRTIO_NET_F_GUEST_CSUM;
}
@@ -2125,86 +2887,207 @@
features &= ~VIRTIO_NET_F_CTRL_VLAN;
}
- return (virtio_reinit(sc->vtnet_dev, features));
+ error = virtio_reinit(dev, features);
+ if (error)
+ device_printf(dev, "virtio reinit error %d\n", error);
+
+ return (error);
}
static void
-vtnet_init_locked(struct vtnet_softc *sc)
+vtnet_init_rx_filters(struct vtnet_softc *sc)
{
+ struct ifnet *ifp;
+
+ ifp = sc->vtnet_ifp;
+
+ if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
+ /* Restore promiscuous and all-multicast modes. */
+ vtnet_rx_filter(sc);
+ /* Restore filtered MAC addresses. */
+ vtnet_rx_filter_mac(sc);
+ }
+
+ if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
+ vtnet_rx_filter_vlan(sc);
+}
+
+static int
+vtnet_init_rx_queues(struct vtnet_softc *sc)
+{
device_t dev;
- struct ifnet *ifp;
+ struct vtnet_rxq *rxq;
+ int i, clsize, error;
+
+ dev = sc->vtnet_dev;
+
+ /*
+ * Use the new cluster size if one has been set (via a MTU
+ * change). Otherwise, use the standard 2K clusters.
+ *
+ * BMV: It might make sense to use page sized clusters as
+ * the default (depending on the features negotiated).
+ */
+ if (sc->vtnet_rx_new_clsize != 0) {
+ clsize = sc->vtnet_rx_new_clsize;
+ sc->vtnet_rx_new_clsize = 0;
+ } else
+ clsize = MCLBYTES;
+
+ sc->vtnet_rx_clsize = clsize;
+ sc->vtnet_rx_nmbufs = VTNET_NEEDED_RX_MBUFS(sc, clsize);
+
+ KASSERT(sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS ||
+ sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs,
+ ("%s: too many rx mbufs %d for %d segments", __func__,
+ sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs));
+
+#ifdef DEV_NETMAP
+ if (vtnet_netmap_init_rx_buffers(sc))
+ return 0;
+#endif /* DEV_NETMAP */
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
+ rxq = &sc->vtnet_rxqs[i];
+
+ /* Hold the lock to satisfy asserts. */
+ VTNET_RXQ_LOCK(rxq);
+ error = vtnet_rxq_populate(rxq);
+ VTNET_RXQ_UNLOCK(rxq);
+
+ if (error) {
+ device_printf(dev,
+ "cannot allocate mbufs for Rx queue %d\n", i);
+ return (error);
+ }
+ }
+
+ return (0);
+}
+
+static int
+vtnet_init_tx_queues(struct vtnet_softc *sc)
+{
+ struct vtnet_txq *txq;
+ int i;
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
+ txq = &sc->vtnet_txqs[i];
+ txq->vtntx_watchdog = 0;
+ }
+
+ return (0);
+}
+
+static int
+vtnet_init_rxtx_queues(struct vtnet_softc *sc)
+{
int error;
+ error = vtnet_init_rx_queues(sc);
+ if (error)
+ return (error);
+
+ error = vtnet_init_tx_queues(sc);
+ if (error)
+ return (error);
+
+ return (0);
+}
+
+static void
+vtnet_set_active_vq_pairs(struct vtnet_softc *sc)
+{
+ device_t dev;
+ int npairs;
+
dev = sc->vtnet_dev;
- ifp = sc->vtnet_ifp;
- VTNET_LOCK_ASSERT(sc);
-
- if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+ if ((sc->vtnet_flags & VTNET_FLAG_MULTIQ) == 0) {
+ sc->vtnet_act_vq_pairs = 1;
return;
+ }
- /* Stop host's adapter, cancel any pending I/O. */
- vtnet_stop(sc);
+ npairs = sc->vtnet_requested_vq_pairs;
- /* Reinitialize the host device. */
- error = vtnet_reinit(sc);
- if (error) {
+ if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) {
device_printf(dev,
- "reinitialization failed, stopping device...\n");
- vtnet_stop(sc);
- return;
+ "cannot set active queue pairs to %d\n", npairs);
+ npairs = 1;
}
- /* Update host with assigned MAC address. */
+ sc->vtnet_act_vq_pairs = npairs;
+}
+
+static int
+vtnet_reinit(struct vtnet_softc *sc)
+{
+ struct ifnet *ifp;
+ int error;
+
+ ifp = sc->vtnet_ifp;
+
+ /* Use the current MAC address. */
bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
vtnet_set_hwaddr(sc);
+ vtnet_set_active_vq_pairs(sc);
+
ifp->if_hwassist = 0;
if (ifp->if_capenable & IFCAP_TXCSUM)
ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;
+ if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
+ ifp->if_hwassist |= VTNET_CSUM_OFFLOAD_IPV6;
if (ifp->if_capenable & IFCAP_TSO4)
- ifp->if_hwassist |= CSUM_TSO;
+ ifp->if_hwassist |= CSUM_IP_TSO;
+ if (ifp->if_capenable & IFCAP_TSO6)
+ ifp->if_hwassist |= CSUM_IP6_TSO;
- error = vtnet_init_rx_vq(sc);
- if (error) {
- device_printf(dev,
- "cannot allocate mbufs for Rx virtqueue\n");
- vtnet_stop(sc);
- return;
- }
+ if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
+ vtnet_init_rx_filters(sc);
- if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
- if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
- /* Restore promiscuous and all-multicast modes. */
- vtnet_rx_filter(sc);
+ error = vtnet_init_rxtx_queues(sc);
+ if (error)
+ return (error);
- /* Restore filtered MAC addresses. */
- vtnet_rx_filter_mac(sc);
- }
+ vtnet_enable_interrupts(sc);
+ ifp->if_drv_flags |= IFF_DRV_RUNNING;
- /* Restore VLAN filters. */
- if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
- vtnet_rx_filter_vlan(sc);
- }
+ return (0);
+}
-#ifdef DEVICE_POLLING
- if (ifp->if_capenable & IFCAP_POLLING) {
- vtnet_disable_rx_intr(sc);
- vtnet_disable_tx_intr(sc);
- } else
-#endif
- {
- vtnet_enable_rx_intr(sc);
- vtnet_enable_tx_intr(sc);
- }
+static void
+vtnet_init_locked(struct vtnet_softc *sc)
+{
+ device_t dev;
+ struct ifnet *ifp;
- ifp->if_drv_flags |= IFF_DRV_RUNNING;
- ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+ dev = sc->vtnet_dev;
+ ifp = sc->vtnet_ifp;
+ VTNET_CORE_LOCK_ASSERT(sc);
+
+ if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+ return;
+
+ vtnet_stop(sc);
+
+ /* Reinitialize with the host. */
+ if (vtnet_virtio_reinit(sc) != 0)
+ goto fail;
+
+ if (vtnet_reinit(sc) != 0)
+ goto fail;
+
virtio_reinit_complete(dev);
vtnet_update_link_status(sc);
callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);
+
+ return;
+
+fail:
+ vtnet_stop(sc);
}
static void
@@ -2214,97 +3097,149 @@
sc = xsc;
- VTNET_LOCK(sc);
+#ifdef DEV_NETMAP
+ if (!NA(sc->vtnet_ifp)) {
+ D("try to attach again");
+ vtnet_netmap_attach(sc);
+ }
+#endif /* DEV_NETMAP */
+
+ VTNET_CORE_LOCK(sc);
vtnet_init_locked(sc);
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
}
static void
+vtnet_free_ctrl_vq(struct vtnet_softc *sc)
+{
+ struct virtqueue *vq;
+
+ vq = sc->vtnet_ctrl_vq;
+
+ /*
+ * The control virtqueue is only polled and therefore it should
+ * already be empty.
+ */
+ KASSERT(virtqueue_empty(vq),
+ ("%s: ctrl vq %p not empty", __func__, vq));
+}
+
+static void
vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
struct sglist *sg, int readable, int writable)
{
struct virtqueue *vq;
- void *c;
vq = sc->vtnet_ctrl_vq;
- VTNET_LOCK_ASSERT(sc);
+ VTNET_CORE_LOCK_ASSERT(sc);
KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
- ("no control virtqueue"));
- KASSERT(virtqueue_empty(vq),
- ("control command already enqueued"));
+ ("%s: CTRL_VQ feature not negotiated", __func__));
+ if (!virtqueue_empty(vq))
+ return;
if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
return;
- virtqueue_notify(vq);
-
/*
- * Poll until the command is complete. Previously, we would
- * sleep until the control virtqueue interrupt handler woke
- * us up, but dropping the VTNET_MTX leads to serialization
- * difficulties.
- *
- * Furthermore, it appears QEMU/KVM only allocates three MSIX
- * vectors. Two of those vectors are needed for the Rx and Tx
- * virtqueues. We do not support sharing both a Vq and config
- * changed notification on the same MSIX vector.
+ * Poll for the response, but the command is likely already
+ * done when we return from the notify.
*/
- c = virtqueue_poll(vq, NULL);
- KASSERT(c == cookie, ("unexpected control command response"));
+ virtqueue_notify(vq);
+ virtqueue_poll(vq, NULL);
}
-static void
-vtnet_rx_filter(struct vtnet_softc *sc)
+static int
+vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
{
- device_t dev;
- struct ifnet *ifp;
+ struct virtio_net_ctrl_hdr hdr __aligned(2);
+ struct sglist_seg segs[3];
+ struct sglist sg;
+ uint8_t ack;
+ int error;
- dev = sc->vtnet_dev;
- ifp = sc->vtnet_ifp;
+ hdr.class = VIRTIO_NET_CTRL_MAC;
+ hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
+ ack = VIRTIO_NET_ERR;
- VTNET_LOCK_ASSERT(sc);
- KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
- ("CTRL_RX feature not negotiated"));
+ sglist_init(&sg, 3, segs);
+ error = 0;
+ error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append(&sg, hwaddr, ETHER_ADDR_LEN);
+ error |= sglist_append(&sg, &ack, sizeof(uint8_t));
+ KASSERT(error == 0 && sg.sg_nseg == 3,
+ ("%s: error %d adding set MAC msg to sglist", __func__, error));
- if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
- device_printf(dev, "cannot %s promiscuous mode\n",
- ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
+ vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
- if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
- device_printf(dev, "cannot %s all-multicast mode\n",
- ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
+ return (ack == VIRTIO_NET_OK ? 0 : EIO);
}
static int
-vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
+vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs)
{
- struct virtio_net_ctrl_hdr hdr;
struct sglist_seg segs[3];
struct sglist sg;
- uint8_t onoff, ack;
+ struct {
+ struct virtio_net_ctrl_hdr hdr;
+ uint8_t pad1;
+ struct virtio_net_ctrl_mq mq;
+ uint8_t pad2;
+ uint8_t ack;
+ } s __aligned(2);
int error;
- if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
- return (ENOTSUP);
+ s.hdr.class = VIRTIO_NET_CTRL_MQ;
+ s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
+ s.mq.virtqueue_pairs = npairs;
+ s.ack = VIRTIO_NET_ERR;
+ sglist_init(&sg, 3, segs);
error = 0;
+ error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq));
+ error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
+ KASSERT(error == 0 && sg.sg_nseg == 3,
+ ("%s: error %d adding MQ message to sglist", __func__, error));
- hdr.class = VIRTIO_NET_CTRL_RX;
- hdr.cmd = cmd;
- onoff = !!on;
- ack = VIRTIO_NET_ERR;
+ vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
+ return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
+}
+
+static int
+vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
+{
+ struct sglist_seg segs[3];
+ struct sglist sg;
+ struct {
+ struct virtio_net_ctrl_hdr hdr;
+ uint8_t pad1;
+ uint8_t onoff;
+ uint8_t pad2;
+ uint8_t ack;
+ } s __aligned(2);
+ int error;
+
+ KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
+ ("%s: CTRL_RX feature not negotiated", __func__));
+
+ s.hdr.class = VIRTIO_NET_CTRL_RX;
+ s.hdr.cmd = cmd;
+ s.onoff = !!on;
+ s.ack = VIRTIO_NET_ERR;
+
sglist_init(&sg, 3, segs);
- error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &onoff, sizeof(uint8_t));
- error |= sglist_append(&sg, &ack, sizeof(uint8_t));
+ error = 0;
+ error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
+ error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
KASSERT(error == 0 && sg.sg_nseg == 3,
- ("error adding Rx filter message to sglist"));
+ ("%s: error %d adding Rx message to sglist", __func__, error));
- vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
+ vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
- return (ack == VIRTIO_NET_OK ? 0 : EIO);
+ return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}
static int
@@ -2321,10 +3256,52 @@
return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
}
+/*
+ * The device defaults to promiscuous mode for backwards compatibility.
+ * Turn it off at attach time if possible.
+ */
static void
+vtnet_attach_disable_promisc(struct vtnet_softc *sc)
+{
+ struct ifnet *ifp;
+
+ ifp = sc->vtnet_ifp;
+
+ VTNET_CORE_LOCK(sc);
+ if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) {
+ ifp->if_flags |= IFF_PROMISC;
+ } else if (vtnet_set_promisc(sc, 0) != 0) {
+ ifp->if_flags |= IFF_PROMISC;
+ device_printf(sc->vtnet_dev,
+ "cannot disable default promiscuous mode\n");
+ }
+ VTNET_CORE_UNLOCK(sc);
+}
+
+static void
+vtnet_rx_filter(struct vtnet_softc *sc)
+{
+ device_t dev;
+ struct ifnet *ifp;
+
+ dev = sc->vtnet_dev;
+ ifp = sc->vtnet_ifp;
+
+ VTNET_CORE_LOCK_ASSERT(sc);
+
+ if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
+ device_printf(dev, "cannot %s promiscuous mode\n",
+ ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
+
+ if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
+ device_printf(dev, "cannot %s all-multicast mode\n",
+ ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
+}
+
+static void
vtnet_rx_filter_mac(struct vtnet_softc *sc)
{
- struct virtio_net_ctrl_hdr hdr;
+ struct virtio_net_ctrl_hdr hdr __aligned(2);
struct vtnet_mac_filter *filter;
struct sglist_seg segs[4];
struct sglist sg;
@@ -2340,11 +3317,10 @@
mcnt = 0;
promisc = 0;
allmulti = 0;
- error = 0;
- VTNET_LOCK_ASSERT(sc);
+ VTNET_CORE_LOCK_ASSERT(sc);
KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
- ("CTRL_RX feature not negotiated"));
+ ("%s: CTRL_RX feature not negotiated", __func__));
/* Unicast MAC addresses: */
if_addr_rlock(ifp);
@@ -2351,8 +3327,13 @@
TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_LINK)
continue;
- else if (ucnt == VTNET_MAX_MAC_ENTRIES)
+ else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
+ sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0)
+ continue;
+ else if (ucnt == VTNET_MAX_MAC_ENTRIES) {
+ promisc = 1;
break;
+ }
bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr),
&filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN);
@@ -2360,10 +3341,8 @@
}
if_addr_runlock(ifp);
- if (ucnt >= VTNET_MAX_MAC_ENTRIES) {
- promisc = 1;
+ if (promisc != 0) {
filter->vmf_unicast.nentries = 0;
-
if_printf(ifp, "more than %d MAC addresses assigned, "
"falling back to promiscuous mode\n",
VTNET_MAX_MAC_ENTRIES);
@@ -2375,8 +3354,10 @@
TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
if (ifma->ifma_addr->sa_family != AF_LINK)
continue;
- else if (mcnt == VTNET_MAX_MAC_ENTRIES)
+ else if (mcnt == VTNET_MAX_MAC_ENTRIES) {
+ allmulti = 1;
break;
+ }
bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
&filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN);
@@ -2384,10 +3365,8 @@
}
if_maddr_runlock(ifp);
- if (mcnt >= VTNET_MAX_MAC_ENTRIES) {
- allmulti = 1;
+ if (allmulti != 0) {
filter->vmf_multicast.nentries = 0;
-
if_printf(ifp, "more than %d multicast MAC addresses "
"assigned, falling back to all-multicast mode\n",
VTNET_MAX_MAC_ENTRIES);
@@ -2394,7 +3373,7 @@
} else
filter->vmf_multicast.nentries = mcnt;
- if (promisc && allmulti)
+ if (promisc != 0 && allmulti != 0)
goto out;
hdr.class = VIRTIO_NET_CTRL_MAC;
@@ -2402,6 +3381,7 @@
ack = VIRTIO_NET_ERR;
sglist_init(&sg, 4, segs);
+ error = 0;
error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
error |= sglist_append(&sg, &filter->vmf_unicast,
sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN);
@@ -2409,7 +3389,7 @@
sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN);
error |= sglist_append(&sg, &ack, sizeof(uint8_t));
KASSERT(error == 0 && sg.sg_nseg == 4,
- ("error adding MAC filtering message to sglist"));
+ ("%s: error %d adding MAC filter msg to sglist", __func__, error));
vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
@@ -2417,111 +3397,99 @@
if_printf(ifp, "error setting host MAC filter table\n");
out:
- if (promisc)
- if (vtnet_set_promisc(sc, 1) != 0)
- if_printf(ifp, "cannot enable promiscuous mode\n");
- if (allmulti)
- if (vtnet_set_allmulti(sc, 1) != 0)
- if_printf(ifp, "cannot enable all-multicast mode\n");
+ if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0)
+ if_printf(ifp, "cannot enable promiscuous mode\n");
+ if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0)
+ if_printf(ifp, "cannot enable all-multicast mode\n");
}
static int
vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
{
- struct virtio_net_ctrl_hdr hdr;
struct sglist_seg segs[3];
struct sglist sg;
- uint8_t ack;
+ struct {
+ struct virtio_net_ctrl_hdr hdr;
+ uint8_t pad1;
+ uint16_t tag;
+ uint8_t pad2;
+ uint8_t ack;
+ } s __aligned(2);
int error;
- hdr.class = VIRTIO_NET_CTRL_VLAN;
- hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
- ack = VIRTIO_NET_ERR;
- error = 0;
+ s.hdr.class = VIRTIO_NET_CTRL_VLAN;
+ s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
+ s.tag = tag;
+ s.ack = VIRTIO_NET_ERR;
sglist_init(&sg, 3, segs);
- error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &tag, sizeof(uint16_t));
- error |= sglist_append(&sg, &ack, sizeof(uint8_t));
+ error = 0;
+ error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
+ error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
KASSERT(error == 0 && sg.sg_nseg == 3,
- ("error adding VLAN control message to sglist"));
+ ("%s: error %d adding VLAN message to sglist", __func__, error));
- vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
+ vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
- return (ack == VIRTIO_NET_OK ? 0 : EIO);
+ return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}
static void
vtnet_rx_filter_vlan(struct vtnet_softc *sc)
{
- device_t dev;
- uint32_t w, mask;
+ uint32_t w;
uint16_t tag;
- int i, nvlans, error;
+ int i, bit;
- VTNET_LOCK_ASSERT(sc);
+ VTNET_CORE_LOCK_ASSERT(sc);
KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
- ("VLAN_FILTER feature not negotiated"));
+ ("%s: VLAN_FILTER feature not negotiated", __func__));
- dev = sc->vtnet_dev;
- nvlans = sc->vtnet_nvlans;
- error = 0;
+ /* Enable the filter for each configured VLAN. */
+ for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) {
+ w = sc->vtnet_vlan_filter[i];
- /* Enable filtering for each configured VLAN. */
- for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) {
- w = sc->vtnet_vlan_shadow[i];
- for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) {
- if ((w & mask) != 0) {
- w &= ~mask;
- nvlans--;
- if (vtnet_exec_vlan_filter(sc, 1, tag) != 0)
- error++;
+ while ((bit = ffs(w) - 1) != -1) {
+ w &= ~(1 << bit);
+ tag = sizeof(w) * CHAR_BIT * i + bit;
+
+ if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) {
+ device_printf(sc->vtnet_dev,
+ "cannot enable VLAN %d filter\n", tag);
}
}
}
-
- KASSERT(nvlans == 0, ("VLAN count incorrect"));
- if (error)
- device_printf(dev, "cannot restore VLAN filter table\n");
}
static void
-vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
+vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
{
struct ifnet *ifp;
int idx, bit;
- KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
- ("VLAN_FILTER feature not negotiated"));
-
- if ((tag == 0) || (tag > 4095))
- return;
-
ifp = sc->vtnet_ifp;
idx = (tag >> 5) & 0x7F;
bit = tag & 0x1F;
- VTNET_LOCK(sc);
+ if (tag == 0 || tag > 4095)
+ return;
- /* Update shadow VLAN table. */
- if (add) {
- sc->vtnet_nvlans++;
- sc->vtnet_vlan_shadow[idx] |= (1 << bit);
- } else {
- sc->vtnet_nvlans--;
- sc->vtnet_vlan_shadow[idx] &= ~(1 << bit);
- }
+ VTNET_CORE_LOCK(sc);
- if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
- if (vtnet_exec_vlan_filter(sc, add, tag) != 0) {
- device_printf(sc->vtnet_dev,
- "cannot %s VLAN %d %s the host filter table\n",
- add ? "add" : "remove", tag,
- add ? "to" : "from");
- }
+ if (add)
+ sc->vtnet_vlan_filter[idx] |= (1 << bit);
+ else
+ sc->vtnet_vlan_filter[idx] &= ~(1 << bit);
+
+ if (ifp->if_capenable & IFCAP_VLAN_HWFILTER &&
+ vtnet_exec_vlan_filter(sc, add, tag) != 0) {
+ device_printf(sc->vtnet_dev,
+ "cannot %s VLAN %d %s the host filter table\n",
+ add ? "add" : "remove", tag, add ? "to" : "from");
}
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
}
static void
@@ -2531,7 +3499,7 @@
if (ifp->if_softc != arg)
return;
- vtnet_set_vlan_filter(arg, 1, tag);
+ vtnet_update_vlan_filter(arg, 1, tag);
}
static void
@@ -2541,10 +3509,50 @@
if (ifp->if_softc != arg)
return;
- vtnet_set_vlan_filter(arg, 0, tag);
+ vtnet_update_vlan_filter(arg, 0, tag);
}
static int
+vtnet_is_link_up(struct vtnet_softc *sc)
+{
+ device_t dev;
+ struct ifnet *ifp;
+ uint16_t status;
+
+ dev = sc->vtnet_dev;
+ ifp = sc->vtnet_ifp;
+
+ if ((ifp->if_capabilities & IFCAP_LINKSTATE) == 0)
+ status = VIRTIO_NET_S_LINK_UP;
+ else
+ status = virtio_read_dev_config_2(dev,
+ offsetof(struct virtio_net_config, status));
+
+ return ((status & VIRTIO_NET_S_LINK_UP) != 0);
+}
+
+static void
+vtnet_update_link_status(struct vtnet_softc *sc)
+{
+ struct ifnet *ifp;
+ int link;
+
+ ifp = sc->vtnet_ifp;
+
+ VTNET_CORE_LOCK_ASSERT(sc);
+ link = vtnet_is_link_up(sc);
+
+ /* Notify if the link status has changed. */
+ if (link != 0 && sc->vtnet_link_active == 0) {
+ sc->vtnet_link_active = 1;
+ if_link_state_change(ifp, LINK_STATE_UP);
+ } else if (link == 0 && sc->vtnet_link_active != 0) {
+ sc->vtnet_link_active = 0;
+ if_link_state_change(ifp, LINK_STATE_DOWN);
+ }
+}
+
+static int
vtnet_ifmedia_upd(struct ifnet *ifp)
{
struct vtnet_softc *sc;
@@ -2569,112 +3577,401 @@
ifmr->ifm_status = IFM_AVALID;
ifmr->ifm_active = IFM_ETHER;
- VTNET_LOCK(sc);
+ VTNET_CORE_LOCK(sc);
if (vtnet_is_link_up(sc) != 0) {
ifmr->ifm_status |= IFM_ACTIVE;
ifmr->ifm_active |= VTNET_MEDIATYPE;
} else
ifmr->ifm_active |= IFM_NONE;
- VTNET_UNLOCK(sc);
+ VTNET_CORE_UNLOCK(sc);
}
static void
-vtnet_add_statistics(struct vtnet_softc *sc)
+vtnet_set_hwaddr(struct vtnet_softc *sc)
{
device_t dev;
- struct vtnet_statistics *stats;
- struct sysctl_ctx_list *ctx;
+ int i;
+
+ dev = sc->vtnet_dev;
+
+ if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) {
+ if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0)
+ device_printf(dev, "unable to set MAC address\n");
+ } else if (sc->vtnet_flags & VTNET_FLAG_MAC) {
+ for (i = 0; i < ETHER_ADDR_LEN; i++) {
+ virtio_write_dev_config_1(dev,
+ offsetof(struct virtio_net_config, mac) + i,
+ sc->vtnet_hwaddr[i]);
+ }
+ }
+}
+
+static void
+vtnet_get_hwaddr(struct vtnet_softc *sc)
+{
+ device_t dev;
+ int i;
+
+ dev = sc->vtnet_dev;
+
+ if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) {
+ /*
+ * Generate a random locally administered unicast address.
+ *
+ * It would be nice to generate the same MAC address across
+ * reboots, but it seems all the hosts currently available
+ * support the MAC feature, so this isn't too important.
+ */
+ sc->vtnet_hwaddr[0] = 0xB2;
+ arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
+ vtnet_set_hwaddr(sc);
+ return;
+ }
+
+ for (i = 0; i < ETHER_ADDR_LEN; i++) {
+ sc->vtnet_hwaddr[i] = virtio_read_dev_config_1(dev,
+ offsetof(struct virtio_net_config, mac) + i);
+ }
+}
+
+static void
+vtnet_vlan_tag_remove(struct mbuf *m)
+{
+ struct ether_vlan_header *evh;
+
+ evh = mtod(m, struct ether_vlan_header *);
+ m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
+ m->m_flags |= M_VLANTAG;
+
+ /* Strip the 802.1Q header. */
+ bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
+ ETHER_HDR_LEN - ETHER_TYPE_LEN);
+ m_adj(m, ETHER_VLAN_ENCAP_LEN);
+}
+
+static void
+vtnet_set_rx_process_limit(struct vtnet_softc *sc)
+{
+ int limit;
+
+ limit = vtnet_tunable_int(sc, "rx_process_limit",
+ vtnet_rx_process_limit);
+ if (limit < 0)
+ limit = INT_MAX;
+ sc->vtnet_rx_process_limit = limit;
+}
+
+static void
+vtnet_set_tx_intr_threshold(struct vtnet_softc *sc)
+{
+ device_t dev;
+ int size, thresh;
+
+ dev = sc->vtnet_dev;
+ size = virtqueue_size(sc->vtnet_txqs[0].vtntx_vq);
+
+ /*
+ * The Tx interrupt is disabled until the queue free count falls
+ * below our threshold. Completed frames are drained from the Tx
+ * virtqueue before transmitting new frames and in the watchdog
+ * callout, so the frequency of Tx interrupts is greatly reduced,
+ * at the cost of not freeing mbufs as quickly as they otherwise
+ * would be.
+ *
+ * N.B. We assume all the Tx queues are the same size.
+ */
+ thresh = size / 4;
+
+ /*
+ * Without indirect descriptors, leave enough room for the most
+ * segments we handle.
+ */
+ if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 &&
+ thresh < sc->vtnet_tx_nsegs)
+ thresh = sc->vtnet_tx_nsegs;
+
+ sc->vtnet_tx_intr_thresh = thresh;
+}
+
+static void
+vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx,
+ struct sysctl_oid_list *child, struct vtnet_rxq *rxq)
+{
+ struct sysctl_oid *node;
+ struct sysctl_oid_list *list;
+ struct vtnet_rxq_stats *stats;
+ char namebuf[16];
+
+ snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id);
+ node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
+ CTLFLAG_RD, NULL, "Receive Queue");
+ list = SYSCTL_CHILDREN(node);
+
+ stats = &rxq->vtnrx_stats;
+
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD,
+ &stats->vrxs_ipackets, "Receive packets");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD,
+ &stats->vrxs_ibytes, "Receive bytes");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD,
+ &stats->vrxs_iqdrops, "Receive drops");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD,
+ &stats->vrxs_ierrors, "Receive errors");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
+ &stats->vrxs_csum, "Receive checksum offloaded");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD,
+ &stats->vrxs_csum_failed, "Receive checksum offload failed");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
+ &stats->vrxs_rescheduled,
+ "Receive interrupt handler rescheduled");
+}
+
+static void
+vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx,
+ struct sysctl_oid_list *child, struct vtnet_txq *txq)
+{
+ struct sysctl_oid *node;
+ struct sysctl_oid_list *list;
+ struct vtnet_txq_stats *stats;
+ char namebuf[16];
+
+ snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id);
+ node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
+ CTLFLAG_RD, NULL, "Transmit Queue");
+ list = SYSCTL_CHILDREN(node);
+
+ stats = &txq->vtntx_stats;
+
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD,
+ &stats->vtxs_opackets, "Transmit packets");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD,
+ &stats->vtxs_obytes, "Transmit bytes");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD,
+ &stats->vtxs_omcasts, "Transmit multicasts");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
+ &stats->vtxs_csum, "Transmit checksum offloaded");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
+ &stats->vtxs_tso, "Transmit segmentation offloaded");
+ SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
+ &stats->vtxs_rescheduled,
+ "Transmit interrupt handler rescheduled");
+}
+
+static void
+vtnet_setup_queue_sysctl(struct vtnet_softc *sc)
+{
+ device_t dev;
+ struct sysctl_ctx_list *ctx;
struct sysctl_oid *tree;
struct sysctl_oid_list *child;
+ int i;
dev = sc->vtnet_dev;
- stats = &sc->vtnet_stats;
ctx = device_get_sysctl_ctx(dev);
tree = device_get_sysctl_tree(dev);
child = SYSCTL_CHILDREN(tree);
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_failed",
+ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
+ vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]);
+ vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]);
+ }
+}
+
+static void
+vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx,
+ struct sysctl_oid_list *child, struct vtnet_softc *sc)
+{
+ struct vtnet_statistics *stats;
+
+ stats = &sc->vtnet_stats;
+
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed",
CTLFLAG_RD, &stats->mbuf_alloc_failed,
"Mbuf cluster allocation failures");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_frame_too_large",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large",
CTLFLAG_RD, &stats->rx_frame_too_large,
"Received frame larger than the mbuf chain");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed",
CTLFLAG_RD, &stats->rx_enq_replacement_failed,
"Enqueuing the replacement receive mbuf failed");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_mergeable_failed",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed",
CTLFLAG_RD, &stats->rx_mergeable_failed,
"Mergeable buffers receive failures");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype",
CTLFLAG_RD, &stats->rx_csum_bad_ethtype,
"Received checksum offloaded buffer with unsupported "
"Ethernet type");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_start",
- CTLFLAG_RD, &stats->rx_csum_bad_start,
- "Received checksum offloaded buffer with incorrect start offset");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
CTLFLAG_RD, &stats->rx_csum_bad_ipproto,
"Received checksum offloaded buffer with incorrect IP protocol");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_bad_offset",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset",
CTLFLAG_RD, &stats->rx_csum_bad_offset,
"Received checksum offloaded buffer with incorrect offset");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_failed",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto",
+ CTLFLAG_RD, &stats->rx_csum_bad_proto,
+ "Received checksum offloaded buffer with incorrect protocol");
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed",
CTLFLAG_RD, &stats->rx_csum_failed,
"Received buffer checksum offload failed");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_csum_offloaded",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded",
CTLFLAG_RD, &stats->rx_csum_offloaded,
"Received buffer checksum offload succeeded");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_task_rescheduled",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled",
CTLFLAG_RD, &stats->rx_task_rescheduled,
"Times the receive interrupt task rescheduled itself");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_offloaded",
- CTLFLAG_RD, &stats->tx_csum_offloaded,
- "Offloaded checksum of transmitted buffer");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_offloaded",
- CTLFLAG_RD, &stats->tx_tso_offloaded,
- "Segmentation offload of transmitted buffer");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
"Aborted transmit of checksum offloaded buffer with unknown "
"Ethernet type");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
"Aborted transmit of TSO buffer with unknown Ethernet type");
- SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_task_rescheduled",
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp",
+ CTLFLAG_RD, &stats->tx_tso_not_tcp,
+ "Aborted transmit of TSO buffer with non TCP protocol");
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged",
+ CTLFLAG_RD, &stats->tx_defragged,
+ "Transmit mbufs defragged");
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed",
+ CTLFLAG_RD, &stats->tx_defrag_failed,
+ "Aborted transmit of buffer because defrag failed");
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded",
+ CTLFLAG_RD, &stats->tx_csum_offloaded,
+ "Offloaded checksum of transmitted buffer");
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded",
+ CTLFLAG_RD, &stats->tx_tso_offloaded,
+ "Segmentation offload of transmitted buffer");
+ SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled",
CTLFLAG_RD, &stats->tx_task_rescheduled,
"Times the transmit interrupt task rescheduled itself");
}
+static void
+vtnet_setup_sysctl(struct vtnet_softc *sc)
+{
+ device_t dev;
+ struct sysctl_ctx_list *ctx;
+ struct sysctl_oid *tree;
+ struct sysctl_oid_list *child;
+
+ dev = sc->vtnet_dev;
+ ctx = device_get_sysctl_ctx(dev);
+ tree = device_get_sysctl_tree(dev);
+ child = SYSCTL_CHILDREN(tree);
+
+ SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs",
+ CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0,
+ "Maximum number of supported virtqueue pairs");
+ SYSCTL_ADD_INT(ctx, child, OID_AUTO, "requested_vq_pairs",
+ CTLFLAG_RD, &sc->vtnet_requested_vq_pairs, 0,
+ "Requested number of virtqueue pairs");
+ SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs",
+ CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0,
+ "Number of active virtqueue pairs");
+
+ vtnet_setup_stat_sysctl(ctx, child, sc);
+}
+
static int
-vtnet_enable_rx_intr(struct vtnet_softc *sc)
+vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)
{
- return (virtqueue_enable_intr(sc->vtnet_rx_vq));
+ return (virtqueue_enable_intr(rxq->vtnrx_vq));
}
static void
-vtnet_disable_rx_intr(struct vtnet_softc *sc)
+vtnet_rxq_disable_intr(struct vtnet_rxq *rxq)
{
- virtqueue_disable_intr(sc->vtnet_rx_vq);
+ virtqueue_disable_intr(rxq->vtnrx_vq);
}
static int
-vtnet_enable_tx_intr(struct vtnet_softc *sc)
+vtnet_txq_enable_intr(struct vtnet_txq *txq)
{
+ struct virtqueue *vq;
-#ifdef VTNET_TX_INTR_MODERATION
+ vq = txq->vtntx_vq;
+
+ if (vtnet_txq_below_threshold(txq) != 0)
+ return (virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG));
+
+ /*
+ * The free count is above our threshold. Keep the Tx interrupt
+ * disabled until the queue is fuller.
+ */
return (0);
-#else
- return (virtqueue_enable_intr(sc->vtnet_tx_vq));
-#endif
}
static void
-vtnet_disable_tx_intr(struct vtnet_softc *sc)
+vtnet_txq_disable_intr(struct vtnet_txq *txq)
{
- virtqueue_disable_intr(sc->vtnet_tx_vq);
+ virtqueue_disable_intr(txq->vtntx_vq);
}
+
+static void
+vtnet_enable_rx_interrupts(struct vtnet_softc *sc)
+{
+ int i;
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
+ vtnet_rxq_enable_intr(&sc->vtnet_rxqs[i]);
+}
+
+static void
+vtnet_enable_tx_interrupts(struct vtnet_softc *sc)
+{
+ int i;
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
+ vtnet_txq_enable_intr(&sc->vtnet_txqs[i]);
+}
+
+static void
+vtnet_enable_interrupts(struct vtnet_softc *sc)
+{
+
+ vtnet_enable_rx_interrupts(sc);
+ vtnet_enable_tx_interrupts(sc);
+}
+
+static void
+vtnet_disable_rx_interrupts(struct vtnet_softc *sc)
+{
+ int i;
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
+ vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]);
+}
+
+static void
+vtnet_disable_tx_interrupts(struct vtnet_softc *sc)
+{
+ int i;
+
+ for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
+ vtnet_txq_disable_intr(&sc->vtnet_txqs[i]);
+}
+
+static void
+vtnet_disable_interrupts(struct vtnet_softc *sc)
+{
+
+ vtnet_disable_rx_interrupts(sc);
+ vtnet_disable_tx_interrupts(sc);
+}
+
+static int
+vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def)
+{
+ char path[64];
+
+ snprintf(path, sizeof(path),
+ "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob);
+ TUNABLE_INT_FETCH(path, &def);
+
+ return (def);
+}
Modified: trunk/sys/dev/virtio/network/if_vtnetvar.h
===================================================================
--- trunk/sys/dev/virtio/network/if_vtnetvar.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/network/if_vtnetvar.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -23,91 +24,189 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/network/if_vtnetvar.h 304081 2016-08-14 15:27:59Z smh $
*/
#ifndef _IF_VTNETVAR_H
#define _IF_VTNETVAR_H
+struct vtnet_softc;
+
struct vtnet_statistics {
- unsigned long mbuf_alloc_failed;
+ uint64_t mbuf_alloc_failed;
- unsigned long rx_frame_too_large;
- unsigned long rx_enq_replacement_failed;
- unsigned long rx_mergeable_failed;
- unsigned long rx_csum_bad_ethtype;
- unsigned long rx_csum_bad_start;
- unsigned long rx_csum_bad_ipproto;
- unsigned long rx_csum_bad_offset;
- unsigned long rx_csum_failed;
- unsigned long rx_csum_offloaded;
- unsigned long rx_task_rescheduled;
+ uint64_t rx_frame_too_large;
+ uint64_t rx_enq_replacement_failed;
+ uint64_t rx_mergeable_failed;
+ uint64_t rx_csum_bad_ethtype;
+ uint64_t rx_csum_bad_ipproto;
+ uint64_t rx_csum_bad_offset;
+ uint64_t rx_csum_bad_proto;
+ uint64_t tx_csum_bad_ethtype;
+ uint64_t tx_tso_bad_ethtype;
+ uint64_t tx_tso_not_tcp;
+ uint64_t tx_defragged;
+ uint64_t tx_defrag_failed;
- unsigned long tx_csum_offloaded;
- unsigned long tx_tso_offloaded;
- unsigned long tx_csum_bad_ethtype;
- unsigned long tx_tso_bad_ethtype;
- unsigned long tx_task_rescheduled;
+ /*
+ * These are accumulated from each Rx/Tx queue.
+ */
+ uint64_t rx_csum_failed;
+ uint64_t rx_csum_offloaded;
+ uint64_t rx_task_rescheduled;
+ uint64_t tx_csum_offloaded;
+ uint64_t tx_tso_offloaded;
+ uint64_t tx_task_rescheduled;
};
+struct vtnet_rxq_stats {
+ uint64_t vrxs_ipackets; /* if_ipackets */
+ uint64_t vrxs_ibytes; /* if_ibytes */
+ uint64_t vrxs_iqdrops; /* if_iqdrops */
+ uint64_t vrxs_ierrors; /* if_ierrors */
+ uint64_t vrxs_csum;
+ uint64_t vrxs_csum_failed;
+ uint64_t vrxs_rescheduled;
+};
+
+struct vtnet_rxq {
+ struct mtx vtnrx_mtx;
+ struct vtnet_softc *vtnrx_sc;
+ struct virtqueue *vtnrx_vq;
+ struct sglist *vtnrx_sg;
+ int vtnrx_id;
+ struct vtnet_rxq_stats vtnrx_stats;
+ struct taskqueue *vtnrx_tq;
+ struct task vtnrx_intrtask;
+ char vtnrx_name[16];
+} __aligned(CACHE_LINE_SIZE);
+
+#define VTNET_RXQ_LOCK(_rxq) mtx_lock(&(_rxq)->vtnrx_mtx)
+#define VTNET_RXQ_UNLOCK(_rxq) mtx_unlock(&(_rxq)->vtnrx_mtx)
+#define VTNET_RXQ_LOCK_ASSERT(_rxq) \
+ mtx_assert(&(_rxq)->vtnrx_mtx, MA_OWNED)
+#define VTNET_RXQ_LOCK_ASSERT_NOTOWNED(_rxq) \
+ mtx_assert(&(_rxq)->vtnrx_mtx, MA_NOTOWNED)
+
+struct vtnet_txq_stats {
+ uint64_t vtxs_opackets; /* if_opackets */
+ uint64_t vtxs_obytes; /* if_obytes */
+ uint64_t vtxs_omcasts; /* if_omcasts */
+ uint64_t vtxs_csum;
+ uint64_t vtxs_tso;
+ uint64_t vtxs_rescheduled;
+};
+
+struct vtnet_txq {
+ struct mtx vtntx_mtx;
+ struct vtnet_softc *vtntx_sc;
+ struct virtqueue *vtntx_vq;
+ struct sglist *vtntx_sg;
+#ifndef VTNET_LEGACY_TX
+ struct buf_ring *vtntx_br;
+#endif
+ int vtntx_id;
+ int vtntx_watchdog;
+ struct vtnet_txq_stats vtntx_stats;
+ struct taskqueue *vtntx_tq;
+ struct task vtntx_intrtask;
+#ifndef VTNET_LEGACY_TX
+ struct task vtntx_defrtask;
+#endif
+ char vtntx_name[16];
+} __aligned(CACHE_LINE_SIZE);
+
+#define VTNET_TXQ_LOCK(_txq) mtx_lock(&(_txq)->vtntx_mtx)
+#define VTNET_TXQ_TRYLOCK(_txq) mtx_trylock(&(_txq)->vtntx_mtx)
+#define VTNET_TXQ_UNLOCK(_txq) mtx_unlock(&(_txq)->vtntx_mtx)
+#define VTNET_TXQ_LOCK_ASSERT(_txq) \
+ mtx_assert(&(_txq)->vtntx_mtx, MA_OWNED)
+#define VTNET_TXQ_LOCK_ASSERT_NOTOWNED(_txq) \
+ mtx_assert(&(_txq)->vtntx_mtx, MA_NOTOWNED)
+
struct vtnet_softc {
device_t vtnet_dev;
struct ifnet *vtnet_ifp;
- struct mtx vtnet_mtx;
+ struct vtnet_rxq *vtnet_rxqs;
+ struct vtnet_txq *vtnet_txqs;
uint32_t vtnet_flags;
-#define VTNET_FLAG_LINK 0x0001
-#define VTNET_FLAG_SUSPENDED 0x0002
+#define VTNET_FLAG_SUSPENDED 0x0001
+#define VTNET_FLAG_MAC 0x0002
#define VTNET_FLAG_CTRL_VQ 0x0004
#define VTNET_FLAG_CTRL_RX 0x0008
-#define VTNET_FLAG_VLAN_FILTER 0x0010
-#define VTNET_FLAG_TSO_ECN 0x0020
-#define VTNET_FLAG_MRG_RXBUFS 0x0040
-#define VTNET_FLAG_LRO_NOMRG 0x0080
+#define VTNET_FLAG_CTRL_MAC 0x0010
+#define VTNET_FLAG_VLAN_FILTER 0x0020
+#define VTNET_FLAG_TSO_ECN 0x0040
+#define VTNET_FLAG_MRG_RXBUFS 0x0080
+#define VTNET_FLAG_LRO_NOMRG 0x0100
+#define VTNET_FLAG_MULTIQ 0x0200
+#define VTNET_FLAG_INDIRECT 0x0400
+#define VTNET_FLAG_EVENT_IDX 0x0800
- struct virtqueue *vtnet_rx_vq;
- struct virtqueue *vtnet_tx_vq;
- struct virtqueue *vtnet_ctrl_vq;
-
+ int vtnet_link_active;
int vtnet_hdr_size;
- int vtnet_tx_size;
- int vtnet_rx_size;
int vtnet_rx_process_limit;
- int vtnet_rx_mbuf_size;
- int vtnet_rx_mbuf_count;
+ int vtnet_rx_nsegs;
+ int vtnet_rx_nmbufs;
+ int vtnet_rx_clsize;
+ int vtnet_rx_new_clsize;
+ int vtnet_tx_intr_thresh;
+ int vtnet_tx_nsegs;
int vtnet_if_flags;
- int vtnet_watchdog_timer;
+ int vtnet_act_vq_pairs;
+ int vtnet_max_vq_pairs;
+ int vtnet_requested_vq_pairs;
+
+ struct virtqueue *vtnet_ctrl_vq;
+ struct vtnet_mac_filter *vtnet_mac_filter;
+ uint32_t *vtnet_vlan_filter;
+
uint64_t vtnet_features;
-
struct vtnet_statistics vtnet_stats;
-
struct callout vtnet_tick_ch;
-
+ struct ifmedia vtnet_media;
eventhandler_tag vtnet_vlan_attach;
eventhandler_tag vtnet_vlan_detach;
- struct ifmedia vtnet_media;
- /*
- * Fake media type; the host does not provide us with
- * any real media information.
- */
-#define VTNET_MEDIATYPE (IFM_ETHER | IFM_1000_T | IFM_FDX)
+ struct mtx vtnet_mtx;
+ char vtnet_mtx_name[16];
char vtnet_hwaddr[ETHER_ADDR_LEN];
+};
- struct vtnet_mac_filter *vtnet_mac_filter;
- /*
- * During reset, the host's VLAN filtering table is lost. The
- * array below is used to restore all the VLANs configured on
- * this interface after a reset.
- */
-#define VTNET_VLAN_SHADOW_SIZE (4096 / 32)
- int vtnet_nvlans;
- uint32_t vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE];
+/*
+ * Maximum number of queue pairs we will autoconfigure to.
+ */
+#define VTNET_MAX_QUEUE_PAIRS 8
- char vtnet_mtx_name[16];
-};
+/*
+ * Additional completed entries can appear in a virtqueue before we can
+ * reenable interrupts. Number of times to retry before scheduling the
+ * taskqueue to process the completed entries.
+ */
+#define VTNET_INTR_DISABLE_RETRIES 4
/*
+ * Similarly, additional completed entries can appear in a virtqueue
+ * between when lasted checked and before notifying the host. Number
+ * of times to retry before scheduling the taskqueue to process the
+ * queue.
+ */
+#define VTNET_NOTIFY_RETRIES 4
+
+/*
+ * Fake the media type. The host does not provide us with any real media
+ * information.
+ */
+#define VTNET_MEDIATYPE (IFM_ETHER | IFM_10G_T | IFM_FDX)
+
+/*
+ * Number of words to allocate for the VLAN shadow table. There is one
+ * bit for each VLAN.
+ */
+#define VTNET_VLAN_FILTER_NWORDS (4096 / 32)
+
+/*
* When mergeable buffers are not negotiated, the vtnet_rx_header structure
* below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
* both keep the VirtIO header and the data non-contiguous and to keep the
@@ -161,9 +260,13 @@
*/
CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
-#define VTNET_WATCHDOG_TIMEOUT 5
+#define VTNET_TX_TIMEOUT 5
#define VTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP | CSUM_SCTP)
+#define VTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | CSUM_SCTP_IPV6)
+#define VTNET_CSUM_ALL_OFFLOAD \
+ (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6 | CSUM_TSO)
+
/* Features desired/implemented by this driver. */
#define VTNET_FEATURES \
(VIRTIO_NET_F_MAC | \
@@ -170,8 +273,10 @@
VIRTIO_NET_F_STATUS | \
VIRTIO_NET_F_CTRL_VQ | \
VIRTIO_NET_F_CTRL_RX | \
+ VIRTIO_NET_F_CTRL_MAC_ADDR | \
VIRTIO_NET_F_CTRL_VLAN | \
VIRTIO_NET_F_CSUM | \
+ VIRTIO_NET_F_GSO | \
VIRTIO_NET_F_HOST_TSO4 | \
VIRTIO_NET_F_HOST_TSO6 | \
VIRTIO_NET_F_HOST_ECN | \
@@ -180,9 +285,18 @@
VIRTIO_NET_F_GUEST_TSO6 | \
VIRTIO_NET_F_GUEST_ECN | \
VIRTIO_NET_F_MRG_RXBUF | \
+ VIRTIO_NET_F_MQ | \
+ VIRTIO_RING_F_EVENT_IDX | \
VIRTIO_RING_F_INDIRECT_DESC)
/*
+ * The VIRTIO_NET_F_HOST_TSO[46] features permit us to send the host
+ * frames larger than 1514 bytes.
+ */
+#define VTNET_TSO_FEATURES (VIRTIO_NET_F_GSO | VIRTIO_NET_F_HOST_TSO4 | \
+ VIRTIO_NET_F_HOST_TSO6 | VIRTIO_NET_F_HOST_ECN)
+
+/*
* The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
* frames larger than 1514 bytes. We do not yet support software LRO
* via tcp_lro_rx().
@@ -195,11 +309,14 @@
/*
* Used to preallocate the Vq indirect descriptors. The first segment
- * is reserved for the header.
+ * is reserved for the header, except for mergeable buffers since the
+ * header is placed inline with the data.
*/
+#define VTNET_MRG_RX_SEGS 1
#define VTNET_MIN_RX_SEGS 2
#define VTNET_MAX_RX_SEGS 34
-#define VTNET_MAX_TX_SEGS 34
+#define VTNET_MIN_TX_SEGS 4
+#define VTNET_MAX_TX_SEGS 64
/*
* Assert we can receive and transmit the maximum with regular
@@ -209,27 +326,34 @@
CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
/*
+ * Number of slots in the Tx bufrings. This value matches most other
+ * multiqueue drivers.
+ */
+#define VTNET_DEFAULT_BUFRING_SIZE 4096
+
+/*
* Determine how many mbufs are in each receive buffer. For LRO without
- * mergeable descriptors, we must allocate an mbuf chain large enough to
+ * mergeable buffers, we must allocate an mbuf chain large enough to
* hold both the vtnet_rx_header and the maximum receivable data.
*/
-#define VTNET_NEEDED_RX_MBUFS(_sc) \
+#define VTNET_NEEDED_RX_MBUFS(_sc, _clsize) \
((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 : \
howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE, \
- (_sc)->vtnet_rx_mbuf_size)
+ (_clsize))
-#define VTNET_MTX(_sc) &(_sc)->vtnet_mtx
-#define VTNET_LOCK(_sc) mtx_lock(VTNET_MTX((_sc)))
-#define VTNET_UNLOCK(_sc) mtx_unlock(VTNET_MTX((_sc)))
-#define VTNET_LOCK_DESTROY(_sc) mtx_destroy(VTNET_MTX((_sc)))
-#define VTNET_LOCK_ASSERT(_sc) mtx_assert(VTNET_MTX((_sc)), MA_OWNED)
-#define VTNET_LOCK_ASSERT_NOTOWNED(_sc) \
- mtx_assert(VTNET_MTX((_sc)), MA_NOTOWNED)
+#define VTNET_CORE_MTX(_sc) &(_sc)->vtnet_mtx
+#define VTNET_CORE_LOCK(_sc) mtx_lock(VTNET_CORE_MTX((_sc)))
+#define VTNET_CORE_UNLOCK(_sc) mtx_unlock(VTNET_CORE_MTX((_sc)))
+#define VTNET_CORE_LOCK_DESTROY(_sc) mtx_destroy(VTNET_CORE_MTX((_sc)))
+#define VTNET_CORE_LOCK_ASSERT(_sc) \
+ mtx_assert(VTNET_CORE_MTX((_sc)), MA_OWNED)
+#define VTNET_CORE_LOCK_ASSERT_NOTOWNED(_sc) \
+ mtx_assert(VTNET_CORE_MTX((_sc)), MA_NOTOWNED)
-#define VTNET_LOCK_INIT(_sc) do { \
+#define VTNET_CORE_LOCK_INIT(_sc) do { \
snprintf((_sc)->vtnet_mtx_name, sizeof((_sc)->vtnet_mtx_name), \
"%s", device_get_nameunit((_sc)->vtnet_dev)); \
- mtx_init(VTNET_MTX((_sc)), (_sc)->vtnet_mtx_name, \
+ mtx_init(VTNET_CORE_MTX((_sc)), (_sc)->vtnet_mtx_name, \
"VTNET Core Lock", MTX_DEF); \
} while (0)
Modified: trunk/sys/dev/virtio/network/virtio_net.h
===================================================================
--- trunk/sys/dev/virtio/network/virtio_net.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/network/virtio_net.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* This header is BSD licensed so anyone can use the definitions to implement
* compatible drivers/servers.
@@ -25,7 +26,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/network/virtio_net.h 255111 2013-09-01 04:23:54Z bryanv $
*/
#ifndef _VIRTIO_NET_H
@@ -50,14 +51,22 @@
#define VIRTIO_NET_F_CTRL_RX 0x40000 /* Control channel RX mode support */
#define VIRTIO_NET_F_CTRL_VLAN 0x80000 /* Control channel VLAN filtering */
#define VIRTIO_NET_F_CTRL_RX_EXTRA 0x100000 /* Extra RX mode control support */
+#define VIRTIO_NET_F_GUEST_ANNOUNCE 0x200000 /* Announce device on network */
+#define VIRTIO_NET_F_MQ 0x400000 /* Device supports RFS */
+#define VIRTIO_NET_F_CTRL_MAC_ADDR 0x800000 /* Set MAC address */
#define VIRTIO_NET_S_LINK_UP 1 /* Link is up */
struct virtio_net_config {
/* The config defining mac address (if VIRTIO_NET_F_MAC) */
- uint8_t mac[ETHER_ADDR_LEN];
+ uint8_t mac[ETHER_ADDR_LEN];
/* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */
uint16_t status;
+ /* Maximum number of each of transmit and receive queues;
+ * see VIRTIO_NET_F_MQ and VIRTIO_NET_CTRL_MQ.
+ * Legal values are between 1 and 0x8000.
+ */
+ uint16_t max_virtqueue_pairs;
} __packed;
/*
@@ -66,6 +75,7 @@
*/
struct virtio_net_hdr {
#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start,csum_offset*/
+#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */
uint8_t flags;
#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */
#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */
@@ -100,8 +110,6 @@
uint8_t cmd;
} __packed;
-typedef uint8_t virtio_net_ctrl_ack;
-
#define VIRTIO_NET_OK 0
#define VIRTIO_NET_ERR 1
@@ -134,6 +142,10 @@
* first sg list contains unicast addresses, the second is for multicast.
* This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
* is available.
+ *
+ * The ADDR_SET command requests one out scatterlist, it contains a
+ * 6 bytes MAC address. This functionality is present if the
+ * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
*/
struct virtio_net_ctrl_mac {
uint32_t entries;
@@ -142,6 +154,7 @@
#define VIRTIO_NET_CTRL_MAC 1
#define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
+#define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
/*
* Control VLAN filtering
@@ -156,4 +169,35 @@
#define VIRTIO_NET_CTRL_VLAN_ADD 0
#define VIRTIO_NET_CTRL_VLAN_DEL 1
+/*
+ * Control link announce acknowledgement
+ *
+ * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that
+ * driver has recevied the notification; device would clear the
+ * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives
+ * this command.
+ */
+#define VIRTIO_NET_CTRL_ANNOUNCE 3
+#define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0
+
+/*
+ * Control Receive Flow Steering
+ *
+ * The command VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET enables Receive Flow
+ * Steering, specifying the number of the transmit and receive queues
+ * that will be used. After the command is consumed and acked by the
+ * device, the device will not steer new packets on receive virtqueues
+ * other than specified nor read from transmit virtqueues other than
+ * specified. Accordingly, driver should not transmit new packets on
+ * virtqueues other than specified.
+ */
+struct virtio_net_ctrl_mq {
+ uint16_t virtqueue_pairs;
+} __packed;
+
+#define VIRTIO_NET_CTRL_MQ 4
+#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
+#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
+#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
+
#endif /* _VIRTIO_NET_H */
Modified: trunk/sys/dev/virtio/pci/virtio_pci.c
===================================================================
--- trunk/sys/dev/virtio/pci/virtio_pci.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/pci/virtio_pci.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -27,7 +28,7 @@
/* Driver for the VirtIO PCI interface. */
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/pci/virtio_pci.c 310080 2016-12-14 16:44:38Z avg $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -169,6 +170,9 @@
#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
+#define VIRTIO_PCI_CONFIG(_sc) \
+ VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
+
/*
* I/O port read/write wrappers.
*/
@@ -272,10 +276,10 @@
return (ENXIO);
}
- if (pci_find_extcap(dev, PCIY_MSI, NULL) != 0)
+ if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
- if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) {
+ if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
rid = PCIR_BAR(1);
sc->vtpci_msix_res = bus_alloc_resource_any(dev,
SYS_RES_MEMORY, &rid, RF_ACTIVE);
@@ -727,7 +731,7 @@
dev = sc->vtpci_dev;
child = sc->vtpci_child_dev;
- if (device_is_attached(child) && bootverbose == 0)
+ if (device_is_attached(child) || bootverbose == 0)
return;
virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
@@ -757,8 +761,10 @@
vtpci_release_child_resources(sc);
/* Reset status for future attempt. */
vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
- } else
+ } else {
vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
+ VIRTIO_ATTACH_COMPLETED(child);
+ }
}
static int
@@ -1082,7 +1088,8 @@
* For shared MSIX, all the virtqueues share the first
* interrupt.
*/
- if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
+ if (!sc->vtpci_vqs[idx].vtv_no_intr &&
+ (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
intr++;
}
Modified: trunk/sys/dev/virtio/pci/virtio_pci.h
===================================================================
--- trunk/sys/dev/virtio/pci/virtio_pci.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/pci/virtio_pci.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright IBM Corp. 2007
*
@@ -30,7 +31,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/pci/virtio_pci.h 268010 2014-06-29 00:37:59Z bryanv $
*/
#ifndef _VIRTIO_PCI_H
@@ -72,8 +73,7 @@
* The remaining space is defined by each driver as the per-driver
* configuration space.
*/
-#define VIRTIO_PCI_CONFIG(sc) \
- (((sc)->vtpci_flags & VTPCI_FLAG_MSIX) ? 24 : 20)
+#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20)
/*
* How many bits to shift physical queue address written to QUEUE_PFN.
Added: trunk/sys/dev/virtio/random/virtio_random.c
===================================================================
--- trunk/sys/dev/virtio/random/virtio_random.c (rev 0)
+++ trunk/sys/dev/virtio/random/virtio_random.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -0,0 +1,232 @@
+/* $MidnightBSD$ */
+/*-
+ * Copyright (c) 2013, Bryan Venteicher <bryanv at FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Driver for VirtIO entropy device. */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/random/virtio_random.c 314667 2017-03-04 13:03:31Z avg $");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/sglist.h>
+#include <sys/callout.h>
+#include <sys/random.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus.h>
+
+#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtqueue.h>
+
+struct vtrnd_softc {
+ device_t vtrnd_dev;
+ uint64_t vtrnd_features;
+ struct callout vtrnd_callout;
+ struct virtqueue *vtrnd_vq;
+};
+
+static int vtrnd_modevent(module_t, int, void *);
+
+static int vtrnd_probe(device_t);
+static int vtrnd_attach(device_t);
+static int vtrnd_detach(device_t);
+
+static void vtrnd_negotiate_features(struct vtrnd_softc *);
+static int vtrnd_alloc_virtqueue(struct vtrnd_softc *);
+static void vtrnd_harvest(struct vtrnd_softc *);
+static void vtrnd_timer(void *);
+
+#define VTRND_FEATURES 0
+
+static struct virtio_feature_desc vtrnd_feature_desc[] = {
+ { 0, NULL }
+};
+
+static device_method_t vtrnd_methods[] = {
+ /* Device methods. */
+ DEVMETHOD(device_probe, vtrnd_probe),
+ DEVMETHOD(device_attach, vtrnd_attach),
+ DEVMETHOD(device_detach, vtrnd_detach),
+
+ DEVMETHOD_END
+};
+
+static driver_t vtrnd_driver = {
+ "vtrnd",
+ vtrnd_methods,
+ sizeof(struct vtrnd_softc)
+};
+static devclass_t vtrnd_devclass;
+
+DRIVER_MODULE(virtio_random, virtio_pci, vtrnd_driver, vtrnd_devclass,
+ vtrnd_modevent, 0);
+MODULE_VERSION(virtio_random, 1);
+MODULE_DEPEND(virtio_random, virtio, 1, 1, 1);
+
+static int
+vtrnd_modevent(module_t mod, int type, void *unused)
+{
+ int error;
+
+ switch (type) {
+ case MOD_LOAD:
+ case MOD_QUIESCE:
+ case MOD_UNLOAD:
+ case MOD_SHUTDOWN:
+ error = 0;
+ break;
+ default:
+ error = EOPNOTSUPP;
+ break;
+ }
+
+ return (error);
+}
+
+static int
+vtrnd_probe(device_t dev)
+{
+
+ if (virtio_get_device_type(dev) != VIRTIO_ID_ENTROPY)
+ return (ENXIO);
+
+ device_set_desc(dev, "VirtIO Entropy Adapter");
+
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtrnd_attach(device_t dev)
+{
+ struct vtrnd_softc *sc;
+ int error;
+
+ sc = device_get_softc(dev);
+ sc->vtrnd_dev = dev;
+
+ callout_init(&sc->vtrnd_callout, 1);
+
+ virtio_set_feature_desc(dev, vtrnd_feature_desc);
+ vtrnd_negotiate_features(sc);
+
+ error = vtrnd_alloc_virtqueue(sc);
+ if (error) {
+ device_printf(dev, "cannot allocate virtqueue\n");
+ goto fail;
+ }
+
+ callout_reset(&sc->vtrnd_callout, 5 * hz, vtrnd_timer, sc);
+
+fail:
+ if (error)
+ vtrnd_detach(dev);
+
+ return (error);
+}
+
+static int
+vtrnd_detach(device_t dev)
+{
+ struct vtrnd_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ callout_drain(&sc->vtrnd_callout);
+
+ return (0);
+}
+
+static void
+vtrnd_negotiate_features(struct vtrnd_softc *sc)
+{
+ device_t dev;
+ uint64_t features;
+
+ dev = sc->vtrnd_dev;
+ features = VTRND_FEATURES;
+
+ sc->vtrnd_features = virtio_negotiate_features(dev, features);
+}
+
+static int
+vtrnd_alloc_virtqueue(struct vtrnd_softc *sc)
+{
+ device_t dev;
+ struct vq_alloc_info vq_info;
+
+ dev = sc->vtrnd_dev;
+
+ VQ_ALLOC_INFO_INIT(&vq_info, 0, NULL, sc, &sc->vtrnd_vq,
+ "%s request", device_get_nameunit(dev));
+
+ return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
+}
+
+static void
+vtrnd_harvest(struct vtrnd_softc *sc)
+{
+ struct sglist_seg segs[1];
+ struct sglist sg;
+ struct virtqueue *vq;
+ uint32_t value;
+ int error;
+
+ vq = sc->vtrnd_vq;
+
+ sglist_init(&sg, 1, segs);
+ error = sglist_append(&sg, &value, sizeof(value));
+ KASSERT(error == 0 && sg.sg_nseg == 1,
+ ("%s: error %d adding buffer to sglist", __func__, error));
+
+ if (!virtqueue_empty(vq))
+ return;
+ if (virtqueue_enqueue(vq, &value, &sg, 0, 1) != 0)
+ return;
+
+ /*
+ * Poll for the response, but the command is likely already
+ * done when we return from the notify.
+ */
+ virtqueue_notify(vq);
+ virtqueue_poll(vq, NULL);
+
+ random_harvest(&value, sizeof(value), sizeof(value) * NBBY / 2,
+ RANDOM_PURE_VIRTIO);
+}
+
+static void
+vtrnd_timer(void *xsc)
+{
+ struct vtrnd_softc *sc;
+
+ sc = xsc;
+
+ vtrnd_harvest(sc);
+ callout_schedule(&sc->vtrnd_callout, 5 * hz);
+}
Property changes on: trunk/sys/dev/virtio/random/virtio_random.c
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Modified: trunk/sys/dev/virtio/scsi/virtio_scsi.c
===================================================================
--- trunk/sys/dev/virtio/scsi/virtio_scsi.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/scsi/virtio_scsi.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2012, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -27,7 +28,7 @@
/* Driver for VirtIO SCSI devices. */
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/scsi/virtio_scsi.c 315813 2017-03-23 06:41:13Z mav $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -75,6 +76,8 @@
static int vtscsi_resume(device_t);
static void vtscsi_negotiate_features(struct vtscsi_softc *);
+static void vtscsi_read_config(struct vtscsi_softc *,
+ struct virtio_scsi_config *);
static int vtscsi_maximum_segments(struct vtscsi_softc *, int);
static int vtscsi_alloc_virtqueues(struct vtscsi_softc *);
static void vtscsi_write_device_config(struct vtscsi_softc *);
@@ -81,7 +84,7 @@
static int vtscsi_reinit(struct vtscsi_softc *);
static int vtscsi_alloc_cam(struct vtscsi_softc *);
-static int vtscsi_register_cam(struct vtscsi_softc *);
+static int vtscsi_register_cam(struct vtscsi_softc *);
static void vtscsi_free_cam(struct vtscsi_softc *);
static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *);
static int vtscsi_register_async(struct vtscsi_softc *);
@@ -91,7 +94,7 @@
static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *,
union ccb *);
-static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *,
+static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *,
union ccb *);
static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *);
static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *);
@@ -99,36 +102,36 @@
static void vtscsi_cam_path_inquiry(struct vtscsi_softc *,
struct cam_sim *, union ccb *);
-static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *,
+static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *,
struct sglist *, struct ccb_scsiio *);
-static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *,
+static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *,
struct vtscsi_request *, int *, int *);
-static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *,
+static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *,
struct vtscsi_request *);
-static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *);
+static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *);
static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *,
struct vtscsi_request *);
-static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *,
+static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *,
struct vtscsi_request *);
static void vtscsi_timedout_scsi_cmd(void *);
static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *);
static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *,
struct ccb_scsiio *, struct virtio_scsi_cmd_resp *);
-static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *,
+static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *,
struct vtscsi_request *);
static void vtscsi_poll_ctrl_req(struct vtscsi_softc *,
struct vtscsi_request *);
-static int vtscsi_execute_ctrl_req(struct vtscsi_softc *,
+static int vtscsi_execute_ctrl_req(struct vtscsi_softc *,
struct vtscsi_request *, struct sglist *, int, int, int);
-static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c,
+static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c,
struct vtscsi_request *);
-static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *,
+static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *,
struct vtscsi_request *);
-static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *,
+static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *,
struct vtscsi_request *);
-static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *);
+static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *);
static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);
static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *,
struct virtio_scsi_cmd_req *);
@@ -135,33 +138,33 @@
static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t,
uintptr_t, struct virtio_scsi_ctrl_tmf_req *);
-static void vtscsi_freeze_simq(struct vtscsi_softc *, int);
+static void vtscsi_freeze_simq(struct vtscsi_softc *, int);
static int vtscsi_thaw_simq(struct vtscsi_softc *, int);
-static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t,
+static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t,
lun_id_t);
-static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t,
+static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t,
lun_id_t);
-static void vtscsi_execute_rescan_bus(struct vtscsi_softc *);
+static void vtscsi_execute_rescan_bus(struct vtscsi_softc *);
-static void vtscsi_handle_event(struct vtscsi_softc *,
+static void vtscsi_handle_event(struct vtscsi_softc *,
struct virtio_scsi_event *);
-static int vtscsi_enqueue_event_buf(struct vtscsi_softc *,
+static int vtscsi_enqueue_event_buf(struct vtscsi_softc *,
struct virtio_scsi_event *);
static int vtscsi_init_event_vq(struct vtscsi_softc *);
-static void vtscsi_reinit_event_vq(struct vtscsi_softc *);
-static void vtscsi_drain_event_vq(struct vtscsi_softc *);
+static void vtscsi_reinit_event_vq(struct vtscsi_softc *);
+static void vtscsi_drain_event_vq(struct vtscsi_softc *);
-static void vtscsi_complete_vqs_locked(struct vtscsi_softc *);
-static void vtscsi_complete_vqs(struct vtscsi_softc *);
-static void vtscsi_drain_vqs(struct vtscsi_softc *);
-static void vtscsi_cancel_request(struct vtscsi_softc *,
+static void vtscsi_complete_vqs_locked(struct vtscsi_softc *);
+static void vtscsi_complete_vqs(struct vtscsi_softc *);
+static void vtscsi_drain_vqs(struct vtscsi_softc *);
+static void vtscsi_cancel_request(struct vtscsi_softc *,
struct vtscsi_request *);
static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *);
static void vtscsi_stop(struct vtscsi_softc *);
static int vtscsi_reset_bus(struct vtscsi_softc *);
-static void vtscsi_init_request(struct vtscsi_softc *,
+static void vtscsi_init_request(struct vtscsi_softc *,
struct vtscsi_request *);
static int vtscsi_alloc_requests(struct vtscsi_softc *);
static void vtscsi_free_requests(struct vtscsi_softc *);
@@ -170,18 +173,18 @@
static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *);
static void vtscsi_complete_request(struct vtscsi_request *);
-static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *);
+static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *);
static void vtscsi_control_vq_intr(void *);
static void vtscsi_event_vq_intr(void *);
static void vtscsi_request_vq_intr(void *);
-static void vtscsi_disable_vqs_intr(struct vtscsi_softc *);
-static void vtscsi_enable_vqs_intr(struct vtscsi_softc *);
+static void vtscsi_disable_vqs_intr(struct vtscsi_softc *);
+static void vtscsi_enable_vqs_intr(struct vtscsi_softc *);
-static void vtscsi_get_tunables(struct vtscsi_softc *);
-static void vtscsi_add_sysctl(struct vtscsi_softc *);
+static void vtscsi_get_tunables(struct vtscsi_softc *);
+static void vtscsi_add_sysctl(struct vtscsi_softc *);
-static void vtscsi_printf_req(struct vtscsi_request *, const char *,
+static void vtscsi_printf_req(struct vtscsi_request *, const char *,
const char *, ...);
/* Global tunables. */
@@ -287,8 +290,7 @@
if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG;
- virtio_read_device_config(dev, 0, &scsicfg,
- sizeof(struct virtio_scsi_config));
+ vtscsi_read_config(sc, &scsicfg);
sc->vtscsi_max_channel = scsicfg.max_channel;
sc->vtscsi_max_target = scsicfg.max_target;
@@ -408,6 +410,35 @@
sc->vtscsi_features = features;
}
+#define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \
+ virtio_read_device_config(_dev, \
+ offsetof(struct virtio_scsi_config, _field), \
+ &(_cfg)->_field, sizeof((_cfg)->_field)) \
+
+static void
+vtscsi_read_config(struct vtscsi_softc *sc,
+ struct virtio_scsi_config *scsicfg)
+{
+ device_t dev;
+
+ dev = sc->vtscsi_dev;
+
+ bzero(scsicfg, sizeof(struct virtio_scsi_config));
+
+ VTSCSI_GET_CONFIG(dev, num_queues, scsicfg);
+ VTSCSI_GET_CONFIG(dev, seg_max, scsicfg);
+ VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg);
+ VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg);
+ VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg);
+ VTSCSI_GET_CONFIG(dev, sense_size, scsicfg);
+ VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg);
+ VTSCSI_GET_CONFIG(dev, max_channel, scsicfg);
+ VTSCSI_GET_CONFIG(dev, max_target, scsicfg);
+ VTSCSI_GET_CONFIG(dev, max_lun, scsicfg);
+}
+
+#undef VTSCSI_GET_CONFIG
+
static int
vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max)
{
@@ -878,7 +909,7 @@
cpi->version_num = 1;
cpi->hba_inquiry = PI_TAG_ABLE;
cpi->target_sprt = 0;
- cpi->hba_misc = PIM_SEQSCAN;
+ cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED;
if (vtscsi_bus_reset_disable != 0)
cpi->hba_misc |= PIM_NOBUSRESET;
cpi->hba_eng_cnt = 0;
@@ -887,9 +918,9 @@
cpi->max_lun = sc->vtscsi_max_lun;
cpi->initiator_id = VTSCSI_INITIATOR_ID;
- strncpy(cpi->sim_vid, "MidnightBSD", SIM_IDLEN);
- strncpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);
- strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
+ strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
+ strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);
+ strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
cpi->unit_number = cam_sim_unit(sim);
cpi->bus_id = cam_sim_bus(sim);
@@ -946,6 +977,9 @@
(vm_paddr_t) dseg->ds_addr, dseg->ds_len);
}
break;
+ case CAM_DATA_BIO:
+ error = sglist_append_bio(sg, (struct bio *) csio->data_ptr);
+ break;
default:
error = EINVAL;
break;
@@ -1054,8 +1088,8 @@
if (ccbh->timeout != CAM_TIME_INFINITY) {
req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET;
- callout_reset(&req->vsr_callout, ccbh->timeout * hz / 1000,
- vtscsi_timedout_scsi_cmd, req);
+ callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout,
+ 0, vtscsi_timedout_scsi_cmd, req, 0);
}
vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n",
@@ -1539,7 +1573,7 @@
lun[0] = 1;
lun[1] = ccbh->target_id;
lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F);
- lun[3] = (ccbh->target_lun >> 8) & 0xFF;
+ lun[3] = ccbh->target_lun & 0xFF;
}
static void
Modified: trunk/sys/dev/virtio/scsi/virtio_scsi.h
===================================================================
--- trunk/sys/dev/virtio/scsi/virtio_scsi.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/scsi/virtio_scsi.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* This header is BSD licensed so anyone can use the definitions to implement
* compatible drivers/servers.
@@ -23,7 +24,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/scsi/virtio_scsi.h 241470 2012-10-11 23:41:18Z grehan $
*/
#ifndef _VIRTIO_SCSI_H
Modified: trunk/sys/dev/virtio/scsi/virtio_scsivar.h
===================================================================
--- trunk/sys/dev/virtio/scsi/virtio_scsivar.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/scsi/virtio_scsivar.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2012, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -23,7 +24,7 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/scsi/virtio_scsivar.h 252707 2013-07-04 17:57:26Z bryanv $
*/
#ifndef _VIRTIO_SCSIVAR_H
Modified: trunk/sys/dev/virtio/virtio.c
===================================================================
--- trunk/sys/dev/virtio/virtio.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtio.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -25,9 +26,8 @@
*/
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/virtio.c 267312 2014-06-10 03:23:35Z bryanv $");
-
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
@@ -41,6 +41,7 @@
#include <sys/rman.h>
#include <dev/virtio/virtio.h>
+#include <dev/virtio/virtio_config.h>
#include <dev/virtio/virtqueue.h>
#include "virtio_bus_if.h"
@@ -148,7 +149,7 @@
if (n > 0)
sbuf_cat(&sb, ">");
-#if __MidnightBSD_version < 4016
+#if __FreeBSD_version < 900020
sbuf_finish(&sb);
if (sbuf_overflowed(&sb) == 0)
#else
Modified: trunk/sys/dev/virtio/virtio.h
===================================================================
--- trunk/sys/dev/virtio/virtio.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtio.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,76 +1,41 @@
+/* $MidnightBSD$ */
/*-
- * This header is BSD licensed so anyone can use the definitions to implement
- * compatible drivers/servers.
+ * Copyright (c) 2014, Bryan Venteicher <bryanv at FreeBSD.org>
+ * All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- * 3. Neither the name of IBM nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
*
- * $MidnightBSD$
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD: stable/10/sys/dev/virtio/virtio.h 268010 2014-06-29 00:37:59Z bryanv $
*/
#ifndef _VIRTIO_H_
#define _VIRTIO_H_
+#include <dev/virtio/virtio_ids.h>
+#include <dev/virtio/virtio_config.h>
+
struct vq_alloc_info;
-/* VirtIO device IDs. */
-#define VIRTIO_ID_NETWORK 0x01
-#define VIRTIO_ID_BLOCK 0x02
-#define VIRTIO_ID_CONSOLE 0x03
-#define VIRTIO_ID_ENTROPY 0x04
-#define VIRTIO_ID_BALLOON 0x05
-#define VIRTIO_ID_IOMEMORY 0x06
-#define VIRTIO_ID_SCSI 0x08
-#define VIRTIO_ID_9P 0x09
-
-/* Status byte for guest to report progress. */
-#define VIRTIO_CONFIG_STATUS_RESET 0x00
-#define VIRTIO_CONFIG_STATUS_ACK 0x01
-#define VIRTIO_CONFIG_STATUS_DRIVER 0x02
-#define VIRTIO_CONFIG_STATUS_DRIVER_OK 0x04
-#define VIRTIO_CONFIG_STATUS_FAILED 0x80
-
/*
- * Generate interrupt when the virtqueue ring is
- * completely used, even if we've suppressed them.
- */
-#define VIRTIO_F_NOTIFY_ON_EMPTY (1 << 24)
-
-/*
- * The guest should never negotiate this feature; it
- * is used to detect faulty drivers.
- */
-#define VIRTIO_F_BAD_FEATURE (1 << 30)
-
-/*
- * Some VirtIO feature bits (currently bits 28 through 31) are
- * reserved for the transport being used (eg. virtio_ring), the
- * rest are per-device feature bits.
- */
-#define VIRTIO_TRANSPORT_F_START 28
-#define VIRTIO_TRANSPORT_F_END 32
-
-/*
* Each virtqueue indirect descriptor list must be physically contiguous.
* To allow us to malloc(9) each list individually, limit the number
* supported to what will fit in one page. With 4KB pages, this is a limit
Modified: trunk/sys/dev/virtio/virtio_bus_if.m
===================================================================
--- trunk/sys/dev/virtio/virtio_bus_if.m 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtio_bus_if.m 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
#-
# Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
# All rights reserved.
@@ -23,7 +24,7 @@
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
-# $FreeBSD$
+# $FreeBSD: stable/10/sys/dev/virtio/virtio_bus_if.m 252707 2013-07-04 17:57:26Z bryanv $
#include <sys/bus.h>
#include <machine/bus.h>
Property changes on: trunk/sys/dev/virtio/virtio_bus_if.m
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Added: trunk/sys/dev/virtio/virtio_config.h
===================================================================
--- trunk/sys/dev/virtio/virtio_config.h (rev 0)
+++ trunk/sys/dev/virtio/virtio_config.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -0,0 +1,68 @@
+/* $MidnightBSD$ */
+/*-
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: stable/10/sys/dev/virtio/virtio_config.h 268010 2014-06-29 00:37:59Z bryanv $
+ */
+
+#ifndef _VIRTIO_CONFIG_H_
+#define _VIRTIO_CONFIG_H_
+
+/* Status byte for guest to report progress. */
+#define VIRTIO_CONFIG_STATUS_RESET 0x00
+#define VIRTIO_CONFIG_STATUS_ACK 0x01
+#define VIRTIO_CONFIG_STATUS_DRIVER 0x03
+#define VIRTIO_CONFIG_STATUS_DRIVER_OK 0x04
+#define VIRTIO_CONFIG_STATUS_FAILED 0x80
+
+/*
+ * Generate interrupt when the virtqueue ring is
+ * completely used, even if we've suppressed them.
+ */
+#define VIRTIO_F_NOTIFY_ON_EMPTY (1 << 24)
+
+/* Support for indirect buffer descriptors. */
+#define VIRTIO_RING_F_INDIRECT_DESC (1 << 28)
+
+/* Support to suppress interrupt until specific index is reached. */
+#define VIRTIO_RING_F_EVENT_IDX (1 << 29)
+
+/*
+ * The guest should never negotiate this feature; it
+ * is used to detect faulty drivers.
+ */
+#define VIRTIO_F_BAD_FEATURE (1 << 30)
+
+/*
+ * Some VirtIO feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. virtio_ring), the
+ * rest are per-device feature bits.
+ */
+#define VIRTIO_TRANSPORT_F_START 28
+#define VIRTIO_TRANSPORT_F_END 32
+
+#endif /* _VIRTIO_CONFIG_H_ */
Property changes on: trunk/sys/dev/virtio/virtio_config.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Added: trunk/sys/dev/virtio/virtio_ids.h
===================================================================
--- trunk/sys/dev/virtio/virtio_ids.h (rev 0)
+++ trunk/sys/dev/virtio/virtio_ids.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -0,0 +1,45 @@
+/* $MidnightBSD$ */
+/*-
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: stable/10/sys/dev/virtio/virtio_ids.h 267312 2014-06-10 03:23:35Z bryanv $
+ */
+
+#ifndef _VIRTIO_IDS_H_
+#define _VIRTIO_IDS_H_
+
+/* VirtIO device IDs. */
+#define VIRTIO_ID_NETWORK 0x01
+#define VIRTIO_ID_BLOCK 0x02
+#define VIRTIO_ID_CONSOLE 0x03
+#define VIRTIO_ID_ENTROPY 0x04
+#define VIRTIO_ID_BALLOON 0x05
+#define VIRTIO_ID_IOMEMORY 0x06
+#define VIRTIO_ID_SCSI 0x08
+#define VIRTIO_ID_9P 0x09
+
+#endif /* _VIRTIO_IDS_H_ */
Property changes on: trunk/sys/dev/virtio/virtio_ids.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Modified: trunk/sys/dev/virtio/virtio_if.m
===================================================================
--- trunk/sys/dev/virtio/virtio_if.m 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtio_if.m 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
#-
# Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
# All rights reserved.
@@ -23,7 +24,7 @@
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
-# $FreeBSD$
+# $FreeBSD: stable/10/sys/dev/virtio/virtio_if.m 255110 2013-09-01 04:20:23Z bryanv $
#include <sys/bus.h>
@@ -31,6 +32,18 @@
CODE {
static int
+ virtio_default_attach_completed(device_t dev)
+ {
+ return (0);
+ }
+};
+
+METHOD int attach_completed {
+ device_t dev;
+} DEFAULT virtio_default_attach_completed;
+
+CODE {
+ static int
virtio_default_config_change(device_t dev)
{
return (0);
Property changes on: trunk/sys/dev/virtio/virtio_if.m
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Modified: trunk/sys/dev/virtio/virtio_ring.h
===================================================================
--- trunk/sys/dev/virtio/virtio_ring.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtio_ring.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright Rusty Russell IBM Corporation 2007.
*
@@ -27,7 +28,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/virtio_ring.h 238360 2012-07-11 02:57:19Z grehan $
*/
#ifndef VIRTIO_RING_H
Modified: trunk/sys/dev/virtio/virtqueue.c
===================================================================
--- trunk/sys/dev/virtio/virtqueue.c 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtqueue.c 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -30,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__MBSDID("$MidnightBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/virtqueue.c 270270 2014-08-21 13:27:05Z bryanv $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -127,7 +128,7 @@
static int vq_ring_use_indirect(struct virtqueue *, int);
static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
struct sglist *, int, int);
-static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
+static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
static int vq_ring_must_notify_host(struct virtqueue *);
static void vq_ring_notify_host(struct virtqueue *);
static void vq_ring_free_chain(struct virtqueue *, uint16_t);
@@ -375,6 +376,13 @@
}
int
+virtqueue_nfree(struct virtqueue *vq)
+{
+
+ return (vq->vq_free_cnt);
+}
+
+int
virtqueue_empty(struct virtqueue *vq)
{
@@ -440,28 +448,38 @@
}
int
-virtqueue_postpone_intr(struct virtqueue *vq)
+virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
{
uint16_t ndesc, avail_idx;
- /*
- * Request the next interrupt be postponed until at least half
- * of the available descriptors have been consumed.
- */
avail_idx = vq->vq_ring.avail->idx;
- ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx) / 2;
+ ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
+ switch (hint) {
+ case VQ_POSTPONE_SHORT:
+ ndesc = ndesc / 4;
+ break;
+ case VQ_POSTPONE_LONG:
+ ndesc = (ndesc * 3) / 4;
+ break;
+ case VQ_POSTPONE_EMPTIED:
+ break;
+ }
+
return (vq_ring_enable_interrupt(vq, ndesc));
}
+/*
+ * Note this is only considered a hint to the host.
+ */
void
virtqueue_disable_intr(struct virtqueue *vq)
{
- /*
- * Note this is only considered a hint to the host.
- */
- if ((vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) == 0)
+ if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
+ vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
+ vq->vq_nentries - 1;
+ } else
vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
}
@@ -531,7 +549,7 @@
used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
uep = &vq->vq_ring.used->ring[used_idx];
- mb();
+ rmb();
desc_idx = (uint16_t) uep->id;
if (len != NULL)
*len = uep->len;
@@ -629,7 +647,7 @@
avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
vq->vq_ring.avail->ring[avail_idx] = desc_idx;
- mb();
+ wmb();
vq->vq_ring.avail->idx++;
/* Keep pending count until virtqueue_notify(). */
Modified: trunk/sys/dev/virtio/virtqueue.h
===================================================================
--- trunk/sys/dev/virtio/virtqueue.h 2018-05-27 22:31:43 UTC (rev 10041)
+++ trunk/sys/dev/virtio/virtqueue.h 2018-05-27 22:33:45 UTC (rev 10042)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2011, Bryan Venteicher <bryanv at FreeBSD.org>
* All rights reserved.
@@ -23,7 +24,7 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * $MidnightBSD$
+ * $FreeBSD: stable/10/sys/dev/virtio/virtqueue.h 270270 2014-08-21 13:27:05Z bryanv $
*/
#ifndef _VIRTIO_VIRTQUEUE_H
@@ -32,15 +33,19 @@
struct virtqueue;
struct sglist;
-/* Support for indirect buffer descriptors. */
-#define VIRTIO_RING_F_INDIRECT_DESC (1 << 28)
-
-/* Support to suppress interrupt until specific index is reached. */
-#define VIRTIO_RING_F_EVENT_IDX (1 << 29)
-
/* Device callback for a virtqueue interrupt. */
typedef void virtqueue_intr_t(void *);
+/*
+ * Hint on how long the next interrupt should be postponed. This is
+ * only used when the EVENT_IDX feature is negotiated.
+ */
+typedef enum {
+ VQ_POSTPONE_SHORT,
+ VQ_POSTPONE_LONG,
+ VQ_POSTPONE_EMPTIED /* Until all available desc are used. */
+} vq_postpone_t;
+
#define VIRTQUEUE_MAX_NAME_SZ 32
/* One for each virtqueue the device wishes to allocate. */
@@ -73,7 +78,7 @@
int virtqueue_intr_filter(struct virtqueue *vq);
void virtqueue_intr(struct virtqueue *vq);
int virtqueue_enable_intr(struct virtqueue *vq);
-int virtqueue_postpone_intr(struct virtqueue *vq);
+int virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint);
void virtqueue_disable_intr(struct virtqueue *vq);
/* Get physical address of the virtqueue ring. */
@@ -82,6 +87,7 @@
int virtqueue_full(struct virtqueue *vq);
int virtqueue_empty(struct virtqueue *vq);
int virtqueue_size(struct virtqueue *vq);
+int virtqueue_nfree(struct virtqueue *vq);
int virtqueue_nused(struct virtqueue *vq);
void virtqueue_notify(struct virtqueue *vq);
void virtqueue_dump(struct virtqueue *vq);
More information about the Midnightbsd-cvs
mailing list