1 /*
2 * Copyright (C) 2014-2018 Vincenzo Maffione, Luigi Rizzo.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 /*
27 */
28
29 #include <net/netmap.h>
30 #include <sys/selinfo.h>
31 #include <vm/vm.h>
32 #include <vm/pmap.h> /* vtophys ? */
33 #include <dev/netmap/netmap_kern.h>
34
35 /* Register and unregister. */
36 static int
vtnet_netmap_reg(struct netmap_adapter * na,int state)37 vtnet_netmap_reg(struct netmap_adapter *na, int state)
38 {
39 struct ifnet *ifp = na->ifp;
40 struct vtnet_softc *sc = ifp->if_softc;
41
42 /*
43 * Trigger a device reinit, asking vtnet_init_locked() to
44 * also enter or exit netmap mode.
45 */
46 VTNET_CORE_LOCK(sc);
47 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
48 vtnet_init_locked(sc, state ? VTNET_INIT_NETMAP_ENTER
49 : VTNET_INIT_NETMAP_EXIT);
50 VTNET_CORE_UNLOCK(sc);
51
52 return (0);
53 }
54
55
56 /* Reconcile kernel and user view of the transmit ring. */
57 static int
vtnet_netmap_txsync(struct netmap_kring * kring,int flags)58 vtnet_netmap_txsync(struct netmap_kring *kring, int flags)
59 {
60 struct netmap_adapter *na = kring->na;
61 struct ifnet *ifp = na->ifp;
62 struct netmap_ring *ring = kring->ring;
63 u_int ring_nr = kring->ring_id;
64 u_int nm_i; /* index into the netmap ring */
65 u_int const lim = kring->nkr_num_slots - 1;
66 u_int const head = kring->rhead;
67
68 /* device-specific */
69 struct vtnet_softc *sc = ifp->if_softc;
70 struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr];
71 struct virtqueue *vq = txq->vtntx_vq;
72 int interrupts = !(kring->nr_kflags & NKR_NOINTR);
73 u_int n;
74
75 /*
76 * First part: process new packets to send.
77 */
78
79 nm_i = kring->nr_hwcur;
80 if (nm_i != head) { /* we have new packets to send */
81 struct sglist *sg = txq->vtntx_sg;
82
83 for (; nm_i != head; nm_i = nm_next(nm_i, lim)) {
84 /* we use an empty header here */
85 struct netmap_slot *slot = &ring->slot[nm_i];
86 u_int len = slot->len;
87 uint64_t paddr;
88 void *addr = PNMB(na, slot, &paddr);
89 int err;
90
91 NM_CHECK_ADDR_LEN(na, addr, len);
92
93 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
94 /* Initialize the scatterlist, expose it to the hypervisor,
95 * and kick the hypervisor (if necessary).
96 */
97 sglist_reset(sg); // cheap
98 err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size);
99 err |= sglist_append_phys(sg, paddr, len);
100 KASSERT(err == 0, ("%s: cannot append to sglist %d",
101 __func__, err));
102 err = virtqueue_enqueue(vq, /*cookie=*/txq, sg,
103 /*readable=*/sg->sg_nseg,
104 /*writeable=*/0);
105 if (unlikely(err)) {
106 if (err != ENOSPC)
107 nm_prerr("virtqueue_enqueue(%s) failed: %d",
108 kring->name, err);
109 break;
110 }
111 }
112
113 virtqueue_notify(vq);
114
115 /* Update hwcur depending on where we stopped. */
116 kring->nr_hwcur = nm_i; /* note we might break early */
117 }
118
119 /* Free used slots. We only consider our own used buffers, recognized
120 * by the token we passed to virtqueue_enqueue.
121 */
122 n = 0;
123 for (;;) {
124 void *token = virtqueue_dequeue(vq, NULL);
125 if (token == NULL)
126 break;
127 if (unlikely(token != (void *)txq))
128 nm_prerr("BUG: TX token mismatch");
129 else
130 n++;
131 }
132 if (n > 0) {
133 kring->nr_hwtail += n;
134 if (kring->nr_hwtail > lim)
135 kring->nr_hwtail -= lim + 1;
136 }
137
138 if (interrupts && virtqueue_nfree(vq) < 32)
139 virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG);
140
141 return 0;
142 }
143
144 /*
145 * Publish 'num 'netmap receive buffers to the host, starting
146 * from the next available one (rx->vtnrx_nm_refill).
147 * Return a positive error code on error, and 0 on success.
148 * If we could not publish all of the buffers that's an error,
149 * since the netmap ring and the virtqueue would go out of sync.
150 */
151 static int
vtnet_netmap_kring_refill(struct netmap_kring * kring,u_int num)152 vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num)
153 {
154 struct netmap_adapter *na = kring->na;
155 struct ifnet *ifp = na->ifp;
156 struct netmap_ring *ring = kring->ring;
157 u_int ring_nr = kring->ring_id;
158 u_int const lim = kring->nkr_num_slots - 1;
159 u_int nm_i;
160
161 /* device-specific */
162 struct vtnet_softc *sc = ifp->if_softc;
163 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
164 struct virtqueue *vq = rxq->vtnrx_vq;
165
166 /* use a local sglist, default might be short */
167 struct sglist_seg ss[2];
168 struct sglist sg = { ss, 0, 0, 2 };
169
170 for (nm_i = rxq->vtnrx_nm_refill; num > 0;
171 nm_i = nm_next(nm_i, lim), num--) {
172 struct netmap_slot *slot = &ring->slot[nm_i];
173 uint64_t paddr;
174 void *addr = PNMB(na, slot, &paddr);
175 int err;
176
177 if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */
178 if (netmap_ring_reinit(kring))
179 return EFAULT;
180 }
181
182 slot->flags &= ~NS_BUF_CHANGED;
183 sglist_reset(&sg);
184 err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size);
185 err |= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
186 KASSERT(err == 0, ("%s: cannot append to sglist %d",
187 __func__, err));
188 /* writable for the host */
189 err = virtqueue_enqueue(vq, /*cookie=*/rxq, &sg,
190 /*readable=*/0, /*writeable=*/sg.sg_nseg);
191 if (unlikely(err)) {
192 nm_prerr("virtqueue_enqueue(%s) failed: %d",
193 kring->name, err);
194 break;
195 }
196 }
197 rxq->vtnrx_nm_refill = nm_i;
198
199 return num == 0 ? 0 : ENOSPC;
200 }
201
202 /*
203 * Publish netmap buffers on a RX virtqueue.
204 * Returns -1 if this virtqueue is not being opened in netmap mode.
205 * If the virtqueue is being opened in netmap mode, return 0 on success and
206 * a positive error code on failure.
207 */
208 static int
vtnet_netmap_rxq_populate(struct vtnet_rxq * rxq)209 vtnet_netmap_rxq_populate(struct vtnet_rxq *rxq)
210 {
211 struct netmap_adapter *na = NA(rxq->vtnrx_sc->vtnet_ifp);
212 struct netmap_kring *kring;
213 struct netmap_slot *slot;
214 int error;
215 int num;
216
217 slot = netmap_reset(na, NR_RX, rxq->vtnrx_id, 0);
218 if (slot == NULL)
219 return -1;
220 kring = na->rx_rings[rxq->vtnrx_id];
221
222 /*
223 * Expose all the RX netmap buffers we can. In case of no indirect
224 * buffers, the number of netmap slots in the RX ring matches the
225 * maximum number of 2-elements sglist that the RX virtqueue can
226 * accommodate. We need to start from kring->nr_hwtail, which is 0
227 * on the first netmap register and may be different from 0 if a
228 * virtio re-init (caused by a netma register or i.e., ifconfig)
229 * happens while the device is in use by netmap.
230 */
231 rxq->vtnrx_nm_refill = kring->nr_hwtail;
232 num = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
233 error = vtnet_netmap_kring_refill(kring, num);
234 virtqueue_notify(rxq->vtnrx_vq);
235
236 return error;
237 }
238
239 /* Reconcile kernel and user view of the receive ring. */
240 static int
vtnet_netmap_rxsync(struct netmap_kring * kring,int flags)241 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
242 {
243 struct netmap_adapter *na = kring->na;
244 struct ifnet *ifp = na->ifp;
245 struct netmap_ring *ring = kring->ring;
246 u_int ring_nr = kring->ring_id;
247 u_int nm_i; /* index into the netmap ring */
248 u_int const lim = kring->nkr_num_slots - 1;
249 u_int const head = kring->rhead;
250 int force_update = (flags & NAF_FORCE_READ) ||
251 (kring->nr_kflags & NKR_PENDINTR);
252 int interrupts = !(kring->nr_kflags & NKR_NOINTR);
253
254 /* device-specific */
255 struct vtnet_softc *sc = ifp->if_softc;
256 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
257 struct virtqueue *vq = rxq->vtnrx_vq;
258
259 /*
260 * First part: import newly received packets.
261 * Only accept our own buffers (matching the token). We should only get
262 * matching buffers. The hwtail should never overrun hwcur, because
263 * we publish only N-1 receive buffers (and not N).
264 * In any case we must not leave this routine with the interrupts
265 * disabled, pending packets in the VQ and hwtail == (hwcur - 1),
266 * otherwise the pending packets could stall.
267 */
268 if (netmap_no_pendintr || force_update) {
269 uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim);
270 void *token;
271
272 vtnet_rxq_disable_intr(rxq);
273
274 nm_i = kring->nr_hwtail;
275 for (;;) {
276 int len;
277 token = virtqueue_dequeue(vq, &len);
278 if (token == NULL) {
279 /*
280 * Enable the interrupts again and double-check
281 * for more work. We can go on until we win the
282 * race condition, since we are not replenishing
283 * in the meanwhile, and thus we will process at
284 * most N-1 slots.
285 */
286 if (interrupts && vtnet_rxq_enable_intr(rxq)) {
287 vtnet_rxq_disable_intr(rxq);
288 continue;
289 }
290 break;
291 }
292 if (unlikely(token != (void *)rxq)) {
293 nm_prerr("BUG: RX token mismatch");
294 } else {
295 if (nm_i == hwtail_lim) {
296 KASSERT(false, ("hwtail would "
297 "overrun hwcur"));
298 }
299
300 /* Skip the virtio-net header. */
301 len -= sc->vtnet_hdr_size;
302 if (unlikely(len < 0)) {
303 nm_prlim(1, "Truncated virtio-net-header, "
304 "missing %d bytes", -len);
305 len = 0;
306 }
307 ring->slot[nm_i].len = len;
308 ring->slot[nm_i].flags = 0;
309 nm_i = nm_next(nm_i, lim);
310 }
311 }
312 kring->nr_hwtail = nm_i;
313 kring->nr_kflags &= ~NKR_PENDINTR;
314 }
315
316 /*
317 * Second part: skip past packets that userspace has released.
318 */
319 nm_i = kring->nr_hwcur; /* netmap ring index */
320 if (nm_i != head) {
321 int released;
322 int error;
323
324 released = head - nm_i;
325 if (released < 0)
326 released += kring->nkr_num_slots;
327 error = vtnet_netmap_kring_refill(kring, released);
328 if (error) {
329 nm_prerr("Failed to replenish RX VQ with %u sgs",
330 released);
331 return error;
332 }
333 kring->nr_hwcur = head;
334 virtqueue_notify(vq);
335 }
336
337 nm_prdis("h %d c %d t %d hwcur %d hwtail %d", kring->rhead,
338 kring->rcur, kring->rtail, kring->nr_hwcur, kring->nr_hwtail);
339
340 return 0;
341 }
342
343
344 /* Enable/disable interrupts on all virtqueues. */
345 static void
vtnet_netmap_intr(struct netmap_adapter * na,int state)346 vtnet_netmap_intr(struct netmap_adapter *na, int state)
347 {
348 struct vtnet_softc *sc = na->ifp->if_softc;
349 int i;
350
351 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
352 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i];
353 struct vtnet_txq *txq = &sc->vtnet_txqs[i];
354 struct virtqueue *txvq = txq->vtntx_vq;
355
356 if (state) {
357 vtnet_rxq_enable_intr(rxq);
358 virtqueue_enable_intr(txvq);
359 } else {
360 vtnet_rxq_disable_intr(rxq);
361 virtqueue_disable_intr(txvq);
362 }
363 }
364 }
365
366 static int
vtnet_netmap_tx_slots(struct vtnet_softc * sc)367 vtnet_netmap_tx_slots(struct vtnet_softc *sc)
368 {
369 int div;
370
371 /* We need to prepend a virtio-net header to each netmap buffer to be
372 * transmitted, therefore calling virtqueue_enqueue() passing sglist
373 * with 2 elements.
374 * TX virtqueues use indirect descriptors if the feature was negotiated
375 * with the host, and if sc->vtnet_tx_nsegs > 1. With indirect
376 * descriptors, a single virtio descriptor is sufficient to reference
377 * each TX sglist. Without them, we need two separate virtio descriptors
378 * for each TX sglist. We therefore compute the number of netmap TX
379 * slots according to these assumptions.
380 */
381 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_tx_nsegs > 1)
382 div = 1;
383 else
384 div = 2;
385
386 return virtqueue_size(sc->vtnet_txqs[0].vtntx_vq) / div;
387 }
388
389 static int
vtnet_netmap_rx_slots(struct vtnet_softc * sc)390 vtnet_netmap_rx_slots(struct vtnet_softc *sc)
391 {
392 int div;
393
394 /* We need to prepend a virtio-net header to each netmap buffer to be
395 * received, therefore calling virtqueue_enqueue() passing sglist
396 * with 2 elements.
397 * RX virtqueues use indirect descriptors if the feature was negotiated
398 * with the host, and if sc->vtnet_rx_nsegs > 1. With indirect
399 * descriptors, a single virtio descriptor is sufficient to reference
400 * each RX sglist. Without them, we need two separate virtio descriptors
401 * for each RX sglist. We therefore compute the number of netmap RX
402 * slots according to these assumptions.
403 */
404 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_rx_nsegs > 1)
405 div = 1;
406 else
407 div = 2;
408
409 return virtqueue_size(sc->vtnet_rxqs[0].vtnrx_vq) / div;
410 }
411
412 static int
vtnet_netmap_config(struct netmap_adapter * na,struct nm_config_info * info)413 vtnet_netmap_config(struct netmap_adapter *na, struct nm_config_info *info)
414 {
415 struct vtnet_softc *sc = na->ifp->if_softc;
416
417 info->num_tx_rings = sc->vtnet_act_vq_pairs;
418 info->num_rx_rings = sc->vtnet_act_vq_pairs;
419 info->num_tx_descs = vtnet_netmap_tx_slots(sc);
420 info->num_rx_descs = vtnet_netmap_rx_slots(sc);
421 info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
422
423 return 0;
424 }
425
426 static void
vtnet_netmap_attach(struct vtnet_softc * sc)427 vtnet_netmap_attach(struct vtnet_softc *sc)
428 {
429 struct netmap_adapter na;
430
431 bzero(&na, sizeof(na));
432
433 na.ifp = sc->vtnet_ifp;
434 na.na_flags = 0;
435 na.num_tx_desc = vtnet_netmap_tx_slots(sc);
436 na.num_rx_desc = vtnet_netmap_rx_slots(sc);
437 na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
438 na.rx_buf_maxsize = 0;
439 na.nm_register = vtnet_netmap_reg;
440 na.nm_txsync = vtnet_netmap_txsync;
441 na.nm_rxsync = vtnet_netmap_rxsync;
442 na.nm_intr = vtnet_netmap_intr;
443 na.nm_config = vtnet_netmap_config;
444
445 netmap_attach(&na);
446
447 nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d",
448 na.num_tx_rings, na.num_tx_desc,
449 na.num_tx_rings, na.num_rx_desc);
450 }
451 /* end of file */
452