1 /*
2 * Copyright (C) 2014 Vincenzo Maffione, Luigi Rizzo. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 /*
27 * $FreeBSD$
28 */
29
30 #include <net/netmap.h>
31 #include <sys/selinfo.h>
32 #include <vm/vm.h>
33 #include <vm/pmap.h> /* vtophys ? */
34 #include <dev/netmap/netmap_kern.h>
35
36
37 #define SOFTC_T vtnet_softc
38
39 /* Free all the unused buffer in all the RX virtqueues.
40 * This function is called when entering and exiting netmap mode.
41 * - buffers queued by the virtio driver return skbuf/mbuf pointer
42 * and need to be freed;
43 * - buffers queued by netmap return the txq/rxq, and do not need work
44 */
45 static void
vtnet_netmap_free_bufs(struct SOFTC_T * sc)46 vtnet_netmap_free_bufs(struct SOFTC_T* sc)
47 {
48 int i, nmb = 0, n = 0, last;
49
50 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
51 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i];
52 struct virtqueue *vq;
53 struct mbuf *m;
54 struct vtnet_txq *txq = &sc->vtnet_txqs[i];
55 struct vtnet_tx_header *txhdr;
56
57 last = 0;
58 vq = rxq->vtnrx_vq;
59 while ((m = virtqueue_drain(vq, &last)) != NULL) {
60 n++;
61 if (m != (void *)rxq)
62 m_freem(m);
63 else
64 nmb++;
65 }
66
67 last = 0;
68 vq = txq->vtntx_vq;
69 while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
70 n++;
71 if (txhdr != (void *)txq) {
72 m_freem(txhdr->vth_mbuf);
73 uma_zfree(vtnet_tx_header_zone, txhdr);
74 } else
75 nmb++;
76 }
77 }
78 D("freed %d mbufs, %d netmap bufs on %d queues",
79 n - nmb, nmb, i);
80 }
81
82 /* Register and unregister. */
83 static int
vtnet_netmap_reg(struct netmap_adapter * na,int onoff)84 vtnet_netmap_reg(struct netmap_adapter *na, int onoff)
85 {
86 struct ifnet *ifp = na->ifp;
87 struct SOFTC_T *sc = ifp->if_softc;
88
89 VTNET_CORE_LOCK(sc);
90 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
91 /* enable or disable flags and callbacks in na and ifp */
92 if (onoff) {
93 nm_set_native_flags(na);
94 } else {
95 nm_clear_native_flags(na);
96 }
97 /* drain queues so netmap and native drivers
98 * do not interfere with each other
99 */
100 vtnet_netmap_free_bufs(sc);
101 vtnet_init_locked(sc); /* also enable intr */
102 VTNET_CORE_UNLOCK(sc);
103 return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
104 }
105
106
107 /* Reconcile kernel and user view of the transmit ring. */
108 static int
vtnet_netmap_txsync(struct netmap_kring * kring,int flags)109 vtnet_netmap_txsync(struct netmap_kring *kring, int flags)
110 {
111 struct netmap_adapter *na = kring->na;
112 struct ifnet *ifp = na->ifp;
113 struct netmap_ring *ring = kring->ring;
114 u_int ring_nr = kring->ring_id;
115 u_int nm_i; /* index into the netmap ring */
116 u_int nic_i; /* index into the NIC ring */
117 u_int n;
118 u_int const lim = kring->nkr_num_slots - 1;
119 u_int const head = kring->rhead;
120
121 /* device-specific */
122 struct SOFTC_T *sc = ifp->if_softc;
123 struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr];
124 struct virtqueue *vq = txq->vtntx_vq;
125
126 /*
127 * First part: process new packets to send.
128 */
129 rmb();
130
131 nm_i = kring->nr_hwcur;
132 if (nm_i != head) { /* we have new packets to send */
133 struct sglist *sg = txq->vtntx_sg;
134
135 nic_i = netmap_idx_k2n(kring, nm_i);
136 for (n = 0; nm_i != head; n++) {
137 /* we use an empty header here */
138 static struct virtio_net_hdr_mrg_rxbuf hdr;
139 struct netmap_slot *slot = &ring->slot[nm_i];
140 u_int len = slot->len;
141 uint64_t paddr;
142 void *addr = PNMB(na, slot, &paddr);
143 int err;
144
145 NM_CHECK_ADDR_LEN(na, addr, len);
146
147 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
148 /* Initialize the scatterlist, expose it to the hypervisor,
149 * and kick the hypervisor (if necessary).
150 */
151 sglist_reset(sg); // cheap
152 // if vtnet_hdr_size > 0 ...
153 err = sglist_append(sg, &hdr, sc->vtnet_hdr_size);
154 // XXX later, support multi segment
155 err = sglist_append_phys(sg, paddr, len);
156 /* use na as the cookie */
157 err = virtqueue_enqueue(vq, txq, sg, sg->sg_nseg, 0);
158 if (unlikely(err < 0)) {
159 D("virtqueue_enqueue failed");
160 break;
161 }
162
163 nm_i = nm_next(nm_i, lim);
164 nic_i = nm_next(nic_i, lim);
165 }
166 /* Update hwcur depending on where we stopped. */
167 kring->nr_hwcur = nm_i; /* note we migth break early */
168
169 /* No more free TX slots? Ask the hypervisor for notifications,
170 * possibly only when a considerable amount of work has been
171 * done.
172 */
173 ND(3,"sent %d packets, hwcur %d", n, nm_i);
174 virtqueue_disable_intr(vq);
175 virtqueue_notify(vq);
176 } else {
177 if (ring->head != ring->tail)
178 ND(5, "pure notify ? head %d tail %d nused %d %d",
179 ring->head, ring->tail, virtqueue_nused(vq),
180 (virtqueue_dump(vq), 1));
181 virtqueue_notify(vq);
182 virtqueue_enable_intr(vq); // like postpone with 0
183 }
184
185
186 /* Free used slots. We only consider our own used buffers, recognized
187 * by the token we passed to virtqueue_add_outbuf.
188 */
189 n = 0;
190 for (;;) {
191 struct vtnet_tx_header *txhdr = virtqueue_dequeue(vq, NULL);
192 if (txhdr == NULL)
193 break;
194 if (likely(txhdr == (void *)txq)) {
195 n++;
196 if (virtqueue_nused(vq) < 32) { // XXX slow release
197 break;
198 }
199 } else { /* leftover from previous transmission */
200 m_freem(txhdr->vth_mbuf);
201 uma_zfree(vtnet_tx_header_zone, txhdr);
202 }
203 }
204 if (n) {
205 kring->nr_hwtail += n;
206 if (kring->nr_hwtail > lim)
207 kring->nr_hwtail -= lim + 1;
208 }
209 if (nm_i != kring->nr_hwtail /* && vtnet_txq_below_threshold(txq) == 0*/) {
210 ND(3, "disable intr, hwcur %d", nm_i);
211 virtqueue_disable_intr(vq);
212 } else {
213 ND(3, "enable intr, hwcur %d", nm_i);
214 virtqueue_postpone_intr(vq, VQ_POSTPONE_SHORT);
215 }
216
217 return 0;
218 }
219
220 static int
vtnet_refill_rxq(struct netmap_kring * kring,u_int nm_i,u_int head)221 vtnet_refill_rxq(struct netmap_kring *kring, u_int nm_i, u_int head)
222 {
223 struct netmap_adapter *na = kring->na;
224 struct ifnet *ifp = na->ifp;
225 struct netmap_ring *ring = kring->ring;
226 u_int ring_nr = kring->ring_id;
227 u_int const lim = kring->nkr_num_slots - 1;
228 u_int n;
229
230 /* device-specific */
231 struct SOFTC_T *sc = ifp->if_softc;
232 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
233 struct virtqueue *vq = rxq->vtnrx_vq;
234
235 /* use a local sglist, default might be short */
236 struct sglist_seg ss[2];
237 struct sglist sg = { ss, 0, 0, 2 };
238
239 for (n = 0; nm_i != head; n++) {
240 static struct virtio_net_hdr_mrg_rxbuf hdr;
241 struct netmap_slot *slot = &ring->slot[nm_i];
242 uint64_t paddr;
243 void *addr = PNMB(na, slot, &paddr);
244 int err = 0;
245
246 if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */
247 if (netmap_ring_reinit(kring))
248 return -1;
249 }
250
251 slot->flags &= ~NS_BUF_CHANGED;
252 sglist_reset(&sg); // cheap
253 err = sglist_append(&sg, &hdr, sc->vtnet_hdr_size);
254 err = sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
255 /* writable for the host */
256 err = virtqueue_enqueue(vq, rxq, &sg, 0, sg.sg_nseg);
257 if (err < 0) {
258 D("virtqueue_enqueue failed");
259 break;
260 }
261 nm_i = nm_next(nm_i, lim);
262 }
263 return nm_i;
264 }
265
266 /* Reconcile kernel and user view of the receive ring. */
267 static int
vtnet_netmap_rxsync(struct netmap_kring * kring,int flags)268 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
269 {
270 struct netmap_adapter *na = kring->na;
271 struct ifnet *ifp = na->ifp;
272 struct netmap_ring *ring = kring->ring;
273 u_int ring_nr = kring->ring_id;
274 u_int nm_i; /* index into the netmap ring */
275 // u_int nic_i; /* index into the NIC ring */
276 u_int n;
277 u_int const lim = kring->nkr_num_slots - 1;
278 u_int const head = kring->rhead;
279 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
280
281 /* device-specific */
282 struct SOFTC_T *sc = ifp->if_softc;
283 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
284 struct virtqueue *vq = rxq->vtnrx_vq;
285
286 /* XXX netif_carrier_ok ? */
287
288 if (head > lim)
289 return netmap_ring_reinit(kring);
290
291 rmb();
292 /*
293 * First part: import newly received packets.
294 * Only accept our
295 * own buffers (matching the token). We should only get
296 * matching buffers, because of vtnet_netmap_free_rx_unused_bufs()
297 * and vtnet_netmap_init_buffers().
298 */
299 if (netmap_no_pendintr || force_update) {
300 uint16_t slot_flags = kring->nkr_slot_flags;
301 struct netmap_adapter *token;
302
303 nm_i = kring->nr_hwtail;
304 n = 0;
305 for (;;) {
306 int len;
307 token = virtqueue_dequeue(vq, &len);
308 if (token == NULL)
309 break;
310 if (likely(token == (void *)rxq)) {
311 ring->slot[nm_i].len = len;
312 ring->slot[nm_i].flags = slot_flags;
313 nm_i = nm_next(nm_i, lim);
314 n++;
315 } else {
316 D("This should not happen");
317 }
318 }
319 kring->nr_hwtail = nm_i;
320 kring->nr_kflags &= ~NKR_PENDINTR;
321 }
322 ND("[B] h %d c %d hwcur %d hwtail %d",
323 ring->head, ring->cur, kring->nr_hwcur,
324 kring->nr_hwtail);
325
326 /*
327 * Second part: skip past packets that userspace has released.
328 */
329 nm_i = kring->nr_hwcur; /* netmap ring index */
330 if (nm_i != head) {
331 int err = vtnet_refill_rxq(kring, nm_i, head);
332 if (err < 0)
333 return 1;
334 kring->nr_hwcur = err;
335 virtqueue_notify(vq);
336 /* After draining the queue may need an intr from the hypervisor */
337 vtnet_rxq_enable_intr(rxq);
338 }
339
340 ND("[C] h %d c %d t %d hwcur %d hwtail %d",
341 ring->head, ring->cur, ring->tail,
342 kring->nr_hwcur, kring->nr_hwtail);
343
344 return 0;
345 }
346
347
348 /* Make RX virtqueues buffers pointing to netmap buffers. */
349 static int
vtnet_netmap_init_rx_buffers(struct SOFTC_T * sc)350 vtnet_netmap_init_rx_buffers(struct SOFTC_T *sc)
351 {
352 struct ifnet *ifp = sc->vtnet_ifp;
353 struct netmap_adapter* na = NA(ifp);
354 unsigned int r;
355
356 if (!nm_native_on(na))
357 return 0;
358 for (r = 0; r < na->num_rx_rings; r++) {
359 struct netmap_kring *kring = &na->rx_rings[r];
360 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[r];
361 struct virtqueue *vq = rxq->vtnrx_vq;
362 struct netmap_slot* slot;
363 int err = 0;
364
365 slot = netmap_reset(na, NR_RX, r, 0);
366 if (!slot) {
367 D("strange, null netmap ring %d", r);
368 return 0;
369 }
370 /* Add up to na>-num_rx_desc-1 buffers to this RX virtqueue.
371 * It's important to leave one virtqueue slot free, otherwise
372 * we can run into ring->cur/ring->tail wraparounds.
373 */
374 err = vtnet_refill_rxq(kring, 0, na->num_rx_desc-1);
375 if (err < 0)
376 return 0;
377 virtqueue_notify(vq);
378 }
379
380 return 1;
381 }
382
383 /* Update the virtio-net device configurations. Number of queues can
384 * change dinamically, by 'ethtool --set-channels $IFNAME combined $N'.
385 * This is actually the only way virtio-net can currently enable
386 * the multiqueue mode.
387 * XXX note that we seem to lose packets if the netmap ring has more
388 * slots than the queue
389 */
390 static int
vtnet_netmap_config(struct netmap_adapter * na,u_int * txr,u_int * txd,u_int * rxr,u_int * rxd)391 vtnet_netmap_config(struct netmap_adapter *na, u_int *txr, u_int *txd,
392 u_int *rxr, u_int *rxd)
393 {
394 struct ifnet *ifp = na->ifp;
395 struct SOFTC_T *sc = ifp->if_softc;
396
397 *txr = *rxr = sc->vtnet_max_vq_pairs;
398 *rxd = 512; // sc->vtnet_rx_nmbufs;
399 *txd = *rxd; // XXX
400 D("vtnet config txq=%d, txd=%d rxq=%d, rxd=%d",
401 *txr, *txd, *rxr, *rxd);
402
403 return 0;
404 }
405
406 static void
vtnet_netmap_attach(struct SOFTC_T * sc)407 vtnet_netmap_attach(struct SOFTC_T *sc)
408 {
409 struct netmap_adapter na;
410
411 bzero(&na, sizeof(na));
412
413 na.ifp = sc->vtnet_ifp;
414 na.num_tx_desc = 1024;// sc->vtnet_rx_nmbufs;
415 na.num_rx_desc = 1024; // sc->vtnet_rx_nmbufs;
416 na.nm_register = vtnet_netmap_reg;
417 na.nm_txsync = vtnet_netmap_txsync;
418 na.nm_rxsync = vtnet_netmap_rxsync;
419 na.nm_config = vtnet_netmap_config;
420 na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
421 D("max rings %d", sc->vtnet_max_vq_pairs);
422 netmap_attach(&na);
423
424 D("virtio attached txq=%d, txd=%d rxq=%d, rxd=%d",
425 na.num_tx_rings, na.num_tx_desc,
426 na.num_tx_rings, na.num_rx_desc);
427 }
428 /* end of file */
429