1 /* $NetBSD: in6_offload.c,v 1.13 2024/07/05 04:31:54 rin Exp $ */
2
3 /*
4 * Copyright (c)2006 YAMAMOTO Takashi,
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: in6_offload.c,v 1.13 2024/07/05 04:31:54 rin Exp $");
31
32 #include <sys/param.h>
33 #include <sys/mbuf.h>
34
35 #include <net/if.h>
36
37 #include <netinet/in.h>
38 #include <netinet/in_systm.h>
39 #include <netinet/ip6.h>
40 #include <netinet/tcp.h>
41 #include <netinet6/in6_var.h>
42 #include <netinet6/ip6_var.h>
43 #include <netinet6/nd6.h>
44 #include <netinet6/in6_offload.h>
45
46 /*
47 * Handle M_CSUM_TSOv6 in software. Split the TCP payload in chunks of
48 * size MSS, and return mbuf chain consists of them.
49 */
50 struct mbuf *
tcp6_segment(struct mbuf * m,int off)51 tcp6_segment(struct mbuf *m, int off)
52 {
53 int mss;
54 int iphlen;
55 int thlen;
56 int hlen;
57 int len;
58 struct ip6_hdr *iph;
59 struct tcphdr *th;
60 uint32_t tcpseq;
61 uint16_t phsum;
62 struct mbuf *hdr = NULL;
63 struct mbuf *m0 = NULL;
64 struct mbuf *prev = NULL;
65 struct mbuf *n, *t;
66 int nsegs;
67
68 KASSERT((m->m_flags & M_PKTHDR) != 0);
69 KASSERT((m->m_pkthdr.csum_flags & M_CSUM_TSOv6) != 0);
70
71 m->m_pkthdr.csum_flags = 0;
72
73 len = m->m_pkthdr.len;
74 KASSERT(len >= off + sizeof(*iph) + sizeof(*th));
75
76 hlen = off + sizeof(*iph);
77 if (m->m_len < hlen) {
78 m = m_pullup(m, hlen);
79 if (m == NULL)
80 goto quit;
81 }
82 iph = (void *)(mtod(m, char *) + off);
83 iphlen = sizeof(*iph);
84 KASSERT((iph->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION);
85 KASSERT(iph->ip6_nxt == IPPROTO_TCP);
86
87 hlen = off + iphlen + sizeof(*th);
88 if (m->m_len < hlen) {
89 m = m_pullup(m, hlen);
90 if (m == NULL)
91 goto quit;
92 }
93 th = (void *)(mtod(m, char *) + off + iphlen);
94 tcpseq = ntohl(th->th_seq);
95 thlen = th->th_off * 4;
96 hlen = off + iphlen + thlen;
97
98 mss = m->m_pkthdr.segsz;
99 KASSERT(mss != 0);
100 KASSERT(len > hlen);
101
102 t = m_split(m, hlen, M_NOWAIT);
103 if (t == NULL)
104 goto quit;
105 hdr = m;
106 m = t;
107
108 len -= hlen;
109 KASSERT(len % mss == 0);
110
111 iph = (void *)(mtod(hdr, char *) + off);
112 iph->ip6_plen = htons(thlen + mss);
113 phsum = in6_cksum_phdr(&iph->ip6_src, &iph->ip6_dst, htonl(thlen + mss),
114 htonl(IPPROTO_TCP));
115
116 for (nsegs = len / mss; nsegs > 0; nsegs--) {
117 if (nsegs > 1) {
118 n = m_dup(hdr, 0, hlen, M_NOWAIT);
119 if (n == NULL)
120 goto quit;
121 } else
122 n = hdr;
123 KASSERT(n->m_len == hlen); /* XXX */
124
125 if (nsegs > 1) {
126 t = m_split(m, mss, M_NOWAIT);
127 if (t == NULL) {
128 m_freem(n);
129 goto quit;
130 }
131 } else
132 t = m;
133 m_cat(n, m);
134 m = t;
135
136 KASSERT(n->m_len >= hlen); /* XXX */
137
138 if (m0 == NULL)
139 m0 = n;
140
141 if (prev != NULL)
142 prev->m_nextpkt = n;
143
144 n->m_pkthdr.len = hlen + mss;
145 n->m_nextpkt = NULL; /* XXX */
146
147 th = (void *)(mtod(n, char *) + off + iphlen);
148 th->th_seq = htonl(tcpseq);
149 th->th_sum = phsum;
150 th->th_sum = in6_cksum(n, 0, off + iphlen, thlen + mss);
151
152 tcpseq += mss;
153 prev = n;
154 }
155 return m0;
156
157 quit:
158 m_freem(hdr);
159 m_freem(m);
160 for (m = m0; m != NULL; m = n) {
161 n = m->m_nextpkt;
162 m_freem(m);
163 }
164
165 return NULL;
166 }
167
168 int
ip6_tso_output(struct ifnet * ifp,struct ifnet * origifp,struct mbuf * m,const struct sockaddr_in6 * dst,struct rtentry * rt)169 ip6_tso_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
170 const struct sockaddr_in6 *dst, struct rtentry *rt)
171 {
172 struct mbuf *n;
173 int error = 0;
174
175 m = tcp6_segment(m, 0);
176 if (m == NULL)
177 return ENOMEM;
178 do {
179 n = m->m_nextpkt;
180 if (error == 0)
181 error = ip6_if_output(ifp, origifp, m, dst, rt);
182 else
183 m_freem(m);
184 m = n;
185 } while (m != NULL);
186 return error;
187 }
188
189 /*
190 * Compute now in software the IP and TCP/UDP checksums. Cancel the
191 * hardware offloading.
192 */
193 void
in6_undefer_cksum(struct mbuf * m,size_t hdrlen,int csum_flags)194 in6_undefer_cksum(struct mbuf *m, size_t hdrlen, int csum_flags)
195 {
196 const size_t ip6_plen_offset =
197 hdrlen + offsetof(struct ip6_hdr, ip6_plen);
198 size_t l4hdroff;
199 size_t l4offset;
200 uint16_t plen;
201 uint16_t csum;
202
203 KASSERT(m->m_flags & M_PKTHDR);
204 KASSERT((m->m_pkthdr.csum_flags & csum_flags) == csum_flags);
205 KASSERT(csum_flags == M_CSUM_UDPv6 || csum_flags == M_CSUM_TCPv6);
206
207 if (__predict_true(hdrlen + sizeof(struct ip6_hdr) <= m->m_len)) {
208 plen = *(uint16_t *)(mtod(m, char *) + ip6_plen_offset);
209 } else {
210 m_copydata(m, ip6_plen_offset, sizeof(plen), &plen);
211 }
212 plen = ntohs(plen);
213
214 l4hdroff = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data);
215 l4offset = hdrlen + l4hdroff;
216 csum = in6_cksum(m, 0, l4offset,
217 plen - (l4hdroff - sizeof(struct ip6_hdr)));
218
219 if (csum == 0 && (csum_flags & M_CSUM_UDPv6) != 0)
220 csum = 0xffff;
221
222 l4offset += M_CSUM_DATA_IPv6_OFFSET(m->m_pkthdr.csum_data);
223
224 if (__predict_true((l4offset + sizeof(uint16_t)) <= m->m_len)) {
225 *(uint16_t *)(mtod(m, char *) + l4offset) = csum;
226 } else {
227 m_copyback(m, l4offset, sizeof(csum), (void *) &csum);
228 }
229
230 m->m_pkthdr.csum_flags ^= csum_flags;
231 }
232
233 /*
234 * Compute now in software the TCP/UDP checksum. Cancel the hardware
235 * offloading.
236 */
237 void
in6_undefer_cksum_tcpudp(struct mbuf * m)238 in6_undefer_cksum_tcpudp(struct mbuf *m)
239 {
240 uint16_t csum, offset;
241
242 KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_UDPv6|M_CSUM_TCPv6)) != 0);
243 KASSERT((~m->m_pkthdr.csum_flags & (M_CSUM_UDPv6|M_CSUM_TCPv6)) != 0);
244 KASSERT((m->m_pkthdr.csum_flags
245 & (M_CSUM_UDPv4|M_CSUM_TCPv4|M_CSUM_TSOv4)) == 0);
246
247 offset = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data);
248 csum = in6_cksum(m, 0, offset, m->m_pkthdr.len - offset);
249 if (csum == 0 && (m->m_pkthdr.csum_flags & M_CSUM_UDPv6) != 0) {
250 csum = 0xffff;
251 }
252
253 offset += M_CSUM_DATA_IPv6_OFFSET(m->m_pkthdr.csum_data);
254 if ((offset + sizeof(csum)) > m->m_len) {
255 m_copyback(m, offset, sizeof(csum), &csum);
256 } else {
257 *(uint16_t *)(mtod(m, char *) + offset) = csum;
258 }
259 }
260