1 /*************************************************************************
2 Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
3 reserved.
4
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are
8 met:
9
10 * Redistributions of source code must retain the above copyright
11 notice, this list of conditions and the following disclaimer.
12
13 * Redistributions in binary form must reproduce the above
14 copyright notice, this list of conditions and the following
15 disclaimer in the documentation and/or other materials provided
16 with the distribution.
17
18 * Neither the name of Cavium Networks nor the names of
19 its contributors may be used to endorse or promote products
20 derived from this software without specific prior written
21 permission.
22
23 This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries.
24
25 TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
26 AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
27
28 *************************************************************************/
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/mbuf.h>
39 #include <sys/socket.h>
40 #include <sys/proc.h>
41 #include <sys/sched.h>
42 #include <sys/smp.h>
43 #include <sys/taskqueue.h>
44
45 #include <net/ethernet.h>
46 #include <net/if.h>
47 #include <net/if_var.h>
48
49 #include "wrapper-cvmx-includes.h"
50 #include "ethernet-headers.h"
51
52 extern int pow_receive_group;
53 extern struct ifnet *cvm_oct_device[];
54
55 static struct task cvm_oct_task;
56 static struct taskqueue *cvm_oct_taskq;
57
58 static int cvm_oct_rx_active;
59
60 /**
61 * Interrupt handler. The interrupt occurs whenever the POW
62 * transitions from 0->1 packets in our group.
63 *
64 * @param cpl
65 * @param dev_id
66 * @param regs
67 * @return
68 */
cvm_oct_do_interrupt(void * dev_id)69 int cvm_oct_do_interrupt(void *dev_id)
70 {
71 /* Acknowledge the interrupt */
72 if (INTERRUPT_LIMIT)
73 cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group);
74 else
75 cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group);
76
77 /*
78 * Schedule task if there isn't one running.
79 */
80 if (atomic_cmpset_int(&cvm_oct_rx_active, 0, 1))
81 taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
82
83 return FILTER_HANDLED;
84 }
85
86
87 /**
88 * This is called on receive errors, and determines if the packet
89 * can be dropped early-on in cvm_oct_tasklet_rx().
90 *
91 * @param work Work queue entry pointing to the packet.
92 * @return Non-zero if the packet can be dropped, zero otherwise.
93 */
cvm_oct_check_rcv_error(cvmx_wqe_t * work)94 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
95 {
96 if ((work->word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) {
97 /* Ignore length errors on min size packets. Some equipment
98 incorrectly pads packets to 64+4FCS instead of 60+4FCS.
99 Note these packets still get counted as frame errors. */
100 } else
101 if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) {
102
103 /* We received a packet with either an alignment error or a
104 FCS error. This may be signalling that we are running
105 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the
106 case we need to parse the packet to determine if we can
107 remove a non spec preamble and generate a correct packet */
108 int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt);
109 int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt);
110 cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl;
111 gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface));
112 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) {
113
114 uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr);
115 int i = 0;
116
117 while (i < work->word1.s.len-1) {
118 if (*ptr != 0x55)
119 break;
120 ptr++;
121 i++;
122 }
123
124 if (*ptr == 0xd5) {
125 /*
126 DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt);
127 */
128 work->packet_ptr.s.addr += i+1;
129 work->word1.s.len -= i+5;
130 } else
131 if ((*ptr & 0xf) == 0xd) {
132 /*
133 DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt);
134 */
135 work->packet_ptr.s.addr += i;
136 work->word1.s.len -= i+4;
137 for (i = 0; i < work->word1.s.len; i++) {
138 *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4);
139 ptr++;
140 }
141 } else {
142 DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt);
143 /*
144 cvmx_helper_dump_packet(work);
145 */
146 cvm_oct_free_work(work);
147 return 1;
148 }
149 }
150 } else {
151 DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code);
152 cvm_oct_free_work(work);
153 return 1;
154 }
155
156 return 0;
157 }
158
159 /**
160 * Tasklet function that is scheduled on a core when an interrupt occurs.
161 *
162 * @param unused
163 */
cvm_oct_tasklet_rx(void * context,int pending)164 void cvm_oct_tasklet_rx(void *context, int pending)
165 {
166 int coreid;
167 uint64_t old_group_mask;
168 int rx_count = 0;
169 int number_to_free;
170 int num_freed;
171 int packet_not_copied;
172
173 sched_pin();
174 coreid = cvmx_get_core_num();
175
176 /* Prefetch cvm_oct_device since we know we need it soon */
177 CVMX_PREFETCH(cvm_oct_device, 0);
178
179 /* Only allow work for our group (and preserve priorities) */
180 old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
181 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
182 (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group);
183
184 while (1) {
185 struct mbuf *m = NULL;
186 int mbuf_in_hw;
187 cvmx_wqe_t *work;
188
189 if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS))
190 work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT);
191 else
192 work = NULL;
193 CVMX_PREFETCH(work, 0);
194 if (work == NULL)
195 break;
196
197 mbuf_in_hw = work->word2.s.bufs == 1;
198 if ((mbuf_in_hw)) {
199 m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *));
200 CVMX_PREFETCH(m, offsetof(struct mbuf, m_data));
201 CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr));
202 }
203 CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0);
204 //CVMX_PREFETCH(m, 0);
205
206
207 rx_count++;
208 /* Immediately throw away all packets with receive errors */
209 if ((work->word2.snoip.rcv_error)) {
210 if (cvm_oct_check_rcv_error(work))
211 continue;
212 }
213
214 /* We can only use the zero copy path if mbufs are in the FPA pool
215 and the packet fits in a single buffer */
216 if ((mbuf_in_hw)) {
217 CVMX_PREFETCH(m->m_data, 0);
218
219 m->m_pkthdr.len = m->m_len = work->word1.s.len;
220
221 packet_not_copied = 1;
222
223 /*
224 * Adjust the data pointer based on the offset
225 * of the packet within the buffer.
226 */
227 m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f);
228 } else {
229
230 /* We have to copy the packet. First allocate an
231 mbuf for it */
232 MGETHDR(m, M_NOWAIT, MT_DATA);
233 if (m == NULL) {
234 DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt);
235 cvm_oct_free_work(work);
236 continue;
237 }
238
239 /* Check if we've received a packet that was entirely
240 stored in the work entry. This is untested */
241 if ((work->word2.s.bufs == 0)) {
242 uint8_t *ptr = work->packet_data;
243
244 if (cvmx_likely(!work->word2.s.not_IP)) {
245 /* The beginning of the packet moves
246 for IP packets */
247 if (work->word2.s.is_v6)
248 ptr += 2;
249 else
250 ptr += 6;
251 }
252 panic("%s: not yet implemented; copy in small packet.", __func__);
253 /* No packet buffers to free */
254 } else {
255 int segments = work->word2.s.bufs;
256 cvmx_buf_ptr_t segment_ptr = work->packet_ptr;
257 int len = work->word1.s.len;
258
259 while (segments--) {
260 cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8);
261 /* Octeon Errata PKI-100: The segment
262 size is wrong. Until it is fixed,
263 calculate the segment size based on
264 the packet pool buffer size. When
265 it is fixed, the following line
266 should be replaced with this one:
267 int segment_size = segment_ptr.s.size; */
268 int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7));
269 /* Don't copy more than what is left
270 in the packet */
271 if (segment_size > len)
272 segment_size = len;
273 /* Copy the data into the packet */
274 panic("%s: not yet implemented; copy in packet segments.", __func__);
275 #if 0
276 memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size);
277 #endif
278 /* Reduce the amount of bytes left
279 to copy */
280 len -= segment_size;
281 segment_ptr = next_ptr;
282 }
283 }
284 packet_not_copied = 0;
285 }
286
287 if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) &&
288 cvm_oct_device[work->word1.cn38xx.ipprt])) {
289 struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt];
290
291 /* Only accept packets for devices
292 that are currently up */
293 if ((ifp->if_flags & IFF_UP)) {
294 m->m_pkthdr.rcvif = ifp;
295
296 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
297 if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error))
298 m->m_pkthdr.csum_flags = 0; /* XXX */
299 else {
300 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
301 m->m_pkthdr.csum_data = 0xffff;
302 }
303 } else {
304 m->m_pkthdr.csum_flags = 0; /* XXX */
305 }
306
307 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
308
309 (*ifp->if_input)(ifp, m);
310 } else {
311 /* Drop any packet received for a device that isn't up */
312 /*
313 DEBUGPRINT("%s: Device not up, packet dropped\n",
314 if_name(ifp));
315 */
316 m_freem(m);
317 }
318 } else {
319 /* Drop any packet received for a device that
320 doesn't exist */
321 DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt);
322 m_freem(m);
323 }
324
325 /* Check to see if the mbuf and work share
326 the same packet buffer */
327 if ((packet_not_copied)) {
328 /* This buffer needs to be replaced, increment
329 the number of buffers we need to free by one */
330 cvmx_fau_atomic_add32(
331 FAU_NUM_PACKET_BUFFERS_TO_FREE, 1);
332
333 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL,
334 DONT_WRITEBACK(1));
335 } else
336 cvm_oct_free_work(work);
337 }
338
339 /*
340 * If we hit our limit, schedule another task while we clean up.
341 */
342 if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) {
343 taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task);
344 } else {
345 /*
346 * No more packets, all done.
347 */
348 if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0))
349 panic("%s: inconsistent rx active state.", __func__);
350 }
351
352 /* Restore the original POW group mask */
353 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask);
354
355 /* Refill the packet buffer pool */
356 number_to_free =
357 cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
358
359 if (number_to_free > 0) {
360 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
361 -number_to_free);
362 num_freed =
363 cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL,
364 CVMX_FPA_PACKET_POOL_SIZE,
365 number_to_free);
366 if (num_freed != number_to_free) {
367 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE,
368 number_to_free - num_freed);
369 }
370 }
371 sched_unpin();
372 }
373
374
375
cvm_oct_rx_initialize(void)376 void cvm_oct_rx_initialize(void)
377 {
378 TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL);
379
380 cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT,
381 taskqueue_thread_enqueue,
382 &cvm_oct_taskq);
383 taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU),
384 PI_NET, "octe taskq");
385 }
386
cvm_oct_rx_shutdown(void)387 void cvm_oct_rx_shutdown(void)
388 {
389 panic("%s: not yet implemented.", __func__);
390 }
391
392