1 /* $NetBSD: hifn7751.c,v 1.82 2023/08/04 07:38:53 riastradh Exp $ */
2 /* $OpenBSD: hifn7751.c,v 1.179 2020/01/11 21:34:03 cheloha Exp $ */
3
4 /*
5 * Invertex AEON / Hifn 7751 driver
6 * Copyright (c) 1999 Invertex Inc. All rights reserved.
7 * Copyright (c) 1999 Theo de Raadt
8 * Copyright (c) 2000-2001 Network Security Technologies, Inc.
9 * http://www.netsec.net
10 * Copyright (c) 2003 Hifn Inc.
11 *
12 * This driver is based on a previous driver by Invertex, for which they
13 * requested: Please send any comments, feedback, bug-fixes, or feature
14 * requests to software@invertex.com.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 *
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. The name of the author may not be used to endorse or promote products
26 * derived from this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 *
39 * Effort sponsored in part by the Defense Advanced Research Projects
40 * Agency (DARPA) and Air Force Research Laboratory, Air Force
41 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
42 *
43 */
44
45 /*
46 * Driver for various Hifn encryption processors.
47 */
48
49 #include <sys/cdefs.h>
50 __KERNEL_RCSID(0, "$NetBSD: hifn7751.c,v 1.82 2023/08/04 07:38:53 riastradh Exp $");
51
52 #include <sys/param.h>
53 #include <sys/cprng.h>
54 #include <sys/device.h>
55 #include <sys/endian.h>
56 #include <sys/errno.h>
57 #include <sys/kernel.h>
58 #include <sys/mbuf.h>
59 #include <sys/module.h>
60 #include <sys/mutex.h>
61 #include <sys/pool.h>
62 #include <sys/proc.h>
63 #include <sys/rndsource.h>
64 #include <sys/sha1.h>
65 #include <sys/systm.h>
66
67 #include <opencrypto/cryptodev.h>
68
69 #include <dev/pci/pcireg.h>
70 #include <dev/pci/pcivar.h>
71 #include <dev/pci/pcidevs.h>
72
73 #include <dev/pci/hifn7751reg.h>
74 #include <dev/pci/hifn7751var.h>
75
76 #undef HIFN_DEBUG
77
78 #ifdef HIFN_DEBUG
79 extern int hifn_debug; /* patchable */
80 int hifn_debug = 1;
81 #endif
82
83 /*
84 * Prototypes and count for the pci_device structure
85 */
86 static int hifn_match(device_t, cfdata_t, void *);
87 static void hifn_attach(device_t, device_t, void *);
88 static int hifn_detach(device_t, int);
89
90 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc),
91 hifn_match, hifn_attach, hifn_detach, NULL);
92
93 static void hifn_reset_board(struct hifn_softc *, int);
94 static void hifn_reset_puc(struct hifn_softc *);
95 static void hifn_puc_wait(struct hifn_softc *);
96 static const char *hifn_enable_crypto(struct hifn_softc *, pcireg_t);
97 static void hifn_set_retry(struct hifn_softc *);
98 static void hifn_init_dma(struct hifn_softc *);
99 static void hifn_init_pci_registers(struct hifn_softc *);
100 static int hifn_sramsize(struct hifn_softc *);
101 static int hifn_dramsize(struct hifn_softc *);
102 static int hifn_ramtype(struct hifn_softc *);
103 static void hifn_sessions(struct hifn_softc *);
104 static int hifn_intr(void *);
105 static u_int hifn_write_command(struct hifn_command *, uint8_t *);
106 static uint32_t hifn_next_signature(uint32_t a, u_int cnt);
107 static int hifn_newsession(void*, uint32_t *, struct cryptoini *);
108 static void hifn_freesession(void*, uint64_t);
109 static int hifn_process(void*, struct cryptop *, int);
110 static void hifn_callback(struct hifn_softc *, struct hifn_command *,
111 uint8_t *);
112 static int hifn_crypto(struct hifn_softc *, struct hifn_command *,
113 struct cryptop*, int);
114 static int hifn_readramaddr(struct hifn_softc *, int, uint8_t *);
115 static int hifn_writeramaddr(struct hifn_softc *, int, uint8_t *);
116 static int hifn_dmamap_aligned(bus_dmamap_t);
117 static int hifn_dmamap_load_src(struct hifn_softc *,
118 struct hifn_command *);
119 static int hifn_dmamap_load_dst(struct hifn_softc *,
120 struct hifn_command *);
121 static int hifn_init_pubrng(struct hifn_softc *);
122 static void hifn_rng(struct hifn_softc *);
123 static void hifn_rng_intr(void *);
124 static void hifn_tick(void *);
125 static void hifn_abort(struct hifn_softc *);
126 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *,
127 int *);
128 static void hifn_write_4(struct hifn_softc *, int, bus_size_t, uint32_t);
129 static uint32_t hifn_read_4(struct hifn_softc *, int, bus_size_t);
130 #ifdef CRYPTO_LZS_COMP
131 static void hifn_compression(struct hifn_softc *, struct cryptop *,
132 struct hifn_command *);
133 static struct mbuf *hifn_mkmbuf_chain(int, struct mbuf *);
134 static int hifn_compress_enter(struct hifn_softc *, struct hifn_command *);
135 static void hifn_callback_comp(struct hifn_softc *, struct hifn_command *,
136 uint8_t *);
137 #endif /* CRYPTO_LZS_COMP */
138
139 struct hifn_stats hifnstats;
140
141 static int
hifn_cmd_ctor(void * vsc,void * vcmd,int pflags)142 hifn_cmd_ctor(void *vsc, void *vcmd, int pflags)
143 {
144 struct hifn_softc *sc = vsc;
145 struct hifn_command *cmd = vcmd;
146 int bflags = pflags & PR_WAITOK ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT;
147 int error;
148
149 memset(cmd, 0, sizeof(*cmd));
150
151 error = bus_dmamap_create(sc->sc_dmat,
152 HIFN_MAX_DMALEN, MAX_SCATTER, HIFN_MAX_SEGLEN,
153 0, bflags, &cmd->src_map);
154 if (error)
155 goto fail0;
156
157 error = bus_dmamap_create(sc->sc_dmat,
158 HIFN_MAX_SEGLEN*MAX_SCATTER, MAX_SCATTER, HIFN_MAX_SEGLEN,
159 0, bflags, &cmd->dst_map_alloc);
160 if (error)
161 goto fail1;
162
163 /* Success! */
164 cmd->dst_map = NULL;
165 return 0;
166
167 fail2: __unused
168 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map_alloc);
169 fail1: bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
170 fail0: return error;
171 }
172
173 static void
hifn_cmd_dtor(void * vsc,void * vcmd)174 hifn_cmd_dtor(void *vsc, void *vcmd)
175 {
176 struct hifn_softc *sc = vsc;
177 struct hifn_command *cmd = vcmd;
178
179 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map_alloc);
180 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
181 }
182
183 static const struct hifn_product {
184 pci_vendor_id_t hifn_vendor;
185 pci_product_id_t hifn_product;
186 int hifn_flags;
187 const char *hifn_name;
188 } hifn_products[] = {
189 { PCI_VENDOR_INVERTEX, PCI_PRODUCT_INVERTEX_AEON,
190 0,
191 "Invertex AEON",
192 },
193
194 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7751,
195 0,
196 "Hifn 7751",
197 },
198 { PCI_VENDOR_NETSEC, PCI_PRODUCT_NETSEC_7751,
199 0,
200 "Hifn 7751 (NetSec)"
201 },
202
203 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7811,
204 HIFN_IS_7811 | HIFN_HAS_RNG | HIFN_HAS_LEDS | HIFN_NO_BURSTWRITE,
205 "Hifn 7811",
206 },
207
208 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7951,
209 HIFN_HAS_RNG | HIFN_HAS_PUBLIC,
210 "Hifn 7951",
211 },
212
213 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7955,
214 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES,
215 "Hifn 7955",
216 },
217
218 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7956,
219 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES,
220 "Hifn 7956",
221 },
222
223 { 0, 0,
224 0,
225 NULL
226 }
227 };
228
229 static const struct hifn_product *
hifn_lookup(const struct pci_attach_args * pa)230 hifn_lookup(const struct pci_attach_args *pa)
231 {
232 const struct hifn_product *hp;
233
234 for (hp = hifn_products; hp->hifn_name != NULL; hp++) {
235 if (PCI_VENDOR(pa->pa_id) == hp->hifn_vendor &&
236 PCI_PRODUCT(pa->pa_id) == hp->hifn_product)
237 return (hp);
238 }
239 return (NULL);
240 }
241
242 static int
hifn_match(device_t parent,cfdata_t match,void * aux)243 hifn_match(device_t parent, cfdata_t match, void *aux)
244 {
245 struct pci_attach_args *pa = aux;
246
247 if (hifn_lookup(pa) != NULL)
248 return 1;
249
250 return 0;
251 }
252
253 static void
hifn_attach(device_t parent,device_t self,void * aux)254 hifn_attach(device_t parent, device_t self, void *aux)
255 {
256 struct hifn_softc *sc = device_private(self);
257 struct pci_attach_args *pa = aux;
258 const struct hifn_product *hp;
259 pci_chipset_tag_t pc = pa->pa_pc;
260 pci_intr_handle_t ih;
261 const char *intrstr = NULL;
262 const char *hifncap;
263 char rbase;
264 uint32_t cmd;
265 uint16_t ena;
266 bus_dma_segment_t seg;
267 bus_dmamap_t dmamap;
268 int rseg;
269 void *kva;
270 char intrbuf[PCI_INTRSTR_LEN];
271
272 hp = hifn_lookup(pa);
273 if (hp == NULL) {
274 printf("\n");
275 panic("hifn_attach: impossible");
276 }
277
278 pci_aprint_devinfo_fancy(pa, "Crypto processor", hp->hifn_name, 1);
279
280 sc->sc_dv = self;
281 sc->sc_pci_pc = pa->pa_pc;
282 sc->sc_pci_tag = pa->pa_tag;
283
284 sc->sc_flags = hp->hifn_flags;
285
286 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
287 cmd |= PCI_COMMAND_MASTER_ENABLE;
288 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
289
290 if (pci_mapreg_map(pa, HIFN_BAR0, PCI_MAPREG_TYPE_MEM, 0,
291 &sc->sc_st0, &sc->sc_sh0, NULL, &sc->sc_iosz0)) {
292 aprint_error_dev(sc->sc_dv, "can't map mem space %d\n", 0);
293 return;
294 }
295
296 if (pci_mapreg_map(pa, HIFN_BAR1, PCI_MAPREG_TYPE_MEM, 0,
297 &sc->sc_st1, &sc->sc_sh1, NULL, &sc->sc_iosz1)) {
298 aprint_error_dev(sc->sc_dv, "can't find mem space %d\n", 1);
299 goto fail_io0;
300 }
301
302 hifn_set_retry(sc);
303
304 if (sc->sc_flags & HIFN_NO_BURSTWRITE) {
305 sc->sc_waw_lastgroup = -1;
306 sc->sc_waw_lastreg = 1;
307 }
308
309 sc->sc_dmat = pa->pa_dmat;
310 if (bus_dmamem_alloc(sc->sc_dmat, sizeof(*sc->sc_dma), PAGE_SIZE, 0,
311 &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
312 aprint_error_dev(sc->sc_dv, "can't alloc DMA buffer\n");
313 goto fail_io1;
314 }
315 if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, sizeof(*sc->sc_dma), &kva,
316 BUS_DMA_NOWAIT)) {
317 aprint_error_dev(sc->sc_dv, "can't map DMA buffers (%lu bytes)\n",
318 (u_long)sizeof(*sc->sc_dma));
319 bus_dmamem_free(sc->sc_dmat, &seg, rseg);
320 goto fail_io1;
321 }
322 if (bus_dmamap_create(sc->sc_dmat, sizeof(*sc->sc_dma), 1,
323 sizeof(*sc->sc_dma), 0, BUS_DMA_NOWAIT, &dmamap)) {
324 aprint_error_dev(sc->sc_dv, "can't create DMA map\n");
325 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma));
326 bus_dmamem_free(sc->sc_dmat, &seg, rseg);
327 goto fail_io1;
328 }
329 if (bus_dmamap_load(sc->sc_dmat, dmamap, kva, sizeof(*sc->sc_dma),
330 NULL, BUS_DMA_NOWAIT)) {
331 aprint_error_dev(sc->sc_dv, "can't load DMA map\n");
332 bus_dmamap_destroy(sc->sc_dmat, dmamap);
333 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma));
334 bus_dmamem_free(sc->sc_dmat, &seg, rseg);
335 goto fail_io1;
336 }
337 sc->sc_dmamap = dmamap;
338 sc->sc_dma = (struct hifn_dma *)kva;
339 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma));
340
341 hifn_reset_board(sc, 0);
342
343 if ((hifncap = hifn_enable_crypto(sc, pa->pa_id)) == NULL) {
344 aprint_error_dev(sc->sc_dv, "crypto enabling failed\n");
345 goto fail_mem;
346 }
347 hifn_reset_puc(sc);
348
349 hifn_init_dma(sc);
350 hifn_init_pci_registers(sc);
351
352 /* XXX can't dynamically determine ram type for 795x; force dram */
353 if (sc->sc_flags & HIFN_IS_7956)
354 sc->sc_drammodel = 1;
355 else if (hifn_ramtype(sc))
356 goto fail_mem;
357
358 if (sc->sc_drammodel == 0)
359 hifn_sramsize(sc);
360 else
361 hifn_dramsize(sc);
362
363 /*
364 * Workaround for NetSec 7751 rev A: half ram size because two
365 * of the address lines were left floating
366 */
367 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETSEC &&
368 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETSEC_7751 &&
369 PCI_REVISION(pa->pa_class) == 0x61)
370 sc->sc_ramsize >>= 1;
371
372 if (pci_intr_map(pa, &ih)) {
373 aprint_error_dev(sc->sc_dv, "couldn't map interrupt\n");
374 goto fail_mem;
375 }
376 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
377 sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, hifn_intr, sc,
378 device_xname(self));
379 if (sc->sc_ih == NULL) {
380 aprint_error_dev(sc->sc_dv, "couldn't establish interrupt\n");
381 if (intrstr != NULL)
382 aprint_error(" at %s", intrstr);
383 aprint_error("\n");
384 goto fail_mem;
385 }
386
387 hifn_sessions(sc);
388
389 rseg = sc->sc_ramsize / 1024;
390 rbase = 'K';
391 if (sc->sc_ramsize >= (1024 * 1024)) {
392 rbase = 'M';
393 rseg /= 1024;
394 }
395 aprint_normal_dev(sc->sc_dv, "%s, %d%cB %cRAM, interrupting at %s\n",
396 hifncap, rseg, rbase,
397 sc->sc_drammodel ? 'D' : 'S', intrstr);
398
399 sc->sc_cid = crypto_get_driverid(0);
400 if (sc->sc_cid < 0) {
401 aprint_error_dev(sc->sc_dv, "couldn't get crypto driver id\n");
402 goto fail_intr;
403 }
404
405 sc->sc_cmd_cache = pool_cache_init(sizeof(struct hifn_command),
406 0, 0, 0, "hifncmd", NULL, IPL_VM,
407 &hifn_cmd_ctor, &hifn_cmd_dtor, sc);
408 pool_cache_prime(sc->sc_cmd_cache, sc->sc_maxses);
409
410 WRITE_REG_0(sc, HIFN_0_PUCNFG,
411 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
412 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
413
414 switch (ena) {
415 case HIFN_PUSTAT_ENA_2:
416 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
417 hifn_newsession, hifn_freesession, hifn_process, sc);
418 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0,
419 hifn_newsession, hifn_freesession, hifn_process, sc);
420 if (sc->sc_flags & HIFN_HAS_AES)
421 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
422 hifn_newsession, hifn_freesession,
423 hifn_process, sc);
424 /*FALLTHROUGH*/
425 case HIFN_PUSTAT_ENA_1:
426 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0,
427 hifn_newsession, hifn_freesession, hifn_process, sc);
428 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0,
429 hifn_newsession, hifn_freesession, hifn_process, sc);
430 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0,
431 hifn_newsession, hifn_freesession, hifn_process, sc);
432 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0,
433 hifn_newsession, hifn_freesession, hifn_process, sc);
434 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
435 hifn_newsession, hifn_freesession, hifn_process, sc);
436 break;
437 }
438
439 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 0,
440 sc->sc_dmamap->dm_mapsize,
441 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
442
443 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM);
444
445 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) {
446 hifn_init_pubrng(sc);
447 }
448
449 callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
450 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
451 return;
452
453 fail_intr:
454 pci_intr_disestablish(pc, sc->sc_ih);
455 fail_mem:
456 bus_dmamap_unload(sc->sc_dmat, dmamap);
457 bus_dmamap_destroy(sc->sc_dmat, dmamap);
458 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma));
459 bus_dmamem_free(sc->sc_dmat, &seg, rseg);
460
461 /* Turn off DMA polling */
462 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
463 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
464
465 fail_io1:
466 bus_space_unmap(sc->sc_st1, sc->sc_sh1, sc->sc_iosz1);
467 fail_io0:
468 bus_space_unmap(sc->sc_st0, sc->sc_sh0, sc->sc_iosz0);
469 }
470
471 static int
hifn_detach(device_t self,int flags)472 hifn_detach(device_t self, int flags)
473 {
474 struct hifn_softc *sc = device_private(self);
475
476 mutex_enter(&sc->sc_mtx);
477 hifn_abort(sc);
478 mutex_exit(&sc->sc_mtx);
479
480 hifn_reset_board(sc, 1);
481
482 pci_intr_disestablish(sc->sc_pci_pc, sc->sc_ih);
483
484 crypto_unregister_all(sc->sc_cid);
485
486 rnd_detach_source(&sc->sc_rnd_source);
487
488 callout_halt(&sc->sc_tickto, NULL);
489 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
490 callout_halt(&sc->sc_rngto, NULL);
491
492 pool_cache_destroy(sc->sc_cmd_cache);
493
494 bus_space_unmap(sc->sc_st1, sc->sc_sh1, sc->sc_iosz1);
495 bus_space_unmap(sc->sc_st0, sc->sc_sh0, sc->sc_iosz0);
496
497 /*
498 * XXX It's not clear if any additional buffers have been
499 * XXX allocated and require free()ing
500 */
501
502 return 0;
503 }
504
505 MODULE(MODULE_CLASS_DRIVER, hifn, "pci,opencrypto");
506
507 #ifdef _MODULE
508 #include "ioconf.c"
509 #endif
510
511 static int
hifn_modcmd(modcmd_t cmd,void * data)512 hifn_modcmd(modcmd_t cmd, void *data)
513 {
514 int error = 0;
515
516 switch (cmd) {
517 case MODULE_CMD_INIT:
518 #ifdef _MODULE
519 error = config_init_component(cfdriver_ioconf_hifn,
520 cfattach_ioconf_hifn, cfdata_ioconf_hifn);
521 #endif
522 return error;
523 case MODULE_CMD_FINI:
524 #ifdef _MODULE
525 error = config_fini_component(cfdriver_ioconf_hifn,
526 cfattach_ioconf_hifn, cfdata_ioconf_hifn);
527 #endif
528 return error;
529 default:
530 return ENOTTY;
531 }
532 }
533
534 static void
hifn_rng_get(size_t bytes,void * priv)535 hifn_rng_get(size_t bytes, void *priv)
536 {
537 struct hifn_softc *sc = priv;
538 struct timeval delta = {0, 400000};
539 struct timeval now, oktime, wait;
540
541 /*
542 * Wait until 0.4 seconds after we start up the RNG to read
543 * anything out of it. If the time hasn't elapsed, schedule a
544 * callout later on.
545 */
546 microtime(&now);
547
548 mutex_enter(&sc->sc_mtx);
549 sc->sc_rng_needbits = MAX(sc->sc_rng_needbits, NBBY*bytes);
550 timeradd(&sc->sc_rngboottime, &delta, &oktime);
551 if (timercmp(&oktime, &now, <=)) {
552 hifn_rng(sc);
553 } else if (!callout_pending(&sc->sc_rngto)) {
554 timersub(&oktime, &now, &wait);
555 callout_schedule(&sc->sc_rngto, MAX(1, tvtohz(&wait)));
556 }
557 mutex_exit(&sc->sc_mtx);
558 }
559
560 static int
hifn_init_pubrng(struct hifn_softc * sc)561 hifn_init_pubrng(struct hifn_softc *sc)
562 {
563 uint32_t r;
564 int i;
565
566 if ((sc->sc_flags & HIFN_IS_7811) == 0) {
567 /* Reset 7951 public key/rng engine */
568 WRITE_REG_1(sc, HIFN_1_PUB_RESET,
569 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
570
571 for (i = 0; i < 100; i++) {
572 DELAY(1000);
573 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
574 HIFN_PUBRST_RESET) == 0)
575 break;
576 }
577
578 if (i == 100) {
579 printf("%s: public key init failed\n",
580 device_xname(sc->sc_dv));
581 return (1);
582 }
583 }
584
585 /* Enable the rng, if available */
586 if (sc->sc_flags & HIFN_HAS_RNG) {
587 if (sc->sc_flags & HIFN_IS_7811) {
588 r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
589 if (r & HIFN_7811_RNGENA_ENA) {
590 r &= ~HIFN_7811_RNGENA_ENA;
591 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
592 }
593 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
594 HIFN_7811_RNGCFG_DEFL);
595 r |= HIFN_7811_RNGENA_ENA;
596 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
597 } else
598 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
599 READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
600 HIFN_RNGCFG_ENA);
601
602 /*
603 * The Hifn RNG documentation states that at their
604 * recommended "conservative" RNG config values,
605 * the RNG must warm up for 0.4s before providing
606 * data that meet their worst-case estimate of 0.06
607 * bits of random data per output register bit.
608 */
609 microtime(&sc->sc_rngboottime);
610 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
611 callout_setfunc(&sc->sc_rngto, hifn_rng_intr, sc);
612 rndsource_setcb(&sc->sc_rnd_source, hifn_rng_get, sc);
613 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dv),
614 RND_TYPE_RNG, RND_FLAG_DEFAULT|RND_FLAG_HASCB);
615 }
616
617 /* Enable public key engine, if available */
618 if (sc->sc_flags & HIFN_HAS_PUBLIC) {
619 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
620 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
621 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
622 }
623
624 return (0);
625 }
626
627 static void
hifn_rng(struct hifn_softc * sc)628 hifn_rng(struct hifn_softc *sc)
629 {
630 uint32_t entropybits;
631
632 KASSERT(mutex_owned(&sc->sc_mtx));
633
634 if (sc->sc_flags & HIFN_IS_7811) {
635 while (sc->sc_rng_needbits) {
636 uint32_t num[2];
637 uint32_t sts;
638
639 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
640 if (sts & HIFN_7811_RNGSTS_UFL) {
641 device_printf(sc->sc_dv, "RNG underflow\n");
642 return;
643 }
644 if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
645 break;
646
647 /*
648 * There are at least two words in the RNG FIFO
649 * at this point.
650 */
651 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
652 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
653 #ifdef HIFN_DEBUG
654 if (hifn_debug >= 2)
655 hexdump(printf, "hifn", num, sizeof num);
656 #endif
657 entropybits = NBBY*sizeof(num)/HIFN_RNG_BITSPER;
658 rnd_add_data_intr(&sc->sc_rnd_source, num, sizeof(num),
659 entropybits);
660 entropybits = MAX(entropybits, 1);
661 entropybits = MIN(entropybits, sc->sc_rng_needbits);
662 sc->sc_rng_needbits -= entropybits;
663 }
664 } else {
665 /*
666 * We must be *extremely* careful here. The Hifn
667 * 795x differ from the published 6500 RNG design
668 * in more ways than the obvious lack of the output
669 * FIFO and LFSR control registers. In fact, there
670 * is only one LFSR, instead of the 6500's two, and
671 * it's 32 bits, not 31.
672 *
673 * Further, a block diagram obtained from Hifn shows
674 * a very curious latching of this register: the LFSR
675 * rotates at a frequency of RNG_Clk / 8, but the
676 * RNG_Data register is latched at a frequency of
677 * RNG_Clk, which means that it is possible for
678 * consecutive reads of the RNG_Data register to read
679 * identical state from the LFSR. The simplest
680 * workaround seems to be to read eight samples from
681 * the register for each one that we use. Since each
682 * read must require at least one PCI cycle, and
683 * RNG_Clk is at least PCI_Clk, this is safe.
684 */
685 while (sc->sc_rng_needbits) {
686 uint32_t num[64];
687 unsigned i;
688
689 for (i = 0; i < 8*__arraycount(num); i++)
690 num[i/8] = READ_REG_1(sc, HIFN_1_RNG_DATA);
691 #ifdef HIFN_DEBUG
692 if (hifn_debug >= 2)
693 hexdump(printf, "hifn", num, sizeof num);
694 #endif
695 entropybits = NBBY*sizeof(num)/HIFN_RNG_BITSPER;
696 rnd_add_data_intr(&sc->sc_rnd_source, num, sizeof num,
697 entropybits);
698 entropybits = MAX(entropybits, 1);
699 entropybits = MIN(entropybits, sc->sc_rng_needbits);
700 sc->sc_rng_needbits -= entropybits;
701 }
702 }
703
704 /* If we still need more, try again in another second. */
705 if (sc->sc_rng_needbits)
706 callout_schedule(&sc->sc_rngto, hz);
707 }
708
709 static void
hifn_rng_intr(void * vsc)710 hifn_rng_intr(void *vsc)
711 {
712 struct hifn_softc *sc = vsc;
713
714 mutex_spin_enter(&sc->sc_mtx);
715 hifn_rng(sc);
716 mutex_spin_exit(&sc->sc_mtx);
717 }
718
719 static void
hifn_puc_wait(struct hifn_softc * sc)720 hifn_puc_wait(struct hifn_softc *sc)
721 {
722 int i;
723
724 for (i = 5000; i > 0; i--) {
725 DELAY(1);
726 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET))
727 break;
728 }
729 if (!i)
730 printf("%s: proc unit did not reset\n", device_xname(sc->sc_dv));
731 }
732
733 /*
734 * Reset the processing unit.
735 */
736 static void
hifn_reset_puc(struct hifn_softc * sc)737 hifn_reset_puc(struct hifn_softc *sc)
738 {
739 /* Reset processing unit */
740 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
741 hifn_puc_wait(sc);
742 }
743
744 static void
hifn_set_retry(struct hifn_softc * sc)745 hifn_set_retry(struct hifn_softc *sc)
746 {
747 uint32_t r;
748
749 r = pci_conf_read(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT);
750 r &= 0xffff0000;
751 pci_conf_write(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT, r);
752 }
753
754 /*
755 * Resets the board. Values in the registers are left as is
756 * from the reset (i.e. initial values are assigned elsewhere).
757 */
758 static void
hifn_reset_board(struct hifn_softc * sc,int full)759 hifn_reset_board(struct hifn_softc *sc, int full)
760 {
761 uint32_t reg;
762
763 /*
764 * Set polling in the DMA configuration register to zero. 0x7 avoids
765 * resetting the board and zeros out the other fields.
766 */
767 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
768 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
769
770 /*
771 * Now that polling has been disabled, we have to wait 1 ms
772 * before resetting the board.
773 */
774 DELAY(1000);
775
776 /* Reset the DMA unit */
777 if (full) {
778 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
779 DELAY(1000);
780 } else {
781 WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
782 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
783 hifn_reset_puc(sc);
784 }
785
786 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma));
787
788 /* Bring dma unit out of reset */
789 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
790 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
791
792 hifn_puc_wait(sc);
793
794 hifn_set_retry(sc);
795
796 if (sc->sc_flags & HIFN_IS_7811) {
797 for (reg = 0; reg < 1000; reg++) {
798 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
799 HIFN_MIPSRST_CRAMINIT)
800 break;
801 DELAY(1000);
802 }
803 if (reg == 1000)
804 printf(": cram init timeout\n");
805 }
806 }
807
808 static uint32_t
hifn_next_signature(uint32_t a,u_int cnt)809 hifn_next_signature(uint32_t a, u_int cnt)
810 {
811 u_int i;
812 uint32_t v;
813
814 for (i = 0; i < cnt; i++) {
815
816 /* get the parity */
817 v = a & 0x80080125;
818 v ^= v >> 16;
819 v ^= v >> 8;
820 v ^= v >> 4;
821 v ^= v >> 2;
822 v ^= v >> 1;
823
824 a = (v & 1) ^ (a << 1);
825 }
826
827 return a;
828 }
829
830 static struct pci2id {
831 u_short pci_vendor;
832 u_short pci_prod;
833 char card_id[13];
834 } const pci2id[] = {
835 {
836 PCI_VENDOR_HIFN,
837 PCI_PRODUCT_HIFN_7951,
838 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
839 0x00, 0x00, 0x00, 0x00, 0x00 }
840 }, {
841 PCI_VENDOR_HIFN,
842 PCI_PRODUCT_HIFN_7955,
843 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
844 0x00, 0x00, 0x00, 0x00, 0x00 }
845 }, {
846 PCI_VENDOR_HIFN,
847 PCI_PRODUCT_HIFN_7956,
848 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
849 0x00, 0x00, 0x00, 0x00, 0x00 }
850 }, {
851 PCI_VENDOR_NETSEC,
852 PCI_PRODUCT_NETSEC_7751,
853 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
854 0x00, 0x00, 0x00, 0x00, 0x00 }
855 }, {
856 PCI_VENDOR_INVERTEX,
857 PCI_PRODUCT_INVERTEX_AEON,
858 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
859 0x00, 0x00, 0x00, 0x00, 0x00 }
860 }, {
861 PCI_VENDOR_HIFN,
862 PCI_PRODUCT_HIFN_7811,
863 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
864 0x00, 0x00, 0x00, 0x00, 0x00 }
865 }, {
866 /*
867 * Other vendors share this PCI ID as well, such as
868 * powercrypt, and obviously they also
869 * use the same key.
870 */
871 PCI_VENDOR_HIFN,
872 PCI_PRODUCT_HIFN_7751,
873 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
874 0x00, 0x00, 0x00, 0x00, 0x00 }
875 },
876 };
877
878 /*
879 * Checks to see if crypto is already enabled. If crypto isn't enable,
880 * "hifn_enable_crypto" is called to enable it. The check is important,
881 * as enabling crypto twice will lock the board.
882 */
883 static const char *
hifn_enable_crypto(struct hifn_softc * sc,pcireg_t pciid)884 hifn_enable_crypto(struct hifn_softc *sc, pcireg_t pciid)
885 {
886 uint32_t dmacfg, ramcfg, encl, addr, i;
887 const char *offtbl = NULL;
888
889 for (i = 0; i < __arraycount(pci2id); i++) {
890 if (pci2id[i].pci_vendor == PCI_VENDOR(pciid) &&
891 pci2id[i].pci_prod == PCI_PRODUCT(pciid)) {
892 offtbl = pci2id[i].card_id;
893 break;
894 }
895 }
896
897 if (offtbl == NULL) {
898 #ifdef HIFN_DEBUG
899 aprint_debug_dev(sc->sc_dv, "Unknown card!\n");
900 #endif
901 return (NULL);
902 }
903
904 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
905 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
906
907 /*
908 * The RAM config register's encrypt level bit needs to be set before
909 * every read performed on the encryption level register.
910 */
911 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
912
913 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
914
915 /*
916 * Make sure we don't re-unlock. Two unlocks kills chip until the
917 * next reboot.
918 */
919 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
920 #ifdef HIFN_DEBUG
921 aprint_debug_dev(sc->sc_dv, "Strong Crypto already enabled!\n");
922 #endif
923 goto report;
924 }
925
926 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
927 #ifdef HIFN_DEBUG
928 aprint_debug_dev(sc->sc_dv, "Unknown encryption level\n");
929 #endif
930 return (NULL);
931 }
932
933 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
934 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
935 DELAY(1000);
936 addr = READ_REG_1(sc, HIFN_1_UNLOCK_SECRET1);
937 DELAY(1000);
938 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, 0);
939 DELAY(1000);
940
941 for (i = 0; i <= 12; i++) {
942 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
943 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, addr);
944
945 DELAY(1000);
946 }
947
948 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
949 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
950
951 #ifdef HIFN_DEBUG
952 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
953 aprint_debug("Encryption engine is permanently locked until next system reset.");
954 else
955 aprint_debug("Encryption engine enabled successfully!");
956 #endif
957
958 report:
959 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
960 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
961
962 switch (encl) {
963 case HIFN_PUSTAT_ENA_0:
964 return ("LZS-only (no encr/auth)");
965
966 case HIFN_PUSTAT_ENA_1:
967 return ("DES");
968
969 case HIFN_PUSTAT_ENA_2:
970 if (sc->sc_flags & HIFN_HAS_AES)
971 return ("3DES/AES");
972 else
973 return ("3DES");
974
975 default:
976 return ("disabled");
977 }
978 /* NOTREACHED */
979 }
980
981 /*
982 * Give initial values to the registers listed in the "Register Space"
983 * section of the HIFN Software Development reference manual.
984 */
985 static void
hifn_init_pci_registers(struct hifn_softc * sc)986 hifn_init_pci_registers(struct hifn_softc *sc)
987 {
988 /* write fixed values needed by the Initialization registers */
989 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
990 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
991 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
992
993 /* write all 4 ring address registers */
994 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
995 offsetof(struct hifn_dma, cmdr[0]));
996 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
997 offsetof(struct hifn_dma, srcr[0]));
998 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
999 offsetof(struct hifn_dma, dstr[0]));
1000 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dmamap->dm_segs[0].ds_addr +
1001 offsetof(struct hifn_dma, resr[0]));
1002
1003 DELAY(2000);
1004
1005 /* write status register */
1006 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1007 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1008 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1009 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1010 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1011 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1012 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1013 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1014 HIFN_DMACSR_S_WAIT |
1015 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1016 HIFN_DMACSR_C_WAIT |
1017 HIFN_DMACSR_ENGINE |
1018 ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1019 HIFN_DMACSR_PUBDONE : 0) |
1020 ((sc->sc_flags & HIFN_IS_7811) ?
1021 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1022
1023 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1024 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1025 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1026 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1027 HIFN_DMAIER_ENGINE |
1028 ((sc->sc_flags & HIFN_IS_7811) ?
1029 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1030 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1031 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1032 CLR_LED(sc, HIFN_MIPSRST_LED0 | HIFN_MIPSRST_LED1 | HIFN_MIPSRST_LED2);
1033
1034 if (sc->sc_flags & HIFN_IS_7956) {
1035 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1036 HIFN_PUCNFG_TCALLPHASES |
1037 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
1038 WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956);
1039 } else {
1040 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1041 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1042 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1043 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1044 }
1045
1046 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1047 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1048 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1049 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1050 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1051 }
1052
1053 /*
1054 * The maximum number of sessions supported by the card
1055 * is dependent on the amount of context ram, which
1056 * encryption algorithms are enabled, and how compression
1057 * is configured. This should be configured before this
1058 * routine is called.
1059 */
1060 static void
hifn_sessions(struct hifn_softc * sc)1061 hifn_sessions(struct hifn_softc *sc)
1062 {
1063 uint32_t pucnfg;
1064 int ctxsize;
1065
1066 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1067
1068 if (pucnfg & HIFN_PUCNFG_COMPSING) {
1069 if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1070 ctxsize = 128;
1071 else
1072 ctxsize = 512;
1073 /*
1074 * 7955/7956 has internal context memory of 32K
1075 */
1076 if (sc->sc_flags & HIFN_IS_7956)
1077 sc->sc_maxses = 32768 / ctxsize;
1078 else
1079 sc->sc_maxses = 1 +
1080 ((sc->sc_ramsize - 32768) / ctxsize);
1081 } else
1082 sc->sc_maxses = sc->sc_ramsize / 16384;
1083
1084 if (sc->sc_maxses > 2048)
1085 sc->sc_maxses = 2048;
1086 }
1087
1088 /*
1089 * Determine ram type (sram or dram). Board should be just out of a reset
1090 * state when this is called.
1091 */
1092 static int
hifn_ramtype(struct hifn_softc * sc)1093 hifn_ramtype(struct hifn_softc *sc)
1094 {
1095 uint8_t data[8], dataexpect[8];
1096 size_t i;
1097
1098 for (i = 0; i < sizeof(data); i++)
1099 data[i] = dataexpect[i] = 0x55;
1100 if (hifn_writeramaddr(sc, 0, data))
1101 return (-1);
1102 if (hifn_readramaddr(sc, 0, data))
1103 return (-1);
1104 if (memcmp(data, dataexpect, sizeof(data)) != 0) {
1105 sc->sc_drammodel = 1;
1106 return (0);
1107 }
1108
1109 for (i = 0; i < sizeof(data); i++)
1110 data[i] = dataexpect[i] = 0xaa;
1111 if (hifn_writeramaddr(sc, 0, data))
1112 return (-1);
1113 if (hifn_readramaddr(sc, 0, data))
1114 return (-1);
1115 if (memcmp(data, dataexpect, sizeof(data)) != 0) {
1116 sc->sc_drammodel = 1;
1117 return (0);
1118 }
1119
1120 return (0);
1121 }
1122
1123 #define HIFN_SRAM_MAX (32 << 20)
1124 #define HIFN_SRAM_STEP_SIZE 16384
1125 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1126
1127 static int
hifn_sramsize(struct hifn_softc * sc)1128 hifn_sramsize(struct hifn_softc *sc)
1129 {
1130 uint32_t a, b;
1131 uint8_t data[8];
1132 uint8_t dataexpect[sizeof(data)];
1133 size_t i;
1134
1135 for (i = 0; i < sizeof(data); i++)
1136 data[i] = dataexpect[i] = i ^ 0x5a;
1137
1138 a = HIFN_SRAM_GRANULARITY * HIFN_SRAM_STEP_SIZE;
1139 b = HIFN_SRAM_GRANULARITY;
1140 for (i = 0; i < HIFN_SRAM_GRANULARITY; ++i) {
1141 a -= HIFN_SRAM_STEP_SIZE;
1142 b -= 1;
1143 le32enc(data, b);
1144 hifn_writeramaddr(sc, a, data);
1145 }
1146
1147 a = 0;
1148 b = 0;
1149 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1150 le32enc(dataexpect, b);
1151 if (hifn_readramaddr(sc, a, data) < 0)
1152 return (0);
1153 if (memcmp(data, dataexpect, sizeof(data)) != 0)
1154 return (0);
1155
1156 a += HIFN_SRAM_STEP_SIZE;
1157 b += 1;
1158 sc->sc_ramsize = a;
1159 }
1160
1161 return (0);
1162 }
1163
1164 /*
1165 * XXX For dram boards, one should really try all of the
1166 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG
1167 * is already set up correctly.
1168 */
1169 static int
hifn_dramsize(struct hifn_softc * sc)1170 hifn_dramsize(struct hifn_softc *sc)
1171 {
1172 uint32_t cnfg;
1173
1174 if (sc->sc_flags & HIFN_IS_7956) {
1175 /*
1176 * 7955/7956 have a fixed internal ram of only 32K.
1177 */
1178 sc->sc_ramsize = 32768;
1179 } else {
1180 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1181 HIFN_PUCNFG_DRAMMASK;
1182 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1183 }
1184 return (0);
1185 }
1186
1187 static void
hifn_alloc_slot(struct hifn_softc * sc,int * cmdp,int * srcp,int * dstp,int * resp)1188 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp,
1189 int *resp)
1190 {
1191 struct hifn_dma *dma = sc->sc_dma;
1192
1193 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1194 dma->cmdi = 0;
1195 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1196 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1197 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1198 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1199 }
1200 *cmdp = dma->cmdi++;
1201 dma->cmdk = dma->cmdi;
1202
1203 if (dma->srci == HIFN_D_SRC_RSIZE) {
1204 dma->srci = 0;
1205 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1206 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1207 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1208 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1209 }
1210 *srcp = dma->srci++;
1211 dma->srck = dma->srci;
1212
1213 if (dma->dsti == HIFN_D_DST_RSIZE) {
1214 dma->dsti = 0;
1215 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1216 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1217 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1218 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1219 }
1220 *dstp = dma->dsti++;
1221 dma->dstk = dma->dsti;
1222
1223 if (dma->resi == HIFN_D_RES_RSIZE) {
1224 dma->resi = 0;
1225 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1226 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1227 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1228 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1229 }
1230 *resp = dma->resi++;
1231 dma->resk = dma->resi;
1232 }
1233
1234 static int
hifn_writeramaddr(struct hifn_softc * sc,int addr,uint8_t * data)1235 hifn_writeramaddr(struct hifn_softc *sc, int addr, uint8_t *data)
1236 {
1237 struct hifn_dma *dma = sc->sc_dma;
1238 struct hifn_base_command wc;
1239 const uint32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1240 int r, cmdi, resi, srci, dsti;
1241
1242 wc.masks = htole16(3 << 13);
1243 wc.session_num = htole16(addr >> 14);
1244 wc.total_source_count = htole16(8);
1245 wc.total_dest_count = htole16(addr & 0x3fff);
1246
1247 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1248
1249 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1250 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1251 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1252
1253 /* build write command */
1254 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND);
1255 *(struct hifn_base_command *)dma->command_bufs[cmdi] = wc;
1256 memcpy(&dma->test_src, data, sizeof(dma->test_src));
1257
1258 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr
1259 + offsetof(struct hifn_dma, test_src));
1260 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr
1261 + offsetof(struct hifn_dma, test_dst));
1262
1263 dma->cmdr[cmdi].l = htole32(16 | masks);
1264 dma->srcr[srci].l = htole32(8 | masks);
1265 dma->dstr[dsti].l = htole32(4 | masks);
1266 dma->resr[resi].l = htole32(4 | masks);
1267
1268 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1269 0, sc->sc_dmamap->dm_mapsize,
1270 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1271
1272 for (r = 10000; r >= 0; r--) {
1273 DELAY(10);
1274 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1275 0, sc->sc_dmamap->dm_mapsize,
1276 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1277 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1278 break;
1279 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1280 0, sc->sc_dmamap->dm_mapsize,
1281 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1282 }
1283 if (r == 0) {
1284 printf("%s: writeramaddr -- "
1285 "result[%d](addr %d) still valid\n",
1286 device_xname(sc->sc_dv), resi, addr);
1287 return (-1);
1288 } else
1289 r = 0;
1290
1291 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1292 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1293 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1294
1295 return (r);
1296 }
1297
1298 static int
hifn_readramaddr(struct hifn_softc * sc,int addr,uint8_t * data)1299 hifn_readramaddr(struct hifn_softc *sc, int addr, uint8_t *data)
1300 {
1301 struct hifn_dma *dma = sc->sc_dma;
1302 struct hifn_base_command rc;
1303 const uint32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1304 int r, cmdi, srci, dsti, resi;
1305
1306 rc.masks = htole16(2 << 13);
1307 rc.session_num = htole16(addr >> 14);
1308 rc.total_source_count = htole16(addr & 0x3fff);
1309 rc.total_dest_count = htole16(8);
1310
1311 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1312
1313 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1314 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1315 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1316
1317 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND);
1318 *(struct hifn_base_command *)dma->command_bufs[cmdi] = rc;
1319
1320 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1321 offsetof(struct hifn_dma, test_src));
1322 dma->test_src = 0;
1323 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1324 offsetof(struct hifn_dma, test_dst));
1325 dma->test_dst = 0;
1326 dma->cmdr[cmdi].l = htole32(8 | masks);
1327 dma->srcr[srci].l = htole32(8 | masks);
1328 dma->dstr[dsti].l = htole32(8 | masks);
1329 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1330
1331 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1332 0, sc->sc_dmamap->dm_mapsize,
1333 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1334
1335 for (r = 10000; r >= 0; r--) {
1336 DELAY(10);
1337 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1338 0, sc->sc_dmamap->dm_mapsize,
1339 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1340 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1341 break;
1342 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1343 0, sc->sc_dmamap->dm_mapsize,
1344 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1345 }
1346 if (r == 0) {
1347 printf("%s: readramaddr -- "
1348 "result[%d](addr %d) still valid\n",
1349 device_xname(sc->sc_dv), resi, addr);
1350 r = -1;
1351 } else {
1352 r = 0;
1353 memcpy(data, &dma->test_dst, sizeof(dma->test_dst));
1354 }
1355
1356 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1357 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1358 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1359
1360 return (r);
1361 }
1362
1363 /*
1364 * Initialize the descriptor rings.
1365 */
1366 static void
hifn_init_dma(struct hifn_softc * sc)1367 hifn_init_dma(struct hifn_softc *sc)
1368 {
1369 struct hifn_dma *dma = sc->sc_dma;
1370 int i;
1371
1372 hifn_set_retry(sc);
1373
1374 /* initialize static pointer values */
1375 for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1376 dma->cmdr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1377 offsetof(struct hifn_dma, command_bufs[i][0]));
1378 for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1379 dma->resr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1380 offsetof(struct hifn_dma, result_bufs[i][0]));
1381
1382 dma->cmdr[HIFN_D_CMD_RSIZE].p =
1383 htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1384 offsetof(struct hifn_dma, cmdr[0]));
1385 dma->srcr[HIFN_D_SRC_RSIZE].p =
1386 htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1387 offsetof(struct hifn_dma, srcr[0]));
1388 dma->dstr[HIFN_D_DST_RSIZE].p =
1389 htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1390 offsetof(struct hifn_dma, dstr[0]));
1391 dma->resr[HIFN_D_RES_RSIZE].p =
1392 htole32(sc->sc_dmamap->dm_segs[0].ds_addr +
1393 offsetof(struct hifn_dma, resr[0]));
1394
1395 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
1396 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
1397 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
1398 }
1399
1400 /*
1401 * Writes out the raw command buffer space. Returns the
1402 * command buffer size.
1403 */
1404 static u_int
hifn_write_command(struct hifn_command * cmd,uint8_t * buf)1405 hifn_write_command(struct hifn_command *cmd, uint8_t *buf)
1406 {
1407 uint8_t *buf_pos;
1408 struct hifn_base_command *base_cmd;
1409 struct hifn_mac_command *mac_cmd;
1410 struct hifn_crypt_command *cry_cmd;
1411 struct hifn_comp_command *comp_cmd;
1412 int using_mac, using_crypt, using_comp, len, ivlen;
1413 uint32_t dlen, slen;
1414
1415 buf_pos = buf;
1416 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1417 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1418 using_comp = cmd->base_masks & HIFN_BASE_CMD_COMP;
1419
1420 base_cmd = (struct hifn_base_command *)buf_pos;
1421 base_cmd->masks = htole16(cmd->base_masks);
1422 slen = cmd->src_map->dm_mapsize;
1423 if (cmd->sloplen)
1424 dlen = cmd->dst_map->dm_mapsize - cmd->sloplen +
1425 sizeof(uint32_t);
1426 else
1427 dlen = cmd->dst_map->dm_mapsize;
1428 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1429 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1430 dlen >>= 16;
1431 slen >>= 16;
1432 base_cmd->session_num = htole16(
1433 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1434 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1435 buf_pos += sizeof(struct hifn_base_command);
1436
1437 if (using_comp) {
1438 comp_cmd = (struct hifn_comp_command *)buf_pos;
1439 dlen = cmd->compcrd->crd_len;
1440 comp_cmd->source_count = htole16(dlen & 0xffff);
1441 dlen >>= 16;
1442 comp_cmd->masks = htole16(cmd->comp_masks |
1443 ((dlen << HIFN_COMP_CMD_SRCLEN_S) & HIFN_COMP_CMD_SRCLEN_M));
1444 comp_cmd->header_skip = htole16(cmd->compcrd->crd_skip);
1445 comp_cmd->reserved = 0;
1446 buf_pos += sizeof(struct hifn_comp_command);
1447 }
1448
1449 if (using_mac) {
1450 mac_cmd = (struct hifn_mac_command *)buf_pos;
1451 dlen = cmd->maccrd->crd_len;
1452 mac_cmd->source_count = htole16(dlen & 0xffff);
1453 dlen >>= 16;
1454 mac_cmd->masks = htole16(cmd->mac_masks |
1455 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1456 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1457 mac_cmd->reserved = 0;
1458 buf_pos += sizeof(struct hifn_mac_command);
1459 }
1460
1461 if (using_crypt) {
1462 cry_cmd = (struct hifn_crypt_command *)buf_pos;
1463 dlen = cmd->enccrd->crd_len;
1464 cry_cmd->source_count = htole16(dlen & 0xffff);
1465 dlen >>= 16;
1466 cry_cmd->masks = htole16(cmd->cry_masks |
1467 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1468 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1469 cry_cmd->reserved = 0;
1470 buf_pos += sizeof(struct hifn_crypt_command);
1471 }
1472
1473 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1474 memcpy(buf_pos, cmd->mac, HIFN_MAC_KEY_LENGTH);
1475 buf_pos += HIFN_MAC_KEY_LENGTH;
1476 }
1477
1478 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1479 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1480 case HIFN_CRYPT_CMD_ALG_3DES:
1481 memcpy(buf_pos, cmd->ck, HIFN_3DES_KEY_LENGTH);
1482 buf_pos += HIFN_3DES_KEY_LENGTH;
1483 break;
1484 case HIFN_CRYPT_CMD_ALG_DES:
1485 memcpy(buf_pos, cmd->ck, HIFN_DES_KEY_LENGTH);
1486 buf_pos += HIFN_DES_KEY_LENGTH;
1487 break;
1488 case HIFN_CRYPT_CMD_ALG_RC4:
1489 len = 256;
1490 do {
1491 int clen;
1492
1493 clen = MIN(cmd->cklen, len);
1494 memcpy(buf_pos, cmd->ck, clen);
1495 len -= clen;
1496 buf_pos += clen;
1497 } while (len > 0);
1498 memset(buf_pos, 0, 4);
1499 buf_pos += 4;
1500 break;
1501 case HIFN_CRYPT_CMD_ALG_AES:
1502 /*
1503 * AES keys are variable 128, 192 and
1504 * 256 bits (16, 24 and 32 bytes).
1505 */
1506 memcpy(buf_pos, cmd->ck, cmd->cklen);
1507 buf_pos += cmd->cklen;
1508 break;
1509 }
1510 }
1511
1512 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1513 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1514 case HIFN_CRYPT_CMD_ALG_AES:
1515 ivlen = HIFN_AES_IV_LENGTH;
1516 break;
1517 default:
1518 ivlen = HIFN_IV_LENGTH;
1519 break;
1520 }
1521 memcpy(buf_pos, cmd->iv, ivlen);
1522 buf_pos += ivlen;
1523 }
1524
1525 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC | HIFN_BASE_CMD_CRYPT |
1526 HIFN_BASE_CMD_COMP)) == 0) {
1527 memset(buf_pos, 0, 8);
1528 buf_pos += 8;
1529 }
1530
1531 return (buf_pos - buf);
1532 }
1533
1534 static int
hifn_dmamap_aligned(bus_dmamap_t map)1535 hifn_dmamap_aligned(bus_dmamap_t map)
1536 {
1537 int i;
1538
1539 for (i = 0; i < map->dm_nsegs; i++) {
1540 if (map->dm_segs[i].ds_addr & 3)
1541 return (0);
1542 if ((i != (map->dm_nsegs - 1)) &&
1543 (map->dm_segs[i].ds_len & 3))
1544 return (0);
1545 }
1546 return (1);
1547 }
1548
1549 static int
hifn_dmamap_load_dst(struct hifn_softc * sc,struct hifn_command * cmd)1550 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1551 {
1552 struct hifn_dma *dma = sc->sc_dma;
1553 bus_dmamap_t map = cmd->dst_map;
1554 uint32_t p, l;
1555 int idx, used = 0, i;
1556
1557 idx = dma->dsti;
1558 for (i = 0; i < map->dm_nsegs - 1; i++) {
1559 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr);
1560 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1561 HIFN_D_MASKDONEIRQ | map->dm_segs[i].ds_len);
1562 HIFN_DSTR_SYNC(sc, idx,
1563 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1564 used++;
1565
1566 if (++idx == HIFN_D_DST_RSIZE) {
1567 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1568 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1569 HIFN_DSTR_SYNC(sc, idx,
1570 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1571 idx = 0;
1572 }
1573 }
1574
1575 if (cmd->sloplen == 0) {
1576 p = map->dm_segs[i].ds_addr;
1577 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1578 map->dm_segs[i].ds_len;
1579 } else {
1580 p = sc->sc_dmamap->dm_segs[0].ds_addr +
1581 offsetof(struct hifn_dma, slop[cmd->slopidx]);
1582 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1583 sizeof(uint32_t);
1584
1585 if ((map->dm_segs[i].ds_len - cmd->sloplen) != 0) {
1586 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr);
1587 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1588 HIFN_D_MASKDONEIRQ |
1589 (map->dm_segs[i].ds_len - cmd->sloplen));
1590 HIFN_DSTR_SYNC(sc, idx,
1591 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1592 used++;
1593
1594 if (++idx == HIFN_D_DST_RSIZE) {
1595 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1596 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1597 HIFN_DSTR_SYNC(sc, idx,
1598 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1599 idx = 0;
1600 }
1601 }
1602 }
1603 dma->dstr[idx].p = htole32(p);
1604 dma->dstr[idx].l = htole32(l);
1605 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1606 used++;
1607
1608 if (++idx == HIFN_D_DST_RSIZE) {
1609 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1610 HIFN_D_MASKDONEIRQ);
1611 HIFN_DSTR_SYNC(sc, idx,
1612 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1613 idx = 0;
1614 }
1615
1616 dma->dsti = idx;
1617 dma->dstu += used;
1618 return (idx);
1619 }
1620
1621 static int
hifn_dmamap_load_src(struct hifn_softc * sc,struct hifn_command * cmd)1622 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1623 {
1624 struct hifn_dma *dma = sc->sc_dma;
1625 bus_dmamap_t map = cmd->src_map;
1626 int idx, i;
1627 uint32_t last = 0;
1628
1629 idx = dma->srci;
1630 for (i = 0; i < map->dm_nsegs; i++) {
1631 if (i == map->dm_nsegs - 1)
1632 last = HIFN_D_LAST;
1633
1634 dma->srcr[idx].p = htole32(map->dm_segs[i].ds_addr);
1635 dma->srcr[idx].l = htole32(map->dm_segs[i].ds_len |
1636 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1637 HIFN_SRCR_SYNC(sc, idx,
1638 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1639
1640 if (++idx == HIFN_D_SRC_RSIZE) {
1641 dma->srcr[idx].l = htole32(HIFN_D_VALID |
1642 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1643 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1644 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1645 idx = 0;
1646 }
1647 }
1648 dma->srci = idx;
1649 dma->srcu += map->dm_nsegs;
1650 return (idx);
1651 }
1652
1653 static int
hifn_crypto(struct hifn_softc * sc,struct hifn_command * cmd,struct cryptop * crp,int hint)1654 hifn_crypto(struct hifn_softc *sc, struct hifn_command *cmd,
1655 struct cryptop *crp, int hint)
1656 {
1657 struct hifn_dma *dma = sc->sc_dma;
1658 uint32_t cmdlen;
1659 int cmdi, resi, err = 0;
1660
1661 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1662 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1663 cmd->srcu.src_m, BUS_DMA_NOWAIT)) {
1664 err = ENOMEM;
1665 goto err_srcmap1;
1666 }
1667 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1668 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1669 cmd->srcu.src_io, BUS_DMA_NOWAIT)) {
1670 err = ENOMEM;
1671 goto err_srcmap1;
1672 }
1673 } else {
1674 err = EINVAL;
1675 goto err_srcmap1;
1676 }
1677
1678 if (hifn_dmamap_aligned(cmd->src_map)) {
1679 cmd->sloplen = cmd->src_map->dm_mapsize & 3;
1680 if (crp->crp_flags & CRYPTO_F_IOV)
1681 cmd->dstu.dst_io = cmd->srcu.src_io;
1682 else if (crp->crp_flags & CRYPTO_F_IMBUF)
1683 cmd->dstu.dst_m = cmd->srcu.src_m;
1684 cmd->dst_map = cmd->src_map;
1685 } else {
1686 if (crp->crp_flags & CRYPTO_F_IOV) {
1687 err = EINVAL;
1688 goto err_srcmap;
1689 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1690 int totlen, len;
1691 struct mbuf *m, *m0, *mlast;
1692
1693 totlen = cmd->src_map->dm_mapsize;
1694 if (cmd->srcu.src_m->m_flags & M_PKTHDR) {
1695 len = MHLEN;
1696 MGETHDR(m0, M_DONTWAIT, MT_DATA);
1697 } else {
1698 len = MLEN;
1699 MGET(m0, M_DONTWAIT, MT_DATA);
1700 }
1701 if (m0 == NULL) {
1702 err = ENOMEM;
1703 goto err_srcmap;
1704 }
1705 if (len == MHLEN)
1706 m_copy_pkthdr(m0, cmd->srcu.src_m);
1707 if (totlen >= MINCLSIZE) {
1708 MCLGET(m0, M_DONTWAIT);
1709 if (m0->m_flags & M_EXT)
1710 len = MCLBYTES;
1711 }
1712 totlen -= len;
1713 m0->m_pkthdr.len = m0->m_len = len;
1714 mlast = m0;
1715
1716 while (totlen > 0) {
1717 MGET(m, M_DONTWAIT, MT_DATA);
1718 if (m == NULL) {
1719 err = ENOMEM;
1720 m_freem(m0);
1721 goto err_srcmap;
1722 }
1723 len = MLEN;
1724 if (totlen >= MINCLSIZE) {
1725 MCLGET(m, M_DONTWAIT);
1726 if (m->m_flags & M_EXT)
1727 len = MCLBYTES;
1728 }
1729
1730 m->m_len = len;
1731 if (m0->m_flags & M_PKTHDR)
1732 m0->m_pkthdr.len += len;
1733 totlen -= len;
1734
1735 mlast->m_next = m;
1736 mlast = m;
1737 }
1738 cmd->dstu.dst_m = m0;
1739 }
1740 cmd->dst_map = cmd->dst_map_alloc;
1741 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1742 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1743 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) {
1744 err = ENOMEM;
1745 goto err_dstmap1;
1746 }
1747 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1748 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1749 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) {
1750 err = ENOMEM;
1751 goto err_dstmap1;
1752 }
1753 }
1754 }
1755
1756 #ifdef HIFN_DEBUG
1757 if (hifn_debug)
1758 printf("%s: Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1759 device_xname(sc->sc_dv),
1760 READ_REG_1(sc, HIFN_1_DMA_CSR),
1761 READ_REG_1(sc, HIFN_1_DMA_IER),
1762 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1763 cmd->src_map->dm_nsegs, cmd->dst_map->dm_nsegs);
1764 #endif
1765
1766 if (cmd->src_map == cmd->dst_map)
1767 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1768 0, cmd->src_map->dm_mapsize,
1769 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
1770 else {
1771 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1772 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1773 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
1774 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1775 }
1776
1777 /*
1778 * need 1 cmd, and 1 res
1779 * need N src, and N dst
1780 */
1781 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
1782 (dma->resu + 1) > HIFN_D_RES_RSIZE) {
1783 err = ENOMEM;
1784 goto err_dstmap;
1785 }
1786 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE ||
1787 (dma->dstu + cmd->dst_map->dm_nsegs + 1) > HIFN_D_DST_RSIZE) {
1788 err = ENOMEM;
1789 goto err_dstmap;
1790 }
1791
1792 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1793 dma->cmdi = 0;
1794 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1795 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1796 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1797 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1798 }
1799 cmdi = dma->cmdi++;
1800 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
1801 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
1802
1803 /* .p for command/result already set */
1804 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
1805 HIFN_D_MASKDONEIRQ);
1806 HIFN_CMDR_SYNC(sc, cmdi,
1807 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1808 dma->cmdu++;
1809 if (sc->sc_c_busy == 0) {
1810 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1811 sc->sc_c_busy = 1;
1812 SET_LED(sc, HIFN_MIPSRST_LED0);
1813 }
1814
1815 /*
1816 * Always enable the command wait interrupt. We are obviously
1817 * missing an interrupt or two somewhere. Enabling the command wait
1818 * interrupt will guarantee we get called periodically until all
1819 * of the queues are drained and thus work around this.
1820 */
1821 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
1822 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1823
1824 hifnstats.hst_ipackets++;
1825 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize;
1826
1827 hifn_dmamap_load_src(sc, cmd);
1828 if (sc->sc_s_busy == 0) {
1829 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1830 sc->sc_s_busy = 1;
1831 SET_LED(sc, HIFN_MIPSRST_LED1);
1832 }
1833
1834 /*
1835 * Unlike other descriptors, we don't mask done interrupt from
1836 * result descriptor.
1837 */
1838 #ifdef HIFN_DEBUG
1839 if (hifn_debug)
1840 printf("load res\n");
1841 #endif
1842 if (dma->resi == HIFN_D_RES_RSIZE) {
1843 dma->resi = 0;
1844 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1845 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1846 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1847 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1848 }
1849 resi = dma->resi++;
1850 dma->hifn_commands[resi] = cmd;
1851 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
1852 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
1853 HIFN_D_VALID | HIFN_D_LAST);
1854 HIFN_RESR_SYNC(sc, resi,
1855 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1856 dma->resu++;
1857 if (sc->sc_r_busy == 0) {
1858 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1859 sc->sc_r_busy = 1;
1860 SET_LED(sc, HIFN_MIPSRST_LED2);
1861 }
1862
1863 if (cmd->sloplen)
1864 cmd->slopidx = resi;
1865
1866 hifn_dmamap_load_dst(sc, cmd);
1867
1868 if (sc->sc_d_busy == 0) {
1869 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1870 sc->sc_d_busy = 1;
1871 }
1872
1873 #ifdef HIFN_DEBUG
1874 if (hifn_debug)
1875 printf("%s: command: stat %8x ier %8x\n",
1876 device_xname(sc->sc_dv),
1877 READ_REG_1(sc, HIFN_1_DMA_CSR),
1878 READ_REG_1(sc, HIFN_1_DMA_IER));
1879 #endif
1880
1881 sc->sc_active = 5;
1882 return (err); /* success */
1883
1884 err_dstmap:
1885 if (cmd->src_map != cmd->dst_map)
1886 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
1887 err_dstmap1:
1888 err_srcmap:
1889 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
1890 err_srcmap1:
1891 return (err);
1892 }
1893
1894 static void
hifn_tick(void * vsc)1895 hifn_tick(void *vsc)
1896 {
1897 struct hifn_softc *sc = vsc;
1898
1899 mutex_spin_enter(&sc->sc_mtx);
1900 if (sc->sc_active == 0) {
1901 struct hifn_dma *dma = sc->sc_dma;
1902 uint32_t r = 0;
1903
1904 if (dma->cmdu == 0 && sc->sc_c_busy) {
1905 sc->sc_c_busy = 0;
1906 r |= HIFN_DMACSR_C_CTRL_DIS;
1907 CLR_LED(sc, HIFN_MIPSRST_LED0);
1908 }
1909 if (dma->srcu == 0 && sc->sc_s_busy) {
1910 sc->sc_s_busy = 0;
1911 r |= HIFN_DMACSR_S_CTRL_DIS;
1912 CLR_LED(sc, HIFN_MIPSRST_LED1);
1913 }
1914 if (dma->dstu == 0 && sc->sc_d_busy) {
1915 sc->sc_d_busy = 0;
1916 r |= HIFN_DMACSR_D_CTRL_DIS;
1917 }
1918 if (dma->resu == 0 && sc->sc_r_busy) {
1919 sc->sc_r_busy = 0;
1920 r |= HIFN_DMACSR_R_CTRL_DIS;
1921 CLR_LED(sc, HIFN_MIPSRST_LED2);
1922 }
1923 if (r)
1924 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
1925 } else
1926 sc->sc_active--;
1927 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
1928 mutex_spin_exit(&sc->sc_mtx);
1929 }
1930
1931 static int
hifn_intr(void * arg)1932 hifn_intr(void *arg)
1933 {
1934 struct hifn_softc *sc = arg;
1935 struct hifn_dma *dma = sc->sc_dma;
1936 uint32_t dmacsr, restart;
1937 int i, u;
1938
1939 mutex_spin_enter(&sc->sc_mtx);
1940
1941 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
1942
1943 #ifdef HIFN_DEBUG
1944 if (hifn_debug)
1945 printf("%s: irq: stat %08x ien %08x u %d/%d/%d/%d\n",
1946 device_xname(sc->sc_dv),
1947 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER),
1948 dma->cmdu, dma->srcu, dma->dstu, dma->resu);
1949 #endif
1950
1951 /* Nothing in the DMA unit interrupted */
1952 if ((dmacsr & sc->sc_dmaier) == 0) {
1953 mutex_spin_exit(&sc->sc_mtx);
1954 return (0);
1955 }
1956
1957 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
1958
1959 if (dmacsr & HIFN_DMACSR_ENGINE)
1960 WRITE_REG_0(sc, HIFN_0_PUISR, READ_REG_0(sc, HIFN_0_PUISR));
1961
1962 if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
1963 (dmacsr & HIFN_DMACSR_PUBDONE))
1964 WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
1965 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
1966
1967 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
1968 if (restart)
1969 printf("%s: overrun %x\n", device_xname(sc->sc_dv), dmacsr);
1970
1971 if (sc->sc_flags & HIFN_IS_7811) {
1972 if (dmacsr & HIFN_DMACSR_ILLR)
1973 printf("%s: illegal read\n", device_xname(sc->sc_dv));
1974 if (dmacsr & HIFN_DMACSR_ILLW)
1975 printf("%s: illegal write\n", device_xname(sc->sc_dv));
1976 }
1977
1978 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
1979 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
1980 if (restart) {
1981 printf("%s: abort, resetting.\n", device_xname(sc->sc_dv));
1982 hifnstats.hst_abort++;
1983 hifn_abort(sc);
1984 goto out;
1985 }
1986
1987 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->resu == 0)) {
1988 /*
1989 * If no slots to process and we receive a "waiting on
1990 * command" interrupt, we disable the "waiting on command"
1991 * (by clearing it).
1992 */
1993 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1994 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1995 }
1996
1997 /* clear the rings */
1998 i = dma->resk;
1999 while (dma->resu != 0) {
2000 HIFN_RESR_SYNC(sc, i,
2001 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2002 if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2003 HIFN_RESR_SYNC(sc, i,
2004 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2005 break;
2006 }
2007
2008 if (i != HIFN_D_RES_RSIZE) {
2009 struct hifn_command *cmd;
2010
2011 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2012 cmd = dma->hifn_commands[i];
2013 KASSERT(cmd != NULL
2014 /*("hifn_intr: null command slot %u", i)*/);
2015 dma->hifn_commands[i] = NULL;
2016
2017 hifn_callback(sc, cmd, dma->result_bufs[i]);
2018 hifnstats.hst_opackets++;
2019 }
2020
2021 if (++i == (HIFN_D_RES_RSIZE + 1))
2022 i = 0;
2023 else
2024 dma->resu--;
2025 }
2026 dma->resk = i;
2027
2028 i = dma->srck; u = dma->srcu;
2029 while (u != 0) {
2030 HIFN_SRCR_SYNC(sc, i,
2031 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2032 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2033 HIFN_SRCR_SYNC(sc, i,
2034 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2035 break;
2036 }
2037 if (++i == (HIFN_D_SRC_RSIZE + 1))
2038 i = 0;
2039 else
2040 u--;
2041 }
2042 dma->srck = i; dma->srcu = u;
2043
2044 i = dma->cmdk; u = dma->cmdu;
2045 while (u != 0) {
2046 HIFN_CMDR_SYNC(sc, i,
2047 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2048 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2049 HIFN_CMDR_SYNC(sc, i,
2050 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2051 break;
2052 }
2053 if (i != HIFN_D_CMD_RSIZE) {
2054 u--;
2055 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2056 }
2057 if (++i == (HIFN_D_CMD_RSIZE + 1))
2058 i = 0;
2059 }
2060 dma->cmdk = i; dma->cmdu = u;
2061
2062 out:
2063 mutex_spin_exit(&sc->sc_mtx);
2064 return (1);
2065 }
2066
2067 /*
2068 * Allocate a new 'session' and return an encoded session id. 'sidp'
2069 * contains our registration id, and should contain an encoded session
2070 * id on successful allocation.
2071 */
2072 static int
hifn_newsession(void * arg,uint32_t * sidp,struct cryptoini * cri)2073 hifn_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri)
2074 {
2075 struct cryptoini *c;
2076 struct hifn_softc *sc = arg;
2077 int i, mac = 0, cry = 0, comp = 0, retval = EINVAL;
2078
2079 mutex_spin_enter(&sc->sc_mtx);
2080 for (i = 0; i < sc->sc_maxses; i++)
2081 if (isclr(sc->sc_sessions, i))
2082 break;
2083 if (i == sc->sc_maxses) {
2084 retval = ENOMEM;
2085 goto out;
2086 }
2087
2088 for (c = cri; c != NULL; c = c->cri_next) {
2089 switch (c->cri_alg) {
2090 case CRYPTO_MD5:
2091 case CRYPTO_SHA1:
2092 case CRYPTO_MD5_HMAC_96:
2093 case CRYPTO_SHA1_HMAC_96:
2094 if (mac) {
2095 goto out;
2096 }
2097 mac = 1;
2098 break;
2099 case CRYPTO_DES_CBC:
2100 case CRYPTO_3DES_CBC:
2101 case CRYPTO_AES_CBC:
2102 case CRYPTO_ARC4:
2103 if (cry) {
2104 goto out;
2105 }
2106 cry = 1;
2107 break;
2108 #ifdef CRYPTO_LZS_COMP
2109 case CRYPTO_LZS_COMP:
2110 if (comp) {
2111 goto out;
2112 }
2113 comp = 1;
2114 break;
2115 #endif
2116 default:
2117 goto out;
2118 }
2119 }
2120 if (mac == 0 && cry == 0 && comp == 0) {
2121 goto out;
2122 }
2123
2124 /*
2125 * XXX only want to support compression without chaining to
2126 * MAC/crypt engine right now
2127 */
2128 if ((comp && mac) || (comp && cry)) {
2129 goto out;
2130 }
2131
2132 *sidp = HIFN_SID(device_unit(sc->sc_dv), i);
2133 setbit(sc->sc_sessions, i);
2134
2135 retval = 0;
2136 out:
2137 mutex_spin_exit(&sc->sc_mtx);
2138 return retval;
2139 }
2140
2141 /*
2142 * Deallocate a session.
2143 * XXX this routine should run a zero'd mac/encrypt key into context ram.
2144 * XXX to blow away any keys already stored there.
2145 */
2146 static void
hifn_freesession(void * arg,uint64_t tid)2147 hifn_freesession(void *arg, uint64_t tid)
2148 {
2149 struct hifn_softc *sc = arg;
2150 int session;
2151 uint32_t sid = ((uint32_t) tid) & 0xffffffff;
2152
2153 mutex_spin_enter(&sc->sc_mtx);
2154 session = HIFN_SESSION(sid);
2155 KASSERTMSG(session >= 0, "session=%d", session);
2156 KASSERTMSG(session < sc->sc_maxses, "session=%d maxses=%d",
2157 session, sc->sc_maxses);
2158 KASSERT(isset(sc->sc_sessions, session));
2159 clrbit(sc->sc_sessions, session);
2160 mutex_spin_exit(&sc->sc_mtx);
2161 }
2162
2163 static int
hifn_process(void * arg,struct cryptop * crp,int hint)2164 hifn_process(void *arg, struct cryptop *crp, int hint)
2165 {
2166 struct hifn_softc *sc = arg;
2167 struct hifn_command *cmd = NULL;
2168 int session, err = 0, ivlen;
2169 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2170
2171 if ((cmd = pool_cache_get(sc->sc_cmd_cache, PR_NOWAIT)) == NULL) {
2172 hifnstats.hst_nomem++;
2173 err = ENOMEM;
2174 goto errout;
2175 }
2176
2177 mutex_spin_enter(&sc->sc_mtx);
2178 session = HIFN_SESSION(crp->crp_sid);
2179 KASSERTMSG(session < sc->sc_maxses, "session=%d maxses=%d",
2180 session, sc->sc_maxses);
2181
2182 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2183 cmd->srcu.src_m = (struct mbuf *)crp->crp_buf;
2184 cmd->dstu.dst_m = (struct mbuf *)crp->crp_buf;
2185 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2186 cmd->srcu.src_io = (struct uio *)crp->crp_buf;
2187 cmd->dstu.dst_io = (struct uio *)crp->crp_buf;
2188 } else {
2189 err = EINVAL;
2190 goto errout; /* XXX we don't handle contiguous buffers! */
2191 }
2192
2193 crd1 = crp->crp_desc;
2194 if (crd1 == NULL) {
2195 err = EINVAL;
2196 goto errout;
2197 }
2198 crd2 = crd1->crd_next;
2199
2200 if (crd2 == NULL) {
2201 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
2202 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2203 crd1->crd_alg == CRYPTO_SHA1 ||
2204 crd1->crd_alg == CRYPTO_MD5) {
2205 maccrd = crd1;
2206 enccrd = NULL;
2207 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2208 crd1->crd_alg == CRYPTO_3DES_CBC ||
2209 crd1->crd_alg == CRYPTO_AES_CBC ||
2210 crd1->crd_alg == CRYPTO_ARC4) {
2211 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2212 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2213 maccrd = NULL;
2214 enccrd = crd1;
2215 #ifdef CRYPTO_LZS_COMP
2216 } else if (crd1->crd_alg == CRYPTO_LZS_COMP) {
2217 hifn_compression(sc, crp, cmd);
2218 mutex_spin_exit(&sc->sc_mtx);
2219 return 0;
2220 #endif
2221 } else {
2222 err = EINVAL;
2223 goto errout;
2224 }
2225 } else {
2226 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
2227 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2228 crd1->crd_alg == CRYPTO_MD5 ||
2229 crd1->crd_alg == CRYPTO_SHA1) &&
2230 (crd2->crd_alg == CRYPTO_DES_CBC ||
2231 crd2->crd_alg == CRYPTO_3DES_CBC ||
2232 crd2->crd_alg == CRYPTO_AES_CBC ||
2233 crd2->crd_alg == CRYPTO_ARC4) &&
2234 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2235 cmd->base_masks = HIFN_BASE_CMD_DECODE;
2236 maccrd = crd1;
2237 enccrd = crd2;
2238 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2239 crd1->crd_alg == CRYPTO_ARC4 ||
2240 crd1->crd_alg == CRYPTO_3DES_CBC ||
2241 crd1->crd_alg == CRYPTO_AES_CBC) &&
2242 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 ||
2243 crd2->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2244 crd2->crd_alg == CRYPTO_MD5 ||
2245 crd2->crd_alg == CRYPTO_SHA1) &&
2246 (crd1->crd_flags & CRD_F_ENCRYPT)) {
2247 enccrd = crd1;
2248 maccrd = crd2;
2249 } else {
2250 /*
2251 * We cannot order the 7751 as requested
2252 */
2253 err = EINVAL;
2254 goto errout;
2255 }
2256 }
2257
2258 if (enccrd) {
2259 cmd->enccrd = enccrd;
2260 cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2261 switch (enccrd->crd_alg) {
2262 case CRYPTO_ARC4:
2263 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2264 break;
2265 case CRYPTO_DES_CBC:
2266 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2267 HIFN_CRYPT_CMD_MODE_CBC |
2268 HIFN_CRYPT_CMD_NEW_IV;
2269 break;
2270 case CRYPTO_3DES_CBC:
2271 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2272 HIFN_CRYPT_CMD_MODE_CBC |
2273 HIFN_CRYPT_CMD_NEW_IV;
2274 break;
2275 case CRYPTO_AES_CBC:
2276 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
2277 HIFN_CRYPT_CMD_MODE_CBC |
2278 HIFN_CRYPT_CMD_NEW_IV;
2279 break;
2280 default:
2281 err = EINVAL;
2282 goto errout;
2283 }
2284 if (enccrd->crd_alg != CRYPTO_ARC4) {
2285 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
2286 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2287 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2288 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2289 memcpy(cmd->iv, enccrd->crd_iv, ivlen);
2290 else
2291 cprng_fast(cmd->iv, ivlen);
2292
2293 if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2294 == 0) {
2295 if (crp->crp_flags & CRYPTO_F_IMBUF)
2296 m_copyback(cmd->srcu.src_m,
2297 enccrd->crd_inject,
2298 ivlen, cmd->iv);
2299 else if (crp->crp_flags & CRYPTO_F_IOV)
2300 cuio_copyback(cmd->srcu.src_io,
2301 enccrd->crd_inject,
2302 ivlen, cmd->iv);
2303 }
2304 } else {
2305 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2306 memcpy(cmd->iv, enccrd->crd_iv, ivlen);
2307 else if (crp->crp_flags & CRYPTO_F_IMBUF)
2308 m_copydata(cmd->srcu.src_m,
2309 enccrd->crd_inject, ivlen, cmd->iv);
2310 else if (crp->crp_flags & CRYPTO_F_IOV)
2311 cuio_copydata(cmd->srcu.src_io,
2312 enccrd->crd_inject,
2313 ivlen, cmd->iv);
2314 }
2315 }
2316
2317 cmd->ck = enccrd->crd_key;
2318 cmd->cklen = enccrd->crd_klen >> 3;
2319 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2320
2321 /*
2322 * Need to specify the size for the AES key in the masks.
2323 */
2324 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
2325 HIFN_CRYPT_CMD_ALG_AES) {
2326 switch (cmd->cklen) {
2327 case 16:
2328 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
2329 break;
2330 case 24:
2331 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
2332 break;
2333 case 32:
2334 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
2335 break;
2336 default:
2337 err = EINVAL;
2338 goto errout;
2339 }
2340 }
2341 }
2342
2343 if (maccrd) {
2344 cmd->maccrd = maccrd;
2345 cmd->base_masks |= HIFN_BASE_CMD_MAC;
2346
2347 switch (maccrd->crd_alg) {
2348 case CRYPTO_MD5:
2349 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2350 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2351 HIFN_MAC_CMD_POS_IPSEC;
2352 break;
2353 case CRYPTO_MD5_HMAC_96:
2354 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2355 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2356 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2357 break;
2358 case CRYPTO_SHA1:
2359 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2360 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2361 HIFN_MAC_CMD_POS_IPSEC;
2362 break;
2363 case CRYPTO_SHA1_HMAC_96:
2364 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2365 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2366 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2367 break;
2368 }
2369
2370 if (maccrd->crd_alg == CRYPTO_SHA1_HMAC_96 ||
2371 maccrd->crd_alg == CRYPTO_MD5_HMAC_96) {
2372 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2373 memcpy(cmd->mac, maccrd->crd_key, maccrd->crd_klen >> 3);
2374 memset(cmd->mac + (maccrd->crd_klen >> 3), 0,
2375 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2376 }
2377 }
2378
2379 cmd->crp = crp;
2380 cmd->session_num = session;
2381 cmd->softc = sc;
2382
2383 err = hifn_crypto(sc, cmd, crp, hint);
2384 if (err == 0) {
2385 mutex_exit(&sc->sc_mtx);
2386 return 0;
2387 } else if (err == ERESTART) {
2388 /*
2389 * There weren't enough resources to dispatch the request
2390 * to the part. Notify the caller so they'll requeue this
2391 * request and resubmit it again soon.
2392 */
2393 #ifdef HIFN_DEBUG
2394 if (hifn_debug)
2395 printf("%s: requeue request\n", device_xname(sc->sc_dv));
2396 #endif
2397 sc->sc_needwakeup |= CRYPTO_SYMQ;
2398 mutex_spin_exit(&sc->sc_mtx);
2399 pool_cache_put(sc->sc_cmd_cache, cmd);
2400 return ERESTART;
2401 }
2402
2403 errout:
2404 if (err == EINVAL)
2405 hifnstats.hst_invalid++;
2406 else
2407 hifnstats.hst_nomem++;
2408 crp->crp_etype = err;
2409 mutex_spin_exit(&sc->sc_mtx);
2410 if (cmd != NULL) {
2411 if (crp->crp_flags & CRYPTO_F_IMBUF &&
2412 cmd->srcu.src_m != cmd->dstu.dst_m)
2413 m_freem(cmd->dstu.dst_m);
2414 cmd->dst_map = NULL;
2415 pool_cache_put(sc->sc_cmd_cache, cmd);
2416 }
2417 crypto_done(crp);
2418 return 0;
2419 }
2420
2421 static void
hifn_abort(struct hifn_softc * sc)2422 hifn_abort(struct hifn_softc *sc)
2423 {
2424 struct hifn_dma *dma = sc->sc_dma;
2425 struct hifn_command *cmd;
2426 struct cryptop *crp;
2427 int i, u;
2428
2429 KASSERT(mutex_owned(&sc->sc_mtx));
2430
2431 i = dma->resk; u = dma->resu;
2432 while (u != 0) {
2433 cmd = dma->hifn_commands[i];
2434 KASSERT(cmd != NULL /*, ("hifn_abort: null cmd slot %u", i)*/);
2435 dma->hifn_commands[i] = NULL;
2436 crp = cmd->crp;
2437
2438 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2439 /* Salvage what we can. */
2440 hifnstats.hst_opackets++;
2441 hifn_callback(sc, cmd, dma->result_bufs[i]);
2442 } else {
2443 if (cmd->src_map == cmd->dst_map) {
2444 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2445 0, cmd->src_map->dm_mapsize,
2446 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2447 } else {
2448 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2449 0, cmd->src_map->dm_mapsize,
2450 BUS_DMASYNC_POSTWRITE);
2451 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2452 0, cmd->dst_map->dm_mapsize,
2453 BUS_DMASYNC_POSTREAD);
2454 }
2455
2456 if (cmd->srcu.src_m != cmd->dstu.dst_m) {
2457 m_freem(cmd->srcu.src_m);
2458 crp->crp_buf = (void *)cmd->dstu.dst_m;
2459 }
2460
2461 /* non-shared buffers cannot be restarted */
2462 if (cmd->src_map != cmd->dst_map) {
2463 /*
2464 * XXX should be EAGAIN, delayed until
2465 * after the reset.
2466 */
2467 crp->crp_etype = ENOMEM;
2468 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2469 } else
2470 crp->crp_etype = ENOMEM;
2471
2472 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2473
2474 cmd->dst_map = NULL;
2475 pool_cache_put(sc->sc_cmd_cache, cmd);
2476
2477 if (crp->crp_etype != EAGAIN)
2478 crypto_done(crp);
2479 }
2480
2481 if (++i == HIFN_D_RES_RSIZE)
2482 i = 0;
2483 u--;
2484 }
2485 dma->resk = i; dma->resu = u;
2486
2487 hifn_reset_board(sc, 1);
2488 hifn_init_dma(sc);
2489 hifn_init_pci_registers(sc);
2490 }
2491
2492 static void
hifn_callback(struct hifn_softc * sc,struct hifn_command * cmd,uint8_t * resbuf)2493 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, uint8_t *resbuf)
2494 {
2495 struct hifn_dma *dma = sc->sc_dma;
2496 struct cryptop *crp = cmd->crp;
2497 struct cryptodesc *crd;
2498 struct mbuf *m;
2499 int totlen, i, u;
2500
2501 KASSERT(mutex_owned(&sc->sc_mtx));
2502
2503 if (cmd->src_map == cmd->dst_map)
2504 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2505 0, cmd->src_map->dm_mapsize,
2506 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2507 else {
2508 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2509 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2510 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2511 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2512 }
2513
2514 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2515 if (cmd->srcu.src_m != cmd->dstu.dst_m) {
2516 crp->crp_buf = (void *)cmd->dstu.dst_m;
2517 totlen = cmd->src_map->dm_mapsize;
2518 for (m = cmd->dstu.dst_m; m != NULL; m = m->m_next) {
2519 if (totlen < m->m_len) {
2520 m->m_len = totlen;
2521 totlen = 0;
2522 } else
2523 totlen -= m->m_len;
2524 }
2525 cmd->dstu.dst_m->m_pkthdr.len =
2526 cmd->srcu.src_m->m_pkthdr.len;
2527 m_freem(cmd->srcu.src_m);
2528 }
2529 }
2530
2531 if (cmd->sloplen != 0) {
2532 if (crp->crp_flags & CRYPTO_F_IMBUF)
2533 m_copyback((struct mbuf *)crp->crp_buf,
2534 cmd->src_map->dm_mapsize - cmd->sloplen,
2535 cmd->sloplen, &dma->slop[cmd->slopidx]);
2536 else if (crp->crp_flags & CRYPTO_F_IOV)
2537 cuio_copyback((struct uio *)crp->crp_buf,
2538 cmd->src_map->dm_mapsize - cmd->sloplen,
2539 cmd->sloplen, &dma->slop[cmd->slopidx]);
2540 }
2541
2542 i = dma->dstk; u = dma->dstu;
2543 while (u != 0) {
2544 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2545 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc),
2546 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2547 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2548 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2549 offsetof(struct hifn_dma, dstr[i]),
2550 sizeof(struct hifn_desc),
2551 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2552 break;
2553 }
2554 if (++i == (HIFN_D_DST_RSIZE + 1))
2555 i = 0;
2556 else
2557 u--;
2558 }
2559 dma->dstk = i; dma->dstu = u;
2560
2561 hifnstats.hst_obytes += cmd->dst_map->dm_mapsize;
2562
2563 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2564 uint8_t *macbuf;
2565
2566 macbuf = resbuf + sizeof(struct hifn_base_result);
2567 if (cmd->base_masks & HIFN_BASE_CMD_COMP)
2568 macbuf += sizeof(struct hifn_comp_result);
2569 macbuf += sizeof(struct hifn_mac_result);
2570
2571 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2572 int len;
2573
2574 if (crd->crd_alg == CRYPTO_MD5)
2575 len = 16;
2576 else if (crd->crd_alg == CRYPTO_SHA1)
2577 len = 20;
2578 else if (crd->crd_alg == CRYPTO_MD5_HMAC_96 ||
2579 crd->crd_alg == CRYPTO_SHA1_HMAC_96)
2580 len = 12;
2581 else
2582 continue;
2583
2584 if (crp->crp_flags & CRYPTO_F_IMBUF)
2585 m_copyback((struct mbuf *)crp->crp_buf,
2586 crd->crd_inject, len, macbuf);
2587 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac)
2588 memcpy(crp->crp_mac, (void *)macbuf, len);
2589 break;
2590 }
2591 }
2592
2593 if (cmd->src_map != cmd->dst_map)
2594 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2595 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2596 cmd->dst_map = NULL;
2597 pool_cache_put(sc->sc_cmd_cache, cmd);
2598 crypto_done(crp);
2599 }
2600
2601 #ifdef CRYPTO_LZS_COMP
2602
2603 static void
hifn_compression(struct hifn_softc * sc,struct cryptop * crp,struct hifn_command * cmd)2604 hifn_compression(struct hifn_softc *sc, struct cryptop *crp,
2605 struct hifn_command *cmd)
2606 {
2607 struct cryptodesc *crd = crp->crp_desc;
2608 int s, err = 0;
2609
2610 cmd->compcrd = crd;
2611 cmd->base_masks |= HIFN_BASE_CMD_COMP;
2612
2613 if ((crp->crp_flags & CRYPTO_F_IMBUF) == 0) {
2614 /*
2615 * XXX can only handle mbufs right now since we can
2616 * XXX dynamically resize them.
2617 */
2618 err = EINVAL;
2619 goto fail;
2620 }
2621
2622 if ((crd->crd_flags & CRD_F_COMP) == 0)
2623 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2624 if (crd->crd_alg == CRYPTO_LZS_COMP)
2625 cmd->comp_masks |= HIFN_COMP_CMD_ALG_LZS |
2626 HIFN_COMP_CMD_CLEARHIST;
2627
2628 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2629 int len;
2630
2631 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
2632 cmd->srcu.src_m, BUS_DMA_NOWAIT)) {
2633 err = ENOMEM;
2634 goto fail;
2635 }
2636
2637 len = cmd->src_map->dm_mapsize / MCLBYTES;
2638 if ((cmd->src_map->dm_mapsize % MCLBYTES) != 0)
2639 len++;
2640 len *= MCLBYTES;
2641
2642 if ((crd->crd_flags & CRD_F_COMP) == 0)
2643 len *= 4;
2644
2645 if (len > HIFN_MAX_DMALEN)
2646 len = HIFN_MAX_DMALEN;
2647
2648 cmd->dstu.dst_m = hifn_mkmbuf_chain(len, cmd->srcu.src_m);
2649 if (cmd->dstu.dst_m == NULL) {
2650 err = ENOMEM;
2651 goto fail;
2652 }
2653
2654 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
2655 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) {
2656 err = ENOMEM;
2657 goto fail;
2658 }
2659 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2660 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
2661 cmd->srcu.src_io, BUS_DMA_NOWAIT)) {
2662 err = ENOMEM;
2663 goto fail;
2664 }
2665 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
2666 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) {
2667 err = ENOMEM;
2668 goto fail;
2669 }
2670 }
2671
2672 if (cmd->src_map == cmd->dst_map)
2673 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2674 0, cmd->src_map->dm_mapsize,
2675 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2676 else {
2677 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2678 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2679 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2680 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2681 }
2682
2683 cmd->crp = crp;
2684 /*
2685 * Always use session 0. The modes of compression we use are
2686 * stateless and there is always at least one compression
2687 * context, zero.
2688 */
2689 cmd->session_num = 0;
2690 cmd->softc = sc;
2691
2692 err = hifn_compress_enter(sc, cmd);
2693 if (err)
2694 goto fail;
2695
2696 return;
2697
2698 fail:
2699 if (cmd->dst_map != NULL) {
2700 if (cmd->dst_map->dm_nsegs > 0)
2701 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2702 }
2703 if (cmd->src_map != NULL) {
2704 if (cmd->src_map->dm_nsegs > 0)
2705 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2706 }
2707 cmd->dst_map = NULL;
2708 pool_cache_put(sc->sc_cmd_cache, cmd);
2709 if (err == EINVAL)
2710 hifnstats.hst_invalid++;
2711 else
2712 hifnstats.hst_nomem++;
2713 crp->crp_etype = err;
2714 crypto_done(crp);
2715 }
2716
2717 static int
hifn_compress_enter(struct hifn_softc * sc,struct hifn_command * cmd)2718 hifn_compress_enter(struct hifn_softc *sc, struct hifn_command *cmd)
2719 {
2720 struct hifn_dma *dma = sc->sc_dma;
2721 int cmdi, resi;
2722 uint32_t cmdlen;
2723
2724 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
2725 (dma->resu + 1) > HIFN_D_CMD_RSIZE)
2726 return (ENOMEM);
2727
2728 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE ||
2729 (dma->dstu + cmd->dst_map->dm_nsegs) > HIFN_D_DST_RSIZE)
2730 return (ENOMEM);
2731
2732 if (dma->cmdi == HIFN_D_CMD_RSIZE) {
2733 dma->cmdi = 0;
2734 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
2735 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2736 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
2737 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2738 }
2739 cmdi = dma->cmdi++;
2740 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
2741 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
2742
2743 /* .p for command/result already set */
2744 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
2745 HIFN_D_MASKDONEIRQ);
2746 HIFN_CMDR_SYNC(sc, cmdi,
2747 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2748 dma->cmdu++;
2749 if (sc->sc_c_busy == 0) {
2750 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
2751 sc->sc_c_busy = 1;
2752 SET_LED(sc, HIFN_MIPSRST_LED0);
2753 }
2754
2755 /*
2756 * Always enable the command wait interrupt. We are obviously
2757 * missing an interrupt or two somewhere. Enabling the command wait
2758 * interrupt will guarantee we get called periodically until all
2759 * of the queues are drained and thus work around this.
2760 */
2761 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
2762 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2763
2764 hifnstats.hst_ipackets++;
2765 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize;
2766
2767 hifn_dmamap_load_src(sc, cmd);
2768 if (sc->sc_s_busy == 0) {
2769 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
2770 sc->sc_s_busy = 1;
2771 SET_LED(sc, HIFN_MIPSRST_LED1);
2772 }
2773
2774 /*
2775 * Unlike other descriptors, we don't mask done interrupt from
2776 * result descriptor.
2777 */
2778 if (dma->resi == HIFN_D_RES_RSIZE) {
2779 dma->resi = 0;
2780 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
2781 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2782 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
2783 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2784 }
2785 resi = dma->resi++;
2786 dma->hifn_commands[resi] = cmd;
2787 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
2788 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2789 HIFN_D_VALID | HIFN_D_LAST);
2790 HIFN_RESR_SYNC(sc, resi,
2791 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2792 dma->resu++;
2793 if (sc->sc_r_busy == 0) {
2794 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
2795 sc->sc_r_busy = 1;
2796 SET_LED(sc, HIFN_MIPSRST_LED2);
2797 }
2798
2799 if (cmd->sloplen)
2800 cmd->slopidx = resi;
2801
2802 hifn_dmamap_load_dst(sc, cmd);
2803
2804 if (sc->sc_d_busy == 0) {
2805 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
2806 sc->sc_d_busy = 1;
2807 }
2808 sc->sc_active = 5;
2809 cmd->cmd_callback = hifn_callback_comp;
2810 return (0);
2811 }
2812
2813 static void
hifn_callback_comp(struct hifn_softc * sc,struct hifn_command * cmd,uint8_t * resbuf)2814 hifn_callback_comp(struct hifn_softc *sc, struct hifn_command *cmd,
2815 uint8_t *resbuf)
2816 {
2817 struct hifn_base_result baseres;
2818 struct cryptop *crp = cmd->crp;
2819 struct hifn_dma *dma = sc->sc_dma;
2820 struct mbuf *m;
2821 int err = 0, i, u;
2822 uint32_t olen;
2823 bus_size_t dstsize;
2824
2825 KASSERT(mutex_owned(&sc->sc_mtx));
2826
2827 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2828 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2829 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2830 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2831
2832 dstsize = cmd->dst_map->dm_mapsize;
2833 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2834
2835 memcpy(&baseres, resbuf, sizeof(struct hifn_base_result));
2836
2837 i = dma->dstk; u = dma->dstu;
2838 while (u != 0) {
2839 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2840 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc),
2841 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2842 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2843 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2844 offsetof(struct hifn_dma, dstr[i]),
2845 sizeof(struct hifn_desc),
2846 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2847 break;
2848 }
2849 if (++i == (HIFN_D_DST_RSIZE + 1))
2850 i = 0;
2851 else
2852 u--;
2853 }
2854 dma->dstk = i; dma->dstu = u;
2855
2856 if (baseres.flags & htole16(HIFN_BASE_RES_DSTOVERRUN)) {
2857 bus_size_t xlen;
2858
2859 xlen = dstsize;
2860
2861 m_freem(cmd->dstu.dst_m);
2862
2863 if (xlen == HIFN_MAX_DMALEN) {
2864 /* We've done all we can. */
2865 err = E2BIG;
2866 goto out;
2867 }
2868
2869 xlen += MCLBYTES;
2870
2871 if (xlen > HIFN_MAX_DMALEN)
2872 xlen = HIFN_MAX_DMALEN;
2873
2874 cmd->dstu.dst_m = hifn_mkmbuf_chain(xlen,
2875 cmd->srcu.src_m);
2876 if (cmd->dstu.dst_m == NULL) {
2877 err = ENOMEM;
2878 goto out;
2879 }
2880 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
2881 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) {
2882 err = ENOMEM;
2883 goto out;
2884 }
2885
2886 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2887 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2888 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2889 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2890
2891 err = hifn_compress_enter(sc, cmd);
2892 if (err != 0)
2893 goto out;
2894 return;
2895 }
2896
2897 olen = dstsize - (letoh16(baseres.dst_cnt) |
2898 (((letoh16(baseres.session) & HIFN_BASE_RES_DSTLEN_M) >>
2899 HIFN_BASE_RES_DSTLEN_S) << 16));
2900
2901 crp->crp_olen = olen - cmd->compcrd->crd_skip;
2902
2903 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2904
2905 m = cmd->dstu.dst_m;
2906 if (m->m_flags & M_PKTHDR)
2907 m->m_pkthdr.len = olen;
2908 crp->crp_buf = (void *)m;
2909 for (; m != NULL; m = m->m_next) {
2910 if (olen >= m->m_len)
2911 olen -= m->m_len;
2912 else {
2913 m->m_len = olen;
2914 olen = 0;
2915 }
2916 }
2917
2918 m_freem(cmd->srcu.src_m);
2919 cmd->dst_map = NULL;
2920 pool_cache_put(sc->sc_cmd_cache, cmd);
2921 crp->crp_etype = 0;
2922 crypto_done(crp);
2923 return;
2924
2925 out:
2926 if (cmd->dst_map != NULL) {
2927 if (cmd->src_map->dm_nsegs != 0)
2928 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2929 }
2930 if (cmd->src_map != NULL) {
2931 if (cmd->src_map->dm_nsegs != 0)
2932 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2933 }
2934 m_freem(cmd->dstu.dst_m);
2935 cmd->dst_map = NULL;
2936 pool_cache_put(sc->sc_cmd_cache, cmd);
2937 crp->crp_etype = err;
2938 crypto_done(crp);
2939 }
2940
2941 static struct mbuf *
hifn_mkmbuf_chain(int totlen,struct mbuf * mtemplate)2942 hifn_mkmbuf_chain(int totlen, struct mbuf *mtemplate)
2943 {
2944 int len;
2945 struct mbuf *m, *m0, *mlast;
2946
2947 if (mtemplate->m_flags & M_PKTHDR) {
2948 len = MHLEN;
2949 MGETHDR(m0, M_DONTWAIT, MT_DATA);
2950 } else {
2951 len = MLEN;
2952 MGET(m0, M_DONTWAIT, MT_DATA);
2953 }
2954 if (m0 == NULL)
2955 return (NULL);
2956 if (len == MHLEN)
2957 m_copy_pkthdr(m0, mtemplate);
2958 MCLGET(m0, M_DONTWAIT);
2959 if (!(m0->m_flags & M_EXT)) {
2960 m_freem(m0);
2961 return (NULL);
2962 }
2963 len = MCLBYTES;
2964
2965 totlen -= len;
2966 m0->m_pkthdr.len = m0->m_len = len;
2967 mlast = m0;
2968
2969 while (totlen > 0) {
2970 MGET(m, M_DONTWAIT, MT_DATA);
2971 if (m == NULL) {
2972 m_freem(m0);
2973 return (NULL);
2974 }
2975 MCLGET(m, M_DONTWAIT);
2976 if (!(m->m_flags & M_EXT)) {
2977 m_free(m);
2978 m_freem(m0);
2979 return (NULL);
2980 }
2981 len = MCLBYTES;
2982 m->m_len = len;
2983 if (m0->m_flags & M_PKTHDR)
2984 m0->m_pkthdr.len += len;
2985 totlen -= len;
2986
2987 mlast->m_next = m;
2988 mlast = m;
2989 }
2990
2991 return (m0);
2992 }
2993 #endif /* CRYPTO_LZS_COMP */
2994
2995 static void
hifn_write_4(struct hifn_softc * sc,int reggrp,bus_size_t reg,uint32_t val)2996 hifn_write_4(struct hifn_softc *sc, int reggrp, bus_size_t reg, uint32_t val)
2997 {
2998 /*
2999 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
3000 * and Group 1 registers; avoid conditions that could create
3001 * burst writes by doing a read in between the writes.
3002 */
3003 if (sc->sc_flags & HIFN_NO_BURSTWRITE) {
3004 if (sc->sc_waw_lastgroup == reggrp &&
3005 sc->sc_waw_lastreg == reg - 4) {
3006 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
3007 }
3008 sc->sc_waw_lastgroup = reggrp;
3009 sc->sc_waw_lastreg = reg;
3010 }
3011 if (reggrp == 0)
3012 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
3013 else
3014 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
3015
3016 }
3017
3018 static uint32_t
hifn_read_4(struct hifn_softc * sc,int reggrp,bus_size_t reg)3019 hifn_read_4(struct hifn_softc *sc, int reggrp, bus_size_t reg)
3020 {
3021 if (sc->sc_flags & HIFN_NO_BURSTWRITE) {
3022 sc->sc_waw_lastgroup = -1;
3023 sc->sc_waw_lastreg = 1;
3024 }
3025 if (reggrp == 0)
3026 return (bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg));
3027 return (bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg));
3028 }
3029