1 /*-
2 * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org>
3 * Copyright (c) 2017 Justin Hibbits <jhibbits@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer,
11 * without modification, immediately at the beginning of the file.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 #include <sys/param.h>
30 #include <sys/module.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/bus.h>
34 #include <sys/conf.h>
35 #include <sys/endian.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/mutex.h>
39 #include <sys/rman.h>
40
41 #include <cam/cam.h>
42 #include <cam/cam_ccb.h>
43 #include <cam/cam_sim.h>
44 #include <cam/cam_xpt_sim.h>
45 #include <cam/cam_debug.h>
46
47 #include <dev/ofw/ofw_bus_subr.h>
48
49 #include <machine/bus.h>
50 #include <machine/resource.h>
51
52 #include "fsl_sata.h"
53
54 struct fsl_sata_channel;
55 struct fsl_sata_slot;
56 enum fsl_sata_err_type;
57 struct fsl_sata_cmd_tab;
58
59 /* local prototypes */
60 static int fsl_sata_init(device_t dev);
61 static int fsl_sata_deinit(device_t dev);
62 static int fsl_sata_suspend(device_t dev);
63 static int fsl_sata_resume(device_t dev);
64 static void fsl_sata_pm(void *arg);
65 static void fsl_sata_intr(void *arg);
66 static void fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus);
67 static void fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb);
68 static void fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
69 static void fsl_sata_execute_transaction(struct fsl_sata_slot *slot);
70 static void fsl_sata_timeout(void *arg);
71 static void fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et);
72 static int fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag);
73 static void fsl_sata_dmainit(device_t dev);
74 static void fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
75 static void fsl_sata_dmafini(device_t dev);
76 static void fsl_sata_slotsalloc(device_t dev);
77 static void fsl_sata_slotsfree(device_t dev);
78 static void fsl_sata_reset(struct fsl_sata_channel *ch);
79 static void fsl_sata_start(struct fsl_sata_channel *ch);
80 static void fsl_sata_stop(struct fsl_sata_channel *ch);
81
82 static void fsl_sata_issue_recovery(struct fsl_sata_channel *ch);
83 static void fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb);
84 static void fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb);
85
86 static void fsl_sataaction(struct cam_sim *sim, union ccb *ccb);
87 static void fsl_satapoll(struct cam_sim *sim);
88
89 static MALLOC_DEFINE(M_FSL_SATA, "FSL SATA driver", "FSL SATA driver data buffers");
90
91 #define recovery_type spriv_field0
92 #define RECOVERY_NONE 0
93 #define RECOVERY_READ_LOG 1
94 #define RECOVERY_REQUEST_SENSE 2
95 #define recovery_slot spriv_field1
96
97 #define FSL_SATA_P_CQR 0x0
98 #define FSL_SATA_P_CAR 0x4
99 #define FSL_SATA_P_CCR 0x10
100 #define FSL_SATA_P_CER 0x18
101 #define FSL_SATA_P_DER 0x20
102 #define FSL_SATA_P_CHBA 0x24
103 #define FSL_SATA_P_HSTS 0x28
104 #define FSL_SATA_P_HSTS_HS_ON 0x80000000
105 #define FSL_SATA_P_HSTS_ME 0x00040000
106 #define FSL_SATA_P_HSTS_DLM 0x00001000
107 #define FSL_SATA_P_HSTS_FOT 0x00000200
108 #define FSL_SATA_P_HSTS_FOR 0x00000100
109 #define FSL_SATA_P_HSTS_FE 0x00000020
110 #define FSL_SATA_P_HSTS_PR 0x00000010
111 #define FSL_SATA_P_HSTS_SNTFU 0x00000004
112 #define FSL_SATA_P_HSTS_DE 0x00000002
113 #define FSL_SATA_P_HCTRL 0x2c
114 #define FSL_SATA_P_HCTRL_HC_ON 0x80000000
115 #define FSL_SATA_P_HCTRL_HC_FORCE_OFF 0x40000000
116 #define FSL_SATA_P_HCTRL_ENT 0x10000000
117 #define FSL_SATA_P_HCTRL_SNOOP 0x00000400
118 #define FSL_SATA_P_HCTRL_PM 0x00000200
119 #define FSL_SATA_P_HCTRL_FATAL 0x00000020
120 #define FSL_SATA_P_HCTRL_PHYRDY 0x00000010
121 #define FSL_SATA_P_HCTRL_SIG 0x00000008
122 #define FSL_SATA_P_HCTRL_SNTFY 0x00000004
123 #define FSL_SATA_P_HCTRL_DE 0x00000002
124 #define FSL_SATA_P_HCTRL_CC 0x00000001
125 #define FSL_SATA_P_HCTRL_INT_MASK 0x0000003f
126 #define FSL_SATA_P_CQPMP 0x30
127 #define FSL_SATA_P_SIG 0x34
128 #define FSL_SATA_P_ICC 0x38
129 #define FSL_SATA_P_ICC_ITC_M 0x1f000000
130 #define FSL_SATA_P_ICC_ITC_S 24
131 #define FSL_SATA_P_ICC_ITTCV_M 0x0007ffff
132 #define FSL_SATA_P_PCC 0x15c
133 #define FSL_SATA_P_PCC_SLUMBER 0x0000000c
134 #define FSL_SATA_P_PCC_PARTIAL 0x0000000a
135 #define FSL_SATA_PCC_LPB_EN 0x0000000e
136
137 #define FSL_SATA_MAX_SLOTS 16
138 /* FSL_SATA register defines */
139
140 #define FSL_SATA_P_SSTS 0x100
141 #define FSL_SATA_P_SERR 0x104
142 #define FSL_SATA_P_SCTL 0x108
143 #define FSL_SATA_P_SNTF 0x10c
144
145 /* Pessimistic prognosis on number of required S/G entries */
146 #define FSL_SATA_SG_ENTRIES 63
147 /* Command list. 16 commands. First, 1Kbyte aligned. */
148 #define FSL_SATA_CL_OFFSET 0
149 #define FSL_SATA_CL_SIZE 16
150 /* Command tables. Up to 32 commands, Each, 4-byte aligned. */
151 #define FSL_SATA_CT_OFFSET (FSL_SATA_CL_OFFSET + FSL_SATA_CL_SIZE * FSL_SATA_MAX_SLOTS)
152 #define FSL_SATA_CT_SIZE (96 + FSL_SATA_SG_ENTRIES * 16)
153 /* Total main work area. */
154 #define FSL_SATA_WORK_SIZE (FSL_SATA_CT_OFFSET + FSL_SATA_CT_SIZE * FSL_SATA_MAX_SLOTS)
155 #define FSL_SATA_MAX_XFER (64 * 1024 * 1024)
156
157 /* Some convenience macros for getting the CTP and CLP */
158 #define FSL_SATA_CTP_BUS(ch, slot) \
159 ((ch->dma.work_bus + FSL_SATA_CT_OFFSET + (FSL_SATA_CT_SIZE * slot->slot)))
160 #define FSL_SATA_PRD_OFFSET(prd) (96 + (prd) * 16)
161 #define FSL_SATA_CTP(ch, slot) \
162 ((struct fsl_sata_cmd_tab *)(ch->dma.work + FSL_SATA_CT_OFFSET + \
163 (FSL_SATA_CT_SIZE * slot->slot)))
164 #define FSL_SATA_CLP(ch, slot) \
165 ((struct fsl_sata_cmd_list *) (ch->dma.work + FSL_SATA_CL_OFFSET + \
166 (FSL_SATA_CL_SIZE * slot->slot)))
167
168 struct fsl_sata_dma_prd {
169 uint32_t dba;
170 uint32_t reserved;
171 uint32_t reserved2;
172 uint32_t dwc_flg; /* 0 based */
173 #define FSL_SATA_PRD_MASK 0x01fffffc /* max 32MB */
174 #define FSL_SATA_PRD_MAX (FSL_SATA_PRD_MASK + 4)
175 #define FSL_SATA_PRD_SNOOP 0x10000000
176 #define FSL_SATA_PRD_EXT 0x80000000
177 } __packed;
178
179 struct fsl_sata_cmd_tab {
180 uint8_t cfis[32];
181 uint8_t sfis[32];
182 uint8_t acmd[16];
183 uint8_t reserved[16];
184 struct fsl_sata_dma_prd prd_tab[FSL_SATA_SG_ENTRIES];
185 #define FSL_SATA_PRD_EXT_INDEX 15
186 #define FSL_SATA_PRD_MAX_DIRECT 16
187 } __packed;
188
189 struct fsl_sata_cmd_list {
190 uint32_t cda; /* word aligned */
191 uint16_t fis_length; /* length in bytes (aligned to words) */
192 uint16_t prd_length; /* PRD entries */
193 uint32_t ttl;
194 uint32_t cmd_flags;
195 #define FSL_SATA_CMD_TAG_MASK 0x001f
196 #define FSL_SATA_CMD_ATAPI 0x0020
197 #define FSL_SATA_CMD_BIST 0x0040
198 #define FSL_SATA_CMD_RESET 0x0080
199 #define FSL_SATA_CMD_QUEUED 0x0100
200 #define FSL_SATA_CMD_SNOOP 0x0200
201 #define FSL_SATA_CMD_VBIST 0x0400
202 #define FSL_SATA_CMD_WRITE 0x0800
203
204 } __packed;
205
206 /* misc defines */
207 #define ATA_IRQ_RID 0
208 #define ATA_INTR_FLAGS (INTR_MPSAFE|INTR_TYPE_BIO|INTR_ENTROPY)
209
210 struct ata_dmaslot {
211 bus_dmamap_t data_map; /* data DMA map */
212 int nsegs; /* Number of segs loaded */
213 };
214
215 /* structure holding DMA related information */
216 struct ata_dma {
217 bus_dma_tag_t work_tag; /* workspace DMA tag */
218 bus_dmamap_t work_map; /* workspace DMA map */
219 uint8_t *work; /* workspace */
220 bus_addr_t work_bus; /* bus address of work */
221 bus_dma_tag_t data_tag; /* data DMA tag */
222 };
223
224 enum fsl_sata_slot_states {
225 FSL_SATA_SLOT_EMPTY,
226 FSL_SATA_SLOT_LOADING,
227 FSL_SATA_SLOT_RUNNING,
228 FSL_SATA_SLOT_EXECUTING
229 };
230
231 struct fsl_sata_slot {
232 struct fsl_sata_channel *ch; /* Channel */
233 uint8_t slot; /* Number of this slot */
234 enum fsl_sata_slot_states state; /* Slot state */
235 union ccb *ccb; /* CCB occupying slot */
236 struct ata_dmaslot dma; /* DMA data of this slot */
237 struct callout timeout; /* Execution timeout */
238 uint32_t ttl;
239 };
240
241 struct fsl_sata_device {
242 int revision;
243 int mode;
244 u_int bytecount;
245 u_int atapi;
246 u_int tags;
247 u_int caps;
248 };
249
250 /* structure describing an ATA channel */
251 struct fsl_sata_channel {
252 device_t dev; /* Device handle */
253 int r_mid; /* Physical channel RID */
254 struct resource *r_mem; /* Memory of this channel */
255 struct resource *r_irq; /* Interrupt of this channel */
256 void *ih; /* Interrupt handle */
257 struct ata_dma dma; /* DMA data */
258 struct cam_sim *sim;
259 struct cam_path *path;
260 uint32_t caps; /* Controller capabilities */
261 int pm_level; /* power management level */
262 int devices; /* What is present */
263 int pm_present; /* PM presence reported */
264
265 union ccb *hold[FSL_SATA_MAX_SLOTS];
266 struct fsl_sata_slot slot[FSL_SATA_MAX_SLOTS];
267 uint32_t oslots; /* Occupied slots */
268 uint32_t rslots; /* Running slots */
269 uint32_t aslots; /* Slots with atomic commands */
270 uint32_t eslots; /* Slots in error */
271 uint32_t toslots; /* Slots in timeout */
272 int lastslot; /* Last used slot */
273 int taggedtarget; /* Last tagged target */
274 int numrslots; /* Number of running slots */
275 int numrslotspd[16];/* Number of running slots per dev */
276 int numtslots; /* Number of tagged slots */
277 int numtslotspd[16];/* Number of tagged slots per dev */
278 int numhslots; /* Number of held slots */
279 int recoverycmd; /* Our READ LOG active */
280 int fatalerr; /* Fatal error happend */
281 int resetting; /* Hard-reset in progress. */
282 int resetpolldiv; /* Hard-reset poll divider. */
283 union ccb *frozen; /* Frozen command */
284 struct callout pm_timer; /* Power management events */
285 struct callout reset_timer; /* Hard-reset timeout */
286
287 struct fsl_sata_device user[16]; /* User-specified settings */
288 struct fsl_sata_device curr[16]; /* Current settings */
289
290 struct mtx_padalign mtx; /* state lock */
291 STAILQ_HEAD(, ccb_hdr) doneq; /* queue of completed CCBs */
292 int batch; /* doneq is in use */
293 };
294
295 enum fsl_sata_err_type {
296 FSL_SATA_ERR_NONE, /* No error */
297 FSL_SATA_ERR_INVALID, /* Error detected by us before submitting. */
298 FSL_SATA_ERR_INNOCENT, /* Innocent victim. */
299 FSL_SATA_ERR_TFE, /* Task File Error. */
300 FSL_SATA_ERR_SATA, /* SATA error. */
301 FSL_SATA_ERR_TIMEOUT, /* Command execution timeout. */
302 FSL_SATA_ERR_NCQ, /* NCQ command error. CCB should be put on hold
303 * until READ LOG executed to reveal error. */
304 };
305
306 /* macros to hide busspace uglyness */
307 #define ATA_INL(res, offset) \
308 bus_read_4((res), (offset))
309 #define ATA_OUTL(res, offset, value) \
310 bus_write_4((res), (offset), (value))
311
312 static int
fsl_sata_probe(device_t dev)313 fsl_sata_probe(device_t dev)
314 {
315
316 if (!ofw_bus_is_compatible(dev, "fsl,pq-sata-v2") &&
317 !ofw_bus_is_compatible(dev, "fsl,pq-sata"))
318 return (ENXIO);
319
320 device_set_desc_copy(dev, "Freescale Integrated SATA Controller");
321 return (BUS_PROBE_DEFAULT);
322 }
323
324 static int
fsl_sata_attach(device_t dev)325 fsl_sata_attach(device_t dev)
326 {
327 struct fsl_sata_channel *ch = device_get_softc(dev);
328 struct cam_devq *devq;
329 int rid, error, i, sata_rev = 0;
330
331 ch->dev = dev;
332 mtx_init(&ch->mtx, "FSL SATA channel lock", NULL, MTX_DEF);
333 ch->pm_level = 0;
334 resource_int_value(device_get_name(dev),
335 device_get_unit(dev), "pm_level", &ch->pm_level);
336 STAILQ_INIT(&ch->doneq);
337 if (ch->pm_level > 3)
338 callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
339 resource_int_value(device_get_name(dev),
340 device_get_unit(dev), "sata_rev", &sata_rev);
341 for (i = 0; i < 16; i++) {
342 ch->user[i].revision = sata_rev;
343 ch->user[i].mode = 0;
344 ch->user[i].bytecount = 8192;
345 ch->user[i].tags = FSL_SATA_MAX_SLOTS;
346 ch->user[i].caps = 0;
347 ch->curr[i] = ch->user[i];
348 if (ch->pm_level) {
349 ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
350 CTS_SATA_CAPS_D_PMREQ;
351 }
352 ch->user[i].caps |= CTS_SATA_CAPS_H_AN;
353 }
354 ch->r_mid = 0;
355 if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
356 &ch->r_mid, RF_ACTIVE)))
357 return (ENXIO);
358 rman_set_bustag(ch->r_mem, &bs_le_tag);
359 fsl_sata_dmainit(dev);
360 fsl_sata_slotsalloc(dev);
361 fsl_sata_init(dev);
362 rid = ATA_IRQ_RID;
363 if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
364 &rid, RF_SHAREABLE | RF_ACTIVE))) {
365 device_printf(dev, "Unable to map interrupt\n");
366 error = ENXIO;
367 goto err0;
368 }
369 if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
370 fsl_sata_intr, ch, &ch->ih))) {
371 device_printf(dev, "Unable to setup interrupt\n");
372 error = ENXIO;
373 goto err1;
374 }
375 mtx_lock(&ch->mtx);
376 /* Create the device queue for our SIM. */
377 devq = cam_simq_alloc(FSL_SATA_MAX_SLOTS);
378 if (devq == NULL) {
379 device_printf(dev, "Unable to allocate simq\n");
380 error = ENOMEM;
381 goto err1;
382 }
383 /* Construct SIM entry */
384 ch->sim = cam_sim_alloc(fsl_sataaction, fsl_satapoll, "fslsata", ch,
385 device_get_unit(dev), (struct mtx *)&ch->mtx, 2, FSL_SATA_MAX_SLOTS,
386 devq);
387 if (ch->sim == NULL) {
388 cam_simq_free(devq);
389 device_printf(dev, "unable to allocate sim\n");
390 error = ENOMEM;
391 goto err1;
392 }
393 if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
394 device_printf(dev, "unable to register xpt bus\n");
395 error = ENXIO;
396 goto err2;
397 }
398 if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
399 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
400 device_printf(dev, "unable to create path\n");
401 error = ENXIO;
402 goto err3;
403 }
404 if (ch->pm_level > 3) {
405 callout_reset(&ch->pm_timer,
406 (ch->pm_level == 4) ? hz / 1000 : hz / 8,
407 fsl_sata_pm, ch);
408 }
409 mtx_unlock(&ch->mtx);
410 return (0);
411
412 err3:
413 xpt_bus_deregister(cam_sim_path(ch->sim));
414 err2:
415 cam_sim_free(ch->sim, /*free_devq*/TRUE);
416 err1:
417 mtx_unlock(&ch->mtx);
418 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
419 err0:
420 bus_release_resource(dev, SYS_RES_MEMORY, ch->r_mid, ch->r_mem);
421 mtx_destroy(&ch->mtx);
422 return (error);
423 }
424
425 static int
fsl_sata_detach(device_t dev)426 fsl_sata_detach(device_t dev)
427 {
428 struct fsl_sata_channel *ch = device_get_softc(dev);
429
430 mtx_lock(&ch->mtx);
431 xpt_async(AC_LOST_DEVICE, ch->path, NULL);
432
433 xpt_free_path(ch->path);
434 xpt_bus_deregister(cam_sim_path(ch->sim));
435 cam_sim_free(ch->sim, /*free_devq*/TRUE);
436 mtx_unlock(&ch->mtx);
437
438 if (ch->pm_level > 3)
439 callout_drain(&ch->pm_timer);
440 bus_teardown_intr(dev, ch->r_irq, ch->ih);
441 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
442
443 fsl_sata_deinit(dev);
444 fsl_sata_slotsfree(dev);
445 fsl_sata_dmafini(dev);
446
447 bus_release_resource(dev, SYS_RES_MEMORY, ch->r_mid, ch->r_mem);
448 mtx_destroy(&ch->mtx);
449 return (0);
450 }
451
452 static int
fsl_sata_wait_register(struct fsl_sata_channel * ch,bus_size_t off,unsigned int mask,unsigned int val,int t)453 fsl_sata_wait_register(struct fsl_sata_channel *ch, bus_size_t off,
454 unsigned int mask, unsigned int val, int t)
455 {
456 int timeout = 0;
457 uint32_t rval;
458
459 while (((rval = ATA_INL(ch->r_mem, off)) & mask) != val) {
460 if (timeout > t) {
461 return (EBUSY);
462 }
463 DELAY(1000);
464 timeout++;
465 }
466 return (0);
467 }
468
469 static int
fsl_sata_init(device_t dev)470 fsl_sata_init(device_t dev)
471 {
472 struct fsl_sata_channel *ch = device_get_softc(dev);
473 uint64_t work;
474 uint32_t r;
475
476 /* Disable port interrupts */
477 r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
478 r &= ~FSL_SATA_P_HCTRL_HC_ON;
479 r |= FSL_SATA_P_HCTRL_HC_FORCE_OFF;
480 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK);
481 fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
482 FSL_SATA_P_HSTS_HS_ON, 0, 1000);
483 /* Setup work areas */
484 work = ch->dma.work_bus + FSL_SATA_CL_OFFSET;
485 ATA_OUTL(ch->r_mem, FSL_SATA_P_CHBA, work);
486 r &= ~FSL_SATA_P_HCTRL_ENT;
487 r &= ~FSL_SATA_P_HCTRL_PM;
488 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r);
489 r = ATA_INL(ch->r_mem, FSL_SATA_P_PCC);
490 ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, r & ~FSL_SATA_PCC_LPB_EN);
491 ATA_OUTL(ch->r_mem, FSL_SATA_P_ICC, (1 << FSL_SATA_P_ICC_ITC_S));
492 fsl_sata_start(ch);
493 return (0);
494 }
495
496 static int
fsl_sata_deinit(device_t dev)497 fsl_sata_deinit(device_t dev)
498 {
499 struct fsl_sata_channel *ch = device_get_softc(dev);
500 uint32_t r;
501
502 /* Disable port interrupts. */
503 r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
504 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK);
505 /* Reset command register. */
506 fsl_sata_stop(ch);
507 /* Allow everything, including partial and slumber modes. */
508 ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, 0);
509 DELAY(100);
510 /* Disable PHY. */
511 ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, ATA_SC_DET_DISABLE);
512 r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
513 /* Turn off the controller. */
514 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_HC_ON);
515 return (0);
516 }
517
518 static int
fsl_sata_suspend(device_t dev)519 fsl_sata_suspend(device_t dev)
520 {
521 struct fsl_sata_channel *ch = device_get_softc(dev);
522
523 mtx_lock(&ch->mtx);
524 xpt_freeze_simq(ch->sim, 1);
525 while (ch->oslots)
526 msleep(ch, &ch->mtx, PRIBIO, "fsl_satasusp", hz/100);
527 fsl_sata_deinit(dev);
528 mtx_unlock(&ch->mtx);
529 return (0);
530 }
531
532 static int
fsl_sata_resume(device_t dev)533 fsl_sata_resume(device_t dev)
534 {
535 struct fsl_sata_channel *ch = device_get_softc(dev);
536
537 mtx_lock(&ch->mtx);
538 fsl_sata_init(dev);
539 fsl_sata_reset(ch);
540 xpt_release_simq(ch->sim, TRUE);
541 mtx_unlock(&ch->mtx);
542 return (0);
543 }
544
545 devclass_t fsl_satach_devclass;
546 static device_method_t fsl_satach_methods[] = {
547 DEVMETHOD(device_probe, fsl_sata_probe),
548 DEVMETHOD(device_attach, fsl_sata_attach),
549 DEVMETHOD(device_detach, fsl_sata_detach),
550 DEVMETHOD(device_suspend, fsl_sata_suspend),
551 DEVMETHOD(device_resume, fsl_sata_resume),
552 DEVMETHOD_END
553 };
554 static driver_t fsl_satach_driver = {
555 "fslsata",
556 fsl_satach_methods,
557 sizeof(struct fsl_sata_channel)
558 };
559 DRIVER_MODULE(fsl_satach, simplebus, fsl_satach_driver, fsl_satach_devclass, NULL, NULL);
560
561 struct fsl_sata_dc_cb_args {
562 bus_addr_t maddr;
563 int error;
564 };
565
566 static void
fsl_sata_dmainit(device_t dev)567 fsl_sata_dmainit(device_t dev)
568 {
569 struct fsl_sata_channel *ch = device_get_softc(dev);
570 struct fsl_sata_dc_cb_args dcba;
571
572 /* Command area. */
573 if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
574 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
575 NULL, NULL, FSL_SATA_WORK_SIZE, 1, FSL_SATA_WORK_SIZE,
576 0, NULL, NULL, &ch->dma.work_tag))
577 goto error;
578 if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work,
579 BUS_DMA_ZERO, &ch->dma.work_map))
580 goto error;
581 if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
582 FSL_SATA_WORK_SIZE, fsl_sata_dmasetupc_cb, &dcba, 0) || dcba.error) {
583 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
584 goto error;
585 }
586 ch->dma.work_bus = dcba.maddr;
587 /* Data area. */
588 if (bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
589 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
590 NULL, NULL, FSL_SATA_MAX_XFER,
591 FSL_SATA_SG_ENTRIES - 1, FSL_SATA_PRD_MAX,
592 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
593 goto error;
594 }
595 if (bootverbose)
596 device_printf(dev, "work area: %p\n", ch->dma.work);
597 return;
598
599 error:
600 device_printf(dev, "WARNING - DMA initialization failed\n");
601 fsl_sata_dmafini(dev);
602 }
603
604 static void
fsl_sata_dmasetupc_cb(void * xsc,bus_dma_segment_t * segs,int nsegs,int error)605 fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
606 {
607 struct fsl_sata_dc_cb_args *dcba = (struct fsl_sata_dc_cb_args *)xsc;
608
609 if (!(dcba->error = error))
610 dcba->maddr = segs[0].ds_addr;
611 }
612
613 static void
fsl_sata_dmafini(device_t dev)614 fsl_sata_dmafini(device_t dev)
615 {
616 struct fsl_sata_channel *ch = device_get_softc(dev);
617
618 if (ch->dma.data_tag) {
619 bus_dma_tag_destroy(ch->dma.data_tag);
620 ch->dma.data_tag = NULL;
621 }
622 if (ch->dma.work_bus) {
623 bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
624 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
625 ch->dma.work_bus = 0;
626 ch->dma.work = NULL;
627 }
628 if (ch->dma.work_tag) {
629 bus_dma_tag_destroy(ch->dma.work_tag);
630 ch->dma.work_tag = NULL;
631 }
632 }
633
634 static void
fsl_sata_slotsalloc(device_t dev)635 fsl_sata_slotsalloc(device_t dev)
636 {
637 struct fsl_sata_channel *ch = device_get_softc(dev);
638 int i;
639
640 /* Alloc and setup command/dma slots */
641 bzero(ch->slot, sizeof(ch->slot));
642 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
643 struct fsl_sata_slot *slot = &ch->slot[i];
644
645 slot->ch = ch;
646 slot->slot = i;
647 slot->state = FSL_SATA_SLOT_EMPTY;
648 slot->ccb = NULL;
649 callout_init_mtx(&slot->timeout, &ch->mtx, 0);
650
651 if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
652 device_printf(ch->dev, "FAILURE - create data_map\n");
653 }
654 }
655
656 static void
fsl_sata_slotsfree(device_t dev)657 fsl_sata_slotsfree(device_t dev)
658 {
659 struct fsl_sata_channel *ch = device_get_softc(dev);
660 int i;
661
662 /* Free all dma slots */
663 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
664 struct fsl_sata_slot *slot = &ch->slot[i];
665
666 callout_drain(&slot->timeout);
667 if (slot->dma.data_map) {
668 bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
669 slot->dma.data_map = NULL;
670 }
671 }
672 }
673
674 static int
fsl_sata_phy_check_events(struct fsl_sata_channel * ch,u_int32_t serr)675 fsl_sata_phy_check_events(struct fsl_sata_channel *ch, u_int32_t serr)
676 {
677
678 if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) ||
679 ((ch->pm_level != 0) && (serr & ATA_SE_EXCHANGED))) {
680 u_int32_t status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS);
681 union ccb *ccb;
682
683 if (bootverbose) {
684 if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE)
685 device_printf(ch->dev, "CONNECT requested\n");
686 else
687 device_printf(ch->dev, "DISCONNECT requested\n");
688 }
689 /* Issue soft reset */
690 xpt_async(AC_BUS_RESET, ch->path, NULL);
691 if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
692 return (0);
693 if (xpt_create_path(&ccb->ccb_h.path, NULL,
694 cam_sim_path(ch->sim),
695 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
696 xpt_free_ccb(ccb);
697 return (0);
698 }
699 xpt_rescan(ccb);
700 return (1);
701 }
702 return (0);
703 }
704
705 static void
fsl_sata_notify_events(struct fsl_sata_channel * ch,u_int32_t status)706 fsl_sata_notify_events(struct fsl_sata_channel *ch, u_int32_t status)
707 {
708 struct cam_path *dpath;
709 int i;
710
711 ATA_OUTL(ch->r_mem, FSL_SATA_P_SNTF, status);
712 if (bootverbose)
713 device_printf(ch->dev, "SNTF 0x%04x\n", status);
714 for (i = 0; i < 16; i++) {
715 if ((status & (1 << i)) == 0)
716 continue;
717 if (xpt_create_path(&dpath, NULL,
718 xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) {
719 xpt_async(AC_SCSI_AEN, dpath, NULL);
720 xpt_free_path(dpath);
721 }
722 }
723 }
724
725 static void
fsl_sata_done(struct fsl_sata_channel * ch,union ccb * ccb)726 fsl_sata_done(struct fsl_sata_channel *ch, union ccb *ccb)
727 {
728
729 mtx_assert(&ch->mtx, MA_OWNED);
730 if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0 ||
731 ch->batch == 0) {
732 xpt_done(ccb);
733 return;
734 }
735
736 STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe);
737 }
738
739 static void
fsl_sata_intr(void * arg)740 fsl_sata_intr(void *arg)
741 {
742 struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg;
743 struct ccb_hdr *ccb_h;
744 uint32_t istatus;
745 STAILQ_HEAD(, ccb_hdr) tmp_doneq = STAILQ_HEAD_INITIALIZER(tmp_doneq);
746
747 /* Read interrupt statuses. */
748 istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & 0x7ffff;
749 if ((istatus & 0x3f) == 0)
750 return;
751
752 mtx_lock(&ch->mtx);
753 ch->batch = 1;
754 fsl_sata_intr_main(ch, istatus);
755 ch->batch = 0;
756 /*
757 * Prevent the possibility of issues caused by processing the queue
758 * while unlocked below by moving the contents to a local queue.
759 */
760 STAILQ_CONCAT(&tmp_doneq, &ch->doneq);
761 mtx_unlock(&ch->mtx);
762 while ((ccb_h = STAILQ_FIRST(&tmp_doneq)) != NULL) {
763 STAILQ_REMOVE_HEAD(&tmp_doneq, sim_links.stqe);
764 xpt_done_direct((union ccb *)ccb_h);
765 }
766 /* Clear interrupt statuses. */
767 ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, istatus & 0x3f);
768
769 }
770
771 static void
fsl_sata_pm(void * arg)772 fsl_sata_pm(void *arg)
773 {
774 struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg;
775 uint32_t work;
776
777 if (ch->numrslots != 0)
778 return;
779 work = ATA_INL(ch->r_mem, FSL_SATA_P_PCC) & ~FSL_SATA_PCC_LPB_EN;
780 if (ch->pm_level == 4)
781 work |= FSL_SATA_P_PCC_PARTIAL;
782 else
783 work |= FSL_SATA_P_PCC_SLUMBER;
784 ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, work);
785 }
786
787 /* XXX: interrupt todo */
788 static void
fsl_sata_intr_main(struct fsl_sata_channel * ch,uint32_t istatus)789 fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus)
790 {
791 uint32_t cer, der, serr = 0, sntf = 0, ok, err;
792 enum fsl_sata_err_type et;
793 int i;
794
795 /* Complete all successful commands. */
796 ok = ATA_INL(ch->r_mem, FSL_SATA_P_CCR);
797 /* Mark all commands complete, to complete the interrupt. */
798 ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, ok);
799 if (ch->aslots == 0 && ok != 0) {
800 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
801 if (((ok >> i) & 1) && ch->slot[i].ccb != NULL)
802 fsl_sata_end_transaction(&ch->slot[i],
803 FSL_SATA_ERR_NONE);
804 }
805 }
806 /* Read command statuses. */
807 if (istatus & FSL_SATA_P_HSTS_SNTFU)
808 sntf = ATA_INL(ch->r_mem, FSL_SATA_P_SNTF);
809 /* XXX: Process PHY events */
810 serr = ATA_INL(ch->r_mem, FSL_SATA_P_SERR);
811 ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, serr);
812 if (istatus & (FSL_SATA_P_HSTS_PR)) {
813 if (serr) {
814 fsl_sata_phy_check_events(ch, serr);
815 }
816 }
817 /* Process command errors */
818 err = (istatus & (FSL_SATA_P_HSTS_FE | FSL_SATA_P_HSTS_DE));
819 cer = ATA_INL(ch->r_mem, FSL_SATA_P_CER);
820 ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, cer);
821 der = ATA_INL(ch->r_mem, FSL_SATA_P_DER);
822 ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, der);
823 /* On error, complete the rest of commands with error statuses. */
824 if (err) {
825 if (ch->frozen) {
826 union ccb *fccb = ch->frozen;
827 ch->frozen = NULL;
828 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
829 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
830 xpt_freeze_devq(fccb->ccb_h.path, 1);
831 fccb->ccb_h.status |= CAM_DEV_QFRZN;
832 }
833 fsl_sata_done(ch, fccb);
834 }
835 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
836 if (ch->slot[i].ccb == NULL)
837 continue;
838 if ((cer & (1 << i)) != 0)
839 et = FSL_SATA_ERR_TFE;
840 else if ((der & (1 << ch->slot[i].ccb->ccb_h.target_id)) != 0)
841 et = FSL_SATA_ERR_SATA;
842 else
843 et = FSL_SATA_ERR_INVALID;
844 fsl_sata_end_transaction(&ch->slot[i], et);
845 }
846 }
847 /* Process NOTIFY events */
848 if (sntf)
849 fsl_sata_notify_events(ch, sntf);
850 }
851
852 /* Must be called with channel locked. */
853 static int
fsl_sata_check_collision(struct fsl_sata_channel * ch,union ccb * ccb)854 fsl_sata_check_collision(struct fsl_sata_channel *ch, union ccb *ccb)
855 {
856 int t = ccb->ccb_h.target_id;
857
858 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
859 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
860 /* Tagged command while we have no supported tag free. */
861 if (((~ch->oslots) & (0xffff >> (16 - ch->curr[t].tags))) == 0)
862 return (1);
863 /* Tagged command while untagged are active. */
864 if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0)
865 return (1);
866 } else {
867 /* Untagged command while tagged are active. */
868 if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0)
869 return (1);
870 }
871 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
872 (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) {
873 /* Atomic command while anything active. */
874 if (ch->numrslots != 0)
875 return (1);
876 }
877 /* We have some atomic command running. */
878 if (ch->aslots != 0)
879 return (1);
880 return (0);
881 }
882
883 /* Must be called with channel locked. */
884 static void
fsl_sata_begin_transaction(struct fsl_sata_channel * ch,union ccb * ccb)885 fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb)
886 {
887 struct fsl_sata_slot *slot;
888 int tag, tags;
889
890 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
891 ("fsl_sata_begin_transaction func_code=0x%x\n", ccb->ccb_h.func_code));
892 /* Choose empty slot. */
893 tags = FSL_SATA_MAX_SLOTS;
894 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
895 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA))
896 tags = ch->curr[ccb->ccb_h.target_id].tags;
897 if (ch->lastslot + 1 < tags)
898 tag = ffs(~(ch->oslots >> (ch->lastslot + 1)));
899 else
900 tag = 0;
901 if (tag == 0 || tag + ch->lastslot >= tags)
902 tag = ffs(~ch->oslots) - 1;
903 else
904 tag += ch->lastslot;
905 ch->lastslot = tag;
906 /* Occupy chosen slot. */
907 slot = &ch->slot[tag];
908 slot->ccb = ccb;
909 slot->ttl = 0;
910 /* Stop PM timer. */
911 if (ch->numrslots == 0 && ch->pm_level > 3)
912 callout_stop(&ch->pm_timer);
913 /* Update channel stats. */
914 ch->oslots |= (1 << tag);
915 ch->numrslots++;
916 ch->numrslotspd[ccb->ccb_h.target_id]++;
917 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
918 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
919 ch->numtslots++;
920 ch->numtslotspd[ccb->ccb_h.target_id]++;
921 ch->taggedtarget = ccb->ccb_h.target_id;
922 }
923 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
924 (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)))
925 ch->aslots |= (1 << tag);
926 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
927 slot->state = FSL_SATA_SLOT_LOADING;
928 bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb,
929 fsl_sata_dmasetprd, slot, 0);
930 } else {
931 slot->dma.nsegs = 0;
932 fsl_sata_execute_transaction(slot);
933 }
934
935 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
936 ("fsl_sata_begin_transaction exit\n"));
937 }
938
939 /* Locked by busdma engine. */
940 static void
fsl_sata_dmasetprd(void * arg,bus_dma_segment_t * segs,int nsegs,int error)941 fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
942 {
943 struct fsl_sata_slot *slot = arg;
944 struct fsl_sata_channel *ch = slot->ch;
945 struct fsl_sata_cmd_tab *ctp;
946 struct fsl_sata_dma_prd *prd;
947 int i, j, len, extlen;
948
949 if (error) {
950 device_printf(ch->dev, "DMA load error %d\n", error);
951 fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID);
952 return;
953 }
954 KASSERT(nsegs <= FSL_SATA_SG_ENTRIES - 1,
955 ("too many DMA segment entries\n"));
956 /* Get a piece of the workspace for this request */
957 ctp = FSL_SATA_CTP(ch, slot);
958 /* Fill S/G table */
959 prd = &ctp->prd_tab[0];
960 for (i = 0, j = 0; i < nsegs; i++, j++) {
961 if (j == FSL_SATA_PRD_EXT_INDEX &&
962 FSL_SATA_PRD_MAX_DIRECT < nsegs) {
963 prd[j].dba = htole32(FSL_SATA_CTP_BUS(ch, slot) +
964 FSL_SATA_PRD_OFFSET(j+1));
965 j++;
966 extlen = 0;
967 }
968 len = segs[i].ds_len;
969 len = roundup2(len, sizeof(uint32_t));
970 prd[j].dba = htole32((uint32_t)segs[i].ds_addr);
971 prd[j].dwc_flg = htole32(FSL_SATA_PRD_SNOOP | len);
972 slot->ttl += len;
973 if (j > FSL_SATA_PRD_MAX_DIRECT)
974 extlen += len;
975 }
976 slot->dma.nsegs = j;
977 if (j > FSL_SATA_PRD_MAX_DIRECT)
978 prd[FSL_SATA_PRD_EXT_INDEX].dwc_flg =
979 htole32(FSL_SATA_PRD_SNOOP | FSL_SATA_PRD_EXT | extlen);
980 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
981 ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
982 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
983 fsl_sata_execute_transaction(slot);
984 }
985
986 /* Must be called with channel locked. */
987 static void
fsl_sata_execute_transaction(struct fsl_sata_slot * slot)988 fsl_sata_execute_transaction(struct fsl_sata_slot *slot)
989 {
990 struct fsl_sata_channel *ch = slot->ch;
991 struct fsl_sata_cmd_tab *ctp;
992 struct fsl_sata_cmd_list *clp;
993 union ccb *ccb = slot->ccb;
994 int port = ccb->ccb_h.target_id & 0x0f;
995 int fis_size, i, softreset;
996 uint32_t tmp;
997 uint32_t cmd_flags = FSL_SATA_CMD_WRITE | FSL_SATA_CMD_SNOOP;
998
999 softreset = 0;
1000 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
1001 ("fsl_sata_execute_transaction func_code=0x%x\n", ccb->ccb_h.func_code));
1002 /* Get a piece of the workspace for this request */
1003 ctp = FSL_SATA_CTP(ch, slot);
1004 /* Setup the FIS for this request */
1005 if (!(fis_size = fsl_sata_setup_fis(ch, ctp, ccb, slot->slot))) {
1006 device_printf(ch->dev, "Setting up SATA FIS failed\n");
1007 fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID);
1008 return;
1009 }
1010 /* Setup the command list entry */
1011 clp = FSL_SATA_CLP(ch, slot);
1012 clp->fis_length = htole16(fis_size);
1013 clp->prd_length = htole16(slot->dma.nsegs);
1014 /* Special handling for Soft Reset command. */
1015 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1016 (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
1017 if (ccb->ataio.cmd.control & ATA_A_RESET) {
1018 softreset = 1;
1019 cmd_flags |= FSL_SATA_CMD_RESET;
1020 } else {
1021 /* Prepare FIS receive area for check. */
1022 for (i = 0; i < 32; i++)
1023 ctp->sfis[i] = 0xff;
1024 softreset = 2;
1025 }
1026 }
1027 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)
1028 cmd_flags |= FSL_SATA_CMD_QUEUED;
1029 clp->cmd_flags = htole32(cmd_flags |
1030 (ccb->ccb_h.func_code == XPT_SCSI_IO ? FSL_SATA_CMD_ATAPI : 0) |
1031 slot->slot);
1032 clp->ttl = htole32(slot->ttl);
1033 clp->cda = htole32(FSL_SATA_CTP_BUS(ch, slot));
1034 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1035 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1036 /* Issue command to the controller. */
1037 slot->state = FSL_SATA_SLOT_RUNNING;
1038 ch->rslots |= (1 << slot->slot);
1039 ATA_OUTL(ch->r_mem, FSL_SATA_P_CQPMP, port);
1040 ATA_OUTL(ch->r_mem, FSL_SATA_P_CQR, (1 << slot->slot));
1041 /* Device reset commands don't interrupt. Poll them. */
1042 if (ccb->ccb_h.func_code == XPT_ATA_IO &&
1043 (ccb->ataio.cmd.command == ATA_DEVICE_RESET || softreset)) {
1044 int count, timeout = ccb->ccb_h.timeout * 100;
1045 enum fsl_sata_err_type et = FSL_SATA_ERR_NONE;
1046
1047 for (count = 0; count < timeout; count++) {
1048 DELAY(10);
1049 tmp = 0;
1050 if (softreset == 2) {
1051 tmp = ATA_INL(ch->r_mem, FSL_SATA_P_SIG);
1052 if (tmp != 0 && tmp != 0xffffffff)
1053 break;
1054 continue;
1055 }
1056 if ((ATA_INL(ch->r_mem, FSL_SATA_P_CCR) & (1 << slot->slot)) != 0)
1057 break;
1058 }
1059
1060 if (timeout && (count >= timeout)) {
1061 device_printf(ch->dev, "Poll timeout on slot %d port %d (round %d)\n",
1062 slot->slot, port, softreset);
1063 device_printf(ch->dev, "hsts %08x cqr %08x ccr %08x ss %08x "
1064 "rs %08x cer %08x der %08x serr %08x car %08x sig %08x\n",
1065 ATA_INL(ch->r_mem, FSL_SATA_P_HSTS),
1066 ATA_INL(ch->r_mem, FSL_SATA_P_CQR),
1067 ATA_INL(ch->r_mem, FSL_SATA_P_CCR),
1068 ATA_INL(ch->r_mem, FSL_SATA_P_SSTS), ch->rslots,
1069 ATA_INL(ch->r_mem, FSL_SATA_P_CER),
1070 ATA_INL(ch->r_mem, FSL_SATA_P_DER),
1071 ATA_INL(ch->r_mem, FSL_SATA_P_SERR),
1072 ATA_INL(ch->r_mem, FSL_SATA_P_CAR),
1073 ATA_INL(ch->r_mem, FSL_SATA_P_SIG));
1074 et = FSL_SATA_ERR_TIMEOUT;
1075 }
1076
1077 fsl_sata_end_transaction(slot, et);
1078 return;
1079 }
1080 /* Start command execution timeout */
1081 callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout / 2,
1082 0, fsl_sata_timeout, slot, 0);
1083 return;
1084 }
1085
1086 /* Must be called with channel locked. */
1087 static void
fsl_sata_process_timeout(struct fsl_sata_channel * ch)1088 fsl_sata_process_timeout(struct fsl_sata_channel *ch)
1089 {
1090 int i;
1091
1092 mtx_assert(&ch->mtx, MA_OWNED);
1093 /* Handle the rest of commands. */
1094 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1095 /* Do we have a running request on slot? */
1096 if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING)
1097 continue;
1098 fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_TIMEOUT);
1099 }
1100 }
1101
1102 /* Must be called with channel locked. */
1103 static void
fsl_sata_rearm_timeout(struct fsl_sata_channel * ch)1104 fsl_sata_rearm_timeout(struct fsl_sata_channel *ch)
1105 {
1106 int i;
1107
1108 mtx_assert(&ch->mtx, MA_OWNED);
1109 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1110 struct fsl_sata_slot *slot = &ch->slot[i];
1111
1112 /* Do we have a running request on slot? */
1113 if (slot->state < FSL_SATA_SLOT_RUNNING)
1114 continue;
1115 if ((ch->toslots & (1 << i)) == 0)
1116 continue;
1117 callout_reset_sbt(&slot->timeout,
1118 SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
1119 fsl_sata_timeout, slot, 0);
1120 }
1121 }
1122
1123 /* Locked by callout mechanism. */
1124 static void
fsl_sata_timeout(void * arg)1125 fsl_sata_timeout(void *arg)
1126 {
1127 struct fsl_sata_slot *slot = arg;
1128 struct fsl_sata_channel *ch = slot->ch;
1129 device_t dev = ch->dev;
1130 uint32_t sstatus;
1131
1132 /* Check for stale timeout. */
1133 if (slot->state < FSL_SATA_SLOT_RUNNING)
1134 return;
1135
1136 /* Check if slot was not being executed last time we checked. */
1137 if (slot->state < FSL_SATA_SLOT_EXECUTING) {
1138 /* Check if slot started executing. */
1139 sstatus = ATA_INL(ch->r_mem, FSL_SATA_P_CAR);
1140 if ((sstatus & (1 << slot->slot)) != 0)
1141 slot->state = FSL_SATA_SLOT_EXECUTING;
1142
1143 callout_reset_sbt(&slot->timeout,
1144 SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
1145 fsl_sata_timeout, slot, 0);
1146 return;
1147 }
1148
1149 device_printf(dev, "Timeout on slot %d port %d\n",
1150 slot->slot, slot->ccb->ccb_h.target_id & 0x0f);
1151
1152 /* Handle frozen command. */
1153 if (ch->frozen) {
1154 union ccb *fccb = ch->frozen;
1155 ch->frozen = NULL;
1156 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1157 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1158 xpt_freeze_devq(fccb->ccb_h.path, 1);
1159 fccb->ccb_h.status |= CAM_DEV_QFRZN;
1160 }
1161 fsl_sata_done(ch, fccb);
1162 }
1163 if (ch->toslots == 0)
1164 xpt_freeze_simq(ch->sim, 1);
1165 ch->toslots |= (1 << slot->slot);
1166 if ((ch->rslots & ~ch->toslots) == 0)
1167 fsl_sata_process_timeout(ch);
1168 else
1169 device_printf(dev, " ... waiting for slots %08x\n",
1170 ch->rslots & ~ch->toslots);
1171 }
1172
1173 /* Must be called with channel locked. */
1174 static void
fsl_sata_end_transaction(struct fsl_sata_slot * slot,enum fsl_sata_err_type et)1175 fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et)
1176 {
1177 struct fsl_sata_channel *ch = slot->ch;
1178 union ccb *ccb = slot->ccb;
1179 struct fsl_sata_cmd_list *clp;
1180 int lastto;
1181 uint32_t sig;
1182
1183 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1184 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1185 clp = FSL_SATA_CLP(ch, slot);
1186 /* Read result registers to the result struct */
1187 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1188 struct ata_res *res = &ccb->ataio.res;
1189
1190 if ((et == FSL_SATA_ERR_TFE) ||
1191 (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
1192 struct fsl_sata_cmd_tab *ctp = FSL_SATA_CTP(ch, slot);
1193 uint8_t *fis = ctp->sfis;
1194
1195 res->status = fis[2];
1196 res->error = fis[3];
1197 res->lba_low = fis[4];
1198 res->lba_mid = fis[5];
1199 res->lba_high = fis[6];
1200 res->device = fis[7];
1201 res->lba_low_exp = fis[8];
1202 res->lba_mid_exp = fis[9];
1203 res->lba_high_exp = fis[10];
1204 res->sector_count = fis[12];
1205 res->sector_count_exp = fis[13];
1206
1207 if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1208 (ccb->ataio.cmd.control & ATA_A_RESET) == 0) {
1209 sig = ATA_INL(ch->r_mem, FSL_SATA_P_SIG);
1210 res->lba_high = sig >> 24;
1211 res->lba_mid = sig >> 16;
1212 res->lba_low = sig >> 8;
1213 res->sector_count = sig;
1214 }
1215 } else
1216 bzero(res, sizeof(*res));
1217 if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 &&
1218 (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1219 ccb->ataio.resid =
1220 ccb->ataio.dxfer_len - le32toh(clp->ttl);
1221 }
1222 } else {
1223 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1224 ccb->csio.resid =
1225 ccb->csio.dxfer_len - le32toh(clp->ttl);
1226 }
1227 }
1228 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1229 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1230 (ccb->ccb_h.flags & CAM_DIR_IN) ?
1231 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1232 bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
1233 }
1234 if (et != FSL_SATA_ERR_NONE)
1235 ch->eslots |= (1 << slot->slot);
1236 /* In case of error, freeze device for proper recovery. */
1237 if ((et != FSL_SATA_ERR_NONE) && (!ch->recoverycmd) &&
1238 !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
1239 xpt_freeze_devq(ccb->ccb_h.path, 1);
1240 ccb->ccb_h.status |= CAM_DEV_QFRZN;
1241 }
1242 /* Set proper result status. */
1243 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1244 switch (et) {
1245 case FSL_SATA_ERR_NONE:
1246 ccb->ccb_h.status |= CAM_REQ_CMP;
1247 if (ccb->ccb_h.func_code == XPT_SCSI_IO)
1248 ccb->csio.scsi_status = SCSI_STATUS_OK;
1249 break;
1250 case FSL_SATA_ERR_INVALID:
1251 ch->fatalerr = 1;
1252 ccb->ccb_h.status |= CAM_REQ_INVALID;
1253 break;
1254 case FSL_SATA_ERR_INNOCENT:
1255 ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1256 break;
1257 case FSL_SATA_ERR_TFE:
1258 case FSL_SATA_ERR_NCQ:
1259 if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1260 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
1261 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1262 } else {
1263 ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
1264 }
1265 break;
1266 case FSL_SATA_ERR_SATA:
1267 ch->fatalerr = 1;
1268 if (!ch->recoverycmd) {
1269 xpt_freeze_simq(ch->sim, 1);
1270 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1271 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1272 }
1273 ccb->ccb_h.status |= CAM_UNCOR_PARITY;
1274 break;
1275 case FSL_SATA_ERR_TIMEOUT:
1276 if (!ch->recoverycmd) {
1277 xpt_freeze_simq(ch->sim, 1);
1278 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1279 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1280 }
1281 ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
1282 break;
1283 default:
1284 ch->fatalerr = 1;
1285 ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1286 }
1287 /* Free slot. */
1288 ch->oslots &= ~(1 << slot->slot);
1289 ch->rslots &= ~(1 << slot->slot);
1290 ch->aslots &= ~(1 << slot->slot);
1291 slot->state = FSL_SATA_SLOT_EMPTY;
1292 slot->ccb = NULL;
1293 /* Update channel stats. */
1294 ch->numrslots--;
1295 ch->numrslotspd[ccb->ccb_h.target_id]--;
1296 ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, 1 << slot->slot);
1297 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1298 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1299 ch->numtslots--;
1300 ch->numtslotspd[ccb->ccb_h.target_id]--;
1301 }
1302 /* Cancel timeout state if request completed normally. */
1303 if (et != FSL_SATA_ERR_TIMEOUT) {
1304 lastto = (ch->toslots == (1 << slot->slot));
1305 ch->toslots &= ~(1 << slot->slot);
1306 if (lastto)
1307 xpt_release_simq(ch->sim, TRUE);
1308 }
1309 /* If it was first request of reset sequence and there is no error,
1310 * proceed to second request. */
1311 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1312 (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1313 (ccb->ataio.cmd.control & ATA_A_RESET) &&
1314 et == FSL_SATA_ERR_NONE) {
1315 ccb->ataio.cmd.control &= ~ATA_A_RESET;
1316 fsl_sata_begin_transaction(ch, ccb);
1317 return;
1318 }
1319 /* If it was our READ LOG command - process it. */
1320 if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
1321 fsl_sata_process_read_log(ch, ccb);
1322 /* If it was our REQUEST SENSE command - process it. */
1323 } else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
1324 fsl_sata_process_request_sense(ch, ccb);
1325 /* If it was NCQ or ATAPI command error, put result on hold. */
1326 } else if (et == FSL_SATA_ERR_NCQ ||
1327 ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
1328 (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) {
1329 ch->hold[slot->slot] = ccb;
1330 ch->numhslots++;
1331 } else
1332 fsl_sata_done(ch, ccb);
1333 /* If we have no other active commands, ... */
1334 if (ch->rslots == 0) {
1335 /* if there was fatal error - reset port. */
1336 if (ch->toslots != 0 || ch->fatalerr) {
1337 fsl_sata_reset(ch);
1338 } else {
1339 /* if we have slots in error, we can reinit port. */
1340 if (ch->eslots != 0) {
1341 fsl_sata_stop(ch);
1342 fsl_sata_start(ch);
1343 }
1344 /* if there commands on hold, we can do READ LOG. */
1345 if (!ch->recoverycmd && ch->numhslots)
1346 fsl_sata_issue_recovery(ch);
1347 }
1348 /* If all the rest of commands are in timeout - give them chance. */
1349 } else if ((ch->rslots & ~ch->toslots) == 0 &&
1350 et != FSL_SATA_ERR_TIMEOUT)
1351 fsl_sata_rearm_timeout(ch);
1352 /* Unfreeze frozen command. */
1353 if (ch->frozen && !fsl_sata_check_collision(ch, ch->frozen)) {
1354 union ccb *fccb = ch->frozen;
1355 ch->frozen = NULL;
1356 fsl_sata_begin_transaction(ch, fccb);
1357 xpt_release_simq(ch->sim, TRUE);
1358 }
1359 /* Start PM timer. */
1360 if (ch->numrslots == 0 && ch->pm_level > 3 &&
1361 (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
1362 callout_schedule(&ch->pm_timer,
1363 (ch->pm_level == 4) ? hz / 1000 : hz / 8);
1364 }
1365 }
1366
1367 static void
fsl_sata_issue_recovery(struct fsl_sata_channel * ch)1368 fsl_sata_issue_recovery(struct fsl_sata_channel *ch)
1369 {
1370 union ccb *ccb;
1371 struct ccb_ataio *ataio;
1372 struct ccb_scsiio *csio;
1373 int i;
1374
1375 /* Find some held command. */
1376 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1377 if (ch->hold[i])
1378 break;
1379 }
1380 ccb = xpt_alloc_ccb_nowait();
1381 if (ccb == NULL) {
1382 device_printf(ch->dev, "Unable to allocate recovery command\n");
1383 completeall:
1384 /* We can't do anything -- complete held commands. */
1385 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1386 if (ch->hold[i] == NULL)
1387 continue;
1388 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1389 ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL;
1390 fsl_sata_done(ch, ch->hold[i]);
1391 ch->hold[i] = NULL;
1392 ch->numhslots--;
1393 }
1394 fsl_sata_reset(ch);
1395 return;
1396 }
1397 ccb->ccb_h = ch->hold[i]->ccb_h; /* Reuse old header. */
1398 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1399 /* READ LOG */
1400 ccb->ccb_h.recovery_type = RECOVERY_READ_LOG;
1401 ccb->ccb_h.func_code = XPT_ATA_IO;
1402 ccb->ccb_h.flags = CAM_DIR_IN;
1403 ccb->ccb_h.timeout = 1000; /* 1s should be enough. */
1404 ataio = &ccb->ataio;
1405 ataio->data_ptr = malloc(512, M_FSL_SATA, M_NOWAIT);
1406 if (ataio->data_ptr == NULL) {
1407 xpt_free_ccb(ccb);
1408 device_printf(ch->dev,
1409 "Unable to allocate memory for READ LOG command\n");
1410 goto completeall;
1411 }
1412 ataio->dxfer_len = 512;
1413 bzero(&ataio->cmd, sizeof(ataio->cmd));
1414 ataio->cmd.flags = CAM_ATAIO_48BIT;
1415 ataio->cmd.command = 0x2F; /* READ LOG EXT */
1416 ataio->cmd.sector_count = 1;
1417 ataio->cmd.sector_count_exp = 0;
1418 ataio->cmd.lba_low = 0x10;
1419 ataio->cmd.lba_mid = 0;
1420 ataio->cmd.lba_mid_exp = 0;
1421 } else {
1422 /* REQUEST SENSE */
1423 ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE;
1424 ccb->ccb_h.recovery_slot = i;
1425 ccb->ccb_h.func_code = XPT_SCSI_IO;
1426 ccb->ccb_h.flags = CAM_DIR_IN;
1427 ccb->ccb_h.status = 0;
1428 ccb->ccb_h.timeout = 1000; /* 1s should be enough. */
1429 csio = &ccb->csio;
1430 csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data;
1431 csio->dxfer_len = ch->hold[i]->csio.sense_len;
1432 csio->cdb_len = 6;
1433 bzero(&csio->cdb_io, sizeof(csio->cdb_io));
1434 csio->cdb_io.cdb_bytes[0] = 0x03;
1435 csio->cdb_io.cdb_bytes[4] = csio->dxfer_len;
1436 }
1437 /* Freeze SIM while doing recovery. */
1438 ch->recoverycmd = 1;
1439 xpt_freeze_simq(ch->sim, 1);
1440 fsl_sata_begin_transaction(ch, ccb);
1441 }
1442
1443 static void
fsl_sata_process_read_log(struct fsl_sata_channel * ch,union ccb * ccb)1444 fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb)
1445 {
1446 uint8_t *data;
1447 struct ata_res *res;
1448 int i;
1449
1450 ch->recoverycmd = 0;
1451
1452 data = ccb->ataio.data_ptr;
1453 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
1454 (data[0] & 0x80) == 0) {
1455 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1456 if (!ch->hold[i])
1457 continue;
1458 if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
1459 continue;
1460 if ((data[0] & 0x1F) == i) {
1461 res = &ch->hold[i]->ataio.res;
1462 res->status = data[2];
1463 res->error = data[3];
1464 res->lba_low = data[4];
1465 res->lba_mid = data[5];
1466 res->lba_high = data[6];
1467 res->device = data[7];
1468 res->lba_low_exp = data[8];
1469 res->lba_mid_exp = data[9];
1470 res->lba_high_exp = data[10];
1471 res->sector_count = data[12];
1472 res->sector_count_exp = data[13];
1473 } else {
1474 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1475 ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
1476 }
1477 fsl_sata_done(ch, ch->hold[i]);
1478 ch->hold[i] = NULL;
1479 ch->numhslots--;
1480 }
1481 } else {
1482 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
1483 device_printf(ch->dev, "Error while READ LOG EXT\n");
1484 else if ((data[0] & 0x80) == 0) {
1485 device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n");
1486 }
1487 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1488 if (!ch->hold[i])
1489 continue;
1490 if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
1491 continue;
1492 fsl_sata_done(ch, ch->hold[i]);
1493 ch->hold[i] = NULL;
1494 ch->numhslots--;
1495 }
1496 }
1497 free(ccb->ataio.data_ptr, M_FSL_SATA);
1498 xpt_free_ccb(ccb);
1499 xpt_release_simq(ch->sim, TRUE);
1500 }
1501
1502 static void
fsl_sata_process_request_sense(struct fsl_sata_channel * ch,union ccb * ccb)1503 fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb)
1504 {
1505 int i;
1506
1507 ch->recoverycmd = 0;
1508
1509 i = ccb->ccb_h.recovery_slot;
1510 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
1511 ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID;
1512 } else {
1513 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1514 ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL;
1515 }
1516 fsl_sata_done(ch, ch->hold[i]);
1517 ch->hold[i] = NULL;
1518 ch->numhslots--;
1519 xpt_free_ccb(ccb);
1520 xpt_release_simq(ch->sim, TRUE);
1521 }
1522
1523 static void
fsl_sata_start(struct fsl_sata_channel * ch)1524 fsl_sata_start(struct fsl_sata_channel *ch)
1525 {
1526 u_int32_t cmd;
1527
1528 /* Clear SATA error register */
1529 ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, 0xFFFFFFFF);
1530 /* Clear any interrupts pending on this channel */
1531 ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, 0x3F);
1532 ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, 0xFFFF);
1533 ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, 0xFFFF);
1534 /* Start operations on this channel */
1535 cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
1536 cmd |= FSL_SATA_P_HCTRL_HC_ON | FSL_SATA_P_HCTRL_SNOOP;
1537 cmd &= ~FSL_SATA_P_HCTRL_HC_FORCE_OFF;
1538 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd |
1539 (ch->pm_present ? FSL_SATA_P_HCTRL_PM : 0));
1540 fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
1541 FSL_SATA_P_HSTS_PR, FSL_SATA_P_HSTS_PR, 500);
1542 ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS,
1543 ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & FSL_SATA_P_HSTS_PR);
1544 }
1545
1546 static void
fsl_sata_stop(struct fsl_sata_channel * ch)1547 fsl_sata_stop(struct fsl_sata_channel *ch)
1548 {
1549 uint32_t cmd;
1550 int i;
1551
1552 /* Kill all activity on this channel */
1553 cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
1554 cmd &= ~FSL_SATA_P_HCTRL_HC_ON;
1555
1556 for (i = 0; i < 2; i++) {
1557 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd);
1558 if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
1559 FSL_SATA_P_HSTS_HS_ON, 0, 500)) {
1560 if (i != 0)
1561 device_printf(ch->dev,
1562 "stopping FSL SATA engine failed\n");
1563 cmd |= FSL_SATA_P_HCTRL_HC_FORCE_OFF;
1564 } else
1565 break;
1566 }
1567 ch->eslots = 0;
1568 }
1569
1570 static void
fsl_sata_reset(struct fsl_sata_channel * ch)1571 fsl_sata_reset(struct fsl_sata_channel *ch)
1572 {
1573 uint32_t ctrl;
1574 int i;
1575
1576 xpt_freeze_simq(ch->sim, 1);
1577 if (bootverbose)
1578 device_printf(ch->dev, "FSL SATA reset...\n");
1579
1580 /* Requeue freezed command. */
1581 if (ch->frozen) {
1582 union ccb *fccb = ch->frozen;
1583 ch->frozen = NULL;
1584 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1585 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1586 xpt_freeze_devq(fccb->ccb_h.path, 1);
1587 fccb->ccb_h.status |= CAM_DEV_QFRZN;
1588 }
1589 fsl_sata_done(ch, fccb);
1590 }
1591 /* Kill the engine and requeue all running commands. */
1592 fsl_sata_stop(ch);
1593 DELAY(1000); /* sleep for 1ms */
1594 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1595 /* Do we have a running request on slot? */
1596 if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING)
1597 continue;
1598 /* XXX; Commands in loading state. */
1599 fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_INNOCENT);
1600 }
1601 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1602 if (!ch->hold[i])
1603 continue;
1604 fsl_sata_done(ch, ch->hold[i]);
1605 ch->hold[i] = NULL;
1606 ch->numhslots--;
1607 }
1608 if (ch->toslots != 0)
1609 xpt_release_simq(ch->sim, TRUE);
1610 ch->eslots = 0;
1611 ch->toslots = 0;
1612 ch->fatalerr = 0;
1613 /* Tell the XPT about the event */
1614 xpt_async(AC_BUS_RESET, ch->path, NULL);
1615 /* Disable port interrupts */
1616 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
1617 ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f);
1618 /* Reset and reconnect PHY, */
1619 fsl_sata_start(ch);
1620 if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 0x08, 0x08, 500)) {
1621 if (bootverbose)
1622 device_printf(ch->dev,
1623 "FSL SATA reset: device not found\n");
1624 ch->devices = 0;
1625 /* Enable wanted port interrupts */
1626 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
1627 ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) | FSL_SATA_P_HCTRL_PHYRDY);
1628 xpt_release_simq(ch->sim, TRUE);
1629 return;
1630 }
1631 if (bootverbose)
1632 device_printf(ch->dev, "FSL SATA reset: device found\n");
1633 ch->devices = 1;
1634 /* Enable wanted port interrupts */
1635 ctrl = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f;
1636 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
1637 ctrl | FSL_SATA_P_HCTRL_FATAL | FSL_SATA_P_HCTRL_PHYRDY |
1638 FSL_SATA_P_HCTRL_SIG | FSL_SATA_P_HCTRL_SNTFY |
1639 FSL_SATA_P_HCTRL_DE | FSL_SATA_P_HCTRL_CC);
1640 xpt_release_simq(ch->sim, TRUE);
1641 }
1642
1643 static int
fsl_sata_setup_fis(struct fsl_sata_channel * ch,struct fsl_sata_cmd_tab * ctp,union ccb * ccb,int tag)1644 fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag)
1645 {
1646 uint8_t *fis = &ctp->cfis[0];
1647
1648 bzero(fis, 32);
1649 fis[0] = 0x27; /* host to device */
1650 fis[1] = (ccb->ccb_h.target_id & 0x0f);
1651 if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1652 fis[1] |= 0x80;
1653 fis[2] = ATA_PACKET_CMD;
1654 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
1655 ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA)
1656 fis[3] = ATA_F_DMA;
1657 else {
1658 fis[5] = ccb->csio.dxfer_len;
1659 fis[6] = ccb->csio.dxfer_len >> 8;
1660 }
1661 fis[7] = ATA_D_LBA;
1662 fis[15] = ATA_A_4BIT;
1663 bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1664 ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes,
1665 ctp->acmd, ccb->csio.cdb_len);
1666 bzero(ctp->acmd + ccb->csio.cdb_len, 32 - ccb->csio.cdb_len);
1667 } else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) {
1668 fis[1] |= 0x80;
1669 fis[2] = ccb->ataio.cmd.command;
1670 fis[3] = ccb->ataio.cmd.features;
1671 fis[4] = ccb->ataio.cmd.lba_low;
1672 fis[5] = ccb->ataio.cmd.lba_mid;
1673 fis[6] = ccb->ataio.cmd.lba_high;
1674 fis[7] = ccb->ataio.cmd.device;
1675 fis[8] = ccb->ataio.cmd.lba_low_exp;
1676 fis[9] = ccb->ataio.cmd.lba_mid_exp;
1677 fis[10] = ccb->ataio.cmd.lba_high_exp;
1678 fis[11] = ccb->ataio.cmd.features_exp;
1679 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1680 fis[12] = tag << 3;
1681 fis[13] = 0;
1682 } else {
1683 fis[12] = ccb->ataio.cmd.sector_count;
1684 fis[13] = ccb->ataio.cmd.sector_count_exp;
1685 }
1686 fis[15] = ATA_A_4BIT;
1687 } else {
1688 fis[15] = ccb->ataio.cmd.control;
1689 }
1690 return (20);
1691 }
1692
1693 static int
fsl_sata_check_ids(struct fsl_sata_channel * ch,union ccb * ccb)1694 fsl_sata_check_ids(struct fsl_sata_channel *ch, union ccb *ccb)
1695 {
1696
1697 if (ccb->ccb_h.target_id > 15) {
1698 ccb->ccb_h.status = CAM_TID_INVALID;
1699 fsl_sata_done(ch, ccb);
1700 return (-1);
1701 }
1702 if (ccb->ccb_h.target_lun != 0) {
1703 ccb->ccb_h.status = CAM_LUN_INVALID;
1704 fsl_sata_done(ch, ccb);
1705 return (-1);
1706 }
1707 return (0);
1708 }
1709
1710 static void
fsl_sataaction(struct cam_sim * sim,union ccb * ccb)1711 fsl_sataaction(struct cam_sim *sim, union ccb *ccb)
1712 {
1713 struct fsl_sata_channel *ch;
1714
1715 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
1716 ("fsl_sataaction func_code=0x%x\n", ccb->ccb_h.func_code));
1717
1718 ch = (struct fsl_sata_channel *)cam_sim_softc(sim);
1719 switch (ccb->ccb_h.func_code) {
1720 /* Common cases first */
1721 case XPT_ATA_IO: /* Execute the requested I/O operation */
1722 case XPT_SCSI_IO:
1723 if (fsl_sata_check_ids(ch, ccb))
1724 return;
1725 if (ch->devices == 0 ||
1726 (ch->pm_present == 0 &&
1727 ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) {
1728 ccb->ccb_h.status = CAM_SEL_TIMEOUT;
1729 break;
1730 }
1731 ccb->ccb_h.recovery_type = RECOVERY_NONE;
1732 /* Check for command collision. */
1733 if (fsl_sata_check_collision(ch, ccb)) {
1734 /* Freeze command. */
1735 ch->frozen = ccb;
1736 /* We have only one frozen slot, so freeze simq also. */
1737 xpt_freeze_simq(ch->sim, 1);
1738 return;
1739 }
1740 fsl_sata_begin_transaction(ch, ccb);
1741 return;
1742 case XPT_ABORT: /* Abort the specified CCB */
1743 /* XXX Implement */
1744 ccb->ccb_h.status = CAM_REQ_INVALID;
1745 break;
1746 case XPT_SET_TRAN_SETTINGS:
1747 {
1748 struct ccb_trans_settings *cts = &ccb->cts;
1749 struct fsl_sata_device *d;
1750
1751 if (fsl_sata_check_ids(ch, ccb))
1752 return;
1753 if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
1754 d = &ch->curr[ccb->ccb_h.target_id];
1755 else
1756 d = &ch->user[ccb->ccb_h.target_id];
1757 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
1758 d->revision = cts->xport_specific.sata.revision;
1759 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
1760 d->mode = cts->xport_specific.sata.mode;
1761 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
1762 d->bytecount = min(8192, cts->xport_specific.sata.bytecount);
1763 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
1764 d->tags = min(FSL_SATA_MAX_SLOTS, cts->xport_specific.sata.tags);
1765 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
1766 ch->pm_present = cts->xport_specific.sata.pm_present;
1767 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
1768 d->atapi = cts->xport_specific.sata.atapi;
1769 ccb->ccb_h.status = CAM_REQ_CMP;
1770 break;
1771 }
1772 case XPT_GET_TRAN_SETTINGS:
1773 /* Get default/user set transfer settings for the target */
1774 {
1775 struct ccb_trans_settings *cts = &ccb->cts;
1776 struct fsl_sata_device *d;
1777 uint32_t status;
1778
1779 if (fsl_sata_check_ids(ch, ccb))
1780 return;
1781 if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
1782 d = &ch->curr[ccb->ccb_h.target_id];
1783 else
1784 d = &ch->user[ccb->ccb_h.target_id];
1785 cts->protocol = PROTO_UNSPECIFIED;
1786 cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
1787 cts->transport = XPORT_SATA;
1788 cts->transport_version = XPORT_VERSION_UNSPECIFIED;
1789 cts->proto_specific.valid = 0;
1790 cts->xport_specific.sata.valid = 0;
1791 if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
1792 (ccb->ccb_h.target_id == 15 ||
1793 (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
1794 status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS) & ATA_SS_SPD_MASK;
1795 if (status & 0x0f0) {
1796 cts->xport_specific.sata.revision =
1797 (status & 0x0f0) >> 4;
1798 cts->xport_specific.sata.valid |=
1799 CTS_SATA_VALID_REVISION;
1800 }
1801 cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D;
1802 if (ch->pm_level) {
1803 cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ;
1804 }
1805 cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN;
1806 cts->xport_specific.sata.caps &=
1807 ch->user[ccb->ccb_h.target_id].caps;
1808 cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
1809 } else {
1810 cts->xport_specific.sata.revision = d->revision;
1811 cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
1812 cts->xport_specific.sata.caps = d->caps;
1813 cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
1814 }
1815 cts->xport_specific.sata.mode = d->mode;
1816 cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
1817 cts->xport_specific.sata.bytecount = d->bytecount;
1818 cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
1819 cts->xport_specific.sata.pm_present = ch->pm_present;
1820 cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
1821 cts->xport_specific.sata.tags = d->tags;
1822 cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
1823 cts->xport_specific.sata.atapi = d->atapi;
1824 cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
1825 ccb->ccb_h.status = CAM_REQ_CMP;
1826 break;
1827 }
1828 case XPT_RESET_BUS: /* Reset the specified SCSI bus */
1829 case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */
1830 fsl_sata_reset(ch);
1831 ccb->ccb_h.status = CAM_REQ_CMP;
1832 break;
1833 case XPT_TERM_IO: /* Terminate the I/O process */
1834 /* XXX Implement */
1835 ccb->ccb_h.status = CAM_REQ_INVALID;
1836 break;
1837 case XPT_PATH_INQ: /* Path routing inquiry */
1838 {
1839 struct ccb_pathinq *cpi = &ccb->cpi;
1840
1841 cpi->version_num = 1; /* XXX??? */
1842 cpi->hba_inquiry = PI_SDTR_ABLE;
1843 cpi->hba_inquiry |= PI_TAG_ABLE;
1844 #if 0
1845 /*
1846 * XXX: CAM tries to reset port 15 if it sees port multiplier
1847 * support. Disable it for now.
1848 */
1849 cpi->hba_inquiry |= PI_SATAPM;
1850 #endif
1851 cpi->target_sprt = 0;
1852 cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED;
1853 cpi->hba_eng_cnt = 0;
1854 /*
1855 * XXX: This should be 15, since hardware *does* support a port
1856 * multiplier. See above.
1857 */
1858 cpi->max_target = 0;
1859 cpi->max_lun = 0;
1860 cpi->initiator_id = 0;
1861 cpi->bus_id = cam_sim_bus(sim);
1862 cpi->base_transfer_speed = 150000;
1863 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1864 strncpy(cpi->hba_vid, "FSL SATA", HBA_IDLEN);
1865 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1866 cpi->unit_number = cam_sim_unit(sim);
1867 cpi->transport = XPORT_SATA;
1868 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
1869 cpi->protocol = PROTO_ATA;
1870 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
1871 cpi->maxio = maxphys;
1872 cpi->ccb_h.status = CAM_REQ_CMP;
1873 break;
1874 }
1875 default:
1876 ccb->ccb_h.status = CAM_REQ_INVALID;
1877 break;
1878 }
1879 fsl_sata_done(ch, ccb);
1880 }
1881
1882 static void
fsl_satapoll(struct cam_sim * sim)1883 fsl_satapoll(struct cam_sim *sim)
1884 {
1885 struct fsl_sata_channel *ch = (struct fsl_sata_channel *)cam_sim_softc(sim);
1886 uint32_t istatus;
1887
1888 /* Read interrupt statuses and process if any. */
1889 istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS);
1890 if (istatus != 0)
1891 fsl_sata_intr_main(ch, istatus);
1892 }
1893 MODULE_VERSION(fsl_sata, 1);
1894 MODULE_DEPEND(fsl_sata, cam, 1, 1, 1);
1895