/* * Copyright (c) 2016-2018 The DragonFly Project. All rights reserved. * * This code is derived from software contributed to The DragonFly Project * by Matthew Dillon * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * 3. Neither the name of The DragonFly Project nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific, prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Primary header file * NVME softc and structural definitions */ #if defined(__DragonFly__) #include "nvme_dragonfly.h" #else #error "build for OS unknown" #endif #include "nvme_fw.h" #include "nvme_log.h" #include "nvme_ident.h" #include "nvme_ns.h" #include "nvme_chipset.h" #include "nvme_ioctl.h" MALLOC_DECLARE(M_NVME); /* * Choose some reasonable limit, even if the hardware supports more. */ #define NVME_MAX_QUEUES 1024 #define NVME_MAX_NAMESPACES 1024 struct nvme_queue; struct nvme_softc; struct nvme_softns; /* * Device matching array for special attach/detach codings. */ typedef struct { pci_vendor_id_t vendor; pci_product_id_t product; int (*attach)(device_t dev); int (*detach)(device_t dev); char *name; } nvme_device_t; /* * Kernel-level request structure. This structure allows the driver to * construct, issue, wait for, and process responses to commands. Each * queue manages its own request bank. * * In order to disconnect the request structure from the hardware queue * mechanism itself, to reduce SMP conflicts and interactions, and allow * command/response processing to block without interfering with queue * operations, this structure embeds a copy of the HW command and response * structures instead of referencing the ones in the actual hardware queues. * These will be copied to/from the actual queue entries by lower-level * chipset code. * * Requests are associated with particular queues, completions can occur on * any queue. Requests associated with the admin queue conveniently include * an additional 4K 'info' block suitable for DMA. */ typedef struct nvme_request { struct nvme_request *next_avail; struct nvme_subqueue *subq; /* which queue is submission on */ struct nvme_comqueue *comq; /* which queue is completion on */ uint32_t state; uint32_t cmd_id; /* reqary[] index */ int waiting; nvme_allcmd_t cmd; /* hw submit structure for entry */ nvme_allres_t res; /* hw completion structure for entry */ nvme_admin_data_t *info; /* DMA data (admin request only) */ bus_addr_t pinfo; /* phys address for PRP */ /* * Aux fields to keep track of bio and other data, depending on * the callback. If the callback is NULL the caller will poll for * completion. */ void (*callback)(struct nvme_request *req, struct lock *lk); struct nvme_softns *nsc; struct bio *bio; } nvme_request_t; #define NVME_REQ_AVAIL 0 #define NVME_REQ_ALLOCATED 1 #define NVME_REQ_SUBMITTED 2 #define NVME_REQ_COMPLETED 3 typedef struct nvme_subqueue { /* * Driver stuff */ struct lock lk; /* queue lock controls access */ struct nvme_softc *sc; nvme_request_t *dump_req; /* one request is set aside */ nvme_request_t *first_avail; nvme_request_t *reqary; uint32_t nqe; /* #of queue entries */ uint16_t qid; /* which queue# are we on? */ uint16_t comqid; /* we are tied to this completion qu */ uint32_t subq_doorbell_reg; uint32_t subq_head; /* start of active requests */ uint32_t subq_tail; /* new requests */ uint32_t unsubmitted; /* #unsubmitted requests */ int unused01; int signal_requeue; /* * DMA resources */ bus_dmamap_t sque_map; bus_dmamap_t prps_map; nvme_allcmd_t *ksubq; /* kernel-mapped addresses */ uint64_t *kprps; /* enough PRPs per request for */ /* MAXPHYS bytes worth of mappings */ bus_addr_t psubq; /* physical addresses */ bus_addr_t pprps; /* * Additional DMA resources for admin queue (A NVME_MAX_ADMIN_BUFFER * sized buffer for each queue entry). */ bus_dmamap_t adm_map; bus_addr_t pdatapgs; nvme_admin_data_t *kdatapgs; } nvme_subqueue_t; typedef struct nvme_comqueue { /* * Driver stuff */ struct lock lk; /* queue lock controls access */ struct nvme_softc *sc; uint32_t nqe; /* #of queue entries */ uint16_t phase; /* phase to match (res->tail.status) */ uint16_t qid; /* which queue# are we on? */ uint32_t comq_doorbell_reg; uint32_t comq_head; /* consume responses */ uint32_t comq_tail; /* * DMA resources */ bus_dmamap_t cque_map; nvme_allres_t *kcomq; bus_addr_t pcomq; } nvme_comqueue_t; typedef struct nvme_softns { struct nvme_softc *sc; nvme_ident_ns_data_t idns; struct bio_queue_head bioq; /* excess BIOs */ struct lock lk; /* mostly for bioq handling */ int state; int unit; uint32_t nsid; uint32_t blksize; struct disk disk; /* disk attachment */ struct devstat stats; cdev_t cdev; /* disk device (cdev) */ } nvme_softns_t; #define NVME_NSC_STATE_UNATTACHED 0 #define NVME_NSC_STATE_ATTACHED 1 typedef struct nvme_softc { TAILQ_ENTRY(nvme_softc) entry; /* global list */ device_t dev; /* core device */ const nvme_device_t *ad; /* quirk detection etc */ thread_t admintd; uint32_t maxqe; uint64_t cap; uint32_t vers; uint32_t dstrd4; /* doorbell stride */ uint32_t entimo; /* enable timeout in ticks */ uint16_t niosubqs; /* #of I/O submission queues */ uint16_t niocomqs; /* #of I/O completion queues */ uint16_t dumpqno; uint16_t eventqno; uint16_t qmap[SMP_MAXCPU][2]; uint32_t flags; nvme_subqueue_t subqueues[NVME_MAX_QUEUES]; nvme_comqueue_t comqueues[NVME_MAX_QUEUES]; int cputovect[SMP_MAXCPU]; /* * bio/disk layer tracking */ ulong opencnt; /* * admin queue irq resources * register map resources */ struct resource *regs; struct resource *bar4; bus_space_tag_t iot; bus_space_handle_t ioh; int rid_regs; int rid_bar4; int nirqs; int irq_type; struct resource *irq[NVME_MAX_QUEUES]; int rid_irq[NVME_MAX_QUEUES]; void *irq_handle[NVME_MAX_QUEUES]; /* * dma tags */ bus_dma_tag_t prps_tag; /* worst-case PRP table(s) per queue */ bus_dma_tag_t sque_tag; /* submission queue */ bus_dma_tag_t cque_tag; /* completion queue */ bus_dma_tag_t adm_tag; /* DMA data buffers for admin cmds */ size_t prp_bytes; size_t cmd_bytes; size_t res_bytes; size_t adm_bytes; /* * Admin thread and controller identify data */ uint32_t admin_signal; struct lock admin_lk; struct lock ioctl_lk; int (*admin_func)(struct nvme_softc *); nvme_ident_ctlr_data_t idctlr; nvme_softns_t *nscary[NVME_MAX_NAMESPACES]; int nscmax; } nvme_softc_t; #define NVME_SC_ATTACHED 0x00000001 #define NVME_SC_UNLOADING 0x00000002 #define ADMIN_SIG_STOP 0x00000001 #define ADMIN_SIG_RUNNING 0x00000002 #define ADMIN_SIG_PROBED 0x00000004 #define ADMIN_SIG_REQUEUE 0x00000008 #define ADMIN_SIG_RUN_MASK (ADMIN_SIG_STOP | ADMIN_SIG_REQUEUE) #define NVME_QMAP_RD 0 #define NVME_QMAP_WR 1 typedef struct nvme_status_buf { char buf[64]; } nvme_status_buf_t; /* * Prototypes */ const nvme_device_t *nvme_lookup_device(device_t dev); void nvme_os_sleep(int ms); int nvme_os_softsleep(void); void nvme_os_hardsleep(int us); u_int32_t nvme_read(nvme_softc_t *sc, bus_size_t r); u_int64_t nvme_read8(nvme_softc_t *sc, bus_size_t r); void nvme_write(nvme_softc_t *sc, bus_size_t r, u_int32_t v); void nvme_write8(nvme_softc_t *sc, bus_size_t r, u_int64_t v); int nvme_enable(nvme_softc_t *sc, int enable); int nvme_alloc_subqueue(nvme_softc_t *sc, uint16_t qid); int nvme_alloc_comqueue(nvme_softc_t *sc, uint16_t qid); void nvme_free_subqueue(nvme_softc_t *sc, uint16_t qid); void nvme_free_comqueue(nvme_softc_t *sc, uint16_t qid); nvme_request_t *nvme_get_admin_request(nvme_softc_t *sc, uint8_t opcode); nvme_request_t *nvme_get_request(nvme_subqueue_t *queue, uint8_t opcode, char *kva, size_t bytes); nvme_request_t *nvme_get_dump_request(nvme_subqueue_t *queue, uint8_t opcode, char *kva, size_t bytes); void nvme_submit_request(nvme_request_t *req); int nvme_wait_request(nvme_request_t *req); int nvme_poll_request(nvme_request_t *req); void nvme_put_request(nvme_request_t *req); void nvme_put_dump_request(nvme_request_t *req); void nvme_poll_completions(nvme_comqueue_t *queue, struct lock *lk); int nvme_start_admin_thread(nvme_softc_t *sc); void nvme_stop_admin_thread(nvme_softc_t *sc); int nvme_create_subqueue(nvme_softc_t *sc, uint16_t qid); int nvme_create_comqueue(nvme_softc_t *sc, uint16_t qid); int nvme_delete_subqueue(nvme_softc_t *sc, uint16_t qid); int nvme_delete_comqueue(nvme_softc_t *sc, uint16_t qid); int nvme_issue_shutdown(nvme_softc_t *sc, int dopoll); void nvme_disk_attach(nvme_softns_t *nsc); void nvme_disk_detach(nvme_softns_t *nsc); void nvme_disk_requeues(nvme_softc_t *sc); int nvme_alloc_disk_unit(void); int nvme_getlog_ioctl(nvme_softc_t *sc, nvme_getlog_ioctl_t *ioc); void nvme_intr(void *arg); size_t string_cleanup(char *str, int domiddle);