| 1 | /* $NetBSD: nvmevar.h,v 1.11 2016/11/01 14:39:38 jdolecek Exp $ */ |
| 2 | /* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ |
| 3 | |
| 4 | /* |
| 5 | * Copyright (c) 2014 David Gwynne <dlg@openbsd.org> |
| 6 | * |
| 7 | * Permission to use, copy, modify, and distribute this software for any |
| 8 | * purpose with or without fee is hereby granted, provided that the above |
| 9 | * copyright notice and this permission notice appear in all copies. |
| 10 | * |
| 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 14 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 16 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 17 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 18 | */ |
| 19 | |
| 20 | #include <sys/bus.h> |
| 21 | #include <sys/cpu.h> |
| 22 | #include <sys/device.h> |
| 23 | #include <sys/mutex.h> |
| 24 | #include <sys/pool.h> |
| 25 | #include <sys/queue.h> |
| 26 | |
| 27 | struct nvme_dmamem { |
| 28 | bus_dmamap_t ndm_map; |
| 29 | bus_dma_segment_t ndm_seg; |
| 30 | size_t ndm_size; |
| 31 | void *ndm_kva; |
| 32 | }; |
| 33 | #define NVME_DMA_MAP(_ndm) ((_ndm)->ndm_map) |
| 34 | #define NVME_DMA_LEN(_ndm) ((_ndm)->ndm_map->dm_segs[0].ds_len) |
| 35 | #define NVME_DMA_DVA(_ndm) ((uint64_t)(_ndm)->ndm_map->dm_segs[0].ds_addr) |
| 36 | #define NVME_DMA_KVA(_ndm) ((void *)(_ndm)->ndm_kva) |
| 37 | |
| 38 | struct nvme_softc; |
| 39 | struct nvme_queue; |
| 40 | |
| 41 | typedef void (*nvme_nnc_done)(void *, struct buf *, uint16_t); |
| 42 | |
| 43 | struct nvme_ccb { |
| 44 | SIMPLEQ_ENTRY(nvme_ccb) ccb_entry; |
| 45 | |
| 46 | /* DMA handles */ |
| 47 | bus_dmamap_t ccb_dmamap; |
| 48 | |
| 49 | bus_addr_t ccb_prpl_off; |
| 50 | uint64_t ccb_prpl_dva; |
| 51 | uint64_t *ccb_prpl; |
| 52 | |
| 53 | /* command context */ |
| 54 | uint16_t ccb_id; |
| 55 | void *ccb_cookie; |
| 56 | #define NVME_CCB_FREE 0xbeefdeed |
| 57 | void (*ccb_done)(struct nvme_queue *, |
| 58 | struct nvme_ccb *, struct nvme_cqe *); |
| 59 | |
| 60 | /* namespace context */ |
| 61 | void *nnc_cookie; |
| 62 | nvme_nnc_done nnc_done; |
| 63 | uint16_t nnc_nsid; |
| 64 | uint16_t nnc_flags; |
| 65 | #define NVME_NS_CTX_F_READ __BIT(0) |
| 66 | #define NVME_NS_CTX_F_POLL __BIT(1) |
| 67 | |
| 68 | struct buf *nnc_buf; |
| 69 | daddr_t nnc_blkno; |
| 70 | size_t nnc_datasize; |
| 71 | int nnc_secsize; |
| 72 | }; |
| 73 | |
| 74 | struct nvme_queue { |
| 75 | struct nvme_softc *q_sc; |
| 76 | kmutex_t q_sq_mtx; |
| 77 | kmutex_t q_cq_mtx; |
| 78 | struct nvme_dmamem *q_sq_dmamem; |
| 79 | struct nvme_dmamem *q_cq_dmamem; |
| 80 | bus_size_t q_sqtdbl; /* submission queue tail doorbell */ |
| 81 | bus_size_t q_cqhdbl; /* completion queue head doorbell */ |
| 82 | uint16_t q_id; |
| 83 | uint32_t q_entries; |
| 84 | uint32_t q_sq_tail; |
| 85 | uint32_t q_cq_head; |
| 86 | uint16_t q_cq_phase; |
| 87 | |
| 88 | kmutex_t q_ccb_mtx; |
| 89 | uint16_t q_nccbs; /* total number of ccbs */ |
| 90 | uint16_t q_nccbs_avail; /* available ccbs */ |
| 91 | struct nvme_ccb *q_ccbs; |
| 92 | SIMPLEQ_HEAD(, nvme_ccb) q_ccb_list; |
| 93 | struct nvme_dmamem *q_ccb_prpls; |
| 94 | }; |
| 95 | |
| 96 | struct nvme_namespace { |
| 97 | struct nvm_identify_namespace *ident; |
| 98 | device_t dev; |
| 99 | uint32_t flags; |
| 100 | #define NVME_NS_F_OPEN __BIT(0) |
| 101 | }; |
| 102 | |
| 103 | struct nvme_softc { |
| 104 | device_t sc_dev; |
| 105 | |
| 106 | bus_space_tag_t sc_iot; |
| 107 | bus_space_handle_t sc_ioh; |
| 108 | bus_size_t sc_ios; |
| 109 | bus_dma_tag_t sc_dmat; |
| 110 | |
| 111 | int (*sc_intr_establish)(struct nvme_softc *, |
| 112 | uint16_t qid, struct nvme_queue *); |
| 113 | int (*sc_intr_disestablish)(struct nvme_softc *, |
| 114 | uint16_t qid); |
| 115 | void **sc_ih; /* interrupt handlers */ |
| 116 | void **sc_softih; /* softintr handlers */ |
| 117 | |
| 118 | u_int sc_rdy_to; /* RDY timeout */ |
| 119 | size_t sc_mps; /* memory page size */ |
| 120 | size_t sc_mdts; /* max data trasfer size */ |
| 121 | u_int sc_max_sgl; /* max S/G segments */ |
| 122 | |
| 123 | struct nvm_identify_controller |
| 124 | sc_identify; |
| 125 | |
| 126 | u_int sc_nn; /* namespace count */ |
| 127 | struct nvme_namespace *sc_namespaces; |
| 128 | |
| 129 | bool sc_use_mq; |
| 130 | u_int sc_nq; /* # of io queue (sc_q) */ |
| 131 | struct nvme_queue *sc_admin_q; |
| 132 | struct nvme_queue **sc_q; |
| 133 | |
| 134 | uint32_t sc_flags; |
| 135 | #define NVME_F_ATTACHED __BIT(0) |
| 136 | #define NVME_F_OPEN __BIT(1) |
| 137 | }; |
| 138 | |
| 139 | #define lemtoh16(p) le16toh(*((uint16_t *)(p))) |
| 140 | #define lemtoh32(p) le32toh(*((uint32_t *)(p))) |
| 141 | #define lemtoh64(p) le64toh(*((uint64_t *)(p))) |
| 142 | #define htolem16(p, x) (*((uint16_t *)(p)) = htole16(x)) |
| 143 | #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) |
| 144 | #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) |
| 145 | |
| 146 | struct nvme_attach_args { |
| 147 | uint16_t naa_nsid; |
| 148 | uint32_t naa_qentries; /* total number of queue slots */ |
| 149 | uint32_t naa_maxphys; /* maximum device transfer size */ |
| 150 | }; |
| 151 | |
| 152 | int nvme_attach(struct nvme_softc *); |
| 153 | int nvme_detach(struct nvme_softc *, int flags); |
| 154 | int nvme_rescan(device_t, const char *, const int *); |
| 155 | void nvme_childdet(device_t, device_t); |
| 156 | int nvme_intr(void *); |
| 157 | void nvme_softintr_intx(void *); |
| 158 | int nvme_intr_msi(void *); |
| 159 | void nvme_softintr_msi(void *); |
| 160 | |
| 161 | static inline struct nvme_queue * |
| 162 | nvme_get_q(struct nvme_softc *sc) |
| 163 | { |
| 164 | return sc->sc_q[cpu_index(curcpu()) % sc->sc_nq]; |
| 165 | } |
| 166 | |
| 167 | /* |
| 168 | * namespace |
| 169 | */ |
| 170 | static inline struct nvme_namespace * |
| 171 | nvme_ns_get(struct nvme_softc *sc, uint16_t nsid) |
| 172 | { |
| 173 | if (nsid == 0 || nsid - 1 >= sc->sc_nn) |
| 174 | return NULL; |
| 175 | return &sc->sc_namespaces[nsid - 1]; |
| 176 | } |
| 177 | |
| 178 | int nvme_ns_identify(struct nvme_softc *, uint16_t); |
| 179 | void nvme_ns_free(struct nvme_softc *, uint16_t); |
| 180 | int nvme_ns_dobio(struct nvme_softc *, uint16_t, void *, |
| 181 | struct buf *, void *, size_t, int, daddr_t, int, nvme_nnc_done); |
| 182 | int nvme_ns_sync(struct nvme_softc *, uint16_t, void *, int, nvme_nnc_done); |
| 183 | |