1 /* 2 * Copyright (c) 2006-2023, RT-Thread Development Team 3 * 4 * SPDX-License-Identifier: Apache-2.0 5 * 6 * Change Logs: 7 * Date Author Notes 8 * 2023-02-25 GuEe-GUI the first version 9 */ 10 11 #ifndef __NVME_H__ 12 #define __NVME_H__ 13 14 #include <rthw.h> 15 #include <rtthread.h> 16 #include <drivers/blk.h> 17 18 #define NVME_RSVD(offset, bytes_size) rt_uint8_t __rsvd##offset[bytes_size] 19 20 enum 21 { 22 /* 23 * Generic Command Status: 24 */ 25 RT_NVME_SC_SUCCESS = 0x0, 26 RT_NVME_SC_INVALID_OPCODE = 0x1, 27 RT_NVME_SC_INVALID_FIELD = 0x2, 28 RT_NVME_SC_CMDID_CONFLICT = 0x3, 29 RT_NVME_SC_DATA_XFER_ERROR = 0x4, 30 RT_NVME_SC_POWER_LOSS = 0x5, 31 RT_NVME_SC_INTERNAL = 0x6, 32 RT_NVME_SC_ABORT_REQ = 0x7, 33 RT_NVME_SC_ABORT_QUEUE = 0x8, 34 RT_NVME_SC_FUSED_FAIL = 0x9, 35 RT_NVME_SC_FUSED_MISSING = 0xa, 36 RT_NVME_SC_INVALID_NS = 0xb, 37 RT_NVME_SC_CMD_SEQ_ERROR = 0xc, 38 RT_NVME_SC_SGL_INVALID_LAST = 0xd, 39 RT_NVME_SC_SGL_INVALID_COUNT = 0xe, 40 RT_NVME_SC_SGL_INVALID_DATA = 0xf, 41 RT_NVME_SC_SGL_INVALID_METADATA = 0x10, 42 RT_NVME_SC_SGL_INVALID_TYPE = 0x11, 43 RT_NVME_SC_CMB_INVALID_USE = 0x12, 44 RT_NVME_SC_PRP_INVALID_OFFSET = 0x13, 45 RT_NVME_SC_ATOMIC_WU_EXCEEDED = 0x14, 46 RT_NVME_SC_OP_DENIED = 0x15, 47 RT_NVME_SC_SGL_INVALID_OFFSET = 0x16, 48 RT_NVME_SC_RESERVED = 0x17, 49 RT_NVME_SC_HOST_ID_INCONSIST = 0x18, 50 RT_NVME_SC_KA_TIMEOUT_EXPIRED = 0x19, 51 RT_NVME_SC_KA_TIMEOUT_INVALID = 0x1a, 52 RT_NVME_SC_ABORTED_PREEMPT_ABORT = 0x1b, 53 RT_NVME_SC_SANITIZE_FAILED = 0x1c, 54 RT_NVME_SC_SANITIZE_IN_PROGRESS = 0x1d, 55 RT_NVME_SC_SGL_INVALID_GRANULARITY = 0x1e, 56 RT_NVME_SC_CMD_NOT_SUP_CMB_QUEUE = 0x1f, 57 RT_NVME_SC_NS_WRITE_PROTECTED = 0x20, 58 RT_NVME_SC_CMD_INTERRUPTED = 0x21, 59 RT_NVME_SC_TRANSIENT_TR_ERR = 0x22, 60 RT_NVME_SC_ADMIN_COMMAND_MEDIA_NOT_READY = 0x24, 61 RT_NVME_SC_INVALID_IO_CMD_SET = 0x2c, 62 63 RT_NVME_SC_LBA_RANGE = 0x80, 64 RT_NVME_SC_CAP_EXCEEDED = 0x81, 65 RT_NVME_SC_NS_NOT_READY = 0x82, 66 RT_NVME_SC_RESERVATION_CONFLICT = 0x83, 67 RT_NVME_SC_FORMAT_IN_PROGRESS = 0x84, 68 69 /* 70 * Command Specific Status: 71 */ 72 RT_NVME_SC_CQ_INVALID = 0x100, 73 RT_NVME_SC_QID_INVALID = 0x101, 74 RT_NVME_SC_QUEUE_SIZE = 0x102, 75 RT_NVME_SC_ABORT_LIMIT = 0x103, 76 RT_NVME_SC_ABORT_MISSING = 0x104, 77 RT_NVME_SC_ASYNC_LIMIT = 0x105, 78 RT_NVME_SC_FIRMWARE_SLOT = 0x106, 79 RT_NVME_SC_FIRMWARE_IMAGE = 0x107, 80 RT_NVME_SC_INVALID_VECTOR = 0x108, 81 RT_NVME_SC_INVALID_LOG_PAGE = 0x109, 82 RT_NVME_SC_INVALID_FORMAT = 0x10a, 83 RT_NVME_SC_FW_NEEDS_CONV_RESET = 0x10b, 84 RT_NVME_SC_INVALID_QUEUE = 0x10c, 85 RT_NVME_SC_FEATURE_NOT_SAVEABLE = 0x10d, 86 RT_NVME_SC_FEATURE_NOT_CHANGEABLE = 0x10e, 87 RT_NVME_SC_FEATURE_NOT_PER_NS = 0x10f, 88 RT_NVME_SC_FW_NEEDS_SUBSYS_RESET = 0x110, 89 RT_NVME_SC_FW_NEEDS_RESET = 0x111, 90 RT_NVME_SC_FW_NEEDS_MAX_TIME = 0x112, 91 RT_NVME_SC_FW_ACTIVATE_PROHIBITED = 0x113, 92 RT_NVME_SC_OVERLAPPING_RANGE = 0x114, 93 RT_NVME_SC_NS_INSUFFICIENT_CAP = 0x115, 94 RT_NVME_SC_NS_ID_UNAVAILABLE = 0x116, 95 RT_NVME_SC_NS_ALREADY_ATTACHED = 0x118, 96 RT_NVME_SC_NS_IS_PRIVATE = 0x119, 97 RT_NVME_SC_NS_NOT_ATTACHED = 0x11a, 98 RT_NVME_SC_THIN_PROV_NOT_SUPP = 0x11b, 99 RT_NVME_SC_CTRL_LIST_INVALID = 0x11c, 100 RT_NVME_SC_SELT_TEST_IN_PROGRESS = 0x11d, 101 RT_NVME_SC_BP_WRITE_PROHIBITED = 0x11e, 102 RT_NVME_SC_CTRL_ID_INVALID = 0x11f, 103 RT_NVME_SC_SEC_CTRL_STATE_INVALID = 0x120, 104 RT_NVME_SC_CTRL_RES_NUM_INVALID = 0x121, 105 RT_NVME_SC_RES_ID_INVALID = 0x122, 106 RT_NVME_SC_PMR_SAN_PROHIBITED = 0x123, 107 RT_NVME_SC_ANA_GROUP_ID_INVALID = 0x124, 108 RT_NVME_SC_ANA_ATTACH_FAILED = 0x125, 109 110 /* 111 * I/O Command Set Specific - NVM commands: 112 */ 113 RT_NVME_SC_BAD_ATTRIBUTES = 0x180, 114 RT_NVME_SC_INVALID_PI = 0x181, 115 RT_NVME_SC_READ_ONLY = 0x182, 116 RT_NVME_SC_ONCS_NOT_SUPPORTED = 0x183, 117 118 /* 119 * I/O Command Set Specific - Fabrics commands: 120 */ 121 RT_NVME_SC_CONNECT_FORMAT = 0x180, 122 RT_NVME_SC_CONNECT_CTRL_BUSY = 0x181, 123 RT_NVME_SC_CONNECT_INVALID_PARAM = 0x182, 124 RT_NVME_SC_CONNECT_RESTART_DISC = 0x183, 125 RT_NVME_SC_CONNECT_INVALID_HOST = 0x184, 126 127 RT_NVME_SC_DISCOVERY_RESTART = 0x190, 128 RT_NVME_SC_AUTH_REQUIRED = 0x191, 129 130 /* 131 * I/O Command Set Specific - Zoned commands: 132 */ 133 RT_NVME_SC_ZONE_BOUNDARY_ERROR = 0x1b8, 134 RT_NVME_SC_ZONE_FULL = 0x1b9, 135 RT_NVME_SC_ZONE_READ_ONLY = 0x1ba, 136 RT_NVME_SC_ZONE_OFFLINE = 0x1bb, 137 RT_NVME_SC_ZONE_INVALID_WRITE = 0x1bc, 138 RT_NVME_SC_ZONE_TOO_MANY_ACTIVE = 0x1bd, 139 RT_NVME_SC_ZONE_TOO_MANY_OPEN = 0x1be, 140 RT_NVME_SC_ZONE_INVALID_TRANSITION = 0x1bf, 141 142 /* 143 * Media and Data Integrity Errors: 144 */ 145 RT_NVME_SC_WRITE_FAULT = 0x280, 146 RT_NVME_SC_READ_ERROR = 0x281, 147 RT_NVME_SC_GUARD_CHECK = 0x282, 148 RT_NVME_SC_APPTAG_CHECK = 0x283, 149 RT_NVME_SC_REFTAG_CHECK = 0x284, 150 RT_NVME_SC_COMPARE_FAILED = 0x285, 151 RT_NVME_SC_ACCESS_DENIED = 0x286, 152 RT_NVME_SC_UNWRITTEN_BLOCK = 0x287, 153 154 /* 155 * Path-related Errors: 156 */ 157 RT_NVME_SC_INTERNAL_PATH_ERROR = 0x300, 158 RT_NVME_SC_ANA_PERSISTENT_LOSS = 0x301, 159 RT_NVME_SC_ANA_INACCESSIBLE = 0x302, 160 RT_NVME_SC_ANA_TRANSITION = 0x303, 161 RT_NVME_SC_CTRL_PATH_ERROR = 0x360, 162 RT_NVME_SC_HOST_PATH_ERROR = 0x370, 163 RT_NVME_SC_HOST_ABORTED_CMD = 0x371, 164 165 RT_NVME_SC_CRD = 0x1800, 166 RT_NVME_SC_MORE = 0x2000, 167 RT_NVME_SC_DNR = 0x4000, 168 }; 169 170 /* Admin commands */ 171 enum 172 { 173 RT_NVME_ADMIN_OPCODE_DELETE_SQ = 0x00, 174 RT_NVME_ADMIN_OPCODE_CREATE_SQ = 0x01, 175 RT_NVME_ADMIN_OPCODE_GET_LOG_PAGE = 0x02, 176 RT_NVME_ADMIN_OPCODE_DELETE_CQ = 0x04, 177 RT_NVME_ADMIN_OPCODE_CREATE_CQ = 0x05, 178 RT_NVME_ADMIN_OPCODE_IDENTIFY = 0x06, 179 RT_NVME_ADMIN_OPCODE_ABORT_CMD = 0x08, 180 RT_NVME_ADMIN_OPCODE_SET_FEATURES = 0x09, 181 RT_NVME_ADMIN_OPCODE_GET_FEATURES = 0x0a, 182 RT_NVME_ADMIN_OPCODE_ASYNC_EVENT = 0x0c, 183 RT_NVME_ADMIN_OPCODE_NS_MGMT = 0x0d, 184 RT_NVME_ADMIN_OPCODE_ACTIVATE_FW = 0x10, 185 RT_NVME_ADMIN_OPCODE_DOWNLOAD_FW = 0x11, 186 RT_NVME_ADMIN_OPCODE_DEV_SELF_TEST = 0x14, 187 RT_NVME_ADMIN_OPCODE_NS_ATTACH = 0x15, 188 RT_NVME_ADMIN_OPCODE_KEEP_ALIVE = 0x18, 189 RT_NVME_ADMIN_OPCODE_DIRECTIVE_SEND = 0x19, 190 RT_NVME_ADMIN_OPCODE_DIRECTIVE_RECV = 0x1a, 191 RT_NVME_ADMIN_OPCODE_VIRTUAL_MGMT = 0x1c, 192 RT_NVME_ADMIN_OPCODE_NVME_MI_SEND = 0x1d, 193 RT_NVME_ADMIN_OPCODE_NVME_MI_RECV = 0x1e, 194 RT_NVME_ADMIN_OPCODE_DBBUF = 0x7c, 195 RT_NVME_ADMIN_OPCODE_FORMAT_NVM = 0x80, 196 RT_NVME_ADMIN_OPCODE_SECURITY_SEND = 0x81, 197 RT_NVME_ADMIN_OPCODE_SECURITY_RECV = 0x82, 198 RT_NVME_ADMIN_OPCODE_SANITIZE_NVM = 0x84, 199 RT_NVME_ADMIN_OPCODE_GET_LBA_STATUS = 0x86, 200 RT_NVME_ADMIN_OPCODE_VENDOR_START = 0xc0, 201 }; 202 203 /* I/O commands */ 204 enum 205 { 206 RT_NVME_CMD_FLUSH = 0x00, 207 RT_NVME_CMD_WRITE = 0x01, 208 RT_NVME_CMD_READ = 0x02, 209 RT_NVME_CMD_WRITE_UNCOR = 0x04, 210 RT_NVME_CMD_COMPARE = 0x05, 211 RT_NVME_CMD_WRITE_ZEROES = 0x08, 212 RT_NVME_CMD_DSM = 0x09, 213 RT_NVME_CMD_VERIFY = 0x0c, 214 RT_NVME_CMD_RESV_REGISTER = 0x0d, 215 RT_NVME_CMD_RESV_REPORT = 0x0e, 216 RT_NVME_CMD_RESV_ACQUIRE = 0x11, 217 RT_NVME_CMD_RESV_RELEASE = 0x15, 218 RT_NVME_CMD_ZONE_MGMT_SEND = 0x79, 219 RT_NVME_CMD_ZONE_MGMT_RECV = 0x7a, 220 RT_NVME_CMD_ZONE_APPEND = 0x7d, 221 RT_NVME_CMD_VENDOR_START = 0x80, 222 }; 223 224 enum 225 { 226 RT_NVME_PSDT_PRP = 0x0, 227 RT_NVME_PSDT_SGL_MPTR_CONTIGUOUS = 0x1, 228 RT_NVME_PSDT_SGL_MPTR_SGL = 0x2, 229 }; 230 231 /* Commands flags */ 232 enum 233 { 234 RT_NVME_CMD_FLAGS_FUSE_SHIFT = 0x00, 235 RT_NVME_CMD_FLAGS_PSDT_SHIFT = 0x06, 236 }; 237 238 struct rt_nvme_command_common 239 { 240 rt_uint8_t opcode; 241 rt_uint8_t flags; 242 rt_uint16_t cmdid; 243 rt_le32_t nsid; 244 rt_le32_t cmd_dw2[2]; 245 rt_le64_t metadata; 246 rt_le64_t prp1; 247 rt_le64_t prp2; 248 rt_le32_t cmd_dw10[6]; 249 }; 250 251 rt_packed(struct rt_nvme_sgl_desc 252 { 253 rt_le64_t adddress; 254 rt_le32_t length; 255 rt_uint8_t reserved[3]; 256 #define SGL_DESC_TYPE_DATA_BLOCK 0x0 257 #define SGL_DESC_TYPE_BIT_BUCKET 0x1 258 #define SGL_DESC_TYPE_SEGMENT 0x2 259 #define SGL_DESC_TYPE_LAST_SEGMENT 0x3 260 #define SGL_DESC_TYPE_KEYED_DATA_BLOCK 0x4 261 #define SGL_DESC_TYPE_VENDOR_SPECIFIC 0xf 262 rt_uint8_t sgl_identify; 263 }); 264 265 struct rt_nvme_command_rw 266 { 267 rt_uint8_t opcode; 268 rt_uint8_t flags; 269 rt_uint16_t cmdid; 270 rt_le32_t nsid; 271 NVME_RSVD(8, 8); 272 rt_le64_t metadata; 273 union 274 { 275 struct 276 { 277 rt_le64_t prp1; 278 rt_le64_t prp2; 279 }; 280 struct rt_nvme_sgl_desc sgl; 281 }; 282 rt_le64_t slba; 283 rt_le16_t length; 284 rt_le16_t control; 285 rt_le32_t dsmgmt; 286 rt_le32_t reftag; 287 rt_le16_t apptag; 288 rt_le16_t appmask; 289 }; 290 291 enum 292 { 293 RT_NVME_RW_LR = 1 << 15, 294 RT_NVME_RW_FUA = 1 << 14, 295 RT_NVME_RW_APPEND_PIREMAP = 1 << 9, 296 RT_NVME_RW_DSM_FREQ_UNSPEC = 0, 297 RT_NVME_RW_DSM_FREQ_TYPICAL = 1, 298 RT_NVME_RW_DSM_FREQ_RARE = 2, 299 RT_NVME_RW_DSM_FREQ_READS = 3, 300 RT_NVME_RW_DSM_FREQ_WRITES = 4, 301 RT_NVME_RW_DSM_FREQ_RW = 5, 302 RT_NVME_RW_DSM_FREQ_ONCE = 6, 303 RT_NVME_RW_DSM_FREQ_PREFETCH = 7, 304 RT_NVME_RW_DSM_FREQ_TEMP = 8, 305 RT_NVME_RW_DSM_LATENCY_NONE = 0 << 4, 306 RT_NVME_RW_DSM_LATENCY_IDLE = 1 << 4, 307 RT_NVME_RW_DSM_LATENCY_NORM = 2 << 4, 308 RT_NVME_RW_DSM_LATENCY_LOW = 3 << 4, 309 RT_NVME_RW_DSM_SEQ_REQ = 1 << 6, 310 RT_NVME_RW_DSM_COMPRESSED = 1 << 7, 311 RT_NVME_RW_PRINFO_PRCHK_REF = 1 << 10, 312 RT_NVME_RW_PRINFO_PRCHK_APP = 1 << 11, 313 RT_NVME_RW_PRINFO_PRCHK_GUARD = 1 << 12, 314 RT_NVME_RW_PRINFO_PRACT = 1 << 13, 315 RT_NVME_RW_DTYPE_STREAMS = 1 << 4, 316 RT_NVME_WZ_DEAC = 1 << 9, 317 }; 318 319 enum 320 { 321 RT_NVME_QUEUE_PHYS_CONTIG = (1 << 0), 322 RT_NVME_CQ_IRQ_ENABLED = (1 << 1), 323 RT_NVME_SQ_PRIO_URGENT = (0 << 1), 324 RT_NVME_SQ_PRIO_HIGH = (1 << 1), 325 RT_NVME_SQ_PRIO_MEDIUM = (2 << 1), 326 RT_NVME_SQ_PRIO_LOW = (3 << 1), 327 RT_NVME_FEAT_ARBITRATION = 0x01, 328 RT_NVME_FEAT_POWER_MGMT = 0x02, 329 RT_NVME_FEAT_LBA_RANGE = 0x03, 330 RT_NVME_FEAT_TEMP_THRESH = 0x04, 331 RT_NVME_FEAT_ERR_RECOVERY = 0x05, 332 RT_NVME_FEAT_VOLATILE_WC = 0x06, 333 RT_NVME_FEAT_NUM_QUEUES = 0x07, 334 RT_NVME_FEAT_IRQ_COALESCE = 0x08, 335 RT_NVME_FEAT_IRQ_CONFIG = 0x09, 336 RT_NVME_FEAT_WRITE_ATOMIC = 0x0a, 337 RT_NVME_FEAT_ASYNC_EVENT = 0x0b, 338 RT_NVME_FEAT_AUTO_PST = 0x0c, 339 RT_NVME_FEAT_SW_PROGRESS = 0x80, 340 RT_NVME_FEAT_HOST_ID = 0x81, 341 RT_NVME_FEAT_RESV_MASK = 0x82, 342 RT_NVME_FEAT_RESV_PERSIST = 0x83, 343 RT_NVME_LOG_ERROR = 0x01, 344 RT_NVME_LOG_SMART = 0x02, 345 RT_NVME_LOG_FW_SLOT = 0x03, 346 RT_NVME_LOG_RESERVATION = 0x80, 347 RT_NVME_FWACT_REPL = (0 << 3), 348 RT_NVME_FWACT_REPL_ACTV = (1 << 3), 349 RT_NVME_FWACT_ACTV = (2 << 3), 350 }; 351 352 struct rt_nvme_command_identify 353 { 354 rt_uint8_t opcode; 355 rt_uint8_t flags; 356 rt_uint16_t cmdid; 357 rt_le32_t nsid; 358 NVME_RSVD(8, 16); 359 rt_le64_t prp1; 360 rt_le64_t prp2; 361 rt_le32_t cns; 362 NVME_RSVD(64, 20); 363 }; 364 365 struct rt_nvme_command_features 366 { 367 rt_uint8_t opcode; 368 rt_uint8_t flags; 369 rt_uint16_t cmdid; 370 rt_le32_t nsid; 371 NVME_RSVD(8, 16); 372 rt_le64_t prp1; 373 rt_le64_t prp2; 374 rt_le32_t fid; 375 rt_le32_t dword11; 376 NVME_RSVD(68, 16); 377 }; 378 379 struct rt_nvme_command_create_cq 380 { 381 rt_uint8_t opcode; 382 rt_uint8_t flags; 383 rt_uint16_t cmdid; 384 NVME_RSVD(4, 20); 385 rt_le64_t prp1; 386 NVME_RSVD(32, 8); 387 rt_le16_t cqid; 388 rt_le16_t qsize; 389 rt_le16_t cq_flags; 390 rt_le16_t irq_vector; 391 NVME_RSVD(104, 16); 392 }; 393 394 struct rt_nvme_command_create_sq 395 { 396 rt_uint8_t opcode; 397 rt_uint8_t flags; 398 rt_uint16_t cmdid; 399 NVME_RSVD(4, 20); 400 rt_le64_t prp1; 401 NVME_RSVD(32, 8); 402 rt_le16_t sqid; 403 rt_le16_t qsize; 404 rt_le16_t sq_flags; 405 rt_le16_t cqid; 406 NVME_RSVD(104, 16); 407 }; 408 409 struct rt_nvme_command_delete_queue 410 { 411 rt_uint8_t opcode; 412 rt_uint8_t flags; 413 rt_uint16_t cmdid; 414 NVME_RSVD(4, 36); 415 rt_le16_t qid; 416 NVME_RSVD(42, 22); 417 }; 418 419 struct rt_nvme_command_write_zeroes 420 { 421 rt_uint8_t opcode; 422 rt_uint8_t flags; 423 rt_uint16_t cmdid; 424 rt_le32_t nsid; 425 NVME_RSVD(8, 8); 426 rt_le64_t metadata; 427 rt_le64_t prp1; 428 rt_le64_t prp2; 429 rt_le64_t slba; 430 rt_le16_t length; 431 rt_le16_t control; 432 rt_le32_t dsmgmt; 433 rt_le32_t reftag; 434 rt_le16_t apptag; 435 rt_le16_t appmask; 436 }; 437 438 struct rt_nvme_command 439 { 440 union 441 { 442 struct rt_nvme_command_common common; 443 struct rt_nvme_command_rw rw; 444 struct rt_nvme_command_identify identify; 445 struct rt_nvme_command_features features; 446 struct rt_nvme_command_create_cq create_cq; 447 struct rt_nvme_command_create_sq create_sq; 448 struct rt_nvme_command_delete_queue delete_queue; 449 struct rt_nvme_command_write_zeroes write_zeroes; 450 }; 451 }; 452 453 struct rt_nvme_completion 454 { 455 union 456 { 457 rt_le16_t u16; 458 rt_le32_t u32; 459 rt_le64_t u64; 460 } result; 461 rt_le16_t sq_head; /* How much of this queue may be reclaimed */ 462 rt_le16_t sq_id; /* Submission queue that generated this entry */ 463 rt_uint16_t cmdid; /* Which command completed */ 464 rt_le16_t status; /* Command status */ 465 }; 466 467 enum 468 { 469 RT_NVME_REG_CAP = 0x0000, /* Controller Capabilities */ 470 RT_NVME_REG_VS = 0x0008, /* Version */ 471 RT_NVME_REG_INTMS = 0x000c, /* Interrupt Mask Set */ 472 RT_NVME_REG_INTMC = 0x0010, /* Interrupt Mask Clear */ 473 RT_NVME_REG_CC = 0x0014, /* Controller Configuration */ 474 RT_NVME_REG_CSTS = 0x001c, /* Controller Status */ 475 RT_NVME_REG_NSSR = 0x0020, /* NVM Subsystem Reset */ 476 RT_NVME_REG_AQA = 0x0024, /* Admin Queue Attributes */ 477 RT_NVME_REG_ASQ = 0x0028, /* Admin SQ Base Address */ 478 RT_NVME_REG_ACQ = 0x0030, /* Admin CQ Base Address */ 479 RT_NVME_REG_CMBLOC = 0x0038, /* Controller Memory Buffer Location */ 480 RT_NVME_REG_CMBSZ = 0x003c, /* Controller Memory Buffer Size */ 481 RT_NVME_REG_BPINFO = 0x0040, /* Boot Partition Information */ 482 RT_NVME_REG_BPRSEL = 0x0044, /* Boot Partition Read Select */ 483 RT_NVME_REG_BPMBL = 0x0048, /* Boot Partition Memory Buffer Location */ 484 RT_NVME_REG_CMBMSC = 0x0050, /* Controller Memory Buffer Memory Space Control */ 485 RT_NVME_REG_CRTO = 0x0068, /* Controller Ready Timeouts */ 486 RT_NVME_REG_PMRCAP = 0x0e00, /* Persistent Memory Capabilities */ 487 RT_NVME_REG_PMRCTL = 0x0e04, /* Persistent Memory Region Control */ 488 RT_NVME_REG_PMRSTS = 0x0e08, /* Persistent Memory Region Status */ 489 RT_NVME_REG_PMREBS = 0x0e0c, /* Persistent Memory Region Elasticity Buffer Size */ 490 RT_NVME_REG_PMRSWTP = 0x0e10, /* Persistent Memory Region Sustained Write Throughput */ 491 RT_NVME_REG_DBS = 0x1000, /* SQ 0 Tail Doorbell */ 492 }; 493 494 #define RT_NVME_CAP_MQES(cap) ((cap) & 0xffff) 495 #define RT_NVME_CAP_TIMEOUT(cap) (((cap) >> 24) & 0xff) 496 #define RT_NVME_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) 497 #define RT_NVME_CAP_MPSMIN(cap) (((cap) >> 48) & 0xf) 498 #define RT_NVME_CAP_MPSMAX(cap) (((cap) >> 52) & 0xf) 499 500 #define RT_NVME_VS(major, minor) (((major) << 16) | ((minor) << 8)) 501 502 #define RT_NVME_AQ_DEPTH 32 503 #define RT_NVME_NR_AEN_COMMANDS 1 504 #define RT_NVME_AQ_BLK_MQ_DEPTH (RT_NVME_AQ_DEPTH - RT_NVME_NR_AEN_COMMANDS) 505 #define RT_NVME_AQ_MQ_TAG_DEPTH (RT_NVME_AQ_BLK_MQ_DEPTH - 1) 506 507 enum 508 { 509 RT_NVME_CC_ENABLE = 1 << 0, 510 RT_NVME_CC_CSS_NVM = 0 << 4, 511 RT_NVME_CC_MPS_SHIFT = 7, 512 RT_NVME_CC_ARB_RR = 0 << 11, 513 RT_NVME_CC_ARB_WRRU = 1 << 11, 514 RT_NVME_CC_ARB_VS = 7 << 11, 515 RT_NVME_CC_SHN_NONE = 0 << 14, 516 RT_NVME_CC_SHN_NORMAL = 1 << 14, 517 RT_NVME_CC_SHN_ABRUPT = 2 << 14, 518 RT_NVME_CC_SHN_MASK = 3 << 14, 519 RT_NVME_CC_IOSQES = 6 << 16, 520 RT_NVME_CC_IOCQES = 4 << 20, 521 RT_NVME_CSTS_RDY = 1 << 0, 522 RT_NVME_CSTS_CFS = 1 << 1, 523 RT_NVME_CSTS_SHST_NORMAL = 0 << 2, 524 RT_NVME_CSTS_SHST_OCCUR = 1 << 2, 525 RT_NVME_CSTS_SHST_CMPLT = 2 << 2, 526 RT_NVME_CSTS_SHST_MASK = 3 << 2, 527 }; 528 529 rt_packed(struct rt_nvme_id_power_state 530 { 531 rt_le16_t mp; /* Maximum Power */ 532 NVME_RSVD(1, 1); 533 rt_uint8_t mxps_nops; /* Max Power Scale, Non-Operational State */ 534 rt_le32_t enlat; /* Entry Latency: microseconds */ 535 rt_le32_t exlat; /* Exit Latency: microseconds */ 536 rt_uint8_t rrt; /* Relative Read Throughput */ 537 rt_uint8_t rrl; /* Relative Read Latency */ 538 rt_uint8_t rwt; /* Relative Write Throughput */ 539 rt_uint8_t rwl; /* Relative Write Latency */ 540 rt_le16_t idlp; /* Idle Power */ 541 rt_uint8_t ips; /* Idle Power Scale */ 542 NVME_RSVD(19, 1); 543 rt_le16_t actp; /* Active Power */ 544 rt_uint8_t apw_aps; /* Active Power Workload, Active Power Scale */ 545 NVME_RSVD(23, 9); 546 }); 547 548 rt_packed(struct rt_nvme_id_ctrl 549 { 550 /* Controller Capabilities and Features */ 551 rt_le16_t vid; /* PCI Vendor ID */ 552 rt_le16_t ssvid; /* PCI Subsystem Vendor */ 553 char sn[20]; /* Serial Number */ 554 char mn[40]; /* Model Number */ 555 char fr[8]; /* Firmware Revision */ 556 rt_uint8_t rab; /* Recommended Arbitration Burst */ 557 rt_uint8_t ieee[3]; /* IEEE OUI Identifier */ 558 rt_uint8_t mic; /* Controller Multi-Path I/O and Namespace Sharing Capabilities */ 559 rt_uint8_t mdts; /* Maximum Data Transfer Size */ 560 rt_uint16_t cntlid; /* Controller ID */ 561 rt_uint32_t ver; /* Version */ 562 rt_uint32_t rtd3r; /* RTD3 Resume Latency */ 563 rt_uint32_t rtd3e; /* RTD3 Entry Latency */ 564 rt_uint32_t oaes; /* Optional Asynchronous Events Supported */ 565 #define RT_NVME_ID_CTRATT_ELBAS 15 /* Extended LBA Formats Supported */ 566 #define RT_NVME_ID_CTRATT_DNVMS 14 /* Delete NVM Set */ 567 #define RT_NVME_ID_CTRATT_DEG 13 /* Delete Endurance Group */ 568 #define RT_NVME_ID_CTRATT_VCM 12 /* Variable Capacity Management */ 569 #define RT_NVME_ID_CTRATT_FCM 11 /* Fixed Capacity Management */ 570 #define RT_NVME_ID_CTRATT_MDS 10 /* Multi-Domain Subsystem */ 571 #define RT_NVME_ID_CTRATT_UUIDL 9 /* UUID List */ 572 #define RT_NVME_ID_CTRATT_SQA 8 /* SQ Associations */ 573 #define RT_NVME_ID_CTRATT_NG 7 /* Namespace Granularity */ 574 #define RT_NVME_ID_CTRATT_TBKAS 6 /* Traffic Based Keep Alive Support */ 575 #define RT_NVME_ID_CTRATT_PLM 5 /* Predictable Latency Mode */ 576 #define RT_NVME_ID_CTRATT_EG 4 /* Endurance Groups */ 577 #define RT_NVME_ID_CTRATT_RRL 3 /* Read Recovery Levels */ 578 #define RT_NVME_ID_CTRATT_NVMS 2 /* NVM Sets */ 579 #define RT_NVME_ID_CTRATT_NOPSPM 1 /* Non-Operational Power State Permissive Mode */ 580 #define RT_NVME_ID_CTRATT_HIS 0 /* Host Identifier Support */ 581 rt_uint32_t ctratt; /* Controller Attributes */ 582 rt_uint16_t rrls; /* Read Recovery Levels Supported */ 583 NVME_RSVD(102, 9); 584 rt_uint8_t cntrltype; /* Controller Type */ 585 rt_uint8_t fguid[16]; /* FRU Globally Unique Identifier */ 586 rt_uint16_t crdt1; /* Command Retry Delay Time 1 */ 587 rt_uint16_t crdt2; /* Command Retry Delay Time 2 */ 588 rt_uint16_t crdt3; /* Command Retry Delay Time 3 */ 589 NVME_RSVD(134, 119); 590 #define RT_NVME_ID_NVMSR_NVMEE 1 /* NVMe Enclosure */ 591 #define RT_NVME_ID_NVMSR_NVMESD 0 /* NVMe Storage Device */ 592 rt_uint8_t nvmsr; /* NVM Subsystem Report */ 593 594 #define RT_NVME_ID_VWCI_VWCRV 7 /* VPD Write Cycles Remaining Valid */ 595 #define RT_NVME_ID_VWCI_VWCR 0 /* VPD Write Cycles Remaining */ 596 rt_uint8_t vwci; /* VPD Write Cycle Information */ 597 #define RT_NVME_ID_MEC_PCIEME 1 /* PCIe Port Management Endpoint */ 598 #define RT_NVME_ID_MEC_SMBUSME 0 /* SMBus/I2C Port Management Endpoint */ 599 rt_uint8_t mec; /* Management Endpoint Capabilities */ 600 601 /* Admin Command Set Attributes & Optional Controller Capabilities */ 602 rt_le16_t oacs; /* Optional Admin Command Support */ 603 rt_uint8_t acl; /* Abort Command Limit */ 604 rt_uint8_t aerl; /* Asynchronous Event Request Limit */ 605 #define RT_NVME_ID_FRMW_SMUD 5 /* Support Multiple Update Detection */ 606 #define RT_NVME_ID_FRMW_FAWR 4 /* Firmware Activation Without Reset */ 607 #define RT_NVME_ID_FRMW_NOFS 1 /* Number Of Firmware Slots */ 608 #define RT_NVME_ID_FRMW_FFSRO 0 /* First Firmware Slot Read Only */ 609 rt_uint8_t frmw; /* Firmware Updates */ 610 rt_uint8_t lpa; /* Log Page Attributes */ 611 rt_uint8_t elpe; /* Error Log Page Entries */ 612 rt_uint8_t npss; /* Number of Power States Support */ 613 rt_uint8_t avscc; /* Admin Vendor Specific Command Configuration */ 614 rt_uint8_t apsta; /* Autonomous Power State Transition Attributes */ 615 rt_le16_t wctemp; /* Warning Composite Temperature Threshold */ 616 rt_le16_t cctemp; /* Critical Composite Temperature Threshold */ 617 rt_uint16_t mtfa; /* Maximum Time for Firmware Activation */ 618 rt_uint32_t hmpre; /* Host Memory Buffer Preferred Size */ 619 rt_uint32_t hmmin; /* Host Memory Buffer Minimum Size */ 620 rt_uint8_t tnvmcap[16]; /* Total NVM Capacity */ 621 rt_uint8_t unvmcap[16]; /* Unallocated NVM Capacity */ 622 #define RT_NVME_ID_RPMBS_ASZ 24 /* Access Size */ 623 #define RT_NVME_ID_RPMBS_TSZ 16 /* Total Size */ 624 #define RT_NVME_ID_RPMBS_AM 3 /* Authentication Method */ 625 #define RT_NVME_ID_RPMBS_NORPMBU 2 /* Number of RPMB Units */ 626 rt_uint32_t rpmbs; /* Replay Protected Memory Block Support */ 627 rt_uint16_t edstt; /* Extended Device Self-test Time */ 628 rt_uint8_t dsto; /* Device Self-test Options */ 629 rt_uint8_t fwug; /* Firmware Update Granularity */ 630 rt_uint16_t kas; /* Keep Alive Support */ 631 rt_uint16_t hctma; /* Host Controlled Thermal Management Attributes */ 632 rt_uint16_t mntmt; /* Minimum Thermal Management Temperature */ 633 rt_uint16_t mxtmt; /* Maximum Thermal Management Temperature */ 634 #define RT_NVME_ID_SANICAP_NODMMAS 30 /* No-Deallocate Modifies Media After Sanitize */ 635 #define RT_NVME_ID_SANICAP_NDI 29 /* No-Deallocate Inhibited */ 636 #define RT_NVME_ID_SANICAP_OWS 2 /* Overwrite Support */ 637 #define RT_NVME_ID_SANICAP_BES 1 /* Block Erase Support */ 638 #define RT_NVME_ID_SANICAP_CES 0 /* Crypto Erase Support */ 639 rt_uint32_t sanicap; /* Sanitize Capabilities */ 640 rt_uint32_t hmminds; /* Host Memory Buffer Minimum Descriptor Entry Size */ 641 rt_uint16_t hmmaxd; /* Host Memory Maximum Descriptors Entries */ 642 rt_uint16_t nsetidmax; /* NVM Set Identifier Maximum */ 643 rt_uint16_t endgidmax; /* Endurance Group Identifier Maximum */ 644 rt_uint8_t anatt; /* ANA Transition Time */ 645 rt_uint8_t anacap; /* Asymmetric Namespace Access Capabilities */ 646 rt_uint32_t anagrpmax; /* ANA Group Identifier Maximum */ 647 rt_uint32_t nanagrpid; /* Number of ANA Group Identifiers */ 648 rt_uint32_t pels; /* Persistent Event Log Size */ 649 rt_uint16_t dmid; /* Domain Identifier */ 650 NVME_RSVD(358, 10); 651 rt_uint8_t megcap[16]; /* Max Endurance Group Capacity */ 652 NVME_RSVD(384, 128); 653 654 /* NVM Command Set Attributes */ 655 rt_uint8_t sqes; /* Submission Queue Entry Size */ 656 rt_uint8_t cqes; /* Completion Queue Entry Size */ 657 rt_le16_t maxcmd; /* Maximum Outstanding Commands */ 658 rt_le32_t nn; /* Number of Namespaces */ 659 rt_le16_t oncs; /* Optional NVM Command Support */ 660 rt_le16_t fuses; /* Fused Operation Support */ 661 rt_uint8_t fna; /* Format NVM Attributes */ 662 rt_uint8_t vwc; /* Volatile Write Cache */ 663 rt_le16_t awun; /* Atomic Write Unit Normal */ 664 rt_le16_t awupf; /* Atomic Write Unit Power Fail */ 665 rt_uint8_t nvscc; /* I/O Command Set Vendor Specific Command Configuration */ 666 rt_uint8_t nwpc; /* Namespace Write Protection Capabilities */ 667 rt_le16_t acwu; /* Atomic Compare & Write Unit */ 668 rt_le16_t cdfs; /* Copy Descriptor Formats Supported */ 669 #define RT_NVME_ID_SGL_SUPPORT_MASK 0x3 670 rt_le32_t sgls; /* SGL Support */ 671 rt_uint32_t mnan; /* Maximum Number of Allowed Namespaces */ 672 char maxdna[16]; /* Maximum Domain Namespace Attachments */ 673 rt_le32_t maxcna; /* Maximum I/O Controller Namespace Attachments */ 674 NVME_RSVD(564, 204); 675 rt_uint8_t subnqn[256]; /* NVM Subsystem NVMe Qualified Name */ 676 NVME_RSVD(1024, 768); 677 rt_le32_t ioccsz; /* I/O Queue Command Capsule Supported Size */ 678 rt_le32_t iorcsz; /* I/O Queue Response Capsule Supported Size */ 679 rt_le16_t icdoff; /* In Capsule Data Offset */ 680 rt_uint8_t ctrattr; /* Fabrics Controller Attributes */ 681 rt_uint8_t msdbd; /* Maximum SGL Data Block Descriptors */ 682 rt_le16_t ofcs; /* Optional Fabric Commands Support */ 683 rt_uint8_t dctype; 684 NVME_RSVD(1807, 241); 685 686 /* Power State Descriptors */ 687 struct rt_nvme_id_power_state psd[32]; 688 689 /* Vendor Specific */ 690 rt_uint8_t vs[1024]; 691 }); 692 693 enum 694 { 695 RT_NVME_CTRL_CMIC_MULTI_PORT = 1 << 0, 696 RT_NVME_CTRL_CMIC_MULTI_CTRL = 1 << 1, 697 RT_NVME_CTRL_CMIC_ANA = 1 << 3, 698 RT_NVME_CTRL_ONCS_COMPARE = 1 << 0, 699 RT_NVME_CTRL_ONCS_WRITE_UNCORRECTABLE = 1 << 1, 700 RT_NVME_CTRL_ONCS_DSM = 1 << 2, 701 RT_NVME_CTRL_ONCS_WRITE_ZEROES = 1 << 3, 702 RT_NVME_CTRL_ONCS_RESERVATIONS = 1 << 5, 703 RT_NVME_CTRL_ONCS_TIMESTAMP = 1 << 6, 704 RT_NVME_CTRL_VWC_PRESENT = 1 << 0, 705 RT_NVME_CTRL_OACS_SEC_SUPP = 1 << 0, 706 RT_NVME_CTRL_OACS_NS_MNGT_SUPP = 1 << 3, 707 RT_NVME_CTRL_OACS_DIRECTIVES = 1 << 5, 708 RT_NVME_CTRL_OACS_DBBUF_SUPP = 1 << 8, 709 RT_NVME_CTRL_LPA_CMD_EFFECTS_LOG = 1 << 1, 710 RT_NVME_CTRL_CTRATT_128_ID = 1 << 0, 711 RT_NVME_CTRL_CTRATT_NON_OP_PSP = 1 << 1, 712 RT_NVME_CTRL_CTRATT_NVM_SETS = 1 << 2, 713 RT_NVME_CTRL_CTRATT_READ_RECV_LVLS = 1 << 3, 714 RT_NVME_CTRL_CTRATT_ENDURANCE_GROUPS = 1 << 4, 715 RT_NVME_CTRL_CTRATT_PREDICTABLE_LAT = 1 << 5, 716 RT_NVME_CTRL_CTRATT_NAMESPACE_GRANULARITY = 1 << 7, 717 RT_NVME_CTRL_CTRATT_UUID_LIST = 1 << 9, 718 }; 719 720 rt_packed(struct rt_nvme_lba_format 721 { 722 rt_le16_t ms; /* Metadata size */ 723 rt_uint8_t ds; /* Data size */ 724 rt_uint8_t rp; /* Relative performance */ 725 }); 726 727 rt_packed(struct rt_nvme_id_ns 728 { 729 rt_le64_t nsze; /* Namespace size */ 730 rt_le64_t ncap; /* Namespace capacity */ 731 rt_le64_t nuse; /* Namespace utilization */ 732 rt_uint8_t nsfeat; /* Namespace features */ 733 rt_uint8_t nlbaf; /* Number of lba formats */ 734 rt_uint8_t flbas; /* Formatted lba size */ 735 rt_uint8_t mc; /* Metadata capabilities */ 736 rt_uint8_t dpc; /* End-to-end data protection capabilities */ 737 rt_uint8_t dps; /* End-to-end data protection type settings */ 738 rt_uint8_t nmic; /* Namespace Multi-path I/O and Namespace Sharing Capabilities */ 739 rt_uint8_t rescap; /* Reservation Capabilities */ 740 rt_uint8_t fpi; /* Format Progress Indicator */ 741 rt_uint8_t dlfeat; /* Deallocate Logical Block Features */ 742 rt_le16_t nawun; /* Namespace Atomic Write Unit Normal */ 743 rt_le16_t nawupf; /* Namespace Atomic Write Unit Power Fail */ 744 rt_le16_t nacwu; /* Namespace Atomic Compare & Write Unit */ 745 rt_le16_t nabsn; /* Namespace Atomic Boundary Size Normal */ 746 rt_le16_t nabo; /* Namespace Atomic Boundary Offset */ 747 rt_le16_t nabspf; /* Namespace Atomic Boundary Size Power Fail */ 748 rt_uint16_t noiob; /* Namespace Optimal IO Boundary */ 749 rt_le64_t nvmcap[2]; /* NVMe Capacity */ 750 rt_uint16_t npwg; /* Namespace Preferred Write Granularity */ 751 rt_uint16_t npwa; /* Namespace Preferred Write Alignment */ 752 rt_uint16_t npdg; /* Namespace Preferred Deallocate Granularity */ 753 rt_uint16_t npda; /* Namespace Preferred Deallocate Alignment */ 754 rt_uint16_t nows; /* Namespace Optimal Write Size */ 755 NVME_RSVD(118, 18); 756 rt_uint32_t anagrpid; /* ANA Group Identifier */ 757 NVME_RSVD(139, 3); 758 rt_uint8_t nsattr; /* Namespace Attributes */ 759 rt_uint16_t nvmsetid; /* NVMe Set Identifier */ 760 rt_uint16_t endgid; /* Endurance Group Identifier */ 761 rt_uint8_t nguid[16]; /* Namespace Globally Unique Identifier */ 762 rt_uint8_t eui64[8]; /* IEEE Extended Unique Identifier */ 763 764 /* Logical Block Address Format */ 765 struct rt_nvme_lba_format lbaf[16]; 766 NVME_RSVD(171, 192); 767 768 /* Vendor specific */ 769 rt_uint8_t vs[3712]; 770 }); 771 772 enum 773 { 774 RT_NVME_NS_FEAT_THIN = 1 << 0, 775 RT_NVME_NS_FLBAS_LBA_MASK = 0xf, 776 RT_NVME_NS_FLBAS_LBA_UMASK = 0x60, 777 RT_NVME_NS_FLBAS_LBA_SHIFT = 1, 778 RT_NVME_NS_FLBAS_META_EXT = 0x10, 779 RT_NVME_LBAF_RP_BEST = 0, 780 RT_NVME_LBAF_RP_BETTER = 1, 781 RT_NVME_LBAF_RP_GOOD = 2, 782 RT_NVME_LBAF_RP_DEGRADED = 3, 783 RT_NVME_NS_DPC_PI_LAST = 1 << 4, 784 RT_NVME_NS_DPC_PI_FIRST = 1 << 3, 785 RT_NVME_NS_DPC_PI_TYPE3 = 1 << 2, 786 RT_NVME_NS_DPC_PI_TYPE2 = 1 << 1, 787 RT_NVME_NS_DPC_PI_TYPE1 = 1 << 0, 788 RT_NVME_NS_DPS_PI_FIRST = 1 << 3, 789 RT_NVME_NS_DPS_PI_MASK = 0x7, 790 RT_NVME_NS_DPS_PI_TYPE1 = 1, 791 RT_NVME_NS_DPS_PI_TYPE2 = 2, 792 RT_NVME_NS_DPS_PI_TYPE3 = 3, 793 }; 794 795 struct rt_nvme_ops; 796 struct rt_nvme_controller; 797 798 /* 799 * An NVM Express queue. Each device has at least two (one for admin commands 800 * and one for I/O commands). 801 */ 802 struct rt_nvme_queue 803 { 804 struct rt_nvme_controller *nvme; 805 struct rt_nvme_command *sq_cmds; 806 struct rt_nvme_completion *cq_entry; 807 808 rt_ubase_t sq_cmds_phy; 809 rt_ubase_t cq_entry_phy; 810 811 rt_uint32_t *doorbell; 812 rt_uint16_t qid; 813 rt_uint16_t depth; 814 rt_uint16_t sq_head; 815 rt_uint16_t sq_tail; 816 rt_uint16_t cq_head; 817 rt_uint16_t cq_phase; 818 819 rt_err_t err; 820 struct rt_nvme_command *cmd; 821 822 struct rt_completion done; 823 struct rt_spinlock lock; 824 }; 825 826 struct rt_nvme_controller 827 { 828 rt_list_t list; 829 struct rt_device *dev; 830 831 int nvme_id; 832 char name[RT_NAME_MAX]; 833 834 void *regs; 835 rt_uint64_t cap; 836 rt_uint32_t page_shift; 837 rt_uint32_t page_size; 838 rt_uint32_t queue_depth; 839 rt_uint32_t io_queue_max; 840 rt_uint32_t ctrl_config; 841 rt_uint32_t max_transfer_shift:8; 842 rt_uint32_t volatile_write_cache:8; 843 rt_uint32_t write_zeroes:1; 844 rt_uint32_t sgl_mode:2; 845 rt_uint32_t doorbell_stride; 846 rt_uint32_t *doorbell_tbl; 847 848 const struct rt_nvme_ops *ops; 849 850 #define RT_USING_NVME_QUEUE (1 + (RT_USING_NVME_IO_QUEUE * RT_CPUS_NR)) 851 int irqs_nr; 852 int irqs[RT_USING_NVME_QUEUE]; 853 union 854 { 855 struct 856 { 857 struct rt_nvme_queue admin_queue; 858 struct rt_nvme_queue io_queues[RT_USING_NVME_IO_QUEUE * RT_CPUS_NR]; 859 }; 860 struct rt_nvme_queue queue[RT_USING_NVME_QUEUE]; 861 }; 862 863 volatile rt_atomic_t cmdid; 864 volatile rt_atomic_t ioqid[RT_CPUS_NR]; 865 866 rt_list_t ns_nodes; 867 }; 868 869 struct rt_nvme_device 870 { 871 struct rt_blk_disk parent; 872 struct rt_nvme_controller *ctrl; 873 874 rt_list_t list; 875 876 rt_uint32_t nsid; 877 rt_uint32_t lba_shift; 878 struct rt_nvme_id_ns id; 879 }; 880 #define rt_disk_to_nvme_device(disk) rt_container_of(disk, struct rt_nvme_device, parent) 881 882 struct rt_nvme_ops 883 { 884 const char *name; 885 886 /* Controller-specific NVM Express queue setup */ 887 rt_err_t (*setup_queue)(struct rt_nvme_queue *queue); 888 /* Controller-specific NVM Express queue cleanup */ 889 rt_err_t (*cleanup_queue)(struct rt_nvme_queue *queue); 890 /* Controller-specific NVM Express command submission */ 891 rt_err_t (*submit_cmd)(struct rt_nvme_queue *queue, struct rt_nvme_command *cmd); 892 /* Controller-specific NVM Express command completion */ 893 void (*complete_cmd)(struct rt_nvme_queue *queue, struct rt_nvme_command *cmd); 894 }; 895 896 rt_err_t rt_nvme_controller_register(struct rt_nvme_controller *nvme); 897 rt_err_t rt_nvme_controller_unregister(struct rt_nvme_controller *nvme); 898 899 #endif /* __NVME_H__ */ 900