Lines Matching refs:mq
181 struct mmc_queue *mq);
249 struct mmc_queue *mq; in power_ro_lock_store() local
260 mq = &md->queue; in power_ro_lock_store()
263 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0); in power_ro_lock_store()
626 struct mmc_queue *mq; in mmc_blk_ioctl_cmd() local
646 mq = &md->queue; in mmc_blk_ioctl_cmd()
647 req = blk_mq_alloc_request(mq->queue, in mmc_blk_ioctl_cmd()
676 struct mmc_queue *mq; in mmc_blk_ioctl_multi_cmd() local
718 mq = &md->queue; in mmc_blk_ioctl_multi_cmd()
719 req = blk_mq_alloc_request(mq->queue, in mmc_blk_ioctl_multi_cmd()
1032 static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req) in mmc_blk_issue_drv_op() argument
1035 struct mmc_card *card = mq->card; in mmc_blk_issue_drv_op()
1036 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_issue_drv_op()
1099 static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req, in mmc_blk_issue_erase_rq() argument
1102 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_issue_erase_rq()
1137 static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_issue_trim_rq() argument
1139 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG); in mmc_blk_issue_trim_rq()
1142 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_issue_discard_rq() argument
1144 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_issue_discard_rq()
1151 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg); in mmc_blk_issue_discard_rq()
1154 static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq, in mmc_blk_issue_secdiscard_rq() argument
1157 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_issue_secdiscard_rq()
1224 static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req) in mmc_blk_issue_flush() argument
1226 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_issue_flush()
1311 static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq, in mmc_blk_data_prep() argument
1315 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_data_prep()
1383 brq->data.blocks = queue_physical_block_size(mq->queue) >> 9; in mmc_blk_data_prep()
1417 brq->data.sg_len = mmc_queue_map_sg(mq, mqrq); in mmc_blk_data_prep()
1447 static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_cqe_complete_rq() argument
1452 struct mmc_host *host = mq->card->host; in mmc_blk_cqe_complete_rq()
1453 enum mmc_issue_type issue_type = mmc_issue_type(mq, req); in mmc_blk_cqe_complete_rq()
1481 spin_lock_irqsave(&mq->lock, flags); in mmc_blk_cqe_complete_rq()
1483 mq->in_flight[issue_type] -= 1; in mmc_blk_cqe_complete_rq()
1485 put_card = (mmc_tot_in_flight(mq) == 0); in mmc_blk_cqe_complete_rq()
1487 mmc_cqe_check_busy(mq); in mmc_blk_cqe_complete_rq()
1489 spin_unlock_irqrestore(&mq->lock, flags); in mmc_blk_cqe_complete_rq()
1491 if (!mq->cqe_busy) in mmc_blk_cqe_complete_rq()
1495 mmc_put_card(mq->card, &mq->ctx); in mmc_blk_cqe_complete_rq()
1498 void mmc_blk_cqe_recovery(struct mmc_queue *mq) in mmc_blk_cqe_recovery() argument
1500 struct mmc_card *card = mq->card; in mmc_blk_cqe_recovery()
1508 mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY); in mmc_blk_cqe_recovery()
1509 mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY); in mmc_blk_cqe_recovery()
1520 struct mmc_queue *mq = q->queuedata; in mmc_blk_cqe_req_done() local
1526 if (mq->in_recovery) in mmc_blk_cqe_req_done()
1527 mmc_blk_cqe_complete_rq(mq, req); in mmc_blk_cqe_req_done()
1553 static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req) in mmc_blk_cqe_issue_flush() argument
1565 return mmc_blk_cqe_start_req(mq->card->host, mrq); in mmc_blk_cqe_issue_flush()
1568 static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_hsq_issue_rw_rq() argument
1571 struct mmc_host *host = mq->card->host; in mmc_blk_hsq_issue_rw_rq()
1574 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq); in mmc_blk_hsq_issue_rw_rq()
1585 static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_cqe_issue_rw_rq() argument
1588 struct mmc_host *host = mq->card->host; in mmc_blk_cqe_issue_rw_rq()
1591 return mmc_blk_hsq_issue_rw_rq(mq, req); in mmc_blk_cqe_issue_rw_rq()
1593 mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL); in mmc_blk_cqe_issue_rw_rq()
1595 return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq); in mmc_blk_cqe_issue_rw_rq()
1601 struct mmc_queue *mq) in mmc_blk_rw_rq_prep() argument
1606 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_rw_rq_prep()
1609 mmc_blk_data_prep(mq, mqrq, recovery_mode, &do_rel_wr, &do_data_tag); in mmc_blk_rw_rq_prep()
1701 static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req) in mmc_blk_read_single() argument
1705 struct mmc_card *card = mq->card; in mmc_blk_read_single()
1708 size_t bytes_per_read = queue_physical_block_size(mq->queue); in mmc_blk_read_single()
1716 mmc_blk_rw_rq_prep(mqrq, card, 1, mq); in mmc_blk_read_single()
1773 struct mmc_queue *mq = req->q->queuedata; in mmc_blk_status_error() local
1776 if (mmc_host_is_spi(mq->card->host)) in mmc_blk_status_error()
1809 static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req) in mmc_blk_mq_rw_recovery() argument
1814 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_mq_rw_recovery()
1815 struct mmc_card *card = mq->card; in mmc_blk_mq_rw_recovery()
1845 if (!mmc_host_is_spi(mq->card->host) && in mmc_blk_mq_rw_recovery()
1847 err = mmc_blk_fix_state(mq->card, req); in mmc_blk_mq_rw_recovery()
1862 if (!mmc_host_is_spi(mq->card->host) && in mmc_blk_mq_rw_recovery()
1892 queue_physical_block_size(mq->queue) >> 9) { in mmc_blk_mq_rw_recovery()
1894 mmc_blk_read_single(mq, req); in mmc_blk_mq_rw_recovery()
1986 static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq, in mmc_blk_rw_reset_success() argument
1991 mmc_blk_reset_success(mq->blkdata, type); in mmc_blk_rw_reset_success()
1994 static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_mq_complete_rq() argument
2009 if (mmc_card_removed(mq->card)) in mmc_blk_mq_complete_rq()
2015 static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq, in mmc_blk_urgent_bkops_needed() argument
2018 return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) && in mmc_blk_urgent_bkops_needed()
2023 static void mmc_blk_urgent_bkops(struct mmc_queue *mq, in mmc_blk_urgent_bkops() argument
2026 if (mmc_blk_urgent_bkops_needed(mq, mqrq)) in mmc_blk_urgent_bkops()
2027 mmc_run_bkops(mq->card); in mmc_blk_urgent_bkops()
2036 struct mmc_queue *mq = q->queuedata; in mmc_blk_hsq_req_done() local
2037 struct mmc_host *host = mq->card->host; in mmc_blk_hsq_req_done()
2041 mmc_blk_urgent_bkops_needed(mq, mqrq)) { in mmc_blk_hsq_req_done()
2042 spin_lock_irqsave(&mq->lock, flags); in mmc_blk_hsq_req_done()
2043 mq->recovery_needed = true; in mmc_blk_hsq_req_done()
2044 mq->recovery_req = req; in mmc_blk_hsq_req_done()
2045 spin_unlock_irqrestore(&mq->lock, flags); in mmc_blk_hsq_req_done()
2049 schedule_work(&mq->recovery_work); in mmc_blk_hsq_req_done()
2053 mmc_blk_rw_reset_success(mq, req); in mmc_blk_hsq_req_done()
2059 if (mq->in_recovery) in mmc_blk_hsq_req_done()
2060 mmc_blk_cqe_complete_rq(mq, req); in mmc_blk_hsq_req_done()
2067 struct mmc_queue *mq = req->q->queuedata; in mmc_blk_mq_complete() local
2068 struct mmc_host *host = mq->card->host; in mmc_blk_mq_complete()
2071 mmc_blk_cqe_complete_rq(mq, req); in mmc_blk_mq_complete()
2073 mmc_blk_mq_complete_rq(mq, req); in mmc_blk_mq_complete()
2076 static void mmc_blk_mq_poll_completion(struct mmc_queue *mq, in mmc_blk_mq_poll_completion() argument
2080 struct mmc_host *host = mq->card->host; in mmc_blk_mq_poll_completion()
2083 mmc_blk_card_busy(mq->card, req)) { in mmc_blk_mq_poll_completion()
2084 mmc_blk_mq_rw_recovery(mq, req); in mmc_blk_mq_poll_completion()
2086 mmc_blk_rw_reset_success(mq, req); in mmc_blk_mq_poll_completion()
2090 mmc_blk_urgent_bkops(mq, mqrq); in mmc_blk_mq_poll_completion()
2093 static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, struct request *req) in mmc_blk_mq_dec_in_flight() argument
2098 spin_lock_irqsave(&mq->lock, flags); in mmc_blk_mq_dec_in_flight()
2100 mq->in_flight[mmc_issue_type(mq, req)] -= 1; in mmc_blk_mq_dec_in_flight()
2102 put_card = (mmc_tot_in_flight(mq) == 0); in mmc_blk_mq_dec_in_flight()
2104 spin_unlock_irqrestore(&mq->lock, flags); in mmc_blk_mq_dec_in_flight()
2107 mmc_put_card(mq->card, &mq->ctx); in mmc_blk_mq_dec_in_flight()
2110 static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req, in mmc_blk_mq_post_req() argument
2115 struct mmc_host *host = mq->card->host; in mmc_blk_mq_post_req()
2123 if (mq->in_recovery) { in mmc_blk_mq_post_req()
2124 mmc_blk_mq_complete_rq(mq, req); in mmc_blk_mq_post_req()
2132 mmc_blk_mq_dec_in_flight(mq, req); in mmc_blk_mq_post_req()
2135 void mmc_blk_mq_recovery(struct mmc_queue *mq) in mmc_blk_mq_recovery() argument
2137 struct request *req = mq->recovery_req; in mmc_blk_mq_recovery()
2138 struct mmc_host *host = mq->card->host; in mmc_blk_mq_recovery()
2141 mq->recovery_req = NULL; in mmc_blk_mq_recovery()
2142 mq->rw_wait = false; in mmc_blk_mq_recovery()
2146 mmc_blk_mq_rw_recovery(mq, req); in mmc_blk_mq_recovery()
2149 mmc_blk_urgent_bkops(mq, mqrq); in mmc_blk_mq_recovery()
2151 mmc_blk_mq_post_req(mq, req, true); in mmc_blk_mq_recovery()
2154 static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq, in mmc_blk_mq_complete_prev_req() argument
2157 if (mmc_host_done_complete(mq->card->host)) in mmc_blk_mq_complete_prev_req()
2160 mutex_lock(&mq->complete_lock); in mmc_blk_mq_complete_prev_req()
2162 if (!mq->complete_req) in mmc_blk_mq_complete_prev_req()
2165 mmc_blk_mq_poll_completion(mq, mq->complete_req); in mmc_blk_mq_complete_prev_req()
2168 *prev_req = mq->complete_req; in mmc_blk_mq_complete_prev_req()
2170 mmc_blk_mq_post_req(mq, mq->complete_req, true); in mmc_blk_mq_complete_prev_req()
2172 mq->complete_req = NULL; in mmc_blk_mq_complete_prev_req()
2175 mutex_unlock(&mq->complete_lock); in mmc_blk_mq_complete_prev_req()
2180 struct mmc_queue *mq = container_of(work, struct mmc_queue, in mmc_blk_mq_complete_work() local
2183 mmc_blk_mq_complete_prev_req(mq, NULL); in mmc_blk_mq_complete_work()
2192 struct mmc_queue *mq = q->queuedata; in mmc_blk_mq_req_done() local
2193 struct mmc_host *host = mq->card->host; in mmc_blk_mq_req_done()
2205 spin_lock_irqsave(&mq->lock, flags); in mmc_blk_mq_req_done()
2206 mq->complete_req = req; in mmc_blk_mq_req_done()
2207 mq->rw_wait = false; in mmc_blk_mq_req_done()
2208 waiting = mq->waiting; in mmc_blk_mq_req_done()
2209 spin_unlock_irqrestore(&mq->lock, flags); in mmc_blk_mq_req_done()
2218 wake_up(&mq->wait); in mmc_blk_mq_req_done()
2220 queue_work(mq->card->complete_wq, &mq->complete_work); in mmc_blk_mq_req_done()
2227 mmc_blk_urgent_bkops_needed(mq, mqrq)) { in mmc_blk_mq_req_done()
2228 spin_lock_irqsave(&mq->lock, flags); in mmc_blk_mq_req_done()
2229 mq->recovery_needed = true; in mmc_blk_mq_req_done()
2230 mq->recovery_req = req; in mmc_blk_mq_req_done()
2231 spin_unlock_irqrestore(&mq->lock, flags); in mmc_blk_mq_req_done()
2232 wake_up(&mq->wait); in mmc_blk_mq_req_done()
2233 schedule_work(&mq->recovery_work); in mmc_blk_mq_req_done()
2237 mmc_blk_rw_reset_success(mq, req); in mmc_blk_mq_req_done()
2239 mq->rw_wait = false; in mmc_blk_mq_req_done()
2240 wake_up(&mq->wait); in mmc_blk_mq_req_done()
2243 mmc_blk_mq_post_req(mq, req, false); in mmc_blk_mq_req_done()
2246 static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err) in mmc_blk_rw_wait_cond() argument
2255 spin_lock_irqsave(&mq->lock, flags); in mmc_blk_rw_wait_cond()
2256 if (mq->recovery_needed) { in mmc_blk_rw_wait_cond()
2260 done = !mq->rw_wait; in mmc_blk_rw_wait_cond()
2262 mq->waiting = !done; in mmc_blk_rw_wait_cond()
2263 spin_unlock_irqrestore(&mq->lock, flags); in mmc_blk_rw_wait_cond()
2268 static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req) in mmc_blk_rw_wait() argument
2272 wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err)); in mmc_blk_rw_wait()
2275 mmc_blk_mq_complete_prev_req(mq, prev_req); in mmc_blk_rw_wait()
2280 static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq, in mmc_blk_mq_issue_rw_rq() argument
2284 struct mmc_host *host = mq->card->host; in mmc_blk_mq_issue_rw_rq()
2288 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq); in mmc_blk_mq_issue_rw_rq()
2294 err = mmc_blk_rw_wait(mq, &prev_req); in mmc_blk_mq_issue_rw_rq()
2298 mq->rw_wait = true; in mmc_blk_mq_issue_rw_rq()
2303 mmc_blk_mq_post_req(mq, prev_req, true); in mmc_blk_mq_issue_rw_rq()
2306 mq->rw_wait = false; in mmc_blk_mq_issue_rw_rq()
2319 static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host) in mmc_blk_wait_for_idle() argument
2324 return mmc_blk_rw_wait(mq, NULL); in mmc_blk_wait_for_idle()
2327 enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req) in mmc_blk_mq_issue_rq() argument
2329 struct mmc_blk_data *md = mq->blkdata; in mmc_blk_mq_issue_rq()
2338 switch (mmc_issue_type(mq, req)) { in mmc_blk_mq_issue_rq()
2340 ret = mmc_blk_wait_for_idle(mq, host); in mmc_blk_mq_issue_rq()
2346 mmc_blk_issue_drv_op(mq, req); in mmc_blk_mq_issue_rq()
2349 mmc_blk_issue_discard_rq(mq, req); in mmc_blk_mq_issue_rq()
2352 mmc_blk_issue_secdiscard_rq(mq, req); in mmc_blk_mq_issue_rq()
2355 mmc_blk_issue_trim_rq(mq, req); in mmc_blk_mq_issue_rq()
2358 mmc_blk_issue_flush(mq, req); in mmc_blk_mq_issue_rq()
2373 ret = mmc_blk_cqe_issue_flush(mq, req); in mmc_blk_mq_issue_rq()
2378 ret = mmc_blk_cqe_issue_rw_rq(mq, req); in mmc_blk_mq_issue_rq()
2380 ret = mmc_blk_mq_issue_rw_rq(mq, req); in mmc_blk_mq_issue_rq()
2802 struct mmc_queue *mq = &md->queue; in mmc_dbg_card_status_get() local
2807 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0); in mmc_dbg_card_status_get()
2831 struct mmc_queue *mq = &md->queue; in mmc_ext_csd_open() local
2843 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0); in mmc_ext_csd_open()