Lines Matching refs:rq
98 static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) in blk_flush_policy() argument
102 if (blk_rq_sectors(rq)) in blk_flush_policy()
106 if (rq->cmd_flags & REQ_PREFLUSH) in blk_flush_policy()
109 (rq->cmd_flags & REQ_FUA)) in blk_flush_policy()
115 static unsigned int blk_flush_cur_seq(struct request *rq) in blk_flush_cur_seq() argument
117 return 1 << ffz(rq->flush.seq); in blk_flush_cur_seq()
120 static void blk_flush_restore_request(struct request *rq) in blk_flush_restore_request() argument
127 rq->bio = rq->biotail; in blk_flush_restore_request()
130 rq->rq_flags &= ~RQF_FLUSH_SEQ; in blk_flush_restore_request()
131 rq->end_io = rq->flush.saved_end_io; in blk_flush_restore_request()
134 static void blk_flush_queue_rq(struct request *rq, bool add_front) in blk_flush_queue_rq() argument
136 blk_mq_add_to_requeue_list(rq, add_front, true); in blk_flush_queue_rq()
139 static void blk_account_io_flush(struct request *rq) in blk_account_io_flush() argument
141 struct block_device *part = rq->rq_disk->part0; in blk_account_io_flush()
146 ktime_get_ns() - rq->start_time_ns); in blk_account_io_flush()
163 static void blk_flush_complete_seq(struct request *rq, in blk_flush_complete_seq() argument
167 struct request_queue *q = rq->q; in blk_flush_complete_seq()
171 BUG_ON(rq->flush.seq & seq); in blk_flush_complete_seq()
172 rq->flush.seq |= seq; in blk_flush_complete_seq()
173 cmd_flags = rq->cmd_flags; in blk_flush_complete_seq()
176 seq = blk_flush_cur_seq(rq); in blk_flush_complete_seq()
186 list_move_tail(&rq->flush.list, pending); in blk_flush_complete_seq()
190 list_move_tail(&rq->flush.list, &fq->flush_data_in_flight); in blk_flush_complete_seq()
191 blk_flush_queue_rq(rq, true); in blk_flush_complete_seq()
201 BUG_ON(!list_empty(&rq->queuelist)); in blk_flush_complete_seq()
202 list_del_init(&rq->flush.list); in blk_flush_complete_seq()
203 blk_flush_restore_request(rq); in blk_flush_complete_seq()
204 blk_mq_end_request(rq, error); in blk_flush_complete_seq()
218 struct request *rq, *n; in flush_end_io() local
255 list_for_each_entry_safe(rq, n, running, flush.list) { in flush_end_io()
256 unsigned int seq = blk_flush_cur_seq(rq); in flush_end_io()
259 blk_flush_complete_seq(rq, fq, seq, error); in flush_end_io()
265 bool is_flush_rq(struct request *rq) in is_flush_rq() argument
267 return rq->end_io == flush_end_io; in is_flush_rq()
349 static void mq_flush_data_end_io(struct request *rq, blk_status_t error) in mq_flush_data_end_io() argument
351 struct request_queue *q = rq->q; in mq_flush_data_end_io()
352 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; in mq_flush_data_end_io()
353 struct blk_mq_ctx *ctx = rq->mq_ctx; in mq_flush_data_end_io()
358 WARN_ON(rq->tag < 0); in mq_flush_data_end_io()
359 blk_mq_put_driver_tag(rq); in mq_flush_data_end_io()
367 blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error); in mq_flush_data_end_io()
382 void blk_insert_flush(struct request *rq) in blk_insert_flush() argument
384 struct request_queue *q = rq->q; in blk_insert_flush()
386 unsigned int policy = blk_flush_policy(fflags, rq); in blk_insert_flush()
387 struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); in blk_insert_flush()
393 rq->cmd_flags &= ~REQ_PREFLUSH; in blk_insert_flush()
395 rq->cmd_flags &= ~REQ_FUA; in blk_insert_flush()
402 rq->cmd_flags |= REQ_SYNC; in blk_insert_flush()
411 blk_mq_end_request(rq, 0); in blk_insert_flush()
415 BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */ in blk_insert_flush()
424 blk_mq_request_bypass_insert(rq, false, true); in blk_insert_flush()
432 memset(&rq->flush, 0, sizeof(rq->flush)); in blk_insert_flush()
433 INIT_LIST_HEAD(&rq->flush.list); in blk_insert_flush()
434 rq->rq_flags |= RQF_FLUSH_SEQ; in blk_insert_flush()
435 rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ in blk_insert_flush()
437 rq->end_io = mq_flush_data_end_io; in blk_insert_flush()
440 blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); in blk_insert_flush()