diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 019a1e3cc0a2..897ade322791 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -32,6 +32,10 @@ #include "nvme.h" #include "pci-dmabuf.h" +static bool repro_arm; +module_param(repro_arm, bool, 0644); +static bool repro_busy; + #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes) #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion)) @@ -1793,6 +1797,9 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, __u16 command_id = READ_ONCE(cqe->command_id); struct request *req; + bool replay = repro_arm && nvmeq->qid && !repro_busy; + u16 tag = nvme_tag_from_cid(command_id); + /* * AEN requests are special as they don't time out and can * survive any kind of queue freeze and often don't respond to @@ -1824,6 +1831,30 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, nvme_req(req)->status != NVME_SC_SUCCESS, nvme_pci_complete_batch)) nvme_pci_complete_rq(req); + + if (replay) { + struct blk_mq_tags *tags = nvme_queue_tagset(nvmeq); + struct request *frq = blk_mq_tag_to_rq(tags, tag); + + repro_arm = false; + if (frq && !frq->mq_hctx) { /* now actually NULL */ + u16 fidx = nvmeq->cq_head; + struct nvme_completion *f = &nvmeq->cqes[fidx]; + + f->result.u64 = 0; + f->sq_head = 0; + f->sq_id = cpu_to_le16(nvmeq->qid); + f->command_id = nvme_cid_install_genctr(nvme_req(frq)->genctr) | tag; + f->status = 0; + + repro_busy = true; + nvme_handle_cqe(nvmeq, iob, fidx); /* -> nvme_find_rq OK -> blk_mq_complete_request_remote -> *NULL */ + repro_busy = false; + } else { + pr_warn("repro: frq=%px mq_hctx=%px — still on the batched free path; set scheduler=mq-deadline\n", + frq, frq ? frq->mq_hctx : NULL); + } + } } static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)