@@ -89,6 +89,10 @@ int set_pages_rw(struct page *page, int numpages);
void clflush_cache_range(void *addr, unsigned int size);
+#ifdef CONFIG_ARCH_HAS_PMEM_API
+#define mmio_wb_range(addr, size) __arch_wb_cache_pmem(addr, size)
+#endif
+
#define mmio_flush_range(addr, size) clflush_cache_range(addr, size)
#ifdef CONFIG_DEBUG_RODATA
@@ -661,6 +661,7 @@ void blk_queue_exit(struct request_queue *q)
{
percpu_ref_put(&q->q_usage_counter);
}
+EXPORT_SYMBOL(blk_queue_exit);
static void blk_queue_usage_counter_release(struct percpu_ref *ref)
{
@@ -74,17 +74,6 @@ bool __blk_end_bidi_request(struct request *rq, int error,
unsigned int nr_bytes, unsigned int bidi_bytes);
void blk_freeze_queue(struct request_queue *q);
-static inline void blk_queue_enter_live(struct request_queue *q)
-{
- /*
- * Given that running in generic_make_request() context
- * guarantees that a live reference against q_usage_counter has
- * been established, further references under that same context
- * need not check that the queue has been frozen (marked dead).
- */
- percpu_ref_get(&q->q_usage_counter);
-}
-
#ifdef CONFIG_BLK_DEV_INTEGRITY
void blk_flush_integrity(void);
#else
@@ -33,6 +33,9 @@
static ASYNC_DOMAIN_EXCLUSIVE(async_pmem);
+#define NUM_FLUSH_THREADS 4
+#define DAX_EXTENT_SHIFT 8
+#define NUM_DAX_EXTENTS (1ULL << DAX_EXTENT_SHIFT)
struct pmem_device {
struct request_queue *pmem_queue;
struct gendisk *pmem_disk;
@@ -45,6 +48,10 @@ struct pmem_device {
unsigned long pfn_flags;
void __pmem *virt_addr;
size_t size;
+ unsigned long size_shift;
+ struct bio *flush_bio;
+ spinlock_t lock;
+ DECLARE_BITMAP(dax_active, NUM_DAX_EXTENTS);
};
static int pmem_major;
@@ -68,6 +75,105 @@ static void pmem_do_bvec(struct pmem_device *pmem, struct page *page,
kunmap_atomic(mem);
}
+struct pmem_flush_ctx {
+ struct pmem_device *pmem;
+ struct block_device *bdev;
+ int id;
+};
+
+static resource_size_t dax_extent_shift(struct pmem_device *pmem)
+{
+ return pmem->size_shift - DAX_EXTENT_SHIFT;
+}
+
+static resource_size_t dax_extent_size(struct pmem_device *pmem)
+{
+ return 1ULL << dax_extent_shift(pmem);
+}
+
+static void pmem_flush(void *data, async_cookie_t cookie)
+{
+ unsigned int i;
+ resource_size_t offset;
+ struct pmem_flush_ctx *ctx = data;
+ struct pmem_device *pmem = ctx->pmem;
+ struct device *dev = part_to_dev(ctx->bdev->bd_part);
+ unsigned long extent = dax_extent_size(pmem) / NUM_FLUSH_THREADS;
+
+ for_each_set_bit(i, pmem->dax_active, NUM_DAX_EXTENTS) {
+ unsigned long flush_len;
+ void *addr;
+
+ offset = dax_extent_size(pmem) * i + extent * ctx->id;
+ if (offset > pmem->size)
+ break;
+ flush_len = min_t(resource_size_t, extent, pmem->size - offset);
+ addr = (void __force *) pmem->virt_addr + offset;
+ dev_dbg(dev, "%s: %p %#lx\n", __func__, addr, flush_len);
+ while (flush_len) {
+ unsigned long len = min_t(unsigned long, flush_len, SZ_1M);
+
+#if defined(mmio_wb_range)
+ mmio_wb_range(addr, len);
+#elif defined(mmio_flush_range)
+ mmio_flush_range(addr, len);
+#else
+ dev_err_once(dev, "%s: failed, no flush method\n",
+ __func__);
+ return;
+#endif
+ flush_len -= len;
+ addr += len;
+ cond_resched();
+ }
+ }
+}
+
+static void __pmem_flush_request(void *data, async_cookie_t cookie)
+{
+ struct pmem_flush_ctx ctx[NUM_FLUSH_THREADS];
+ struct pmem_device *pmem = data;
+ struct bio *bio;
+ int i;
+
+ spin_lock(&pmem->lock);
+ bio = pmem->flush_bio;
+ pmem->flush_bio = bio->bi_next;
+ bio->bi_next = NULL;
+ spin_unlock(&pmem->lock);
+
+ for (i = 0; i < NUM_FLUSH_THREADS; i++) {
+ ctx[i].bdev = bio->bi_bdev;
+ ctx[i].pmem = pmem;
+ ctx[i].id = i;
+ cookie = async_schedule_domain(pmem_flush, &ctx[i], &async_pmem);
+ }
+ async_synchronize_cookie_domain(cookie, &async_pmem);
+ wmb_pmem();
+ bio_endio(bio);
+ blk_queue_exit(pmem->pmem_queue);
+}
+
+static void pmem_flush_request(struct pmem_device *pmem, struct bio *bio)
+{
+ int do_flush = 1;
+
+ spin_lock(&pmem->lock);
+ if (bitmap_weight(pmem->dax_active, NUM_DAX_EXTENTS) == 0) {
+ do_flush = 0;
+ } else {
+ bio->bi_next = pmem->flush_bio;
+ pmem->flush_bio = bio;
+ }
+ spin_unlock(&pmem->lock);
+
+ if (do_flush) {
+ blk_queue_enter_live(pmem->pmem_queue);
+ async_schedule(__pmem_flush_request, pmem);
+ } else
+ bio_endio(bio);
+}
+
static void pmem_make_request(struct request_queue *q, struct bio *bio)
{
bool do_acct;
@@ -87,7 +193,11 @@ static void pmem_make_request(struct request_queue *q, struct bio *bio)
if (bio_data_dir(bio))
wmb_pmem();
- bio_endio(bio);
+ /* we're always durable unless/until dax is activated */
+ if (bio->bi_rw & REQ_FLUSH)
+ pmem_flush_request(pmem, bio);
+ else
+ bio_endio(bio);
}
static int pmem_rw_page(struct block_device *bdev, sector_t sector,
@@ -112,6 +222,27 @@ static long pmem_direct_access(struct block_device *bdev,
dax->addr = pmem->virt_addr + offset;
dax->pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
+ if (dax->flags & BLKDAX_F_DIRTY) {
+ unsigned long start = offset >> dax_extent_shift(pmem);
+ unsigned long len;
+ size_t size;
+
+ size = min_t(size_t, pmem->size - offset, dax->size);
+ size = ALIGN(size, dax_extent_size(pmem));
+ len = max_t(unsigned long, 1, size >> dax_extent_shift(pmem));
+
+ /*
+ * Any flush initiated after the lock is dropped observes new
+ * dirty state
+ */
+ spin_lock(&pmem->lock);
+ bitmap_set(pmem->dax_active, start, len);
+ spin_unlock(&pmem->lock);
+
+ dev_dbg(part_to_dev(bdev->bd_part), "dax active %lx +%lx\n",
+ start, len);
+ }
+
return pmem->size - offset;
}
@@ -132,8 +263,12 @@ static struct pmem_device *pmem_alloc(struct device *dev,
if (!pmem)
return ERR_PTR(-ENOMEM);
+ spin_lock_init(&pmem->lock);
pmem->phys_addr = res->start;
pmem->size = resource_size(res);
+ pmem->size_shift = ilog2(pmem->size);
+ if (1ULL << pmem->size_shift < pmem->size)
+ pmem->size_shift++;
if (!arch_has_wmb_pmem())
dev_warn(dev, "unable to guarantee persistence of writes\n");
@@ -217,6 +352,8 @@ static int pmem_attach_disk(struct device *dev,
blk_queue_max_hw_sectors(pmem->pmem_queue, UINT_MAX);
blk_queue_bounce_limit(pmem->pmem_queue, BLK_BOUNCE_ANY);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, pmem->pmem_queue);
+ /* every write via pmem_make_request has FUA semantics by default */
+ blk_queue_flush(pmem->pmem_queue, REQ_FLUSH | REQ_FUA);
disk = alloc_disk_node(0, nid);
if (!disk) {
@@ -793,6 +793,17 @@ extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
struct scsi_ioctl_command __user *);
+static inline void blk_queue_enter_live(struct request_queue *q)
+{
+ /*
+ * Given that running in generic_make_request() context
+ * guarantees that a live reference against q_usage_counter has
+ * been established, further references under that same context
+ * need not check that the queue has been frozen (marked dead).
+ */
+ percpu_ref_get(&q->q_usage_counter);
+}
+
extern int blk_queue_enter(struct request_queue *q, gfp_t gfp);
extern void blk_queue_exit(struct request_queue *q);
extern void blk_start_queue(struct request_queue *q);
For the normal (make_request) I/O path writes are always synchronously flushed through to media. However, when DAX is in use it is possible that userspace leaves dirty data in the cache. Ideally userspace uses cache-writeback and persistent-commit instructions directly to flush writes to media. If instead userspace uses fsync()/msync() for consistency guarantees then the driver needs to flush the cpu cache manually. Ideally an architecture would provide a single instruction to write-back all dirty lines in the cache. In the absence of that the driver resorts to flushing line by line. Introduce mmio_wb_range() as the non-invalidating version of mmio_flush_range() and arrange for a small number of flusher threads to parallelize the work. The flush is a nop until a userspace mapping, BLKDAX_F_DIRTY request, arrives and we reduce the amount of work per-flush by tracking open active dax extents. Finer granularity 'dax_active' tracking and clearing mapped extents will be a subject of future experiments. For now this enables moderately cheap fsync/msync without per-fs and mm enabling. Signed-off-by: Dan Williams <dan.j.williams@intel.com> --- arch/x86/include/asm/cacheflush.h | 4 + block/blk-core.c | 1 block/blk.h | 11 --- drivers/nvdimm/pmem.c | 139 +++++++++++++++++++++++++++++++++++++ include/linux/blkdev.h | 11 +++ 5 files changed, 154 insertions(+), 12 deletions(-)