@@ -25,6 +25,7 @@
* @align: allocation and mapping alignment for child dax devices
* @res: physical address range of the region
* @pfn_flags: identify whether the pfns are paged back or not
+ * @pgmap: backing page map for the device address range
*/
struct dax_region {
int id;
@@ -35,6 +36,7 @@ struct dax_region {
unsigned int align;
struct resource res;
unsigned long pfn_flags;
+ struct dev_pagemap *pgmap;
};
/**
@@ -19,7 +19,7 @@ struct dax_region;
void dax_region_put(struct dax_region *dax_region);
struct dax_region *alloc_dax_region(struct device *parent,
int region_id, struct resource *res, unsigned int align,
- void *addr, unsigned long flags);
+ void *addr, struct dev_pagemap *pgmap, unsigned long flags);
struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
int id, struct resource *res, int count);
#endif /* __DEVICE_DAX_H__ */
@@ -10,6 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
+#include <linux/memmap_async.h>
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/device.h>
@@ -101,7 +102,7 @@ static void dax_region_unregister(void *region)
struct dax_region *alloc_dax_region(struct device *parent, int region_id,
struct resource *res, unsigned int align, void *addr,
- unsigned long pfn_flags)
+ struct dev_pagemap *pgmap, unsigned long pfn_flags)
{
struct dax_region *dax_region;
@@ -130,6 +131,7 @@ struct dax_region *alloc_dax_region(struct device *parent, int region_id,
dax_region->id = region_id;
ida_init(&dax_region->ida);
dax_region->align = align;
+ dax_region->pgmap = pgmap;
dax_region->dev = parent;
dax_region->base = addr;
if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
@@ -244,6 +246,15 @@ __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
return -1;
}
+static void dax_pfn_sync(struct dax_region *dax_region, pfn_t pfn,
+ unsigned long size)
+{
+ struct dev_pagemap *pgmap = dax_region->pgmap;
+ struct memmap_async_state *async = pgmap->async;
+
+ memmap_sync(pfn, PHYS_PFN(size), async);
+}
+
static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
{
struct device *dev = &dev_dax->dev;
@@ -273,6 +284,7 @@ static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
}
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+ dax_pfn_sync(dax_region, pfn, PAGE_SIZE);
rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
@@ -328,6 +340,7 @@ static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
}
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+ dax_pfn_sync(dax_region, pfn, PMD_SIZE);
return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
vmf->flags & FAULT_FLAG_WRITE);
@@ -379,6 +392,7 @@ static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
}
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+ dax_pfn_sync(dax_region, pfn, PUD_SIZE);
return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
vmf->flags & FAULT_FLAG_WRITE);
@@ -11,6 +11,7 @@
* General Public License for more details.
*/
#include <linux/percpu-refcount.h>
+#include <linux/memmap_async.h>
#include <linux/memremap.h>
#include <linux/module.h>
#include <linux/pfn_t.h>
@@ -110,6 +111,7 @@ static int dax_pmem_probe(struct device *dev)
return rc;
dax_pmem->pgmap.ref = &dax_pmem->ref;
+ dax_pmem->pgmap.async = &nd_pfn->async;
addr = devm_memremap_pages(dev, &dax_pmem->pgmap, dax_pmem_percpu_kill);
if (IS_ERR(addr))
return PTR_ERR(addr);
@@ -123,7 +125,8 @@ static int dax_pmem_probe(struct device *dev)
return -EINVAL;
dax_region = alloc_dax_region(dev, region_id, &res,
- le32_to_cpu(pfn_sb->align), addr, PFN_DEV|PFN_MAP);
+ le32_to_cpu(pfn_sb->align), addr, &dax_pmem->pgmap,
+ PFN_DEV|PFN_MAP);
if (!dax_region)
return -ENOMEM;
Given that the nd_dax shares device data with nd_pfn devices, arrange for the nd_pfn memmap_async_state instance to be registered by the devm_memremap_pages() call in the dax-pmem driver. Then, provide the generic dev_pagemap instance to the device-dax driver so that it can utilize memmap_sync() before dax-mapping pfns. Cc: Dave Jiang <dave.jiang@intel.com> Cc: Ross Zwisler <ross.zwisler@linux.intel.com> Cc: Vishal Verma <vishal.l.verma@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com> --- drivers/dax/dax-private.h | 2 ++ drivers/dax/device-dax.h | 2 +- drivers/dax/device.c | 16 +++++++++++++++- drivers/dax/pmem.c | 5 ++++- 4 files changed, 22 insertions(+), 3 deletions(-)