mm, dax, pmem: introduce pfn_t

For the purpose of communicating the optional presence of a 'struct
page' for the pfn returned from ->direct_access(), introduce a type that
encapsulates a page-frame-number plus flags.  These flags contain the
historical "page_link" encoding for a scatterlist entry, but can also
denote "device memory".  Where "device memory" is a set of pfns that are
not part of the kernel's linear mapping by default, but are accessed via
the same memory controller as ram.

The motivation for this new type is large capacity persistent memory
that needs struct page entries in the 'memmap' to support 3rd party DMA
(i.e.  O_DIRECT I/O with a persistent memory source/target).  However,
we also need it in support of maintaining a list of mapped inodes which
need to be unmapped at driver teardown or freeze_bdev() time.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Dave Hansen <dave@sr71.net>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c
index c713b349d967946fe1b95e8ae9ff1ec18f7761bc..0d112b94d91d3ef00d6ac69a3abf61a238ddcb76 100644 (file)
--- a/arch/powerpc/sysdev/axonram.c
+++ b/arch/powerpc/sysdev/axonram.c
@@ -43,6 +43,7 @@
 #include <linux/types.h>
 #include <linux/of_device.h>
 #include <linux/of_platform.h>
+#include <linux/pfn_t.h>
 
 #include <asm/page.h>
 #include <asm/prom.h>
@@ -142,15 +143,13 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio)
  */
 static long
 axon_ram_direct_access(struct block_device *device, sector_t sector,
-                      void __pmem **kaddr, unsigned long *pfn)
+                      void __pmem **kaddr, pfn_t *pfn)
 {
        struct axon_ram_bank *bank = device->bd_disk->private_data;
        loff_t offset = (loff_t)sector << AXON_RAM_SECTOR_SHIFT;
-       void *addr = (void *)(bank->ph_addr + offset);
-
-       *kaddr = (void __pmem *)addr;
-       *pfn = virt_to_phys(addr) >> PAGE_SHIFT;
 
+       *kaddr = (void __pmem __force *) bank->io_addr + offset;
+       *pfn = phys_to_pfn_t(bank->ph_addr + offset, PFN_DEV);
        return bank->size - offset;
 }
 

diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index a5880f4ab40eb069bda60d6403b223d61ff1db80..cb27190e9f395f94a4cac646e3a4beb12b7fa6fc 100644 (file)
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -19,6 +19,9 @@
 #include <linux/radix-tree.h>
 #include <linux/fs.h>
 #include <linux/slab.h>
+#ifdef CONFIG_BLK_DEV_RAM_DAX
+#include <linux/pfn_t.h>
+#endif
 
 #include <asm/uaccess.h>
 
@@ -378,7 +381,7 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector,
 
 #ifdef CONFIG_BLK_DEV_RAM_DAX
 static long brd_direct_access(struct block_device *bdev, sector_t sector,
-                       void __pmem **kaddr, unsigned long *pfn)
+                       void __pmem **kaddr, pfn_t *pfn)
 {
        struct brd_device *brd = bdev->bd_disk->private_data;
        struct page *page;
@@ -389,7 +392,7 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector,
        if (!page)
                return -ENOSPC;
        *kaddr = (void __pmem *)page_address(page);
-       *pfn = page_to_pfn(page);
+       *pfn = page_to_pfn_t(page);
 
        return PAGE_SIZE;
 }

diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index b493ff3fccb2dcc85643711a5b02bc3bfc954836..5def7f4ddbd2ab65c2427f6f57374c5141b95877 100644 (file)
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -25,6 +25,7 @@
 #include <linux/moduleparam.h>
 #include <linux/badblocks.h>
 #include <linux/vmalloc.h>
+#include <linux/pfn_t.h>
 #include <linux/slab.h>
 #include <linux/pmem.h>
 #include <linux/nd.h>
@@ -40,6 +41,7 @@ struct pmem_device {
        phys_addr_t             phys_addr;
        /* when non-zero this device is hosting a 'pfn' instance */
        phys_addr_t             data_offset;
+       unsigned long           pfn_flags;
        void __pmem             *virt_addr;
        size_t                  size;
        struct badblocks        bb;
@@ -135,13 +137,13 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector,
 }
 
 static long pmem_direct_access(struct block_device *bdev, sector_t sector,
-                     void __pmem **kaddr, unsigned long *pfn)
+                     void __pmem **kaddr, pfn_t *pfn)
 {
        struct pmem_device *pmem = bdev->bd_disk->private_data;
        resource_size_t offset = sector * 512 + pmem->data_offset;
 
        *kaddr = pmem->virt_addr + offset;
-       *pfn = (pmem->phys_addr + offset) >> PAGE_SHIFT;
+       *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
 
        return pmem->size - offset;
 }
@@ -174,9 +176,11 @@ static struct pmem_device *pmem_alloc(struct device *dev,
                return ERR_PTR(-EBUSY);
        }
 
-       if (pmem_should_map_pages(dev))
+       pmem->pfn_flags = PFN_DEV;
+       if (pmem_should_map_pages(dev)) {
                pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, res);
-       else
+               pmem->pfn_flags |= PFN_MAP;
+       } else
                pmem->virt_addr = (void __pmem *) devm_memremap(dev,
                                pmem->phys_addr, pmem->size,
                                ARCH_MEMREMAP_PMEM);
@@ -384,6 +388,7 @@ static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns)
        pmem = dev_get_drvdata(dev);
        devm_memunmap(dev, (void __force *) pmem->virt_addr);
        pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &nsio->res);
+       pmem->pfn_flags |= PFN_MAP;
        if (IS_ERR(pmem->virt_addr)) {
                rc = PTR_ERR(pmem->virt_addr);
                goto err;

diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 94a8f4ab57bc4c5fa6ccf3fca0002daa25e6a010..ce7b7018174051998663377b0168aa221cc33185 100644 (file)
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -17,6 +17,7 @@
 #include <linux/completion.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <linux/pfn_t.h>
 #include <asm/extmem.h>
 #include <asm/io.h>
 
@@ -30,7 +31,7 @@ static void dcssblk_release(struct gendisk *disk, fmode_t mode);
 static blk_qc_t dcssblk_make_request(struct request_queue *q,
                                                struct bio *bio);
 static long dcssblk_direct_access(struct block_device *bdev, sector_t secnum,
-                        void __pmem **kaddr, unsigned long *pfn);
+                        void __pmem **kaddr, pfn_t *pfn);
 
 static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0";
 
@@ -883,20 +884,18 @@ fail:
 
 static long
 dcssblk_direct_access (struct block_device *bdev, sector_t secnum,
-                       void __pmem **kaddr, unsigned long *pfn)
+                       void __pmem **kaddr, pfn_t *pfn)
 {
        struct dcssblk_dev_info *dev_info;
        unsigned long offset, dev_sz;
-       void *addr;
 
        dev_info = bdev->bd_disk->private_data;
        if (!dev_info)
                return -ENODEV;
        dev_sz = dev_info->end - dev_info->start;
        offset = secnum * 512;
-       addr = (void *) (dev_info->start + offset);
-       *pfn = virt_to_phys(addr) >> PAGE_SHIFT;
-       *kaddr = (void __pmem *) addr;
+       *kaddr = (void __pmem *) (dev_info->start + offset);
+       *pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset), PFN_DEV);
 
        return dev_sz - offset;
 }

diff --git a/fs/dax.c b/fs/dax.c
index 3220da70ee2072679dee6f72309bdb4a3c1829a1..6b13d6cd9a9a3b649b04a243045318e8f01e30cd 100644 (file)
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -28,6 +28,7 @@
 #include <linux/sched.h>
 #include <linux/uio.h>
 #include <linux/vmstat.h>
+#include <linux/pfn_t.h>
 #include <linux/sizes.h>
 
 static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
@@ -362,7 +363,7 @@ static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh,
        }
        dax_unmap_atomic(bdev, &dax);
 
-       error = vm_insert_mixed(vma, vaddr, dax.pfn);
+       error = vm_insert_mixed(vma, vaddr, pfn_t_to_pfn(dax.pfn));
 
  out:
        i_mmap_unlock_read(mapping);
@@ -667,7 +668,8 @@ int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address,
                        result = VM_FAULT_SIGBUS;
                        goto out;
                }
-               if ((length < PMD_SIZE) || (dax.pfn & PG_PMD_COLOUR)) {
+               if (length < PMD_SIZE
+                               || (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR)) {
                        dax_unmap_atomic(bdev, &dax);
                        goto fallback;
                }
@@ -676,7 +678,7 @@ int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address,
                 * TODO: teach vmf_insert_pfn_pmd() to support
                 * 'pte_special' for pmds
                 */
-               if (pfn_valid(dax.pfn)) {
+               if (pfn_t_has_page(dax.pfn)) {
                        dax_unmap_atomic(bdev, &dax);
                        goto fallback;
                }
@@ -690,7 +692,8 @@ int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address,
                }
                dax_unmap_atomic(bdev, &dax);
 
-               result |= vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write);
+               result |= vmf_insert_pfn_pmd(vma, address, pmd,
+                               pfn_t_to_pfn(dax.pfn), write);
        }
 
  out:

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 88821fa26f198b297b0fdb0db8840a3c182082ee..bfb64d672e1976142aec9171da9bcf388505bd53 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -15,6 +15,7 @@
 #include <linux/backing-dev-defs.h>
 #include <linux/wait.h>
 #include <linux/mempool.h>
+#include <linux/pfn.h>
 #include <linux/bio.h>
 #include <linux/stringify.h>
 #include <linux/gfp.h>
@@ -1628,7 +1629,7 @@ struct blk_dax_ctl {
        sector_t sector;
        void __pmem *addr;
        long size;
-       unsigned long pfn;
+       pfn_t pfn;
 };
 
 struct block_device_operations {
@@ -1638,7 +1639,7 @@ struct block_device_operations {
        int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
        int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
        long (*direct_access)(struct block_device *, sector_t, void __pmem **,
-                       unsigned long *pfn);
+                       pfn_t *);
        unsigned int (*check_events) (struct gendisk *disk,
                                      unsigned int clearing);
        /* ->media_changed() is DEPRECATED, use ->check_events() instead */

diff --git a/include/linux/pfn.h b/include/linux/pfn.h
index 97f3e88aead4acbb7dcb5e3ac4cf2c9e03885e6a..2d8e49711b6392a591ee352b12c162096d4c89f6 100644 (file)
--- a/include/linux/pfn.h
+++ b/include/linux/pfn.h
@@ -3,6 +3,15 @@
 
 #ifndef __ASSEMBLY__
 #include <linux/types.h>
+
+/*
+ * pfn_t: encapsulates a page-frame number that is optionally backed
+ * by memmap (struct page).  Whether a pfn_t has a 'struct page'
+ * backing is indicated by flags in the high bits of the value.
+ */
+typedef struct {
+       unsigned long val;
+} pfn_t;
 #endif
 
 #define PFN_ALIGN(x)   (((unsigned long)(x) + (PAGE_SIZE - 1)) & PAGE_MASK)

diff --git a/include/linux/pfn_t.h b/include/linux/pfn_t.h
new file mode 100644 (file)
index 0000000..c557a0e
--- /dev/null
+++ b/include/linux/pfn_t.h
@@ -0,0 +1,67 @@
+#ifndef _LINUX_PFN_T_H_
+#define _LINUX_PFN_T_H_
+#include <linux/mm.h>
+
+/*
+ * PFN_FLAGS_MASK - mask of all the possible valid pfn_t flags
+ * PFN_SG_CHAIN - pfn is a pointer to the next scatterlist entry
+ * PFN_SG_LAST - pfn references a page and is the last scatterlist entry
+ * PFN_DEV - pfn is not covered by system memmap by default
+ * PFN_MAP - pfn has a dynamic page mapping established by a device driver
+ */
+#define PFN_FLAGS_MASK (((unsigned long) ~PAGE_MASK) \
+               << (BITS_PER_LONG - PAGE_SHIFT))
+#define PFN_SG_CHAIN (1UL << (BITS_PER_LONG - 1))
+#define PFN_SG_LAST (1UL << (BITS_PER_LONG - 2))
+#define PFN_DEV (1UL << (BITS_PER_LONG - 3))
+#define PFN_MAP (1UL << (BITS_PER_LONG - 4))
+
+static inline pfn_t __pfn_to_pfn_t(unsigned long pfn, unsigned long flags)
+{
+       pfn_t pfn_t = { .val = pfn | (flags & PFN_FLAGS_MASK), };
+
+       return pfn_t;
+}
+
+/* a default pfn to pfn_t conversion assumes that @pfn is pfn_valid() */
+static inline pfn_t pfn_to_pfn_t(unsigned long pfn)
+{
+       return __pfn_to_pfn_t(pfn, 0);
+}
+
+extern pfn_t phys_to_pfn_t(dma_addr_t addr, unsigned long flags);
+
+static inline bool pfn_t_has_page(pfn_t pfn)
+{
+       return (pfn.val & PFN_MAP) == PFN_MAP || (pfn.val & PFN_DEV) == 0;
+}
+
+static inline unsigned long pfn_t_to_pfn(pfn_t pfn)
+{
+       return pfn.val & ~PFN_FLAGS_MASK;
+}
+
+static inline struct page *pfn_t_to_page(pfn_t pfn)
+{
+       if (pfn_t_has_page(pfn))
+               return pfn_to_page(pfn_t_to_pfn(pfn));
+       return NULL;
+}
+
+static inline dma_addr_t pfn_t_to_phys(pfn_t pfn)
+{
+       return PFN_PHYS(pfn_t_to_pfn(pfn));
+}
+
+static inline void *pfn_t_to_virt(pfn_t pfn)
+{
+       if (pfn_t_has_page(pfn))
+               return __va(pfn_t_to_phys(pfn));
+       return NULL;
+}
+
+static inline pfn_t page_to_pfn_t(struct page *page)
+{
+       return pfn_to_pfn_t(page_to_pfn(page));
+}
+#endif /* _LINUX_PFN_T_H_ */

diff --git a/kernel/memremap.c b/kernel/memremap.c
index 7658d32c5c78aa6343ac8cd669070084d1cd3805..449cb6a5d9a1e4408753a50988dadccefadb6e78 100644 (file)
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -12,6 +12,7 @@
  */
 #include <linux/device.h>
 #include <linux/types.h>
+#include <linux/pfn_t.h>
 #include <linux/io.h>
 #include <linux/mm.h>
 #include <linux/memory_hotplug.h>
@@ -147,6 +148,12 @@ void devm_memunmap(struct device *dev, void *addr)
 }
 EXPORT_SYMBOL(devm_memunmap);
 
+pfn_t phys_to_pfn_t(dma_addr_t addr, unsigned long flags)
+{
+       return __pfn_to_pfn_t(addr >> PAGE_SHIFT, flags);
+}
+EXPORT_SYMBOL(phys_to_pfn_t);
+
 #ifdef CONFIG_ZONE_DEVICE
 struct page_map {
        struct resource res;