Skip to content

Commit df6ad69

Browse files
Jérôme Glissetorvalds
Jérôme Glisse
authored and
torvalds
committed
mm/device-public-memory: device memory cache coherent with CPU
Platform with advance system bus (like CAPI or CCIX) allow device memory to be accessible from CPU in a cache coherent fashion. Add a new type of ZONE_DEVICE to represent such memory. The use case are the same as for the un-addressable device memory but without all the corners cases. Link: http://lkml.kernel.org/r/20170817000548.32038-19-jglisse@redhat.com Signed-off-by: Jérôme Glisse <jglisse@redhat.com> Cc: Aneesh Kumar <aneesh.kumar@linux.vnet.ibm.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Ross Zwisler <ross.zwisler@linux.intel.com> Cc: Balbir Singh <bsingharora@gmail.com> Cc: David Nellans <dnellans@nvidia.com> Cc: Evgeny Baskakov <ebaskakov@nvidia.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: John Hubbard <jhubbard@nvidia.com> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Mark Hairgrove <mhairgrove@nvidia.com> Cc: Michal Hocko <mhocko@kernel.org> Cc: Sherry Cheung <SCheung@nvidia.com> Cc: Subhash Gutti <sgutti@nvidia.com> Cc: Vladimir Davydov <vdavydov.dev@gmail.com> Cc: Bob Liu <liubo95@huawei.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 8315ada commit df6ad69

14 files changed

Lines changed: 159 additions & 47 deletions

File tree

‎fs/proc/task_mmu.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1267,7 +1267,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
12671267
if (pm->show_pfn)
12681268
frame = pte_pfn(pte);
12691269
flags |= PM_PRESENT;
1270-
page = vm_normal_page(vma, addr, pte);
1270+
page = _vm_normal_page(vma, addr, pte, true);
12711271
if (pte_soft_dirty(pte))
12721272
flags |= PM_SOFT_DIRTY;
12731273
} else if (is_swap_pte(pte)) {

‎include/linux/hmm.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ int hmm_vma_fault(struct vm_area_struct *vma,
327327
#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
328328

329329

330-
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
330+
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
331331
struct hmm_devmem;
332332

333333
struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma,
@@ -494,7 +494,7 @@ struct hmm_device {
494494
*/
495495
struct hmm_device *hmm_device_new(void *drvdata);
496496
void hmm_device_put(struct hmm_device *hmm_device);
497-
#endif /* IS_ENABLED(CONFIG_DEVICE_PRIVATE) */
497+
#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */
498498

499499

500500
/* Below are for HMM internal use only! Not to be used by device driver! */

‎include/linux/ioport.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ enum {
131131
IORES_DESC_PERSISTENT_MEMORY = 4,
132132
IORES_DESC_PERSISTENT_MEMORY_LEGACY = 5,
133133
IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,
134+
IORES_DESC_DEVICE_PUBLIC_MEMORY = 7,
134135
};
135136

136137
/* helpers to define resources */

‎include/linux/memremap.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,18 @@ static inline struct vmem_altmap *to_vmem_altmap(unsigned long memmap_start)
5757
*
5858
* A more complete discussion of unaddressable memory may be found in
5959
* include/linux/hmm.h and Documentation/vm/hmm.txt.
60+
*
61+
* MEMORY_DEVICE_PUBLIC:
62+
* Device memory that is cache coherent from device and CPU point of view. This
63+
* is use on platform that have an advance system bus (like CAPI or CCIX). A
64+
* driver can hotplug the device memory using ZONE_DEVICE and with that memory
65+
* type. Any page of a process can be migrated to such memory. However no one
66+
* should be allow to pin such memory so that it can always be evicted.
6067
*/
6168
enum memory_type {
6269
MEMORY_DEVICE_HOST = 0,
6370
MEMORY_DEVICE_PRIVATE,
71+
MEMORY_DEVICE_PUBLIC,
6472
};
6573

6674
/*
@@ -92,6 +100,8 @@ enum memory_type {
92100
* The page_free() callback is called once the page refcount reaches 1
93101
* (ZONE_DEVICE pages never reach 0 refcount unless there is a refcount bug.
94102
* This allows the device driver to implement its own memory management.)
103+
*
104+
* For MEMORY_DEVICE_PUBLIC only the page_free() callback matter.
95105
*/
96106
typedef int (*dev_page_fault_t)(struct vm_area_struct *vma,
97107
unsigned long addr,
@@ -134,6 +144,12 @@ static inline bool is_device_private_page(const struct page *page)
134144
return is_zone_device_page(page) &&
135145
page->pgmap->type == MEMORY_DEVICE_PRIVATE;
136146
}
147+
148+
static inline bool is_device_public_page(const struct page *page)
149+
{
150+
return is_zone_device_page(page) &&
151+
page->pgmap->type == MEMORY_DEVICE_PUBLIC;
152+
}
137153
#else
138154
static inline void *devm_memremap_pages(struct device *dev,
139155
struct resource *res, struct percpu_ref *ref,
@@ -157,6 +173,11 @@ static inline bool is_device_private_page(const struct page *page)
157173
{
158174
return false;
159175
}
176+
177+
static inline bool is_device_public_page(const struct page *page)
178+
{
179+
return false;
180+
}
160181
#endif
161182

162183
/**

‎include/linux/mm.h‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -800,15 +800,16 @@ static inline bool is_zone_device_page(const struct page *page)
800800
}
801801
#endif
802802

803-
#ifdef CONFIG_DEVICE_PRIVATE
804-
void put_zone_device_private_page(struct page *page);
803+
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
804+
void put_zone_device_private_or_public_page(struct page *page);
805805
#else
806-
static inline void put_zone_device_private_page(struct page *page)
806+
static inline void put_zone_device_private_or_public_page(struct page *page)
807807
{
808808
}
809-
#endif
809+
#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */
810810

811811
static inline bool is_device_private_page(const struct page *page);
812+
static inline bool is_device_public_page(const struct page *page);
812813

813814
DECLARE_STATIC_KEY_FALSE(device_private_key);
814815

@@ -834,8 +835,9 @@ static inline void put_page(struct page *page)
834835
* include/linux/memremap.h and HMM for details.
835836
*/
836837
if (static_branch_unlikely(&device_private_key) &&
837-
unlikely(is_device_private_page(page))) {
838-
put_zone_device_private_page(page);
838+
unlikely(is_device_private_page(page) ||
839+
is_device_public_page(page))) {
840+
put_zone_device_private_or_public_page(page);
839841
return;
840842
}
841843

@@ -1224,8 +1226,10 @@ struct zap_details {
12241226
pgoff_t last_index; /* Highest page->index to unmap */
12251227
};
12261228

1227-
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1228-
pte_t pte);
1229+
struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1230+
pte_t pte, bool with_public_device);
1231+
#define vm_normal_page(vma, addr, pte) _vm_normal_page(vma, addr, pte, false)
1232+
12291233
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
12301234
pmd_t pmd);
12311235

‎kernel/memremap.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -501,8 +501,8 @@ struct vmem_altmap *to_vmem_altmap(unsigned long memmap_start)
501501
#endif /* CONFIG_ZONE_DEVICE */
502502

503503

504-
#ifdef CONFIG_DEVICE_PRIVATE
505-
void put_zone_device_private_page(struct page *page)
504+
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
505+
void put_zone_device_private_or_public_page(struct page *page)
506506
{
507507
int count = page_ref_dec_return(page);
508508

@@ -522,5 +522,5 @@ void put_zone_device_private_page(struct page *page)
522522
} else if (!count)
523523
__put_page(page);
524524
}
525-
EXPORT_SYMBOL(put_zone_device_private_page);
526-
#endif /* CONFIG_DEVICE_PRIVATE */
525+
EXPORT_SYMBOL(put_zone_device_private_or_public_page);
526+
#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */

‎mm/Kconfig‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -720,12 +720,23 @@ config HMM_MIRROR
720720
config DEVICE_PRIVATE
721721
bool "Unaddressable device memory (GPU memory, ...)"
722722
depends on ARCH_HAS_HMM
723+
select HMM
723724

724725
help
725726
Allows creation of struct pages to represent unaddressable device
726727
memory; i.e., memory that is only accessible from the device (or
727728
group of devices). You likely also want to select HMM_MIRROR.
728729

730+
config DEVICE_PUBLIC
731+
bool "Addressable device memory (like GPU memory)"
732+
depends on ARCH_HAS_HMM
733+
select HMM
734+
735+
help
736+
Allows creation of struct pages to represent addressable device
737+
memory; i.e., memory that is accessible from both the device and
738+
the CPU
739+
729740
config FRAME_VECTOR
730741
bool
731742

‎mm/gup.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,13 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
456456
if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
457457
goto unmap;
458458
*page = pte_page(*pte);
459+
460+
/*
461+
* This should never happen (a device public page in the gate
462+
* area).
463+
*/
464+
if (is_device_public_page(*page))
465+
goto unmap;
459466
}
460467
get_page(*page);
461468
out:

‎mm/hmm.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -737,7 +737,7 @@ EXPORT_SYMBOL(hmm_vma_fault);
737737
#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
738738

739739

740-
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
740+
#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
741741
struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma,
742742
unsigned long addr)
743743
{
@@ -1177,4 +1177,4 @@ static int __init hmm_init(void)
11771177
}
11781178

11791179
device_initcall(hmm_init);
1180-
#endif /* IS_ENABLED(CONFIG_DEVICE_PRIVATE) */
1180+
#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */

‎mm/madvise.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
355355
continue;
356356
}
357357

358-
page = vm_normal_page(vma, addr, ptent);
358+
page = _vm_normal_page(vma, addr, ptent, true);
359359
if (!page)
360360
continue;
361361

0 commit comments

Comments
 (0)