mm: do not allocate page->ptl dynamically, if spinlock_t fits to long

kiryl · torvalds · commit 597d795a2a78 · 2013-12-20T12:25:45.000-08:00
In struct page we have enough space to fit long-size page-&gt;ptl there,
but we use dynamically-allocated page-&gt;ptl if size(spinlock_t) is larger
than sizeof(int).

It hurts 64-bit architectures with CONFIG_GENERIC_LOCKBREAK, where
sizeof(spinlock_t) == 8, but it easily fits into struct page.

Signed-off-by: Kirill A. Shutemov &lt;kirill.shutemov@linux.intel.com&gt;
Acked-by: Hugh Dickins &lt;hughd@google.com&gt;
Signed-off-by: Linus Torvalds &lt;torvalds@linux-foundation.org&gt;
diff --git a/include/linux/lockref.h b/include/linux/lockref.h
@@ -19,7 +19,7 @@
 
 #define USE_CMPXCHG_LOCKREF \
 	(IS_ENABLED(CONFIG_ARCH_USE_CMPXCHG_LOCKREF) && \
-	 IS_ENABLED(CONFIG_SMP) && !BLOATED_SPINLOCKS)
+	 IS_ENABLED(CONFIG_SMP) && SPINLOCK_SIZE <= 4)
 
 struct lockref {
 	union {
diff --git a/include/linux/mm.h b/include/linux/mm.h
@@ -1317,15 +1317,15 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
 #endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
 
 #if USE_SPLIT_PTE_PTLOCKS
-#if BLOATED_SPINLOCKS
+#if ALLOC_SPLIT_PTLOCKS
 extern bool ptlock_alloc(struct page *page);
 extern void ptlock_free(struct page *page);
 
 static inline spinlock_t *ptlock_ptr(struct page *page)
 {
 	return page->ptl;
 }
-#else /* BLOATED_SPINLOCKS */
+#else /* ALLOC_SPLIT_PTLOCKS */
 static inline bool ptlock_alloc(struct page *page)
 {
 	return true;
@@ -1339,7 +1339,7 @@ static inline spinlock_t *ptlock_ptr(struct page *page)
 {
 	return &page->ptl;
 }
-#endif /* BLOATED_SPINLOCKS */
+#endif /* ALLOC_SPLIT_PTLOCKS */
 
 static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
 {
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
@@ -26,6 +26,7 @@ struct address_space;
 #define USE_SPLIT_PTE_PTLOCKS	(NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
 #define USE_SPLIT_PMD_PTLOCKS	(USE_SPLIT_PTE_PTLOCKS && \
 		IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
+#define ALLOC_SPLIT_PTLOCKS	(SPINLOCK_SIZE > BITS_PER_LONG/8)
 
 /*
  * Each physical page in the system has a struct page associated with
@@ -155,7 +156,7 @@ struct page {
 						 * system if PG_buddy is set.
 						 */
 #if USE_SPLIT_PTE_PTLOCKS
-#if BLOATED_SPINLOCKS
+#if ALLOC_SPLIT_PTLOCKS
 		spinlock_t *ptl;
 #else
 		spinlock_t ptl;
diff --git a/kernel/bounds.c b/kernel/bounds.c
@@ -22,6 +22,6 @@ void foo(void)
 #ifdef CONFIG_SMP
 	DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
 #endif
-	DEFINE(BLOATED_SPINLOCKS, sizeof(spinlock_t) > sizeof(int));
+	DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
 	/* End of constants */
 }
diff --git a/mm/memory.c b/mm/memory.c
@@ -4271,7 +4271,7 @@ void copy_user_huge_page(struct page *dst, struct page *src,
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
 
-#if USE_SPLIT_PTE_PTLOCKS && BLOATED_SPINLOCKS
+#if ALLOC_SPLIT_PTLOCKS
 bool ptlock_alloc(struct page *page)
 {
 	spinlock_t *ptl;

Original file line number	Diff line number	Diff line change
`@@ -1317,15 +1317,15 @@ static inline pmd_t pmd_alloc(struct mm_struct mm, pud_t *pud, unsigned long a`
`1317`	`1317`	`#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */`
`1318`	`1318`
`1319`	`1319`	`#if USE_SPLIT_PTE_PTLOCKS`
`1320`		`-#if BLOATED_SPINLOCKS`
	`1320`	`+#if ALLOC_SPLIT_PTLOCKS`
`1321`	`1321`	`extern bool ptlock_alloc(struct page *page);`
`1322`	`1322`	`extern void ptlock_free(struct page *page);`
`1323`	`1323`
`1324`	`1324`	`static inline spinlock_t ptlock_ptr(struct page page)`
`1325`	`1325`	`{`
`1326`	`1326`	`return page->ptl;`
`1327`	`1327`	`}`
`1328`		`-#else /* BLOATED_SPINLOCKS */`
	`1328`	`+#else /* ALLOC_SPLIT_PTLOCKS */`
`1329`	`1329`	`static inline bool ptlock_alloc(struct page *page)`
`1330`	`1330`	`{`
`1331`	`1331`	`return true;`
`@@ -1339,7 +1339,7 @@ static inline spinlock_t ptlock_ptr(struct page page)`
`1339`	`1339`	`{`
`1340`	`1340`	`return &page->ptl;`
`1341`	`1341`	`}`
`1342`		`-#endif /* BLOATED_SPINLOCKS */`
	`1342`	`+#endif /* ALLOC_SPLIT_PTLOCKS */`
`1343`	`1343`
`1344`	`1344`	`static inline spinlock_t pte_lockptr(struct mm_struct mm, pmd_t *pmd)`
`1345`	`1345`	`{`
Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,6 @@ void foo(void)`
`22`	`22`	`#ifdef CONFIG_SMP`
`23`	`23`	`DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));`
`24`	`24`	`#endif`
`25`		`- DEFINE(BLOATED_SPINLOCKS, sizeof(spinlock_t) > sizeof(int));`
	`25`	`+ DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));`
`26`	`26`	`/* End of constants */`
`27`	`27`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4271,7 +4271,7 @@ void copy_user_huge_page(struct page dst, struct page src,`
`4271`	`4271`	`}`
`4272`	`4272`	`#endif /* CONFIG_TRANSPARENT_HUGEPAGE \|\| CONFIG_HUGETLBFS */`
`4273`	`4273`
`4274`		`-#if USE_SPLIT_PTE_PTLOCKS && BLOATED_SPINLOCKS`
	`4274`	`+#if ALLOC_SPLIT_PTLOCKS`
`4275`	`4275`	`bool ptlock_alloc(struct page *page)`
`4276`	`4276`	`{`
`4277`	`4277`	`spinlock_t *ptl;`