Merge branch 'x86-uaccess-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

torvalds · torvalds · commit c2136301e43c · 2013-11-12T11:46:06.000+09:00
Pull x86 uaccess changes from Ingo Molnar:
 "A single change that micro-optimizes __copy_*_user_inatomic(), used by
  the futex code"

* 'x86-uaccess-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86: Add 1/2/4/8 byte optimization to 64bit __copy_{from,to}_user_inatomic
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
@@ -49,11 +49,10 @@ __must_check unsigned long
 copy_in_user(void __user *to, const void __user *from, unsigned len);
 
 static __always_inline __must_check
-int __copy_from_user(void *dst, const void __user *src, unsigned size)
+int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
 {
 	int ret = 0;
 
-	might_fault();
 	if (!__builtin_constant_p(size))
 		return copy_user_generic(dst, (__force void *)src, size);
 	switch (size) {
@@ -93,11 +92,17 @@ int __copy_from_user(void *dst, const void __user *src, unsigned size)
 }
 
 static __always_inline __must_check
-int __copy_to_user(void __user *dst, const void *src, unsigned size)
+int __copy_from_user(void *dst, const void __user *src, unsigned size)
+{
+	might_fault();
+	return __copy_from_user_nocheck(dst, src, size);
+}
+
+static __always_inline __must_check
+int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
 {
 	int ret = 0;
 
-	might_fault();
 	if (!__builtin_constant_p(size))
 		return copy_user_generic((__force void *)dst, src, size);
 	switch (size) {
@@ -136,6 +141,13 @@ int __copy_to_user(void __user *dst, const void *src, unsigned size)
 	}
 }
 
+static __always_inline __must_check
+int __copy_to_user(void __user *dst, const void *src, unsigned size)
+{
+	might_fault();
+	return __copy_to_user_nocheck(dst, src, size);
+}
+
 static __always_inline __must_check
 int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 {
@@ -192,13 +204,13 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 static __must_check __always_inline int
 __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
 {
-	return copy_user_generic(dst, (__force const void *)src, size);
+	return __copy_from_user_nocheck(dst, (__force const void *)src, size);
 }
 
 static __must_check __always_inline int
 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
 {
-	return copy_user_generic((__force void *)dst, src, size);
+	return __copy_to_user_nocheck((__force void *)dst, src, size);
 }
 
 extern long __copy_user_nocache(void *dst, const void __user *src,

Original file line number	Diff line number	Diff line change
`@@ -49,11 +49,10 @@ __must_check unsigned long`
`49`	`49`	`copy_in_user(void __user to, const void __user from, unsigned len);`
`50`	`50`
`51`	`51`	`static __always_inline __must_check`
`52`		`-int __copy_from_user(void dst, const void __user src, unsigned size)`
	`52`	`+int __copy_from_user_nocheck(void dst, const void __user src, unsigned size)`
`53`	`53`	`{`
`54`	`54`	`int ret = 0;`
`55`	`55`
`56`		`- might_fault();`
`57`	`56`	`if (!__builtin_constant_p(size))`
`58`	`57`	`return copy_user_generic(dst, (__force void *)src, size);`
`59`	`58`	`switch (size) {`
`@@ -93,11 +92,17 @@ int __copy_from_user(void dst, const void __user src, unsigned size)`
`93`	`92`	`}`
`94`	`93`
`95`	`94`	`static __always_inline __must_check`
`96`		`-int __copy_to_user(void __user dst, const void src, unsigned size)`
	`95`	`+int __copy_from_user(void dst, const void __user src, unsigned size)`
	`96`	`+{`
	`97`	`+ might_fault();`
	`98`	`+ return __copy_from_user_nocheck(dst, src, size);`
	`99`	`+}`
	`100`	`+`
	`101`	`+static __always_inline __must_check`
	`102`	`+int __copy_to_user_nocheck(void __user dst, const void src, unsigned size)`
`97`	`103`	`{`
`98`	`104`	`int ret = 0;`
`99`	`105`
`100`		`- might_fault();`
`101`	`106`	`if (!__builtin_constant_p(size))`
`102`	`107`	`return copy_user_generic((__force void *)dst, src, size);`
`103`	`108`	`switch (size) {`
`@@ -136,6 +141,13 @@ int __copy_to_user(void __user dst, const void src, unsigned size)`
`136`	`141`	`}`
`137`	`142`	`}`
`138`	`143`
	`144`	`+static __always_inline __must_check`
	`145`	`+int __copy_to_user(void __user dst, const void src, unsigned size)`
	`146`	`+{`
	`147`	`+ might_fault();`
	`148`	`+ return __copy_to_user_nocheck(dst, src, size);`
	`149`	`+}`
	`150`	`+`
`139`	`151`	`static __always_inline __must_check`
`140`	`152`	`int __copy_in_user(void __user dst, const void __user src, unsigned size)`
`141`	`153`	`{`
`@@ -192,13 +204,13 @@ int __copy_in_user(void __user dst, const void __user src, unsigned size)`
`192`	`204`	`static __must_check __always_inline int`
`193`	`205`	`__copy_from_user_inatomic(void dst, const void __user src, unsigned size)`
`194`	`206`	`{`
`195`		`- return copy_user_generic(dst, (__force const void *)src, size);`
	`207`	`+ return __copy_from_user_nocheck(dst, (__force const void *)src, size);`
`196`	`208`	`}`
`197`	`209`
`198`	`210`	`static __must_check __always_inline int`
`199`	`211`	`__copy_to_user_inatomic(void __user dst, const void src, unsigned size)`
`200`	`212`	`{`
`201`		`- return copy_user_generic((__force void *)dst, src, size);`
	`213`	`+ return __copy_to_user_nocheck((__force void *)dst, src, size);`
`202`	`214`	`}`
`203`	`215`
`204`	`216`	`extern long __copy_user_nocache(void dst, const void __user src,`