diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 500580822da409faf4f9f40c9360f9be5caddb2b..61597611a5ec7c0e566b9da6332b447d62684b23 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -308,15 +308,25 @@ do { \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ } while (0) +/* + * We must not call into the scheduler between uaccess_ttbr0_enable() and + * uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions, + * we must evaluate these outside of the critical section. + */ #define __raw_get_user(x, ptr, err) \ do { \ + __typeof__(*(ptr)) __user *__rgu_ptr = (ptr); \ + __typeof__(x) __rgu_val; \ __chk_user_ptr(ptr); \ + \ uaccess_enable_not_uao(); \ if (get_fs() == KERNEL_DS) \ - __raw_get_mem(x, ptr, err, K); \ + __raw_get_mem(__rgu_val, __rgu_ptr, err, K); \ else \ - __raw_get_mem(x, ptr, err, U); \ + __raw_get_mem(__rgu_val, __rgu_ptr, err, U); \ uaccess_disable_not_uao(); \ + \ + (x) = __rgu_val; \ } while (0) #define __get_user_error(x, ptr, err) \ @@ -379,14 +389,22 @@ do { \ } \ } while (0) +/* + * We must not call into the scheduler between uaccess_ttbr0_enable() and + * uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions, + * we must evaluate these outside of the critical section. + */ #define __raw_put_user(x, ptr, err) \ do { \ - __chk_user_ptr(ptr); \ + __typeof__(*(ptr)) __user *__rpu_ptr = (ptr); \ + __typeof__(*(ptr)) __rpu_val = (x); \ + __chk_user_ptr(__rpu_ptr); \ + \ uaccess_enable_not_uao(); \ if (get_fs() == KERNEL_DS) \ - __raw_put_mem(x, ptr, err, K); \ + __raw_put_mem(__rpu_val, __rpu_ptr, err, K); \ else \ - __raw_put_mem(x, ptr, err, U); \ + __raw_put_mem(__rpu_val, __rpu_ptr, err, U); \ uaccess_disable_not_uao(); \ } while (0)