Skip to content

Commit dc0ef0d

Browse files
Michal Hockotorvalds
Michal Hocko
authored and
torvalds
committed
mm: make mmap_sem for write waits killable for mm syscalls
This is a follow up work for oom_reaper [1]. As the async OOM killing depends on oom_sem for read we would really appreciate if a holder for write didn't stood in the way. This patchset is changing many of down_write calls to be killable to help those cases when the writer is blocked and waiting for readers to release the lock and so help __oom_reap_task to process the oom victim. Most of the patches are really trivial because the lock is help from a shallow syscall paths where we can return EINTR trivially and allow the current task to die (note that EINTR will never get to the userspace as the task has fatal signal pending). Others seem to be easy as well as the callers are already handling fatal errors and bail and return to userspace which should be sufficient to handle the failure gracefully. I am not familiar with all those code paths so a deeper review is really appreciated. As this work is touching more areas which are not directly connected I have tried to keep the CC list as small as possible and people who I believed would be familiar are CCed only to the specific patches (all should have received the cover though). This patchset is based on linux-next and it depends on down_write_killable for rw_semaphores which got merged into tip locking/rwsem branch and it is merged into this next tree. I guess it would be easiest to route these patches via mmotm because of the dependency on the tip tree but if respective maintainers prefer other way I have no objections. I haven't covered all the mmap_write(mm->mmap_sem) instances here $ git grep "down_write(.*\<mmap_sem\>)" next/master | wc -l 98 $ git grep "down_write(.*\<mmap_sem\>)" | wc -l 62 I have tried to cover those which should be relatively easy to review in this series because this alone should be a nice improvement. Other places can be changed on top. [0] http://lkml.kernel.org/r/1456752417-9626-1-git-send-email-mhocko@kernel.org [1] http://lkml.kernel.org/r/1452094975-551-1-git-send-email-mhocko@kernel.org [2] http://lkml.kernel.org/r/1456750705-7141-1-git-send-email-mhocko@kernel.org This patch (of 18): This is the first step in making mmap_sem write waiters killable. It focuses on the trivial ones which are taking the lock early after entering the syscall and they are not changing state before. Therefore it is very easy to change them to use down_write_killable and immediately return with -EINTR. This will allow the waiter to pass away without blocking the mmap_sem which might be required to make a forward progress. E.g. the oom reaper will need the lock for reading to dismantle the OOM victim address space. The only tricky function in this patch is vm_mmap_pgoff which has many call sites via vm_mmap. To reduce the risk keep vm_mmap with the original non-killable semantic for now. vm_munmap callers do not bother checking the return value so open code it into the munmap syscall path for now for simplicity. Signed-off-by: Michal Hocko <mhocko@suse.com> Acked-by: Vlastimil Babka <vbabka@suse.cz> Cc: Mel Gorman <mgorman@suse.de> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Konstantin Khlebnikov <koct9i@gmail.com> Cc: Hugh Dickins <hughd@google.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: David Rientjes <rientjes@google.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent e10af13 commit dc0ef0d

8 files changed

Lines changed: 55 additions & 21 deletions

File tree

‎mm/internal.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -442,9 +442,10 @@ extern u64 hwpoison_filter_flags_value;
442442
extern u64 hwpoison_filter_memcg;
443443
extern u32 hwpoison_filter_enable;
444444

445-
extern unsigned long vm_mmap_pgoff(struct file *, unsigned long,
445+
extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long,
446446
unsigned long, unsigned long,
447-
unsigned long, unsigned long);
447+
unsigned long, unsigned long,
448+
bool);
448449

449450
extern void set_pageblock_order(void);
450451
unsigned long reclaim_clean_pages_from_list(struct zone *zone,

‎mm/madvise.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -707,10 +707,12 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
707707
return error;
708708

709709
write = madvise_need_mmap_write(behavior);
710-
if (write)
711-
down_write(&current->mm->mmap_sem);
712-
else
710+
if (write) {
711+
if (down_write_killable(&current->mm->mmap_sem))
712+
return -EINTR;
713+
} else {
713714
down_read(&current->mm->mmap_sem);
715+
}
714716

715717
/*
716718
* If the interval [start,end) covers some unmapped address

‎mm/mlock.c‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -617,7 +617,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len,
617617
return error;
618618
}
619619

620-
static int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
620+
static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
621621
{
622622
unsigned long locked;
623623
unsigned long lock_limit;
@@ -635,7 +635,8 @@ static int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
635635
lock_limit >>= PAGE_SHIFT;
636636
locked = len >> PAGE_SHIFT;
637637

638-
down_write(&current->mm->mmap_sem);
638+
if (down_write_killable(&current->mm->mmap_sem))
639+
return -EINTR;
639640

640641
locked += current->mm->locked_vm;
641642

@@ -678,7 +679,8 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
678679
len = PAGE_ALIGN(len + (offset_in_page(start)));
679680
start &= PAGE_MASK;
680681

681-
down_write(&current->mm->mmap_sem);
682+
if (down_write_killable(&current->mm->mmap_sem))
683+
return -EINTR;
682684
ret = apply_vma_lock_flags(start, len, 0);
683685
up_write(&current->mm->mmap_sem);
684686

@@ -748,9 +750,10 @@ SYSCALL_DEFINE1(mlockall, int, flags)
748750
lock_limit = rlimit(RLIMIT_MEMLOCK);
749751
lock_limit >>= PAGE_SHIFT;
750752

751-
ret = -ENOMEM;
752-
down_write(&current->mm->mmap_sem);
753+
if (down_write_killable(&current->mm->mmap_sem))
754+
return -EINTR;
753755

756+
ret = -ENOMEM;
754757
if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
755758
capable(CAP_IPC_LOCK))
756759
ret = apply_mlockall_flags(flags);
@@ -765,7 +768,8 @@ SYSCALL_DEFINE0(munlockall)
765768
{
766769
int ret;
767770

768-
down_write(&current->mm->mmap_sem);
771+
if (down_write_killable(&current->mm->mmap_sem))
772+
return -EINTR;
769773
ret = apply_mlockall_flags(0);
770774
up_write(&current->mm->mmap_sem);
771775
return ret;

‎mm/mmap.c‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
178178
unsigned long min_brk;
179179
bool populate;
180180

181-
down_write(&mm->mmap_sem);
181+
if (down_write_killable(&mm->mmap_sem))
182+
return -EINTR;
182183

183184
#ifdef CONFIG_COMPAT_BRK
184185
/*
@@ -1332,7 +1333,7 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
13321333

13331334
flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
13341335

1335-
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1336+
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff, true);
13361337
out_fput:
13371338
if (file)
13381339
fput(file);
@@ -2493,6 +2494,10 @@ int vm_munmap(unsigned long start, size_t len)
24932494
int ret;
24942495
struct mm_struct *mm = current->mm;
24952496

2497+
/*
2498+
* XXX convert to down_write_killable as soon as all users are able
2499+
* to handle the error.
2500+
*/
24962501
down_write(&mm->mmap_sem);
24972502
ret = do_munmap(mm, start, len);
24982503
up_write(&mm->mmap_sem);
@@ -2502,8 +2507,15 @@ EXPORT_SYMBOL(vm_munmap);
25022507

25032508
SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
25042509
{
2510+
int ret;
2511+
struct mm_struct *mm = current->mm;
2512+
25052513
profile_munmap(addr);
2506-
return vm_munmap(addr, len);
2514+
if (down_write_killable(&mm->mmap_sem))
2515+
return -EINTR;
2516+
ret = do_munmap(mm, addr, len);
2517+
up_write(&mm->mmap_sem);
2518+
return ret;
25072519
}
25082520

25092521

@@ -2535,7 +2547,9 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
25352547
if (pgoff + (size >> PAGE_SHIFT) < pgoff)
25362548
return ret;
25372549

2538-
down_write(&mm->mmap_sem);
2550+
if (down_write_killable(&mm->mmap_sem))
2551+
return -EINTR;
2552+
25392553
vma = find_vma(mm, start);
25402554

25412555
if (!vma || !(vma->vm_flags & VM_SHARED))
@@ -2700,6 +2714,11 @@ unsigned long vm_brk(unsigned long addr, unsigned long len)
27002714
unsigned long ret;
27012715
bool populate;
27022716

2717+
/*
2718+
* XXX not all users are chcecking the return value, convert
2719+
* to down_write_killable after they are able to cope with
2720+
* error
2721+
*/
27032722
down_write(&mm->mmap_sem);
27042723
ret = do_brk(addr, len);
27052724
populate = ((mm->def_flags & VM_LOCKED) != 0);

‎mm/mprotect.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,8 @@ SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
379379

380380
reqprot = prot;
381381

382-
down_write(&current->mm->mmap_sem);
382+
if (down_write_killable(&current->mm->mmap_sem))
383+
return -EINTR;
383384

384385
vma = find_vma(current->mm, start);
385386
error = -ENOMEM;

‎mm/mremap.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
503503
if (!new_len)
504504
return ret;
505505

506-
down_write(&current->mm->mmap_sem);
506+
if (down_write_killable(&current->mm->mmap_sem))
507+
return -EINTR;
507508

508509
if (flags & MREMAP_FIXED) {
509510
ret = mremap_to(addr, old_len, new_addr, new_len,

‎mm/nommu.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1446,7 +1446,7 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
14461446

14471447
flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
14481448

1449-
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1449+
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff, true);
14501450

14511451
if (file)
14521452
fput(file);

‎mm/util.c‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -289,15 +289,20 @@ EXPORT_SYMBOL_GPL(get_user_pages_fast);
289289

290290
unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
291291
unsigned long len, unsigned long prot,
292-
unsigned long flag, unsigned long pgoff)
292+
unsigned long flag, unsigned long pgoff, bool killable)
293293
{
294294
unsigned long ret;
295295
struct mm_struct *mm = current->mm;
296296
unsigned long populate;
297297

298298
ret = security_mmap_file(file, prot, flag);
299299
if (!ret) {
300-
down_write(&mm->mmap_sem);
300+
if (killable) {
301+
if (down_write_killable(&mm->mmap_sem))
302+
return -EINTR;
303+
} else {
304+
down_write(&mm->mmap_sem);
305+
}
301306
ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff,
302307
&populate);
303308
up_write(&mm->mmap_sem);
@@ -307,6 +312,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
307312
return ret;
308313
}
309314

315+
/* XXX are all callers checking an error */
310316
unsigned long vm_mmap(struct file *file, unsigned long addr,
311317
unsigned long len, unsigned long prot,
312318
unsigned long flag, unsigned long offset)
@@ -316,7 +322,7 @@ unsigned long vm_mmap(struct file *file, unsigned long addr,
316322
if (unlikely(offset_in_page(offset)))
317323
return -EINVAL;
318324

319-
return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
325+
return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT, false);
320326
}
321327
EXPORT_SYMBOL(vm_mmap);
322328

0 commit comments

Comments
 (0)