[RFC,24/24] do_mmap: implement easiest cases of fine grained locking

Commit Message

Michel Lespinasse Feb. 24, 2020, 8:30 p.m. UTC

Use a range lock in the easiest possible mmap case:
- the mmap address is known;
- there are no existing vmas within the mmap range;
- there is no file being mapped.

When these conditions are met, we can trivially support a fine grained
range lock by just holding the mm_vma_lock accross the entire mmap
operation. This is safe because the mmap only registers the new
mapping using O(log N) operations, and does not have to call back into
arbitrary code (such as file mmap handlers) or iterate over existing
vmas and mapped pages.

Signed-off-by: Michel Lespinasse <[e-mail]>
---
 mm/mmap.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

Patch

diff --git mm/mmap.c mm/mmap.c
index 75755f1cbd0b..5fa23f300e72 100644
--- mm/mmap.c
+++ mm/mmap.c
@@ -1372,6 +1372,7 @@  unsigned long do_mmap(struct file *file, unsigned long addr,
 			unsigned long pgoff, bool locked,
 			unsigned long *populate, struct list_head *uf)
 {
+	struct mm_lock_range mmap_range, *range = NULL;
 	struct mm_struct *mm = current->mm;
 	int pkey = 0;
 
@@ -1406,8 +1407,18 @@  unsigned long do_mmap(struct file *file, unsigned long addr,
 	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
 		return -EOVERFLOW;
 
-	if (!locked && mm_write_lock_killable(mm))
-		return -EINTR;
+	if (!locked) {
+		if (addr && !file) {
+			mm_init_lock_range(&mmap_range, addr, addr + len);
+			range = &mmap_range;
+		} else
+			range = mm_coarse_lock_range();
+	retry:
+		if (mm_write_range_lock_killable(mm, range))
+			return -EINTR;
+		if (!mm_range_is_coarse(range))
+			mm_vma_lock(mm);
+	}
 
 	/* Too many mappings? */
 	if (mm->map_count > sysctl_max_map_count) {
@@ -1422,12 +1433,20 @@  unsigned long do_mmap(struct file *file, unsigned long addr,
 	if (IS_ERR_VALUE(addr))
 		goto unlock;
 
-	if (flags & MAP_FIXED_NOREPLACE) {
+	if ((flags & MAP_FIXED_NOREPLACE) ||
+	    (!locked && !mm_range_is_coarse(range))) {
 		struct vm_area_struct *vma = find_vma(mm, addr);
 
 		if (vma && vma->vm_start < addr + len) {
-			addr = -EEXIST;
-			goto unlock;
+			if (flags & MAP_FIXED_NOREPLACE) {
+				addr = -EEXIST;
+				goto unlock;
+			} else {
+				mm_vma_unlock(mm);
+				mm_write_range_unlock(mm, range);
+				range = mm_coarse_lock_range();
+				goto retry;
+			}
 		}
 	}
 
@@ -1587,8 +1606,11 @@  unsigned long do_mmap(struct file *file, unsigned long addr,
 		*populate = len;
 
 unlock:
-	if (!locked)
-		mm_write_unlock(mm);
+	if (!locked) {
+		if (!mm_range_is_coarse(range))
+			mm_vma_unlock(mm);
+		mm_write_range_unlock(mm, range);
+	}
 	return addr;
 }