[PATCH 2/2] make slowpath

[KOSAKI Motohiro]

Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@xxxxxxxxxxxxxx>
---
 kernel/futex.c |   60 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/kernel/futex.c b/kernel/futex.c
index 1a91ea2..5382c05 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -213,6 +213,58 @@ static void drop_futex_key_refs(union futex_key *key)
 	}
 }
 
+static noinline int
+get_futex_key_slowpath(unsigned long address, union futex_key *key, int rw)
+{
+	int err;
+	struct mm_struct *mm = current->mm;
+	struct page *page;
+	struct vm_area_struct *vma;
+
+	down_read(&mm->mmap_sem);
+
+	vma = find_extend_vma(mm, address);
+	err = -EFAULT;
+	if (unlikely(!vma))
+		goto err_out;
+
+	err = -EACCES;
+	if (!(vma->vm_flags & (rw==VERIFY_READ ? VM_READ : VM_WRITE)))
+		goto err_out;
+
+	err = 0;
+	if (likely(!(vma->vm_flags & VM_MAYSHARE))) {
+		key->both.offset |= FUT_OFF_MMSHARED;
+		key->private.mm = mm;
+		key->private.address = address;
+		goto out;
+	}
+
+	if (likely(!(vma->vm_flags & VM_NONLINEAR))) {
+		key->shared.inode = vma->vm_file->f_path.dentry->d_inode;
+		key->both.offset |= FUT_OFF_INODE;
+		key->shared.pgoff = (((address - vma->vm_start) >> PAGE_SHIFT)
+				     + vma->vm_pgoff);
+		goto out;
+	}
+
+	err = get_user_pages(current, mm, address, 1, 0, 0, &page, NULL);
+	if (err < 0)
+		goto err_out;
+
+	key->shared.inode = vma->vm_file->f_path.dentry->d_inode;
+	key->both.offset |= FUT_OFF_INODE;
+	key->shared.pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
+	put_page(page);
+
+ out:
+	get_futex_key_refs(key);
+ err_out:
+	up_read(&current->mm->mmap_sem);
+
+	return err;
+}
+
 /**
  * get_futex_key() - Get parameters which are the keys for a futex
  * @uaddr:	virtual address of the futex
@@ -236,7 +288,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
 	unsigned long address = (unsigned long)uaddr;
 	struct mm_struct *mm = current->mm;
 	struct page *page, *page_head;
-	int err;
+	int ret;
 
 	/*
 	 * The futex address must be "naturally" aligned.
@@ -263,9 +315,9 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
 	}
 
 again:
-	err = get_user_pages_fast(address, 1, rw == VERIFY_WRITE, &page);
-	if (err < 0)
-		return err;
+	ret = __get_user_pages_fast(address, 1, 1, &page);
+	if (unlikely(!ret))
+		return get_futex_key_slowpath(address, key, rw);
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	page_head = page;
-- 
1.7.3.1


--------------020007070804060709010708
Content-Type: text/plain;
 name="performance.txt"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename="performance.txt"

<without patch>

futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=1
Result: 39683 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=2
Result: 12804 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=3
Result: 12626 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=4
Result: 12453 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=5
Result: 12453 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=6
Result: 12484 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=8
Result: 12531 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=10
Result: 12392 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=12
Result: 12610 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=16
Result: 12469 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=24
Result: 12422 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=32
Result: 12500 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=64
Result: 12346 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=128
Result: 12484 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=256
Result: 12330 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=512
Result: 11628 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=1024
Result: 8285 Kiter/s


<with patch>

futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=1
Result: 39683 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=2
Result: 12500 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=3
Result: 13055 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=4
Result: 12438 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=5
Result: 12453 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=6
Result: 12755 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=8
Result: 12547 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=10
Result: 12516 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=12
Result: 12407 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=16
Result: 12469 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=24
Result: 12453 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=32
Result: 12453 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=64
Result: 12330 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=128
Result: 12407 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=256
Result: 12346 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=512
Result: 11876 Kiter/s
futex_wait: Measure FUTEX_WAIT operations per second
        Arguments: iterations=100000000 threads=1024
Result: 9107 Kiter/s


--------------020007070804060709010708--

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/