[PATCH] draft of per-cpu flush flag

[Tycho Andersen]

Signed-off-by: Tycho Andersen <tycho@xxxxxxxxxx>
---
 mm/xpfo.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 70 insertions(+), 8 deletions(-)

diff --git a/mm/xpfo.c b/mm/xpfo.c
index 0b178ad5a37e..5f9aeaaa40d2 100644
--- a/mm/xpfo.c
+++ b/mm/xpfo.c
@@ -31,6 +31,8 @@ struct xpfo {
 	bool inited;		/* Map counter and lock initialized */
 	atomic_t mapcount;	/* Counter for balancing map/unmap requests */
 	spinlock_t maplock;	/* Lock to serialize map/unmap requests */
+	void *mapped;           /* per-cpu variable to indicate whether this
+	                           CPU has mapped this page or not */
 };
 
 DEFINE_STATIC_KEY_FALSE(xpfo_inited);
@@ -78,6 +80,43 @@ static inline struct xpfo *lookup_xpfo(struct page *page)
 	return (void *)page_ext + page_xpfo_ops.offset;
 }
 
+/*
+ * Return the map status of this page. Note that the cpu needs to be pinned,
+ * either via get_cpu() or a spin lock.
+ */
+static bool xpfo_test_unmapped(struct xpfo *xpfo)
+{
+	if (!xpfo->mapped) {
+		return test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+	} else {
+		return *(bool *)per_cpu_ptr(xpfo->mapped,
+					    smp_processor_id());
+	}
+}
+
+/*
+ * Set the unmapped status of this page. Returns the previous state. Note that
+ * the cpu needs to be pinnned, either via get_cpu() or a spin lock.
+ */
+static bool xpfo_test_set_unmapped(struct xpfo *xpfo, bool unmapped)
+{
+	if (!xpfo->mapped) {
+		if (unmapped)
+			return test_and_set_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+		else
+			return test_and_clear_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+	} else {
+		bool *p, prev;
+
+		p = per_cpu_ptr(xpfo->mapped, smp_processor_id());
+		prev = *p;
+		*p = unmapped;
+
+		return prev;
+	}
+}
+
+
 void xpfo_alloc_pages(struct page *page, int order, gfp_t gfp)
 {
 	int i, flush_tlb = 0;
@@ -91,7 +130,7 @@ void xpfo_alloc_pages(struct page *page, int order, gfp_t gfp)
 		if (!xpfo)
 			continue;
 
-		WARN(test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags),
+		WARN(xpfo_test_unmapped(xpfo),
 		     "xpfo: unmapped page being allocated\n");
 
 		/* Initialize the map lock and map counter */
@@ -99,7 +138,9 @@ void xpfo_alloc_pages(struct page *page, int order, gfp_t gfp)
 			spin_lock_init(&xpfo->maplock);
 			atomic_set(&xpfo->mapcount, 0);
 			xpfo->inited = true;
+			xpfo->mapped = NULL;
 		}
+
 		WARN(atomic_read(&xpfo->mapcount),
 		     "xpfo: already mapped page being allocated\n");
 
@@ -168,13 +209,19 @@ void xpfo_kmap(void *kaddr, struct page *page)
 		return;
 
 	spin_lock(&xpfo->maplock);
+	if (!xpfo->mapped) {
+		xpfo->mapped = alloc_percpu(bool);
+		if (!xpfo->mapped)
+			WARN_ON("xpfo: percpu flag allocation failed\n");
+	}
 
 	/*
 	 * The page was previously allocated to user space, so map it back
 	 * into the kernel. No TLB flush required.
 	 */
-	if ((atomic_inc_return(&xpfo->mapcount) == 1) &&
-	    test_and_clear_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags))
+	xpfo_test_set_unmapped(xpfo, false);
+
+	if (atomic_inc_return(&xpfo->mapcount) == 1)
 		set_kpte(kaddr, page, PAGE_KERNEL);
 
 	spin_unlock(&xpfo->maplock);
@@ -205,10 +252,25 @@ void xpfo_kunmap(void *kaddr, struct page *page)
 	 * The page is to be allocated back to user space, so unmap it from the
 	 * kernel, flush the TLB and tag it as a user page.
 	 */
-	if (atomic_dec_return(&xpfo->mapcount) == 0) {
-		WARN(test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags),
-		     "xpfo: unmapping already unmapped page\n");
-		set_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+	if (xpfo->mapped) {
+		/*
+		 * We have a per-cpu map, and we know it is mapped on this
+		 * process, so let's flush our local TLB.
+		 */
+		xpfo_test_set_unmapped(xpfo, true);
+
+		/*
+		 * I think this is incorrect -- the page should still be mapped
+		 * by the other cpus, it's just the TLB entry here is a bit stale.
+		 */
+		set_kpte(kaddr, page, __pgprot(0));
+		__flush_tlb_one((unsigned long) kaddr);
+	} else if (atomic_dec_return(&xpfo->mapcount) == 0) {
+		/*
+		 * No per-cpu map, so let's just do a best effort and
+		 * unmap/flush all the TLBs when the count reaches 0.
+		 */
+		xpfo_test_set_unmapped(xpfo, true);
 		set_kpte(kaddr, page, __pgprot(0));
 		flush_tlb_kernel_range((unsigned long) kaddr,
 				       (unsigned long) kaddr + PAGE_SIZE);
@@ -229,7 +291,7 @@ bool xpfo_page_is_unmapped(struct page *page)
 	if (unlikely(!xpfo) && !xpfo->inited)
 		return false;
 
-	return test_bit(XPFO_PAGE_UNMAPPED, &xpfo->flags);
+	return xpfo_test_unmapped(xpfo);
 }
 EXPORT_SYMBOL(xpfo_page_is_unmapped);
 
-- 
2.11.0