Re: [PATCH 2.6.20-rc5 1/1] MM: enhance Linux swap subsystem

[yunfeng zhang]

You can apply my previous patch on 2.6.20 by changing

-#define VM_PURE_PRIVATE	0x04000000	/* Is the vma is only belonging to a mm,
to
+#define VM_PURE_PRIVATE	0x08000000	/* Is the vma is only belonging to a mm,

New revision is based on 2.6.20 with my previous patch, major changelogs are
1) pte_unmap pairs on shrink_pvma_scan_ptes and pps_swapoff_scan_ptes.
2) Now, kppsd can be woke up by kswapd.
3) New global variable accelerate_kppsd is appended to accelerate the
  reclamation process when a memory inode is low.


Signed-off-by: Yunfeng Zhang <zyf.zeroos@xxxxxxxxx>

Index: linux-2.6.19/Documentation/vm_pps.txt
===================================================================
--- linux-2.6.19.orig/Documentation/vm_pps.txt	2007-02-12
12:45:07.000000000 +0800
+++ linux-2.6.19/Documentation/vm_pps.txt	2007-02-12 15:30:16.490797672 +0800
@@ -143,23 +143,32 @@
2) mm/memory.c   do_wp_page, handle_pte_fault::unmapped_pte, do_anonymous_page,
   do_swap_page (page-fault)
3) mm/memory.c   get_user_pages (sometimes core need share PrivatePage with us)
+4) mm/vmscan.c   balance_pgdat  (kswapd/x can do stage 5 of its node pages,
+   while kppsd can do stage 1-4)
+5) mm/vmscan.c   kppsd          (new core daemon -- kppsd, see below)

There isn't new lock order defined in pps, that is, it's compliable to Linux
-lock order.
+lock order. Locks in shrink_private_vma copied from shrink_list of 2.6.16.29
+(my initial version).
// }])>

// Others about pps <([{
A new kernel thread -- kppsd is introduced in mm/vmscan.c, its task is to
-execute the stages of pps periodically, note an appropriate timeout ticks is
-necessary so we can give application a chance to re-map back its PrivatePage
-from UnmappedPTE to PTE, that is, show their conglomeration affinity.
-
-kppsd can be controlled by new fields -- scan_control::may_reclaim/reclaim_node
-may_reclaim = 1 means starting reclamation (stage 5).  reclaim_node = (node
-number) is used when a memory node is low. Caller should set them to wakeup_sc,
-then wake up kppsd (vmscan.c:balance_pgdat). Note, if kppsd is started due to
-timeout, it doesn't do stage 5 at all (vmscan.c:kppsd). Other alive legacy
-fields are gfp_mask, may_writepage and may_swap.
+execute the stage 1 - 4 of pps periodically, note an appropriate timeout ticks
+(current 2 seconds) is necessary so we can give application a chance to re-map
+back its PrivatePage from UnmappedPTE to PTE, that is, show their
+conglomeration affinity.
+
+shrink_private_vma can be controlled by new fields -- may_reclaim, reclaim_node
+and is_kppsd of scan_control.  may_reclaim = 1 means starting reclamation
+(stage 5). reclaim_node = (node number, -1 means all memory inode) is used when
+a memory node is low. Caller (kswapd/x), typically, set reclaim_node to start
+shrink_private_vma (vmscan.c:balance_pgdat). Note, only to kppsd is_kppsd = 1.
+Other alive legacy fields to pps are gfp_mask, may_writepage and may_swap.
+
+When a memory inode is low, kswapd/x can wake up kppsd by increasing global
+variable accelerate_kppsd (balance_pgdat), which accelerate stage 1 - 4, and
+call shrink_private_vma to do stage 5.

PPS statistic data is appended to /proc/meminfo entry, its prototype is in
include/linux/mm.h.
Index: linux-2.6.19/mm/swapfile.c
===================================================================
--- linux-2.6.19.orig/mm/swapfile.c	2007-02-12 12:45:07.000000000 +0800
+++ linux-2.6.19/mm/swapfile.c	2007-02-12 12:45:21.000000000 +0800
@@ -569,6 +569,7 @@
			}
		}
	} while (pte++, addr += PAGE_SIZE, addr != end);
+	pte_unmap(pte);
	return 0;
}

Index: linux-2.6.19/mm/vmscan.c
===================================================================
--- linux-2.6.19.orig/mm/vmscan.c	2007-02-12 12:45:07.000000000 +0800
+++ linux-2.6.19/mm/vmscan.c	2007-02-12 15:48:59.217292888 +0800
@@ -70,6 +70,7 @@
	/* pps control command. See Documentation/vm_pps.txt. */
	int may_reclaim;
	int reclaim_node;
+	int is_kppsd;
};

/*
@@ -1101,9 +1102,9 @@
	return ret;
}

-// pps fields.
+// pps fields, see Documentation/vm_pps.txt.
static wait_queue_head_t kppsd_wait;
-static struct scan_control wakeup_sc;
+static int accelerate_kppsd = 0;
struct pps_info pps_info = {
	.total = ATOMIC_INIT(0),
	.pte_count = ATOMIC_INIT(0), // stage 1 and 2.
@@ -1118,24 +1119,22 @@
	struct page* pages[MAX_SERIES_LENGTH];
	int series_length;
	int series_stage;
-} series;
+};

-static int get_series_stage(pte_t* pte, int index)
+static int get_series_stage(struct series_t* series, pte_t* pte, int index)
{
-	series.orig_ptes[index] = *pte;
-	series.ptes[index] = pte;
-	if (pte_present(series.orig_ptes[index])) {
-		struct page* page = pfn_to_page(pte_pfn(series.orig_ptes[index]));
-		series.pages[index] = page;
+	series->orig_ptes[index] = *pte;
+	series->ptes[index] = pte;
+	struct page* page = pfn_to_page(pte_pfn(series->orig_ptes[index]));
+	series->pages[index] = page;
+	if (pte_present(series->orig_ptes[index])) {
		if (page == ZERO_PAGE(addr)) // reserved page is exclusive from us.
			return 7;
-		if (pte_young(series.orig_ptes[index])) {
+		if (pte_young(series->orig_ptes[index])) {
			return 1;
		} else
			return 2;
-	} else if (pte_unmapped(series.orig_ptes[index])) {
-		struct page* page = pfn_to_page(pte_pfn(series.orig_ptes[index]));
-		series.pages[index] = page;
+	} else if (pte_unmapped(series->orig_ptes[index])) {
		if (!PageSwapCache(page))
			return 3;
		else {
@@ -1148,19 +1147,20 @@
		return 6;
}

-static void find_series(pte_t** start, unsigned long* addr, unsigned long end)
+static void find_series(struct series_t* series, pte_t** start, unsigned long*
+		addr, unsigned long end)
{
	int i;
-	int series_stage = get_series_stage((*start)++, 0);
+	int series_stage = get_series_stage(series, (*start)++, 0);
	*addr += PAGE_SIZE;

	for (i = 1; i < MAX_SERIES_LENGTH && *addr < end; i++, (*start)++,
		*addr += PAGE_SIZE) {
-		if (series_stage != get_series_stage(*start, i))
+		if (series_stage != get_series_stage(series, *start, i))
			break;
	}
-	series.series_stage = series_stage;
-	series.series_length = i;
+	series->series_stage = series_stage;
+	series->series_length = i;
}

struct delay_tlb_task delay_tlb_tasks[32] = { [0 ... 31] = {0} };
@@ -1284,9 +1284,9 @@
	goto fill_it;
}

-static void shrink_pvma_scan_ptes(struct scan_control* sc, struct mm_struct*
-		mm, struct vm_area_struct* vma, pmd_t* pmd, unsigned long addr,
-		unsigned long end)
+static unsigned long shrink_pvma_scan_ptes(struct scan_control* sc, struct
+		mm_struct* mm, struct vm_area_struct* vma, pmd_t* pmd, unsigned
+		long addr, unsigned long end)
{
	int i, statistic;
	spinlock_t* ptl = pte_lockptr(mm, pmd);
@@ -1295,32 +1295,43 @@
	struct pagevec freed_pvec;
	int may_enter_fs = (sc->gfp_mask & (__GFP_FS | __GFP_IO));
	struct address_space* mapping = &swapper_space;
+	unsigned long nr_reclaimed = 0;
+	struct series_t series;

	pagevec_init(&freed_pvec, 1);
	do {
		memset(&series, 0, sizeof(struct series_t));
-		find_series(&pte, &addr, end);
+		find_series(&series, &pte, &addr, end);
		if (sc->may_reclaim == 0 && series.series_stage == 5)
			continue;
+		if (!sc->is_kppsd && series.series_stage != 5)
+			continue;
		switch (series.series_stage) {
		case 1: // PTE -- untouched PTE.
		for (i = 0; i < series.series_length; i++) {
			struct page* page = series.pages[i];
-			lock_page(page);
+			if (TestSetPageLocked(page))
+				continue;
			spin_lock(ptl);
-			if (unlikely(pte_same(*series.ptes[i],
-					series.orig_ptes[i]))) {
-				if (pte_dirty(*series.ptes[i]))
-				    set_page_dirty(page);
-				set_pte_at(mm, addr + i * PAGE_SIZE,
-					series.ptes[i],
-					pte_mkold(pte_mkclean(*series.ptes[i])));
+			// To get dirty bit from pte safely, using the idea of
+			// dftlb of stage 2.
+			pte_t pte_new = series.orig_ptes[i];
+			pte_new = pte_mkold(pte_mkclean(series.orig_ptes[i]));
+			if (cmpxchg(&series.ptes[i]->pte_low,
+						series.orig_ptes[i].pte_low,
+						pte_new.pte_low) !=
+				series.orig_ptes[i].pte_low) {
+				spin_unlock(ptl);
+				unlock_page(page);
+				continue;
			}
+			if (pte_dirty(series.orig_ptes[i]))
+				set_page_dirty(page);
			spin_unlock(ptl);
			unlock_page(page);
		}
		fill_in_tlb_tasks(vma, addr, addr + (PAGE_SIZE *
-			    series.series_length));
+					series.series_length));
		break;
		case 2: // untouched PTE -- UnmappedPTE.
		/*
@@ -1335,37 +1346,39 @@
		spin_lock(ptl);
		statistic = 0;
		for (i = 0; i < series.series_length; i++) {
-			if (unlikely(pte_same(*series.ptes[i],
-					series.orig_ptes[i]))) {
-				pte_t pte_unmapped = series.orig_ptes[i];
-				pte_unmapped.pte_low &= ~_PAGE_PRESENT;
-				pte_unmapped.pte_low |= _PAGE_UNMAPPED;
-				if (cmpxchg(&series.ptes[i]->pte_low,
-					    series.orig_ptes[i].pte_low,
-					    pte_unmapped.pte_low) !=
-					series.orig_ptes[i].pte_low)
-					continue;
-				page_remove_rmap(series.pages[i], vma);
-				anon_rss--;
-				statistic++;
-			}
+			pte_t pte_unmapped = series.orig_ptes[i];
+			pte_unmapped.pte_low &= ~_PAGE_PRESENT;
+			pte_unmapped.pte_low |= _PAGE_UNMAPPED;
+			if (cmpxchg(&series.ptes[i]->pte_low,
+						series.orig_ptes[i].pte_low,
+						pte_unmapped.pte_low) !=
+				series.orig_ptes[i].pte_low)
+				continue;
+			page_remove_rmap(series.pages[i], vma);
+			anon_rss--;
+			statistic++;
		}
		atomic_add(statistic, &pps_info.unmapped_count);
		atomic_sub(statistic, &pps_info.pte_count);
		spin_unlock(ptl);
-		break;
+		if (!accelerate_kppsd)
+			break;
		case 3: // Attach SwapPage to PrivatePage.
		/*
		 * A better arithmetic should be applied to Linux SwapDevice to
		 * allocate fake continual SwapPages which are close to each
		 * other, the offset between two close SwapPages is less than 8.
+		 *
+		 * We can re-allocate SwapPages here if process private pages
+		 * are pure private.
		 */
		if (sc->may_swap) {
			for (i = 0; i < series.series_length; i++) {
-				lock_page(series.pages[i]);
+				if (TestSetPageLocked(series.pages[i]))
+					continue;
				if (!PageSwapCache(series.pages[i])) {
					if (!add_to_swap(series.pages[i],
-						    GFP_ATOMIC)) {
+								GFP_ATOMIC)) {
						unlock_page(series.pages[i]);
						break;
					}
@@ -1373,45 +1386,49 @@
				unlock_page(series.pages[i]);
			}
		}
-		break;
+		if (!accelerate_kppsd)
+			break;
		case 4: // SwapPage isn't consistent with PrivatePage.
		/*
		 * A mini version pageout().
		 *
		 * Current swap space can't commit multiple pages together:(
		 */
-		if (sc->may_writepage && may_enter_fs) {
-			for (i = 0; i < series.series_length; i++) {
-				struct page* page = series.pages[i];
-				int res;
+		if (!(sc->may_writepage && may_enter_fs))
+			break;
+		for (i = 0; i < series.series_length; i++) {
+			struct page* page = series.pages[i];
+			int res;

-				if (!may_write_to_queue(mapping->backing_dev_info))
-					break;
-				lock_page(page);
-				if (!PageDirty(page) || PageWriteback(page)) {
-					unlock_page(page);
-					continue;
-				}
-				clear_page_dirty_for_io(page);
-				struct writeback_control wbc = {
-					.sync_mode = WB_SYNC_NONE,
-					.nr_to_write = SWAP_CLUSTER_MAX,
-					.nonblocking = 1,
-					.for_reclaim = 1,
-				};
-				page_cache_get(page);
-				SetPageReclaim(page);
-				res = swap_writepage(page, &wbc);
-				if (res < 0) {
-					handle_write_error(mapping, page, res);
-					ClearPageReclaim(page);
-					page_cache_release(page);
-					break;
-				}
-				if (!PageWriteback(page))
-					ClearPageReclaim(page);
+			if (!may_write_to_queue(mapping->backing_dev_info))
+				break;
+			if (TestSetPageLocked(page))
+				continue;
+			if (!PageDirty(page) || PageWriteback(page)) {
+				unlock_page(page);
+				continue;
+			}
+			clear_page_dirty_for_io(page);
+			struct writeback_control wbc = {
+				.sync_mode = WB_SYNC_NONE,
+				.nr_to_write = SWAP_CLUSTER_MAX,
+				.range_start = 0,
+				.range_end = LLONG_MAX,
+				.nonblocking = 1,
+				.for_reclaim = 1,
+			};
+			page_cache_get(page);
+			SetPageReclaim(page);
+			res = swap_writepage(page, &wbc);
+			if (res < 0) {
+				handle_write_error(mapping, page, res);
+				ClearPageReclaim(page);
				page_cache_release(page);
+				break;
			}
+			if (!PageWriteback(page))
+				ClearPageReclaim(page);
+			page_cache_release(page);
		}
		break;
		case 5: // UnmappedPTE -- SwappedPTE, reclaim PrivatePage.
@@ -1419,10 +1436,11 @@
		for (i = 0; i < series.series_length; i++) {
			struct page* page = series.pages[i];
			if (!(page_to_nid(page) == sc->reclaim_node ||
-				    sc->reclaim_node == -1))
+						sc->reclaim_node == -1))
				continue;

-			lock_page(page);
+			if (TestSetPageLocked(page))
+				continue;
			spin_lock(ptl);
			if (!pte_same(*series.ptes[i], series.orig_ptes[i]) ||
					/* We're racing with get_user_pages. */
@@ -1449,6 +1467,7 @@
		atomic_add(statistic, &pps_info.swapped_count);
		atomic_sub(statistic, &pps_info.unmapped_count);
		atomic_sub(statistic, &pps_info.total);
+		nr_reclaimed += statistic;
		break;
		case 6:
		// NULL operation!
@@ -1456,58 +1475,67 @@
		}
	} while (addr < end);
	add_mm_counter(mm, anon_rss, anon_rss);
+	pte_unmap(pte);
	if (pagevec_count(&freed_pvec))
		__pagevec_release_nonlru(&freed_pvec);
+	return nr_reclaimed;
}

-static void shrink_pvma_pmd_range(struct scan_control* sc, struct mm_struct*
-		mm, struct vm_area_struct* vma, pud_t* pud, unsigned long addr,
-		unsigned long end)
+static unsigned long shrink_pvma_pmd_range(struct scan_control* sc, struct
+		mm_struct* mm, struct vm_area_struct* vma, pud_t* pud, unsigned
+		long addr, unsigned long end)
{
	unsigned long next;
+	unsigned long nr_reclaimed = 0;
	pmd_t* pmd = pmd_offset(pud, addr);
	do {
		next = pmd_addr_end(addr, end);
		if (pmd_none_or_clear_bad(pmd))
			continue;
-		shrink_pvma_scan_ptes(sc, mm, vma, pmd, addr, next);
+		nr_reclaimed += shrink_pvma_scan_ptes(sc, mm, vma, pmd, addr, next);
	} while (pmd++, addr = next, addr != end);
+	return nr_reclaimed;
}

-static void shrink_pvma_pud_range(struct scan_control* sc, struct mm_struct*
-		mm, struct vm_area_struct* vma, pgd_t* pgd, unsigned long addr,
-		unsigned long end)
+static unsigned long shrink_pvma_pud_range(struct scan_control* sc, struct
+		mm_struct* mm, struct vm_area_struct* vma, pgd_t* pgd, unsigned
+		long addr, unsigned long end)
{
	unsigned long next;
+	unsigned long nr_reclaimed = 0;
	pud_t* pud = pud_offset(pgd, addr);
	do {
		next = pud_addr_end(addr, end);
		if (pud_none_or_clear_bad(pud))
			continue;
-		shrink_pvma_pmd_range(sc, mm, vma, pud, addr, next);
+		nr_reclaimed += shrink_pvma_pmd_range(sc, mm, vma, pud, addr, next);
	} while (pud++, addr = next, addr != end);
+	return nr_reclaimed;
}

-static void shrink_pvma_pgd_range(struct scan_control* sc, struct mm_struct*
-		mm, struct vm_area_struct* vma)
+static unsigned long shrink_pvma_pgd_range(struct scan_control* sc, struct
+		mm_struct* mm, struct vm_area_struct* vma)
{
	unsigned long next;
	unsigned long addr = vma->vm_start;
	unsigned long end = vma->vm_end;
+	unsigned long nr_reclaimed = 0;
	pgd_t* pgd = pgd_offset(mm, addr);
	do {
		next = pgd_addr_end(addr, end);
		if (pgd_none_or_clear_bad(pgd))
			continue;
-		shrink_pvma_pud_range(sc, mm, vma, pgd, addr, next);
+		nr_reclaimed += shrink_pvma_pud_range(sc, mm, vma, pgd, addr, next);
	} while (pgd++, addr = next, addr != end);
+	return nr_reclaimed;
}

-static void shrink_private_vma(struct scan_control* sc)
+static unsigned long shrink_private_vma(struct scan_control* sc)
{
	struct vm_area_struct* vma;
	struct list_head *pos;
	struct mm_struct *prev, *mm;
+	unsigned long nr_reclaimed = 0;

	prev = mm = &init_mm;
	pos = &init_mm.mmlist;
@@ -1520,22 +1548,25 @@
		spin_unlock(&mmlist_lock);
		mmput(prev);
		prev = mm;
-		start_tlb_tasks(mm);
+		if (sc->is_kppsd)
+			start_tlb_tasks(mm);
		if (down_read_trylock(&mm->mmap_sem)) {
			for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
				if (!(vma->vm_flags & VM_PURE_PRIVATE))
					continue;
				if (vma->vm_flags & VM_LOCKED)
					continue;
-				shrink_pvma_pgd_range(sc, mm, vma);
+				nr_reclaimed += shrink_pvma_pgd_range(sc, mm, vma);
			}
			up_read(&mm->mmap_sem);
		}
-		end_tlb_tasks();
+		if (sc->is_kppsd)
+			end_tlb_tasks();
		spin_lock(&mmlist_lock);
	}
	spin_unlock(&mmlist_lock);
	mmput(prev);
+	return nr_reclaimed;
}

/*
@@ -1585,10 +1616,12 @@
	sc.may_writepage = !laptop_mode;
	count_vm_event(PAGEOUTRUN);

-	wakeup_sc = sc;
-	wakeup_sc.may_reclaim = 1;
-	wakeup_sc.reclaim_node = pgdat->node_id;
-	wake_up_interruptible(&kppsd_wait);
+	accelerate_kppsd++;
+	wake_up(&kppsd_wait);
+	sc.may_reclaim = 1;
+	sc.reclaim_node = pgdat->node_id;
+	sc.is_kppsd = 0;
+	nr_reclaimed += shrink_private_vma(&sc);

	for (i = 0; i < pgdat->nr_zones; i++)
		temp_priority[i] = DEF_PRIORITY;
@@ -2173,26 +2206,24 @@
static int kppsd(void* p)
{
	struct task_struct *tsk = current;
-	int timeout;
	DEFINE_WAIT(wait);
	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE;
	struct scan_control default_sc;
	default_sc.gfp_mask = GFP_KERNEL;
-	default_sc.may_writepage = 1;
	default_sc.may_swap = 1;
	default_sc.may_reclaim = 0;
	default_sc.reclaim_node = -1;
+	default_sc.is_kppsd = 1;

	while (1) {
		try_to_freeze();
-		prepare_to_wait(&kppsd_wait, &wait, TASK_INTERRUPTIBLE);
-		timeout = schedule_timeout(2000);
-		finish_wait(&kppsd_wait, &wait);
-
-		if (timeout)
-			shrink_private_vma(&wakeup_sc);
-		else
-			shrink_private_vma(&default_sc);
+		accelerate_kppsd >>= 1;
+		wait_event_timeout(kppsd_wait, accelerate_kppsd != 0,
+				msecs_to_jiffies(2000));
+		default_sc.may_writepage = !laptop_mode;
+		if (accelerate_kppsd)
+			default_sc.may_writepage = 1;
+		shrink_private_vma(&default_sc);
	}
	return 0;
}
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/