Re: [PATCH] add a clear_pages function to clear pages of higherorder
From: Christoph Lameter
Date: Thu Mar 24 2005 - 21:52:10 EST
On Thu, 24 Mar 2005, David S. Miller wrote:
> Erm... were any of your test builds done with the new CONFIG_CLEAR_COLD
> option enabled? :-)
These were all fixed but I failed to do a "quilt refresh" .... sigh... The
email issues are also fixed now .... sigh. What a day.
> Next, replace your arch/sparc64/lib/clear_page.S diff with this one and
> things would be working and we'll be using the proper temporal vs.
> non-temporal stores on that platform.
Thanks.
Here is the patch with your changes and a "quilt refresh" ;-)
---------------------------------------------------------------------
Introduces a new function clear_cold(void *pageaddress, int order) to clear
pages of an arbitrary size with non temporal stores. Cold clearing is typically
faster than hot clearing. Hot clearing is beneficial when the data is to be used soon.
(Will also work well with the new hot and cold aware prezeroing daemon)
Use cold clearing for huge pages.
For ia64 also make clear_page uses temporal stores.
Patch needs fixes to work properly on i386 and x86_64.
Signed-off-by: Christoph Lameter <clameter@xxxxxxx>
Index: linux-2.6.11/mm/hugetlb.c
===================================================================
--- linux-2.6.11.orig/mm/hugetlb.c 2005-03-01 23:38:12.000000000 -0800
+++ linux-2.6.11/mm/hugetlb.c 2005-03-24 14:12:53.000000000 -0800
@@ -78,7 +78,6 @@ void free_huge_page(struct page *page)
struct page *alloc_huge_page(void)
{
struct page *page;
- int i;
spin_lock(&hugetlb_lock);
page = dequeue_huge_page();
@@ -89,8 +88,7 @@ struct page *alloc_huge_page(void)
spin_unlock(&hugetlb_lock);
set_page_count(page, 1);
page[1].mapping = (void *)free_huge_page;
- for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
- clear_highpage(&page[i]);
+ prep_zero_page(page, HUGETLB_PAGE_ORDER, GFP_HIGHUSER | __GFP_COLD);
return page;
}
Index: linux-2.6.11/mm/page_alloc.c
===================================================================
--- linux-2.6.11.orig/mm/page_alloc.c 2005-03-24 13:15:40.000000000 -0800
+++ linux-2.6.11/mm/page_alloc.c 2005-03-24 18:39:22.000000000 -0800
@@ -633,11 +633,17 @@ void fastcall free_cold_page(struct page
free_hot_cold_page(page, 1);
}
-static inline void prep_zero_page(struct page *page, int order, int gfp_flags)
+void prep_zero_page(struct page *page, unsigned int order, int gfp_flags)
{
int i;
BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM);
+
+#ifdef CONFIG_CLEAR_COLD
+ if ((gfp_flags & __GFP_COLD) && !PageHighMem(page))
+ clear_cold(page_address(page), order);
+ else
+#endif
for(i = 0; i < (1 << order); i++)
clear_highpage(page + i);
}
Index: linux-2.6.11/include/linux/gfp.h
===================================================================
--- linux-2.6.11.orig/include/linux/gfp.h 2005-03-01 23:37:50.000000000 -0800
+++ linux-2.6.11/include/linux/gfp.h 2005-03-24 14:16:44.000000000 -0800
@@ -131,4 +131,5 @@ extern void FASTCALL(free_cold_page(stru
void page_alloc_init(void);
+void prep_zero_page(struct page *, unsigned int order, int gfp_flags);
#endif /* __LINUX_GFP_H */
Index: linux-2.6.11/arch/ia64/Kconfig
===================================================================
--- linux-2.6.11.orig/arch/ia64/Kconfig 2005-03-01 23:38:26.000000000 -0800
+++ linux-2.6.11/arch/ia64/Kconfig 2005-03-24 14:12:53.000000000 -0800
@@ -46,6 +46,10 @@ config GENERIC_IOMAP
bool
default y
+config CLEAR_COLD
+ bool
+ default y
+
choice
prompt "System type"
default IA64_GENERIC
Index: linux-2.6.11/include/asm-ia64/page.h
===================================================================
--- linux-2.6.11.orig/include/asm-ia64/page.h 2005-03-01 23:37:48.000000000 -0800
+++ linux-2.6.11/include/asm-ia64/page.h 2005-03-24 14:12:53.000000000 -0800
@@ -57,6 +57,8 @@
# define STRICT_MM_TYPECHECKS
extern void clear_page (void *page);
+/* Clear arbitrary order page using nontemporal writes */
+extern void clear_cold (void *page, unsigned int order);
extern void copy_page (void *to, void *from);
/*
Index: linux-2.6.11/arch/ia64/kernel/ia64_ksyms.c
===================================================================
--- linux-2.6.11.orig/arch/ia64/kernel/ia64_ksyms.c 2005-03-01 23:38:08.000000000 -0800
+++ linux-2.6.11/arch/ia64/kernel/ia64_ksyms.c 2005-03-24 14:12:53.000000000 -0800
@@ -39,6 +39,7 @@ EXPORT_SYMBOL(__up);
#include <asm/page.h>
EXPORT_SYMBOL(clear_page);
+EXPORT_SYMBOL(clear_cold);
#ifdef CONFIG_VIRTUAL_MEM_MAP
#include <linux/bootmem.h>
Index: linux-2.6.11/arch/ia64/lib/clear_page.S
===================================================================
--- linux-2.6.11.orig/arch/ia64/lib/clear_page.S 2005-03-01 23:37:47.000000000 -0800
+++ linux-2.6.11/arch/ia64/lib/clear_page.S 2005-03-24 14:24:29.000000000 -0800
@@ -7,6 +7,8 @@
* 1/06/01 davidm Tuned for Itanium.
* 2/12/02 kchen Tuned for both Itanium and McKinley
* 3/08/02 davidm Some more tweaking
+ * 24/3/04 clameter Make clear_page use temporal stores
+ add clear_cold using nontemporal stores
*/
#include <linux/config.h>
@@ -53,6 +55,59 @@ GLOBAL_ENTRY(clear_page)
;;
#ifdef CONFIG_ITANIUM
// Optimized for Itanium
+1: stf.spill [dst1] = f0, 64
+ stf.spill [dst2] = f0, 64
+ cmp.lt p8,p0=dst_fetch, dst_last
+ ;;
+#else
+ // Optimized for McKinley
+1: stf.spill [dst1] = f0, 64
+ stf.spill [dst2] = f0, 64
+ stf.spill [dst3] = f0, 64
+ stf.spill [dst4] = f0, 128
+ cmp.lt p8,p0=dst_fetch, dst_last
+ ;;
+ stf.spill [dst1] = f0, 64
+ stf.spill [dst2] = f0, 64
+#endif
+ stf.spill [dst3] = f0, 64
+(p8) stf.spill [dst_fetch] = f0, L3_LINE_SIZE
+ br.cloop.sptk.few 1b
+ ;;
+ mov ar.lc = saved_lc // restore lc
+ br.ret.sptk.many rp
+END(clear_page)
+
+#define totsize r14
+
+GLOBAL_ENTRY(clear_cold)
+ .prologue
+ .regstk 2,0,0,0
+ mov r16 = PAGE_SIZE/L3_LINE_SIZE // main loop count
+ mov totsize = PAGE_SIZE
+ .save ar.lc, saved_lc
+ mov saved_lc = ar.lc
+ ;;
+ .body
+ adds dst1 = 16, in0
+ mov ar.lc = (PREFETCH_LINES - 1)
+ mov dst_fetch = in0
+ adds dst2 = 32, in0
+ shl r16 = r16, in1
+ shl totsize = totsize, in1
+ ;;
+.fetc: stf.spill.nta [dst_fetch] = f0, L3_LINE_SIZE
+ adds dst3 = 48, in0 // executing this multiple times is harmless
+ br.cloop.sptk.few .fetc
+ add r16 = -1,r16
+ add dst_last = totsize, dst_fetch
+ adds dst4 = 64, in0
+ ;;
+ mov ar.lc = r16 // one L3 line per iteration
+ adds dst_last = -PREFETCH_LINES*L3_LINE_SIZE, dst_last
+ ;;
+#ifdef CONFIG_ITANIUM
+ // Optimized for Itanium
1: stf.spill.nta [dst1] = f0, 64
stf.spill.nta [dst2] = f0, 64
cmp.lt p8,p0=dst_fetch, dst_last
@@ -74,4 +129,4 @@ GLOBAL_ENTRY(clear_page)
;;
mov ar.lc = saved_lc // restore lc
br.ret.sptk.many rp
-END(clear_page)
+END(clear_cold)
Index: linux-2.6.11/arch/i386/Kconfig
===================================================================
--- linux-2.6.11.orig/arch/i386/Kconfig 2005-03-24 13:15:36.000000000 -0800
+++ linux-2.6.11/arch/i386/Kconfig 2005-03-24 14:12:53.000000000 -0800
@@ -33,6 +33,10 @@ config GENERIC_IOMAP
bool
default y
+config CLEAR_COLD
+ bool
+ default y
+
source "init/Kconfig"
menu "Processor type and features"
Index: linux-2.6.11/include/asm-i386/page.h
===================================================================
--- linux-2.6.11.orig/include/asm-i386/page.h 2005-03-01 23:37:49.000000000 -0800
+++ linux-2.6.11/include/asm-i386/page.h 2005-03-24 14:12:53.000000000 -0800
@@ -19,6 +19,7 @@
#include <asm/mmx.h>
#define clear_page(page) mmx_clear_page((void *)(page))
+#define clear_cold(page, order) mmx_clear_cold((void *)(page), order)
#define copy_page(to,from) mmx_copy_page(to,from)
#else
@@ -29,6 +30,8 @@
*/
#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
+/* Clear arbitrary order page with nontemporal stores... is memset temporal?? */
+#define clear_cold(page, order) memset((void *)(page), 0, PAGE_SIZE << order)
#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE)
#endif
Index: linux-2.6.11/include/asm-i386/mmx.h
===================================================================
--- linux-2.6.11.orig/include/asm-i386/mmx.h 2005-03-01 23:38:09.000000000 -0800
+++ linux-2.6.11/include/asm-i386/mmx.h 2005-03-24 14:12:53.000000000 -0800
@@ -9,6 +9,7 @@
extern void *_mmx_memcpy(void *to, const void *from, size_t size);
extern void mmx_clear_page(void *page);
+extern void mmx_clear_cold(void *page, unsigned int order);
extern void mmx_copy_page(void *to, void *from);
#endif
Index: linux-2.6.11/arch/i386/lib/mmx.c
===================================================================
--- linux-2.6.11.orig/arch/i386/lib/mmx.c 2005-03-01 23:38:09.000000000 -0800
+++ linux-2.6.11/arch/i386/lib/mmx.c 2005-03-24 14:12:53.000000000 -0800
@@ -397,3 +397,14 @@ void mmx_copy_page(void *to, void *from)
else
fast_copy_page(to, from);
}
+
+/* FIXME: Make this a real cold zeroing function */
+void mmx_clear_cold(void *page, int order)
+{
+ int i;
+
+ for(i=0; i < (1 << order); i++) {
+ mmx_clear_page(page);
+ page += PAGE_SIZE;
+ }
+}
Index: linux-2.6.11/arch/x86_64/Kconfig
===================================================================
--- linux-2.6.11.orig/arch/x86_64/Kconfig 2005-03-24 13:15:37.000000000 -0800
+++ linux-2.6.11/arch/x86_64/Kconfig 2005-03-24 14:12:53.000000000 -0800
@@ -78,6 +78,10 @@ config GENERIC_IOMAP
bool
default y
+config CLEAR_COLD
+ bool
+ default y
+
source "init/Kconfig"
Index: linux-2.6.11/include/asm-x86_64/page.h
===================================================================
--- linux-2.6.11.orig/include/asm-x86_64/page.h 2005-03-01 23:37:47.000000000 -0800
+++ linux-2.6.11/include/asm-x86_64/page.h 2005-03-24 14:12:53.000000000 -0800
@@ -33,6 +33,8 @@
#ifndef __ASSEMBLY__
void clear_page(void *);
+/* Clear arbitrary order page using non-temporal writes */
+void clear_cold(void *, int order);
void copy_page(void *, void *);
#define clear_user_page(page, vaddr, pg) clear_page(page)
Index: linux-2.6.11/arch/x86_64/kernel/x8664_ksyms.c
===================================================================
--- linux-2.6.11.orig/arch/x86_64/kernel/x8664_ksyms.c 2005-03-24 13:15:37.000000000 -0800
+++ linux-2.6.11/arch/x86_64/kernel/x8664_ksyms.c 2005-03-24 14:12:53.000000000 -0800
@@ -108,6 +108,7 @@ EXPORT_SYMBOL(pci_mem_start);
EXPORT_SYMBOL(copy_page);
EXPORT_SYMBOL(clear_page);
+EXPORT_SYMBOL(clear_cold);
EXPORT_SYMBOL(cpu_pda);
#ifdef CONFIG_SMP
Index: linux-2.6.11/arch/x86_64/lib/clear_page.S
===================================================================
--- linux-2.6.11.orig/arch/x86_64/lib/clear_page.S 2005-03-01 23:38:08.000000000 -0800
+++ linux-2.6.11/arch/x86_64/lib/clear_page.S 2005-03-24 14:12:53.000000000 -0800
@@ -48,3 +48,57 @@ clear_page_c:
ret
clear_page_c_end:
.previous
+
+
+/*
+ * Zero a page cold.
+ * rdi page
+ * rsi order
+ */
+ .globl clear_cold
+ .p2align 4
+clear_cold:
+ movl $4096/64,%eax
+ movl %esi, %ecx
+ shll %cl, %eax
+ movl %eax, %ecx
+ xorl %eax,%eax
+ .p2align 4
+.Lcloop:
+ decl %ecx
+#define PUTC(x) movq %rax,x*8(%rdi)
+ movq %rax,(%rdi)
+ PUTC(1)
+ PUTC(2)
+ PUTC(3)
+ PUTC(4)
+ PUTC(5)
+ PUTC(6)
+ PUTC(7)
+ leaq 64(%rdi),%rdi
+ jnz .Lcloop
+ nop
+ ret
+clear_cold_end:
+
+ .section .altinstructions,"a"
+ .align 8
+ .quad clear_cold
+ .quad clear_cold_c
+ .byte X86_FEATURE_K8_C
+ .byte clear_cold_end-clear_cold
+ .byte clear_cold_c_end-clear_cold_c
+ .previous
+
+ .section .altinstr_replacement,"ax"
+clear_cold_c:
+ movl $4096/8,%eax
+ movl %esi, %ecx
+ shll %cl, %eax
+ movl %eax, %ecx
+ xorl %eax,%eax
+ rep
+ stosq
+ ret
+clear_cold_c_end:
+ .previous
Index: linux-2.6.11/arch/sparc64/lib/clear_page.S
===================================================================
--- linux-2.6.11.orig/arch/sparc64/lib/clear_page.S 2005-03-01 23:38:17.000000000 -0800
+++ linux-2.6.11/arch/sparc64/lib/clear_page.S 2005-03-24 18:39:44.000000000 -0800
@@ -72,13 +72,127 @@ clear_user_page: /* %o0=dest, %o1=vaddr
mov 1, %o4
clear_page_common:
+ membar #StoreLoad | #StoreStore | #LoadStore
+ sethi %hi(PAGE_SIZE/64), %o1
+ mov %o0, %g1 ! remember vaddr for tlbflush
+ or %o1, %lo(PAGE_SIZE/64), %o1
+
+#define PREFETCH(x, y) prefetch x, y
+#define PREFETCH_CODE 2
+
+ PREFETCH([%o0 + 0x000], PREFETCH_CODE)
+ PREFETCH([%o0 + 0x040], PREFETCH_CODE)
+ PREFETCH([%o0 + 0x080], PREFETCH_CODE)
+ PREFETCH([%o0 + 0x0c0], PREFETCH_CODE)
+ PREFETCH([%o0 + 0x100], PREFETCH_CODE)
+ PREFETCH([%o0 + 0x140], PREFETCH_CODE)
+ PREFETCH([%o0 + 0x180], PREFETCH_CODE)
+1:
+ stx %g0, [%o0 + 0x00]
+ stx %g0, [%o0 + 0x08]
+ stx %g0, [%o0 + 0x10]
+ stx %g0, [%o0 + 0x18]
+ stx %g0, [%o0 + 0x20]
+ stx %g0, [%o0 + 0x28]
+ stx %g0, [%o0 + 0x30]
+ stx %g0, [%o0 + 0x38]
+ PREFETCH([%o0 + 0x1c0], PREFETCH_CODE)
+ subcc %o1, 1, %o1
+ bne,pt %icc, 1b
+ add %o0, 0x40, %o0
+
+ brz,pn %o4, out
+ nop
+
+ stxa %g0, [%g1] ASI_DMMU_DEMAP
+ membar #Sync
+ stw %o2, [%g6 + TI_PRE_COUNT]
+
+out: retl
+ nop
+
+ .globl clear_cold
+clear_cold: /* %o0=dest, %o1=order */
+ sethi %hi(PAGE_SIZE/64), %o2
+ clr %o4
+ or %o2, %lo(PAGE_SIZE/64), %o2
+ sllx %o2, %o1, %o1
VISEntryHalf
membar #StoreLoad | #StoreStore | #LoadStore
fzero %f0
+ fzero %f2
+ faddd %f0, %f2, %f4
+ fmuld %f0, %f2, %f6
+ faddd %f0, %f2, %f8
+ fmuld %f0, %f2, %f10
+
+ faddd %f0, %f2, %f12
+ fmuld %f0, %f2, %f14
+2: stda %f0, [%o0 + %g0] ASI_BLK_P
+ subcc %o1, 1, %o1
+ bne,pt %icc, 2b
+ add %o0, 0x40, %o0
+ membar #Sync
+ VISExitHalf
+
+ retl
+ nop
+
+ .globl clear_cold
+clear_cold: /* %o0=dest, %o1=order */
+ sethi %hi(PAGE_SIZE/64), %o2
+ clr %o4
+ or %o2, %lo(PAGE_SIZE/64), %o2
+ ba,pt %xcc, clear_cold_common
+ sllx %o2, %o1, %o1
+
+ /* This thing is pretty important, it shows up
+ * on the profiles via do_anonymous_page().
+ */
+ .align 32
+ .globl clear_cold_page
+clear_cold_user_page: /* %o0=dest, %o1=vaddr */
+ lduw [%g6 + TI_PRE_COUNT], %o2
+ sethi %uhi(PAGE_OFFSET), %g2
+ sethi %hi(PAGE_SIZE), %o4
+
+ sllx %g2, 32, %g2
+ sethi %uhi(TTE_BITS_TOP), %g3
+
+ sllx %g3, 32, %g3
+ sub %o0, %g2, %g1 ! paddr
+
+ or %g3, TTE_BITS_BOTTOM, %g3
+ and %o1, %o4, %o0 ! vaddr D-cache alias bit
+
+ or %g1, %g3, %g1 ! TTE data
+ sethi %hi(TLBTEMP_BASE), %o3
+
+ add %o2, 1, %o4
+ add %o0, %o3, %o0 ! TTE vaddr
+
+ /* Disable preemption. */
+ mov TLB_TAG_ACCESS, %g3
+ stw %o4, [%g6 + TI_PRE_COUNT]
+
+ /* Load TLB entry. */
+ rdpr %pstate, %o4
+ wrpr %o4, PSTATE_IE, %pstate
+ stxa %o0, [%g3] ASI_DMMU
+ stxa %g1, [%g0] ASI_DTLB_DATA_IN
+ flush %g6
+ wrpr %o4, 0x0, %pstate
+
sethi %hi(PAGE_SIZE/64), %o1
+ mov 1, %o4
+ or %o1, %lo(PAGE_SIZE/64), %o1
+
+clear_cold_common:
+ VISEntryHalf
+ membar #StoreLoad | #StoreStore | #LoadStore
+ fzero %f0
mov %o0, %g1 ! remember vaddr for tlbflush
fzero %f2
- or %o1, %lo(PAGE_SIZE/64), %o1
faddd %f0, %f2, %f4
fmuld %f0, %f2, %f6
faddd %f0, %f2, %f8
@@ -86,20 +200,20 @@ clear_page_common:
faddd %f0, %f2, %f12
fmuld %f0, %f2, %f14
-1: stda %f0, [%o0 + %g0] ASI_BLK_P
+2: stda %f0, [%o0 + %g0] ASI_BLK_P
subcc %o1, 1, %o1
- bne,pt %icc, 1b
+ bne,pt %icc, 2b
add %o0, 0x40, %o0
membar #Sync
VISExitHalf
- brz,pn %o4, out
+ brz,pn %o4, outcold
nop
stxa %g0, [%g1] ASI_DMMU_DEMAP
membar #Sync
stw %o2, [%g6 + TI_PRE_COUNT]
-out: retl
+outcold: retl
nop
Index: linux-2.6.11/include/asm-sparc64/page.h
===================================================================
--- linux-2.6.11.orig/include/asm-sparc64/page.h 2005-03-01 23:38:07.000000000 -0800
+++ linux-2.6.11/include/asm-sparc64/page.h 2005-03-24 14:12:53.000000000 -0800
@@ -16,6 +16,8 @@
extern void _clear_page(void *page);
#define clear_page(X) _clear_page((void *)(X))
+/* Non temporal clear an arbitrary order page */
+extern void clear_cold(void *page, unsigned int order);
struct page;
extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page);
#define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE)
Index: linux-2.6.11/arch/sparc64/Kconfig
===================================================================
--- linux-2.6.11.orig/arch/sparc64/Kconfig 2005-03-01 23:38:25.000000000 -0800
+++ linux-2.6.11/arch/sparc64/Kconfig 2005-03-24 14:12:53.000000000 -0800
@@ -16,6 +16,10 @@ config TIME_INTERPOLATION
bool
default y
+config CLEAR_COLD
+ bool
+ default y
+
source "init/Kconfig"
config SYSVIPC_COMPAT
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/