[parisc-linux] Possibly working for/exec speedup patch
James Bottomley
James.Bottomley at steeleye.com
Sun Apr 18 11:12:40 MDT 2004
This one has degraded slightly from the previous (still showing a 50%
decrease in fork/exec, though).
This one, I can't actually seem to break under stress (although I'm sure
someone else will manage...).
James
# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher.
# This patch includes the following deltas:
# ChangeSet 1.1792 -> 1.1799
# mm/fremap.c 1.16 -> 1.18
# include/asm-parisc/cache.h 1.6 -> 1.8
# arch/parisc/kernel/pacache.S 1.5 -> 1.8
# include/asm-parisc/page.h 1.5 -> 1.8
# arch/parisc/kernel/entry.S 1.14 -> 1.17
# arch/parisc/kernel/cache.c 1.12 -> 1.15
# mm/memory.c 1.154 -> 1.157
# include/asm-parisc/tlb.h 1.1 -> 1.2
# arch/parisc/kernel/sys_parisc.c 1.17 -> 1.18
# include/asm-parisc/pgtable.h 1.16 -> 1.18
# include/asm-parisc/cacheflush.h 1.9 -> 1.14
# arch/parisc/kernel/signal.c 1.14 -> 1.15
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 04/04/14 jejb at raven.il.steeleye.com 1.1790.1.1
# ICI
# --------------------------------------------
# 04/04/14 jejb at raven.il.steeleye.com 1.1793
# hand
# --------------------------------------------
# 04/04/15 jejb at raven.il.steeleye.com 1.1794
# First draft of working stingy cache flushing
# --------------------------------------------
# 04/04/16 jejb at raven.il.steeleye.com 1.1795
# More clean ups
# --------------------------------------------
# 04/04/16 jejb at raven.il.steeleye.com 1.1796
# Fix flushing
#
# - eliminate flush_icache_page()
# - make copy_user_page do correct flushing
# --------------------------------------------
# 04/04/18 jejb at raven.il.steeleye.com 1.1797
# Add missing page alignment to cache flush arguments
# --------------------------------------------
# 04/04/18 jejb at raven.il.steeleye.com 1.1798
# Add kernel dcache flush limitation
# --------------------------------------------
# 04/04/18 jejb at raven.il.steeleye.com 1.1799
# Eliminate the PG_dcache_dirty bit
#
# We now rely on the FLUSH_PAGE bit telling us if
# the page needs flushing or not
# --------------------------------------------
#
diff -Nru a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
--- a/arch/parisc/kernel/cache.c Sun Apr 18 12:07:31 2004
+++ b/arch/parisc/kernel/cache.c Sun Apr 18 12:07:31 2004
@@ -68,12 +68,14 @@
{
struct page *page = pte_page(pte);
- if (VALID_PAGE(page) && page->mapping &&
- test_bit(PG_dcache_dirty, &page->flags)) {
-
+ if (VALID_PAGE(page))
+ /* We used to check to see if the page needed flushing here.
+ * Now this API automatically detects whether the page
+ * needs flushing or not. This is used to defer calls to
+ * kernel page flushing until mmu update time.
+ *
+ * DO NOT REMOVE OR CONDITIONALISE THIS */
flush_kernel_dcache_page(page_address(page));
- clear_bit(PG_dcache_dirty, &page->flags);
- }
}
void
@@ -230,6 +232,7 @@
void __flush_dcache_page(struct page *page)
{
struct list_head *l;
+ pte_t *pte;
flush_kernel_dcache_page(page_address(page));
@@ -260,9 +263,10 @@
* have to find a congruent address with an existing
* translation */
- if (!translation_exists(mpnt, addr))
+ if (!(pte = translation_exists(mpnt, addr)))
continue;
+ pte_clear_flush(pte);
__flush_cache_page(mpnt, addr);
/* If we find an address to flush, that will also
@@ -293,15 +297,205 @@
/* This is just for speed. If the page translation isn't
* there there's no point exciting the nadtlb handler into
* a nullification frenzy */
- if(!translation_exists(mpnt, addr))
+ if(!(pte = translation_exists(mpnt, addr)))
continue;
+ pte_clear_flush(pte);
__flush_cache_page(mpnt, addr);
return;
}
}
EXPORT_SYMBOL(__flush_dcache_page);
+
+/* set to max pages to flush before a full flush. Zero means no limit */
+#define MAX_FLUSH_PAGES 0
+#undef DEBUG_PAGE_FLUSHING
+
+#ifdef DEBUG_PAGE_FLUSHING
+#define DBG(a...) printk(a)
+#else
+#define DBG(...)
+#endif
+
+#if (MAX_FLUSH_PAGES != 0)
+
+/* we get to use the bottom 12 bits of the addr for flags since the
+ * address must be page aligned */
+#define ICACHE_FLUSH_FLAG 0x1
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+ unsigned long count = 0, actual_count = 0;
+ unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+ unsigned long *pages;
+
+ preempt_disable();
+ if(mm != current->active_mm) {
+ DBG("flush_tlb_mm: current MM is not active ");
+ /* FIXME: awful hack: move the process the mm belongs
+ * to temporarily to being the active one. This only
+ * works because we can never get back into user
+ * context from here. */
+ mtctl(__pa(mm->pgd), 25);
+ mtsp(mm->context, 3);
+ }
+
+ pages = kmalloc(MAX_FLUSH_PAGES * sizeof(unsigned long), GFP_ATOMIC);
+ if(!pages) {
+ printk(KERN_ERR "flush_tlb_mm: allocation failed: full flush\n");
+ goto full_flush;
+ }
+
+ for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ unsigned long start;
+
+ pmd_t *pmd;
+ pgd_t *pgd;
+ pte_t *pte;
+
+ count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+ for (start = vma->vm_start; start < vma->vm_end;
+ start += PAGE_SIZE) {
+ pgd = pgd_offset(mm, start);
+
+ if (pgd_none(*pgd)) {
+ start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+ continue;
+ }
+
+ pmd = pmd_offset(pgd, start);
+ if (pmd_none(*pmd)) {
+ start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+ continue;
+ }
+ pte = pte_offset_map(pmd, start);
+ if(pte_val(*pte)==0 || !pte_present(*pte))
+ continue;
+
+ /* FIXME: Here we could also skip over any
+ * shared mapping page (i.e. equivalently
+ * aliased) with at least one other user */
+
+ pages[actual_count] = start;
+
+ if (vma->vm_flags & VM_EXEC)
+ pages[actual_count] |= ICACHE_FLUSH_FLAG;
+ if(++actual_count >= MAX_FLUSH_PAGES)
+ goto full_flush_free;
+
+ }
+ }
+
+ DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+ for(count = 0; count < actual_count; count++) {
+ unsigned long addr = pages[count] & PAGE_MASK;
+ flush_user_dcache_page(addr);
+ if(pages[count] & ICACHE_FLUSH_FLAG) {
+ flush_user_icache_page(addr);
+ pitlb_user(addr);
+ }
+ pdtlb_user(addr);
+ }
+ out_free:
+ kfree(pages);
+ out:
+ mtsp(sr3, 3);
+ mtctl(cr25, 25);
+ preempt_enable();
+ return;
+
+ full_flush_free:
+ DBG("flush_cache_mm: over max pages %ld (count %ld), flushing everything\n", actual_count, count);
+ flush_cache_all();
+ goto out_free;
+
+ full_flush:
+ flush_cache_all();
+ goto out;
+}
+
+#else
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+ unsigned long count = 0, actual_count = 0;
+ unsigned long sr3 = mfsp(3), cr25 = mfctl(25);
+#if 1
+ static int flushed = 0;
+
+ if(unlikely(!flushed)) {
+ printk("flush_cache_mm: INIT FLUSH ALL\n");
+ flushed = 1;
+ flush_cache_all();
+ return;
+ }
+#endif
+ preempt_disable();
+ if(mm != current->active_mm) {
+ DBG("flush_tlb_mm: current MM is not active ");
+ /* FIXME: awful hack: move the process the mm belongs
+ * to temporarily to being the active one. This only
+ * works because we can never get back into user
+ * context from here. */
+ mtctl(__pa(mm->pgd), 25);
+ mtsp(mm->context, 3);
+ }
+
+ for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ unsigned long start;
+
+ pmd_t *pmd;
+ pgd_t *pgd;
+ pte_t *pte;
+
+ count += (vma->vm_end - vma->vm_start)/PAGE_SIZE;
+
+ for (start = vma->vm_start; start < vma->vm_end;
+ start += PAGE_SIZE) {
+ pgd = pgd_offset(mm, start);
+
+ if (pgd_none(*pgd)) {
+ start = (start & PGDIR_MASK) + PGDIR_SIZE - PAGE_SIZE;
+ continue;
+ }
+
+ pmd = pmd_offset(pgd, start);
+ if (pmd_none(*pmd)) {
+ start = (start & PMD_MASK) + PMD_SIZE - PAGE_SIZE;
+ continue;
+ }
+ pte = pte_offset_map(pmd, start);
+ if(pte_val(*pte)==0 || !pte_present(*pte))
+ continue;
+
+ if(!pte_flush(*pte))
+ continue;
+
+ /* FIXME: Here we could also skip over any
+ * shared mapping page (i.e. equivalently
+ * aliased) with at least one other user */
+ pte_clear_flush(pte);
+ flush_user_dcache_page(start);
+ if (vma->vm_flags & VM_EXEC) {
+ flush_user_icache_page(start);
+ pitlb_user(start);
+ }
+ pdtlb_user(start);
+ actual_count++;
+ }
+ }
+ mtsp(sr3, 3);
+ mtctl(cr25, 25);
+ preempt_enable();
+ DBG("FLUSHED %lu (actual %lu)\n", count, actual_count);
+}
+#endif
+
+EXPORT_SYMBOL(flush_cache_mm);
/* Defined in arch/parisc/kernel/pacache.S */
EXPORT_SYMBOL(flush_kernel_dcache_range_asm);
diff -Nru a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
--- a/arch/parisc/kernel/entry.S Sun Apr 18 12:07:31 2004
+++ b/arch/parisc/kernel/entry.S Sun Apr 18 12:07:31 2004
@@ -49,7 +49,6 @@
.level 2.0
#endif
- .import pa_dbit_lock,data
/* space_to_prot macro creates a prot id from a space id */
@@ -1017,6 +1016,10 @@
or t1,pte,t0 /* t0 has R bit set */
and,*<> t1,pte,%r0 /* test and nullify if already set */
std t0,0(ptp) /* write back pte */
+ ldi _PAGE_FLUSH,t1
+ or t1,t0,t0
+ and,*<> t1,pte,%r0
+ std t0,0(ptp)
space_to_prot spc prot /* create prot id from space */
depd pte,8,7,prot /* add in prot bits from pte */
@@ -1093,7 +1096,7 @@
shladd t0,3,ptp,ptp
ldi _PAGE_ACCESSED,t1
ldd 0(ptp),pte
- bb,>=,n pte,_PAGE_PRESENT_BIT,nadtlb_check_flush_20w
+ bb,>=,n pte,_PAGE_PRESENT_BIT,nadtlb_emulate
space_to_prot spc prot /* create prot id from space */
depd pte,8,7,prot /* add in prot bits from pte */
@@ -1112,23 +1115,6 @@
rfir
nop
-nadtlb_check_flush_20w:
- bb,>=,n pte,_PAGE_FLUSH_BIT,nadtlb_emulate
-
- /* Insert a "flush only" translation */
-
- depdi,z 7,7,3,prot
- depdi 1,10,1,prot
-
- /* Get rid of prot bits and convert to page addr for idtlbt */
-
- depdi 0,63,12,pte
- extrd,u pte,56,52,pte
- idtlbt pte,prot
-
- rfir
- nop
-
#else
dtlb_miss_11:
@@ -1514,6 +1500,10 @@
or t1,pte,t0 /* t0 has R bit set */
and,*<> t1,pte,%r0 /* test and nullify if already set */
std t0,0(ptp) /* write back pte */
+ ldi _PAGE_FLUSH,t1
+ or t1,t0,t0
+ and,*<> t1,pte,%r0
+ std t0,0(ptp)
space_to_prot spc prot /* create prot id from space */
depd pte,8,7,prot /* add in prot bits from pte */
diff -Nru a/arch/parisc/kernel/pacache.S b/arch/parisc/kernel/pacache.S
--- a/arch/parisc/kernel/pacache.S Sun Apr 18 12:07:31 2004
+++ b/arch/parisc/kernel/pacache.S Sun Apr 18 12:07:31 2004
@@ -308,6 +308,24 @@
1:
+#ifdef __LP64__
+ ldd 0(%r25),%r19
+ ldd 8(%r25),%r20
+ ldd 16(%r25),%r21
+ ldd 24(%r25),%r22
+ std %r19,0(%r26)
+ std %r20,8(%r26)
+ std %r21,16(%r26)
+ std %r22,24(%r26)
+ ldd 32(%r25),%r19
+ ldd 40(%r25),%r20
+ ldd 48(%r25),%r21
+ ldd 56(%r25),%r22
+ std %r19,32(%r26)
+ std %r20,40(%r26)
+ std %r21,48(%r26)
+ std %r22,56(%r26)
+#else
ldw 0(%r25),%r19
ldw 4(%r25),%r20
ldw 8(%r25),%r21
@@ -340,6 +358,7 @@
stw %r20,52(%r26)
stw %r21,56(%r26)
stw %r22,60(%r26)
+#endif
ldo 64(%r26),%r26
ADDIB> -1,%r1,1b
ldo 64(%r25),%r25
@@ -375,7 +394,6 @@
* %r23 physical page (shifted for tlb insert) of "from" translation
*/
-#if 0
/*
* We can't do this since copy_user_page is used to bring in
@@ -390,9 +408,9 @@
* lobby for such a change.
*/
- .export copy_user_page_asm,code
+ .export copy_user_page_vaddr_asm,code
-copy_user_page_asm:
+copy_user_page_vaddr_asm:
.proc
.callinfo NO_CALLS
.entry
@@ -436,6 +454,24 @@
1:
+#ifdef __LP64__
+ ldd 0(%r29),%r19
+ ldd 8(%r29),%r20
+ ldd 16(%r29),%r21
+ ldd 24(%r29),%r22
+ std %r19,0(%r28)
+ std %r20,8(%r28)
+ std %r21,16(%r28)
+ std %r22,24(%r28)
+ ldd 32(%r29),%r19
+ ldd 40(%r29),%r20
+ ldd 48(%r29),%r21
+ ldd 56(%r29),%r22
+ std %r19,32(%r28)
+ std %r20,40(%r28)
+ std %r21,48(%r28)
+ std %r22,56(%r28)
+#else
ldw 0(%r29),%r19
ldw 4(%r29),%r20
ldw 8(%r29),%r21
@@ -468,6 +504,7 @@
stw %r20,52(%r28)
stw %r21,56(%r28)
stw %r22,60(%r28)
+#endif
ldo 64(%r28),%r28
ADDIB> -1,%r1,1b
ldo 64(%r29),%r29
@@ -477,7 +514,6 @@
.exit
.procend
-#endif
.export clear_user_page_asm,code
@@ -506,6 +542,16 @@
ldi 64,%r1
1:
+#ifdef __LP64__
+ std %r0,0(%r28)
+ std %r0,8(%r28)
+ std %r0,16(%r28)
+ std %r0,24(%r28)
+ std %r0,32(%r28)
+ std %r0,40(%r28)
+ std %r0,48(%r28)
+ std %r0,56(%r28)
+#else
stw %r0,0(%r28)
stw %r0,4(%r28)
stw %r0,8(%r28)
@@ -522,6 +568,7 @@
stw %r0,52(%r28)
stw %r0,56(%r28)
stw %r0,60(%r28)
+#endif
ADDIB> -1,%r1,1b
ldo 64(%r28),%r28
@@ -531,9 +578,9 @@
.procend
- .export flush_kernel_dcache_page
+ .export __flush_kernel_dcache_page
-flush_kernel_dcache_page:
+__flush_kernel_dcache_page:
.proc
.callinfo NO_CALLS
.entry
@@ -542,9 +589,11 @@
ldw R%dcache_stride(%r1),%r23
#ifdef __LP64__
- depdi,z 1,63-PAGE_SHIFT,1,%r25
+ depdi,z 1,63-PAGE_SHIFT,1,%r25 /* PAGE_SIZE */
+ depdi 0,63,12,%r26 /* page align */
#else
- depwi,z 1,31-PAGE_SHIFT,1,%r25
+ depwi,z 1,31-PAGE_SHIFT,1,%r25 /* PAGE_SIZE */
+ depdi 0,31,12,%r26 /* page align */
#endif
add %r26,%r25,%r25
sub %r25,%r23,%r25
@@ -586,9 +635,11 @@
ldw R%dcache_stride(%r1),%r23
#ifdef __LP64__
- depdi,z 1,63-PAGE_SHIFT,1,%r25
+ depdi,z 1,63-PAGE_SHIFT,1,%r25 /* PAGE_SIZE into %r25 */
+ depdi 0,63,12,%r26 /* page align argument */
#else
- depwi,z 1,31-PAGE_SHIFT,1,%r25
+ depwi,z 1,31-PAGE_SHIFT,1,%r25 /* PAGE_SIZE */
+ depwi 0,31,12,%r26 /* Page align */
#endif
add %r26,%r25,%r25
sub %r25,%r23,%r25
@@ -631,8 +682,10 @@
#ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r25
+ depdi 0,63,12,%r26
#else
depwi,z 1,31-PAGE_SHIFT,1,%r25
+ depwi 0,31,12,%r26
#endif
add %r26,%r25,%r25
sub %r25,%r23,%r25
diff -Nru a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c
--- a/arch/parisc/kernel/signal.c Sun Apr 18 12:07:31 2004
+++ b/arch/parisc/kernel/signal.c Sun Apr 18 12:07:31 2004
@@ -375,10 +375,9 @@
}
#endif
- flush_user_dcache_range((unsigned long) &frame->tramp[0],
- (unsigned long) &frame->tramp[TRAMP_SIZE]);
- flush_user_icache_range((unsigned long) &frame->tramp[0],
- (unsigned long) &frame->tramp[TRAMP_SIZE]);
+ __flush_cache_range(current->active_mm,
+ (unsigned long) &frame->tramp[0],
+ (unsigned long) &frame->tramp[TRAMP_SIZE]);
/* TRAMP Words 0-4, Lenght 5 = SIGRESTARTBLOCK_TRAMP
* TRAMP Words 5-9, Length 4 = SIGRETURN_TRAMP
diff -Nru a/include/asm-parisc/cache.h b/include/asm-parisc/cache.h
--- a/include/asm-parisc/cache.h Sun Apr 18 12:07:31 2004
+++ b/include/asm-parisc/cache.h Sun Apr 18 12:07:31 2004
@@ -46,7 +46,7 @@
extern void flush_kernel_icache_range_asm(unsigned long, unsigned long);
extern void flush_user_dcache_range_asm(unsigned long, unsigned long);
extern void flush_kernel_dcache_range_asm(unsigned long, unsigned long);
-extern void flush_kernel_dcache_page(void *);
+extern void __flush_kernel_dcache_page(void *);
extern void flush_kernel_icache_page(void *);
extern void disable_sr_hashing(void); /* turns off space register hashing */
extern void disable_sr_hashing_asm(int); /* low level support for above */
@@ -63,9 +63,11 @@
extern int icache_stride;
extern struct pdc_cache_info cache_info;
-#define pdtlb(addr) asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr));
-#define pitlb(addr) asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr));
-#define pdtlb_kernel(addr) asm volatile("pdtlb 0(%0)" : : "r" (addr));
+#define pdtlb(addr) asm volatile("pdtlb 0(%%sr1,%0)" : : "r" (addr))
+#define pitlb(addr) asm volatile("pitlb 0(%%sr1,%0)" : : "r" (addr))
+#define pdtlb_kernel(addr) asm volatile("pdtlb 0(%0)" : : "r" (addr))
+#define pdtlb_user(addr) asm volatile("pdtlb 0(%%sr3,%0)" : : "r" (addr))
+#define pitlb_user(addr) asm volatile("pitlb 0(%%sr3,%0)" : : "r" (addr))
#endif /* ! __ASSEMBLY__ */
diff -Nru a/include/asm-parisc/cacheflush.h b/include/asm-parisc/cacheflush.h
--- a/include/asm-parisc/cacheflush.h Sun Apr 18 12:07:31 2004
+++ b/include/asm-parisc/cacheflush.h Sun Apr 18 12:07:31 2004
@@ -9,14 +9,7 @@
/* Cache flush operations */
-#ifdef CONFIG_SMP
-#define flush_cache_mm(mm) flush_cache_all()
-#else
-#define flush_cache_mm(mm) flush_cache_all_local()
-#endif
-
-#define flush_kernel_dcache_range(start,size) \
- flush_kernel_dcache_range_asm((start), (start)+(size));
+extern void flush_cache_mm(struct mm_struct *);
extern void flush_cache_all_local(void);
@@ -33,52 +26,6 @@
#define flush_cache_vmap(start, end) flush_cache_all()
#define flush_cache_vunmap(start, end) flush_cache_all()
-/* The following value needs to be tuned and probably scaled with the
- * cache size.
- */
-
-#define FLUSH_THRESHOLD 0x80000
-
-static inline void
-flush_user_dcache_range(unsigned long start, unsigned long end)
-{
-#ifdef CONFIG_SMP
- flush_user_dcache_range_asm(start,end);
-#else
- if ((end - start) < FLUSH_THRESHOLD)
- flush_user_dcache_range_asm(start,end);
- else
- flush_data_cache();
-#endif
-}
-
-static inline void
-flush_user_icache_range(unsigned long start, unsigned long end)
-{
-#ifdef CONFIG_SMP
- flush_user_icache_range_asm(start,end);
-#else
- if ((end - start) < FLUSH_THRESHOLD)
- flush_user_icache_range_asm(start,end);
- else
- flush_instruction_cache();
-#endif
-}
-
-extern void __flush_dcache_page(struct page *page);
-
-static inline void flush_dcache_page(struct page *page)
-{
- if (page->mapping && list_empty(&page->mapping->i_mmap) &&
- list_empty(&page->mapping->i_mmap_shared)) {
- set_bit(PG_dcache_dirty, &page->flags);
- } else {
- __flush_dcache_page(page);
- }
-}
-
-#define flush_icache_page(vma,page) do { flush_kernel_dcache_page(page_address(page)); flush_kernel_icache_page(page_address(page)); } while (0)
-
#define flush_icache_range(s,e) do { flush_kernel_dcache_range_asm(s,e); flush_kernel_icache_range_asm(s,e); } while (0)
#define copy_to_user_page(vma, page, vaddr, dst, src, len) \
@@ -88,49 +35,31 @@
#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
memcpy(dst, src, len)
-static inline void flush_cache_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
-{
- int sr3;
-
- if (!vma->vm_mm->context) {
- BUG();
- return;
- }
-
- sr3 = mfsp(3);
- if (vma->vm_mm->context == sr3) {
- flush_user_dcache_range(start,end);
- flush_user_icache_range(start,end);
- } else {
- flush_cache_all();
- }
-}
-
/* Simple function to work out if we have an existing address translation
* for a user space vma. */
-static inline int translation_exists(struct vm_area_struct *vma,
- unsigned long addr)
+static inline pte_t *__translation_exists(struct mm_struct *mm,
+ unsigned long addr)
{
- pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
+ pgd_t *pgd = pgd_offset(mm, addr);
pmd_t *pmd;
pte_t *pte;
if(pgd_none(*pgd))
- return 0;
+ return NULL;
pmd = pmd_offset(pgd, addr);
if(pmd_none(*pmd) || pmd_bad(*pmd))
- return 0;
+ return NULL;
pte = pte_offset_map(pmd, addr);
/* The PA flush mappings show up as pte_none, but they're
* valid none the less */
if(pte_none(*pte) && ((pte_val(*pte) & _PAGE_FLUSH) == 0))
- return 0;
- return 1;
+ return NULL;
+ return pte;
}
+#define translation_exists(vma, addr) __translation_exists((vma)->vm_mm, addr)
/* Private function to flush a page from the cache of a non-current
@@ -157,8 +86,11 @@
mtsp(vma->vm_mm->context, 3);
flush_user_dcache_page(vmaddr);
- if(vma->vm_flags & VM_EXEC)
+ if(vma->vm_flags & VM_EXEC) {
flush_user_icache_page(vmaddr);
+ pitlb_user(vmaddr);
+ }
+ pdtlb_user(vmaddr);
/* put the old current process back */
mtsp(space, 3);
@@ -171,21 +103,153 @@
{
if (likely(vma->vm_mm->context == mfsp(3))) {
flush_user_dcache_page(vmaddr);
- if (vma->vm_flags & VM_EXEC)
+ if (vma->vm_flags & VM_EXEC) {
flush_user_icache_page(vmaddr);
+ pitlb_user(vmaddr);
+ }
+ pdtlb_user(vmaddr);
} else {
flush_user_cache_page_non_current(vma, vmaddr);
}
}
+/* The following value needs to be tuned and probably scaled with the
+ * cache size.
+ */
+
+#define FLUSH_THRESHOLD 0x80000
+
+static inline void
+__flush_cache_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end)
+{
+ unsigned long vaddr;
+ pte_t *pte;
+
+ /* small range, don't bother to flush the whole page (and
+ * thus don't mark the page as flushed */
+ if (likely(end - start < PAGE_SIZE)) {
+ flush_user_dcache_range_asm(start,end);
+ flush_user_icache_range_asm(start,end);
+ return;
+ }
+
+ for (vaddr = start & PAGE_MASK; vaddr < end; vaddr += PAGE_SIZE) {
+ if(unlikely(!(pte = __translation_exists(mm, vaddr))))
+ continue;
+
+ if (unlikely(!pte_flush(*pte)))
+ continue;
+
+ pte_clear_flush(pte);
+ flush_user_dcache_page(vaddr);
+ flush_user_icache_page(vaddr);
+ pdtlb_user(vaddr);
+ pitlb_user(vaddr);
+ }
+}
+
+static inline void flush_cache_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ unsigned long sr3 = mfsp(3);
+
+ BUG_ON(!vma->vm_mm->context);
+
+ if (likely(vma->vm_mm->context == sr3)) {
+ __flush_cache_range(vma->vm_mm, start, end);
+ } else {
+ unsigned long pgd = mfctl(25);
+ /* we don't mind taking interrups since they may not
+ * do anything with user space, but we can't
+ * be preempted here */
+ preempt_disable();
+
+ /* make us current */
+ mtctl(__pa(vma->vm_mm->pgd), 25);
+ mtsp(vma->vm_mm->context, 3);
+
+ __flush_cache_range(vma->vm_mm, start, end);
+
+ /* put the old current process back */
+ mtsp(sr3, 3);
+ mtctl(pgd, 25);
+ preempt_enable();
+
+ }
+}
+
+extern void __flush_dcache_page(struct page *page);
+
+static inline void flush_dcache_page(struct page *page)
+{
+ if (!(page->mapping && list_empty(&page->mapping->i_mmap) &&
+ list_empty(&page->mapping->i_mmap_shared)))
+ __flush_dcache_page(page);
+}
+
+static inline void flush_kernel_dcache_page(void *vaddr)
+{
+ pte_t *pte = __translation_exists(&init_mm, (unsigned long)vaddr);
+
+ if (likely(pte && pte_flush(*pte))) {
+ pte_clear_flush(pte);
+ __flush_kernel_dcache_page(vaddr);
+ pdtlb_kernel(vaddr);
+ }
+}
+
static inline void
flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
+ pte_t *pte = translation_exists(vma, vmaddr);
+
BUG_ON(!vma->vm_mm->context);
- if(likely(translation_exists(vma, vmaddr)))
+ if (likely(pte && pte_flush(*pte))) {
+ pte_clear_flush(pte);
__flush_cache_page(vma, vmaddr);
+ }
+
+}
+
+static inline void
+flush_kernel_dcache_range(unsigned long start, unsigned long end)
+{
+ unsigned long vaddr;
+
+ /* small range, don't bother to flush the whole page (and
+ * thus don't mark the page as flushed */
+ if (likely(end - start < PAGE_SIZE)) {
+ flush_kernel_dcache_range_asm(start,end);
+ return;
+ }
+ for (vaddr = start & PAGE_MASK; vaddr < end; vaddr += PAGE_SIZE)
+ flush_kernel_dcache_page((void *)vaddr);
}
+
+
+static inline void
+flush_icache_page(struct vm_area_struct *vma, struct page *page)
+{
+}
+
+static inline void
+copy_user_page(void *vto, void *vfrom, unsigned long vaddr, struct page *pg)
+{
+ pte_t *pte;
+ copy_user_page_asm(vto, vfrom);
+ if(likely((pte = __translation_exists(current->active_mm, vaddr)) &&
+ pte_flush(*pte))) {
+ /* no point clearing flush here, it would be set again
+ * when the user accesses the page, likewise, don't
+ * purge the TLB entries */
+ flush_user_dcache_page(vaddr);
+ flush_user_icache_page(vaddr);
+ }
+
+}
+
#endif
diff -Nru a/include/asm-parisc/page.h b/include/asm-parisc/page.h
--- a/include/asm-parisc/page.h Sun Apr 18 12:07:31 2004
+++ b/include/asm-parisc/page.h Sun Apr 18 12:07:31 2004
@@ -19,15 +19,10 @@
extern void purge_kernel_dcache_page(unsigned long);
extern void copy_user_page_asm(void *to, void *from);
+extern void copy_user_page_vaddr_asm(void *to, void *from, unsigned long vaddr);
extern void clear_user_page_asm(void *page, unsigned long vaddr);
-static inline void
-copy_user_page(void *vto, void *vfrom, unsigned long vaddr, struct page *pg)
-{
- copy_user_page_asm(vto, vfrom);
- flush_kernel_dcache_page(vto);
- /* XXX: ppc flushes icache too, should we? */
-}
+/* see cacheflush.h for copy_user_page */
static inline void
clear_user_page(void *page, unsigned long vaddr, struct page *pg)
diff -Nru a/include/asm-parisc/pgtable.h b/include/asm-parisc/pgtable.h
--- a/include/asm-parisc/pgtable.h Sun Apr 18 12:07:31 2004
+++ b/include/asm-parisc/pgtable.h Sun Apr 18 12:07:31 2004
@@ -29,15 +29,6 @@
*/
#define kern_addr_valid(addr) (1)
-/* Certain architectures need to do special things when PTEs
- * within a page table are directly modified. Thus, the following
- * hook is made available.
- */
-#define set_pte(pteptr, pteval) \
- do{ \
- *(pteptr) = (pteval); \
- } while(0)
-
#endif /* !__ASSEMBLY__ */
#define pte_ERROR(e) \
@@ -213,6 +204,15 @@
#define __S110 PAGE_RWX
#define __S111 PAGE_RWX
+/* Certain architectures need to do special things when PTEs
+ * within a page table are directly modified. Thus, the following
+ * hook is made available.
+ */
+static inline void set_pte(pte_t *pteptr, pte_t pteval)
+{
+ *pteptr = pteval;
+}
+
extern pgd_t swapper_pg_dir[]; /* declared in init_task.c */
/* initial page tables for 0-8MB for kernel */
@@ -230,17 +230,16 @@
#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-#define pte_none(x) ((pte_val(x) == 0) || (pte_val(x) & _PAGE_FLUSH))
+#define pte_none(x) ((pte_val(x) == 0))
#define pte_present(x) (pte_val(x) & _PAGE_PRESENT)
-#define pte_clear(xp) do { pte_val(*(xp)) = 0; } while (0)
+#define pte_clear_flush(xp) do { pte_val(*(xp)) &= ~ _PAGE_FLUSH; } while(0)
+#define pte_flush(x) (pte_val(x) & _PAGE_FLUSH)
#define pmd_none(x) (!pmd_val(x))
#define pmd_bad(x) ((pmd_val(x) & ~PAGE_MASK) != _PAGE_TABLE)
#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT)
#define pmd_clear(xp) do { pmd_val(*(xp)) = 0; } while (0)
-
-
#ifdef __LP64__
#define pgd_page(pgd) ((unsigned long) __va(pgd_val(pgd) & PAGE_MASK))
@@ -282,6 +281,13 @@
extern inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
extern inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; return pte; }
+static inline void pte_clear(pte_t *xp) {
+ WARN_ON(pte_flush(*xp) && pte_user(*xp));
+ pte_val(*xp) = 0;
+}
+
+
+
/*
* Conversion functions: convert a page and protection to a page entry,
* and a page entry and page directory to the page they refer to.
@@ -358,10 +364,6 @@
extern void paging_init (void);
-/* Used for deferring calls to flush_dcache_page() */
-
-#define PG_dcache_dirty PG_arch_1
-
struct vm_area_struct; /* forward declaration (include/linux/mm.h) */
extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
@@ -402,23 +404,12 @@
#endif
}
-#ifdef CONFIG_SMP
-extern spinlock_t pa_dbit_lock;
-#else
-static int pa_dbit_lock; /* dummy to keep the compilers happy */
-#endif
-
static inline pte_t ptep_get_and_clear(pte_t *ptep)
{
pte_t old_pte;
- pte_t pte;
- spin_lock(&pa_dbit_lock);
- pte = old_pte = *ptep;
- pte_val(pte) &= ~_PAGE_PRESENT;
- pte_val(pte) |= _PAGE_FLUSH;
- set_pte(ptep,pte);
- spin_unlock(&pa_dbit_lock);
+ old_pte = *ptep;
+ pte_clear(ptep);
return old_pte;
}
diff -Nru a/include/asm-parisc/tlb.h b/include/asm-parisc/tlb.h
--- a/include/asm-parisc/tlb.h Sun Apr 18 12:07:31 2004
+++ b/include/asm-parisc/tlb.h Sun Apr 18 12:07:31 2004
@@ -7,7 +7,7 @@
} while (0)
#define tlb_start_vma(tlb, vma) \
-do { if (!(tlb)->fullmm) \
+do { \
flush_cache_range(vma, vma->vm_start, vma->vm_end); \
} while (0)
More information about the parisc-linux
mailing list