@@ -15,8 +15,6 @@ DECLARE_STATIC_KEY_TRUE(parisc_has_cache);
DECLARE_STATIC_KEY_TRUE(parisc_has_dcache);
DECLARE_STATIC_KEY_TRUE(parisc_has_icache);
-#define flush_cache_dup_mm(mm) flush_cache_mm(mm)
-
void flush_user_icache_range_asm(unsigned long, unsigned long);
void flush_kernel_icache_range_asm(unsigned long, unsigned long);
void flush_user_dcache_range_asm(unsigned long, unsigned long);
@@ -30,6 +28,7 @@ void flush_kernel_icache_page(void *);
void flush_cache_all_local(void);
void flush_cache_all(void);
void flush_cache_mm(struct mm_struct *mm);
+void flush_cache_dup_mm(struct mm_struct *mm);
void flush_kernel_dcache_page_addr(void *addr);
@@ -11,10 +11,15 @@
* loading restrictions). If you place this region anywhere above
* __PAGE_OFFSET, you must adjust the memory map accordingly */
-/* The alias region is used in kernel space to do copy/clear to or
- * from areas congruently mapped with user space. It is 8MB large
- * and must be 16MB aligned */
-#define TMPALIAS_MAP_START ((__PAGE_OFFSET) - 16*1024*1024)
+/* The alias region is used in kernel space to copy/clear/flush data
+ * from pages congruently mapped with user space. It is comprised of a
+ * pair of 4 MB regions aligned on a 8 MB boundary. The size of
+ * these regions is determined by the largest cache aliasing boundary
+ * for machines that support equivalent aliasing. As far as I can tell,
+ * machines with PA8800/PA8900 processors do not support equivalent
+ * aliasing for cache flushes. I tried boundaries up to 128 MB. */
+#define TMPALIAS_MAP_START ((__PAGE_OFFSET) - 8*1024*1024)
+#define TMPALIAS_SIZE_BITS 22 /* 4 MB */
#define FIXMAP_SIZE (FIX_BITMAP_COUNT << PAGE_SHIFT)
#define FIXMAP_START (TMPALIAS_MAP_START - FIXMAP_SIZE)
@@ -27,6 +27,7 @@
#include <asm/processor.h>
#include <asm/sections.h>
#include <asm/shmparam.h>
+#include <asm/mmu_context.h>
int split_tlb __ro_after_init;
int dcache_stride __ro_after_init;
@@ -91,7 +92,7 @@ static inline void flush_data_cache(void)
}
-/* Virtual address of pfn. */
+/* Kernel virtual address of pfn. */
#define pfn_va(pfn) __va(PFN_PHYS(pfn))
void
@@ -124,11 +125,13 @@ show_cache_info(struct seq_file *m)
cache_info.ic_size/1024 );
if (cache_info.dc_loop != 1)
snprintf(buf, 32, "%lu-way associative", cache_info.dc_loop);
- seq_printf(m, "D-cache\t\t: %ld KB (%s%s, %s)\n",
+ seq_printf(m, "D-cache\t\t: %ld KB (%s%s, %s, alias=%d)\n",
cache_info.dc_size/1024,
(cache_info.dc_conf.cc_wt ? "WT":"WB"),
(cache_info.dc_conf.cc_sh ? ", shared I/D":""),
- ((cache_info.dc_loop == 1) ? "direct mapped" : buf));
+ ((cache_info.dc_loop == 1) ? "direct mapped" : buf),
+ cache_info.dc_conf.cc_alias
+ );
seq_printf(m, "ITLB entries\t: %ld\n" "DTLB entries\t: %ld%s\n",
cache_info.it_size,
cache_info.dt_size,
@@ -317,6 +320,7 @@ __flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr,
{
if (!static_branch_likely(&parisc_has_cache))
return;
+
preempt_disable();
flush_dcache_page_asm(physaddr, vmaddr);
if (vma->vm_flags & VM_EXEC)
@@ -337,6 +341,70 @@ __purge_cache_page(struct vm_area_struct *vma, unsigned long vmaddr,
preempt_enable();
}
+static void flush_user_cache_page(struct vm_area_struct *vma, unsigned long vmaddr)
+{
+ unsigned long flags, space, pgd, prot;
+#ifdef CONFIG_TLB_PTLOCK
+ unsigned long pgd_lock;
+#endif
+
+ vmaddr &= PAGE_MASK;
+
+ preempt_disable();
+
+ /* Set context for flush */
+ local_irq_save(flags);
+ prot = mfctl(8);
+ space = mfsp(SR_USER);
+ pgd = mfctl(25);
+#ifdef CONFIG_TLB_PTLOCK
+ pgd_lock = mfctl(28);
+#endif
+ switch_mm_irqs_off(NULL, vma->vm_mm, NULL);
+ local_irq_restore(flags);
+
+ flush_user_dcache_range_asm(vmaddr, vmaddr + PAGE_SIZE);
+ if (vma->vm_flags & VM_EXEC)
+ flush_user_icache_range_asm(vmaddr, vmaddr + PAGE_SIZE);
+ flush_tlb_page(vma, vmaddr);
+
+ /* Restore previous context */
+ local_irq_save(flags);
+#ifdef CONFIG_TLB_PTLOCK
+ mtctl(pgd_lock, 28);
+#endif
+ mtctl(pgd, 25);
+ mtsp(space, SR_USER);
+ mtctl(prot, 8);
+ local_irq_restore(flags);
+
+ preempt_enable();
+}
+
+static inline pte_t *get_ptep(struct mm_struct *mm, unsigned long addr)
+{
+ pte_t *ptep = NULL;
+ pgd_t *pgd = mm->pgd;
+
+ if (!pgd_none(*pgd)) {
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ if (!p4d_none(*p4d)) {
+ pud_t *pud = pud_offset(p4d, addr);
+ if (!pud_none(*pud)) {
+ pmd_t *pmd = pmd_offset(pud, addr);
+ if (!pmd_none(*pmd))
+ ptep = pte_offset_map(pmd, addr);
+ }
+ }
+ }
+ return ptep;
+}
+
+static inline bool pte_flush(pte_t pte)
+{
+ return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_NO_CACHE)) == (_PAGE_PRESENT | _PAGE_ACCESSED );
+}
+
void flush_dcache_page(struct page *page)
{
struct address_space *mapping = page_mapping_file(page);
@@ -357,31 +425,43 @@ void flush_dcache_page(struct page *page)
pgoff = page->index;
- /* We have carefully arranged in arch_get_unmapped_area() that
+ /*
+ * We have carefully arranged in arch_get_unmapped_area() that
* *any* mappings of a file are always congruently mapped (whether
* declared as MAP_PRIVATE or MAP_SHARED), so we only need
- * to flush one address here for them all to become coherent */
-
+ * to flush one address here for them all to become coherent
+ * on machines that support equivalent aliasing
+ */
flush_dcache_mmap_lock(mapping);
vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
addr = mpnt->vm_start + offset;
- /* The TLB is the engine of coherence on parisc: The
- * CPU is entitled to speculate any page with a TLB
- * mapping, so here we kill the mapping then flush the
- * page along a special flush only alias mapping.
- * This guarantees that the page is no-longer in the
- * cache for any process and nor may it be
- * speculatively read in (until the user or kernel
- * specifically accesses it, of course) */
-
- flush_tlb_page(mpnt, addr);
- if (old_addr == 0 || (old_addr & (SHM_COLOUR - 1))
- != (addr & (SHM_COLOUR - 1))) {
- __flush_cache_page(mpnt, addr, page_to_phys(page));
- if (parisc_requires_coherency() && old_addr)
- printk(KERN_ERR "INEQUIVALENT ALIASES 0x%lx and 0x%lx in file %pD\n", old_addr, addr, mpnt->vm_file);
+ /* PA8800/PA8900 don't support equivalent aliasing */
+ if (parisc_requires_coherency()) {
+ pte_t *ptep;
+
+ ptep = get_ptep(mpnt->vm_mm, addr);
+ if (ptep && pte_flush(*ptep))
+ flush_user_cache_page(mpnt, addr);
+ } else {
+ /*
+ * The TLB is the engine of coherence on parisc:
+ * The CPU is entitled to speculate any page
+ * with a TLB mapping, so here we kill the
+ * mapping then flush the page along a special
+ * flush only alias mapping. This guarantees that
+ * the page is no-longer in the cache for any
+ * process and nor may it be speculatively read
+ * in (until the user or kernel specifically
+ * accesses it, of course)
+ */
+ flush_tlb_page(mpnt, addr);
+ if (old_addr == 0 || (old_addr & (SHM_COLOUR - 1)) != (addr & (SHM_COLOUR - 1))) {
+ __flush_cache_page(mpnt, addr, page_to_phys(page));
+ if (old_addr)
+ printk(KERN_ERR "INEQUIVALENT ALIASES 0x%lx and 0x%lx in file %pD\n", old_addr, addr, mpnt->vm_file);
+ }
old_addr = addr;
}
}
@@ -542,35 +622,54 @@ static inline unsigned long mm_total_size(struct mm_struct *mm)
return usize;
}
-static inline pte_t *get_ptep(pgd_t *pgd, unsigned long addr)
+static void flush_cache_pages(struct vm_area_struct *vma, struct mm_struct *mm,
+ unsigned long start, unsigned long end)
{
- pte_t *ptep = NULL;
+ unsigned long addr, pfn;
+ pte_t *ptep;
- if (!pgd_none(*pgd)) {
- p4d_t *p4d = p4d_offset(pgd, addr);
- if (!p4d_none(*p4d)) {
- pud_t *pud = pud_offset(p4d, addr);
- if (!pud_none(*pud)) {
- pmd_t *pmd = pmd_offset(pud, addr);
- if (!pmd_none(*pmd))
- ptep = pte_offset_map(pmd, addr);
+ for (addr = start; addr < end; addr += PAGE_SIZE) {
+ /*
+ * The vma can contain pages that aren't present. Although
+ * the pte search is expensive, we need the pte to find the
+ * page pfn. We also need to check it to avoid faulting
+ * when we call flush_user_cache_page()
+ */
+ ptep = get_ptep(mm, addr);
+ if (ptep && pte_flush(*ptep)) {
+ if (parisc_requires_coherency()) {
+ flush_user_cache_page(vma, addr);
+ } else {
+ pfn = pte_pfn(*ptep);
+ if (pfn_valid(pfn))
+ __flush_cache_page(vma, addr, PFN_PHYS(pfn));
}
}
}
- return ptep;
}
-static void flush_cache_pages(struct vm_area_struct *vma, struct mm_struct *mm,
+static void purge_cache_pages(struct vm_area_struct *vma, struct mm_struct *mm,
unsigned long start, unsigned long end)
{
unsigned long addr, pfn;
pte_t *ptep;
for (addr = start; addr < end; addr += PAGE_SIZE) {
- ptep = get_ptep(mm->pgd, addr);
- if (ptep) {
- pfn = pte_pfn(*ptep);
- flush_cache_page(vma, addr, pfn);
+ /*
+ * The vma can contain pages that aren't present. Although
+ * the pte search is expensive, we need the pte to find the
+ * page pfn. We also need to check it to avoid faulting
+ * when we call flush_user_cache_page()
+ */
+ ptep = get_ptep(mm, addr);
+ if (ptep && pte_flush(*ptep)) {
+ if (parisc_requires_coherency()) {
+ flush_user_cache_page(vma, addr);
+ } else {
+ pfn = pte_pfn(*ptep);
+ if (pfn_valid(pfn))
+ __purge_cache_page(vma, addr, PFN_PHYS(pfn));
+ }
}
}
}
@@ -583,12 +682,30 @@ void flush_cache_mm(struct mm_struct *mm)
rp3440, etc. So, avoid it if the mm isn't too big. */
if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
mm_total_size(mm) >= parisc_cache_flush_threshold) {
- if (mm->context.space_id)
- flush_tlb_all();
+ flush_tlb_all();
+ flush_cache_all();
+ return;
+ }
+
+ /* Purge mm */
+ for (vma = mm->mmap; vma; vma = vma->vm_next)
+ purge_cache_pages(vma, mm, vma->vm_start, vma->vm_end);
+}
+
+void flush_cache_dup_mm(struct mm_struct *mm)
+{
+ struct vm_area_struct *vma;
+
+ /* Flushing the whole cache on each cpu takes forever on
+ rp3440, etc. So, avoid it if the mm isn't too big. */
+ if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
+ mm_total_size(mm) >= parisc_cache_flush_threshold) {
+ flush_tlb_all();
flush_cache_all();
return;
}
+ /* Flush mm */
for (vma = mm->mmap; vma; vma = vma->vm_next)
flush_cache_pages(vma, mm, vma->vm_start, vma->vm_end);
}
@@ -596,10 +713,11 @@ void flush_cache_mm(struct mm_struct *mm)
void flush_cache_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
+ struct mm_struct *mm = vma->vm_mm;
+
if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
- end - start >= parisc_cache_flush_threshold) {
- if (vma->vm_mm->context.space_id)
- flush_tlb_range(vma, start, end);
+ mm_total_size(mm) >= parisc_cache_flush_threshold) {
+ flush_tlb_range(vma, start, end);
flush_cache_all();
return;
}
@@ -607,17 +725,17 @@ void flush_cache_range(struct vm_area_struct *vma,
flush_cache_pages(vma, vma->vm_mm, start, end);
}
-void
-flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn)
+void flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn)
{
- if (pfn_valid(pfn)) {
- flush_tlb_page(vma, vmaddr);
- if (likely(vma->vm_mm->context.space_id)) {
- __flush_cache_page(vma, vmaddr, PFN_PHYS(pfn));
- } else {
- __purge_cache_page(vma, vmaddr, PFN_PHYS(pfn));
- }
+ if (parisc_requires_coherency()) {
+
+ /* So far, we don't need to check pte */
+ flush_user_cache_page(vma, vmaddr);
+ return;
}
+
+ if (pfn_valid(pfn))
+ __flush_cache_page(vma, vmaddr, PFN_PHYS(pfn));
}
void flush_kernel_vmap_range(void *vaddr, int size)
@@ -554,9 +554,9 @@
extrd,s \pte,63,25,\pte
.endm
- /* The alias region is an 8MB aligned 16MB to do clear and
- * copy user pages at addresses congruent with the user
- * virtual address.
+ /* The alias region is 8MB aligned to 8MB to do clear and
+ * copy user page at addresses congruent with the user
+ * virtual address. It is comprised of a pair of 4MB regions.
*
* To use the alias page, you set %r26 up with the to TLB
* entry (identifying the physical page) and %r23 up with
@@ -571,7 +571,7 @@
depdi 0,31,32,\tmp
#endif
copy \va,\tmp1
- depi 0,31,23,\tmp1
+ depi_safe 0,31,TMPALIAS_SIZE_BITS+1,\tmp1
cmpb,COND(<>),n \tmp,\tmp1,\fault
mfctl %cr19,\tmp /* iir */
/* get the opcode (first six bits) into \tmp */
@@ -605,9 +605,9 @@
* Check "subtle" note in pacache.S re: r23/r26.
*/
#ifdef CONFIG_64BIT
- extrd,u,*= \va,41,1,%r0
+ extrd,u,*= \va,63-TMPALIAS_SIZE_BITS,1,%r0
#else
- extrw,u,= \va,9,1,%r0
+ extrw,u,= \va,31-TMPALIAS_SIZE_BITS,1,%r0
#endif
or,COND(tr) %r23,%r0,\pte
or %r26,%r0,\pte
@@ -366,6 +366,7 @@ ENTRY_CFI(clear_page_asm)
addib,COND(>),n -1, %r1, 1b
ldo 64(%r26), %r26
#endif
+ sync
bv %r0(%r2)
nop
ENDPROC_CFI(clear_page_asm)
@@ -477,6 +478,7 @@ ENTRY_CFI(copy_page_asm)
addib,COND(>),n -1, %r1, 1b
ldw 0(%r25), %r19
#endif
+ sync
bv %r0(%r2)
nop
ENDPROC_CFI(copy_page_asm)
@@ -487,6 +489,8 @@ ENDPROC_CFI(copy_page_asm)
* parisc chip designers that there will not ever be a parisc
* chip with a larger alias boundary (Never say never :-) ).
*
+ * Yah, what about the PA8800 and PA8900?
+ *
* Subtle: the dtlb miss handlers support the temp alias region by
* "knowing" that if a dtlb miss happens within the temp alias
* region it must have occurred while in clear_user_page. Since
@@ -545,17 +549,17 @@ ENTRY_CFI(copy_user_page_asm)
#endif
convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
convert_phys_for_tlb_insert20 %r23 /* convert phys addr to tlb insert format */
- depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
+ depd %r24,63,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
copy %r28, %r29
- depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
+ depdi 1, 63-TMPALIAS_SIZE_BITS,1, %r29 /* Form aliased virtual address 'from' */
#else
extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
- depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
+ depw %r24, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
copy %r28, %r29
- depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
+ depwi 1, 31-TMPALIAS_SIZE_BITS,1, %r29 /* Form aliased virtual address 'from' */
#endif
/* Purge any old translations */
@@ -678,6 +682,7 @@ ENTRY_CFI(copy_user_page_asm)
ldo 64(%r29), %r29
#endif
+ sync
bv %r0(%r2)
nop
ENDPROC_CFI(copy_user_page_asm)
@@ -691,11 +696,11 @@ ENTRY_CFI(clear_user_page_asm)
depdi 0, 31,32, %r28 /* clear any sign extension */
#endif
convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
- depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
+ depd %r25, 63,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
#else
extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
- depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
+ depw %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
#endif
@@ -756,6 +761,7 @@ ENTRY_CFI(clear_user_page_asm)
ldo 64(%r28), %r28
#endif /* CONFIG_64BIT */
+ sync
bv %r0(%r2)
nop
ENDPROC_CFI(clear_user_page_asm)
@@ -767,11 +773,11 @@ ENTRY_CFI(flush_dcache_page_asm)
depdi 0, 31,32, %r28 /* clear any sign extension */
#endif
convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
- depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
+ depd %r25, 63,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
#else
extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
- depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
+ depw %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
#endif
@@ -826,11 +832,11 @@ ENTRY_CFI(purge_dcache_page_asm)
depdi 0, 31,32, %r28 /* clear any sign extension */
#endif
convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
- depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
+ depd %r25, 63,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
#else
extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
- depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
+ depw %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
#endif
@@ -885,11 +891,11 @@ ENTRY_CFI(flush_icache_page_asm)
depdi 0, 31,32, %r28 /* clear any sign extension */
#endif
convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
- depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
+ depd %r25, 63,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
#else
extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
- depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
+ depw %r25, 31,TMPALIAS_SIZE_BITS, %r28 /* Form aliased virtual address 'to' */
depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
#endif
Originally, I was convinced that we needed to use tmpalias flushes everwhere, for both user and kernel flushes. However, when I modified flush_kernel_dcache_page_addr, to use a tmpalias flush, my c8000 would crash quite early when booting. The PDC returns alias values of 0 for the icache and dcache. This indicates that either the alias boundary is greater than 16MB or equivalent aliasing doesn't work. I modified the tmpalias code to make it easy to try alternate boundaries. I tried boundaries up to 128MB but still kernel tmpalias flushes didn't work on c8000. This led me to conclude that tmpalias flushes don't work on PA8800 and PA8900 machines, and that we needed to flush directly using the virtual address of user and kernel pages. This is likely the major cause of instability on the c8000 and rp34xx machines. Flushing user pages requires doing a temporary context switch as we have to flush pages that don't belong to the current context. Further, we have to deal with pages that aren't present. If a page isn't present, the flush instructions fault on every line. Other code has been rearranged and simplified based on testing. For example, I introduced a flush_cache_dup_mm routine. flush_cache_mm and flush_cache_dup_mm differ in that flush_cache_mm calls purge_cache_pages and flush_cache_dup_mm calls flush_cache_pages. In some implementations, pdc is more efficient than fdc. Based on my testing, I don't believe there's any performance benefit on the c8000. Signed-off-by: John David Anglin <dave.anglin@bell.net> ---