@@ -116,6 +116,9 @@ struct cpu_cache_fns {
void (*dma_unmap_area)(const void *, size_t, int);
void (*dma_flush_range)(const void *, const void *);
+
+ void (*force_dcache_clean)(void *, size_t);
+ void (*force_dcache_invalidate)(void *, size_t);
};
/*
@@ -133,6 +136,8 @@ extern struct cpu_cache_fns cpu_cache;
#define __cpuc_coherent_kern_range cpu_cache.coherent_kern_range
#define __cpuc_coherent_user_range cpu_cache.coherent_user_range
#define __cpuc_flush_dcache_area cpu_cache.flush_kern_dcache_area
+#define __cpuc_force_dcache_clean cpu_cache.force_dcache_clean
+#define __cpuc_force_dcache_invalidate cpu_cache.force_dcache_invalidate
/*
* These are private to the dma-mapping API. Do not use directly.
@@ -152,6 +157,8 @@ extern void __cpuc_flush_user_range(unsigned long, unsigned long, unsigned int);
extern void __cpuc_coherent_kern_range(unsigned long, unsigned long);
extern int __cpuc_coherent_user_range(unsigned long, unsigned long);
extern void __cpuc_flush_dcache_area(void *, size_t);
+extern void __cpuc_force_dcache_clean(void *, size_t);
+extern void __cpuc_force_dcache_invalidate(void *, size_t);
/*
* These are private to the dma-mapping API. Do not use directly.
@@ -518,4 +525,8 @@ static inline void secure_flush_area(const void *addr, size_t size)
outer_flush_range(phys, phys + size);
}
+#define ARCH_HAS_FORCE_CACHE 1
+void kernel_force_cache_clean(struct page *page, size_t size);
+void kernel_force_cache_invalidate(struct page *page, size_t size);
+
#endif
@@ -157,6 +157,8 @@ static inline void nop_dma_unmap_area(const void *s, size_t l, int f) { }
#define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range)
#define __cpuc_coherent_user_range __glue(_CACHE,_coherent_user_range)
#define __cpuc_flush_dcache_area __glue(_CACHE,_flush_kern_dcache_area)
+#define __cpuc_force_dcache_clean __glue(_CACHE,_force_dcache_clean)
+#define __cpuc_force_dcache_invalidate __glue(_CACHE,_force_dcache_invalidate)
#define dmac_flush_range __glue(_CACHE,_dma_flush_range)
#endif
@@ -3,7 +3,7 @@
#
obj-y := dma-mapping.o extable.o fault.o init.o \
- iomap.o
+ iomap.o cacheflush.o
obj-$(CONFIG_MMU) += fault-armv.o flush.o idmap.o ioremap.o \
mmap.o pgd.o mmu.o pageattr.o
@@ -244,6 +244,14 @@ ENDPROC(fa_dma_unmap_area)
.globl fa_flush_kern_cache_louis
.equ fa_flush_kern_cache_louis, fa_flush_kern_cache_all
+ENTRY(fa_force_dcache_invalidate)
+ ret lr
+ENDPROC(fa_force_dcache_invalidate)
+
+ENTRY(fa_force_dcache_clean)
+ ret lr
+ENDPROC(fa_force_dcache_clean)
+
__INITDATA
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
@@ -45,6 +45,12 @@ ENDPROC(nop_coherent_user_range)
.globl nop_dma_unmap_area
.equ nop_dma_unmap_area, nop_flush_icache_all
+ .globl nop_force_dcache_clean
+ .equ nop_force_dcache_clean, nop_flush_icache_all
+
+ .globl nop_force_dcache_invalidate
+ .equ nop_force_dcache_invalidate, nop_flush_icache_all
+
__INITDATA
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
@@ -144,6 +144,16 @@ ENDPROC(v4_dma_map_area)
.globl v4_flush_kern_cache_louis
.equ v4_flush_kern_cache_louis, v4_flush_kern_cache_all
+
+ENTRY(v4_force_dcache_invalidate)
+ ret lr
+ENDPROC(v4_force_dcache_invalidate)
+
+ENTRY(v4_force_dcache_clean)
+ ret lr
+ENDPROC(v4_force_dcache_clean)
+
+
__INITDATA
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
@@ -255,6 +255,14 @@ ENDPROC(v4wb_dma_unmap_area)
.globl v4wb_flush_kern_cache_louis
.equ v4wb_flush_kern_cache_louis, v4wb_flush_kern_cache_all
+ENTRY(v4wb_force_dcache_invalidate)
+ ret lr
+ENDPROC(v4wb_force_dcache_invalidate)
+
+ENTRY(v4wb_force_dcache_clean)
+ ret lr
+ENDPROC(v4wb_force_dcache_clean)
+
__INITDATA
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
@@ -200,6 +200,14 @@ ENDPROC(v4wt_dma_map_area)
.globl v4wt_flush_kern_cache_louis
.equ v4wt_flush_kern_cache_louis, v4wt_flush_kern_cache_all
+ENTRY(v4wt_force_dcache_invalidate)
+ ret lr
+ENDPROC(v4wt_force_dcache_invalidate)
+
+ENTRY(v4wt_force_dcache_clean)
+ ret lr
+ENDPROC(v4wt_force_dcache_clean)
+
__INITDATA
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
@@ -333,3 +333,11 @@ ENDPROC(v6_dma_unmap_area)
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
define_cache_functions v6
+
+ENTRY(v6_force_dcache_invalidate)
+ ret lr
+ENDPROC(v6_force_dcache_invalidate)
+
+ENTRY(v6_force_dcache_clean)
+ ret lr
+ENDPROC(v6_force_dcache_clean)
@@ -442,6 +442,19 @@ ENTRY(v7_dma_unmap_area)
ret lr
ENDPROC(v7_dma_unmap_area)
+
+ENTRY(v7_force_dcache_invalidate)
+ add r1, r1, r0
+ b v7_dma_inv_range
+ ret lr
+ENDPROC(v7_force_dcache_invalidate)
+
+ENTRY(v7_force_dcache_clean)
+ add r1, r1, r0
+ b v7_dma_clean_range
+ ret lr
+ENDPROC(v7_force_dcache_clean)
+
__INITDATA
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
new file mode 100644
@@ -0,0 +1,71 @@
+/*
+ * Based on arch/arm/mm/dma-mapping.c which is
+ * Copyright (C) 2000-2004 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+
+#include <asm/highmem.h>
+#include <asm/cacheflush.h>
+
+static void __force_cache_op(struct page *page, size_t size,
+ void (*op)(void *start, size_t size))
+{
+ unsigned long pfn;
+ size_t left = size;
+
+ pfn = page_to_pfn(page);
+
+ do {
+ size_t len = left;
+ void *vaddr;
+
+ page = pfn_to_page(pfn);
+
+ if (PageHighMem(page)) {
+ if (len > PAGE_SIZE)
+ len = PAGE_SIZE;
+ if (cache_is_vipt_nonaliasing()) {
+ vaddr = kmap_atomic(page);
+ op(vaddr, len);
+ kunmap_atomic(vaddr);
+ } else {
+ vaddr = kmap_high_get(page);
+ if (vaddr) {
+ op(vaddr, len);
+ kunmap_high(page);
+ }
+ }
+ } else {
+
+ op(page_address(page), len);
+ }
+ pfn++;
+ left -= len;
+ } while(left);
+}
+
+void kernel_force_cache_clean(struct page *page, size_t size)
+{
+ phys_addr_t paddr;
+
+ paddr = page_to_phys(page);
+ __force_cache_op(page, size, __cpuc_force_dcache_clean);
+ outer_clean_range(paddr, paddr + size);
+}
+
+void kernel_force_cache_invalidate(struct page *page, size_t size)
+{
+ phys_addr_t paddr;
+
+ paddr = page_to_phys(page);
+ __force_cache_op(page, size, __cpuc_force_dcache_invalidate);
+ outer_inv_range(paddr, paddr + size);
+}
@@ -334,6 +334,14 @@ ENTRY(cpu_arm920_dcache_clean_area)
bhi 1b
ret lr
+ENTRY(arm920_force_dcache_invalidate)
+ ret lr
+ENDPROC(arm920_force_dcache_invalidate)
+
+ENTRY(arm920_force_dcache_clean)
+ ret lr
+ENDPROC(arm920_force_dcache_clean)
+
/* =============================== PageTable ============================== */
/*
@@ -338,6 +338,14 @@ ENTRY(cpu_arm922_dcache_clean_area)
#endif
ret lr
+ENTRY(arm922_force_dcache_invalidate)
+ ret lr
+ENDPROC(arm922_force_dcache_invalidate)
+
+ENTRY(arm922_force_dcache_clean)
+ ret lr
+ENDPROC(arm922_force_dcache_clean)
+
/* =============================== PageTable ============================== */
/*
@@ -392,6 +392,14 @@ ENTRY(cpu_arm925_dcache_clean_area)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
ret lr
+ENTRY(arm925_force_dcache_invalidate)
+ ret lr
+ENDPROC(arm925_force_dcache_invalidate)
+
+ENTRY(arm925_force_dcache_clean)
+ ret lr
+ENDPROC(arm925_force_dcache_clean)
+
/* =============================== PageTable ============================== */
/*
@@ -355,6 +355,14 @@ ENTRY(cpu_arm926_dcache_clean_area)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
ret lr
+ENTRY(arm926_force_dcache_invalidate)
+ ret lr
+ENDPROC(arm926_force_dcache_invalidate)
+
+ENTRY(arm926_force_dcache_clean)
+ ret lr
+ENDPROC(arm926_force_dcache_clean)
+
/* =============================== PageTable ============================== */
/*
@@ -439,6 +439,8 @@ ENDPROC(feroceon_dma_unmap_area)
range_alias coherent_kern_range
range_alias coherent_user_range
range_alias dma_unmap_area
+ range_alias force_dcache_clean
+ range_alias force_dcache_invalidate
define_cache_functions feroceon_range
@@ -463,6 +465,15 @@ ENTRY(cpu_feroceon_dcache_clean_area)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
ret lr
+ENTRY(feroceon_force_dcache_invalidate)
+ ret lr
+ENDPROC(feroceon_force_dcache_invalidate)
+
+ENTRY(feroceon_force_dcache_clean)
+ ret lr
+ENDPROC(feroceon_force_dcache_clean)
+
+
/* =============================== PageTable ============================== */
/*
@@ -310,6 +310,8 @@ ENTRY(\name\()_cache_fns)
.long \name\()_dma_map_area
.long \name\()_dma_unmap_area
.long \name\()_dma_flush_range
+ .long \name\()_force_dcache_clean
+ .long \name\()_force_dcache_invalidate
.size \name\()_cache_fns, . - \name\()_cache_fns
.endm
@@ -343,6 +343,15 @@ ENDPROC(xsc3_dma_unmap_area)
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
define_cache_functions xsc3
+ENTRY(xsc3_force_dcache_clean)
+ ret lr
+ENDPROC(xsc3_force_dcache_clean)
+
+ENTRY(xsc3_force_dcache_invalidate)
+ ret lr
+ENDPROC(xsc3_force_dcache_invalidate)
+
+
ENTRY(cpu_xsc3_dcache_clean_area)
1: mcr p15, 0, r0, c7, c10, 1 @ clean L1 D line
add r0, r0, #CACHELINESIZE
@@ -449,6 +449,8 @@ ENDPROC(xscale_dma_unmap_area)
a0_alias flush_kern_dcache_area
a0_alias dma_flush_range
a0_alias dma_unmap_area
+ a0_alias force_dcache_clean
+ a0_alias force_dcache_invalidate
@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
define_cache_functions xscale_80200_A0_A1
@@ -460,6 +462,13 @@ ENTRY(cpu_xscale_dcache_clean_area)
bhi 1b
ret lr
+ENTRY(xscale_force_dcache_invalidate)
+ ret lr
+ENDPROC(xscale_force_dcache_invalidate)
+
+ENTRY(xscale_force_dcache_clean)
+ ret lr
+ENDPROC(xscale_force_dcache_clean)
/* =============================== PageTable ============================== */
/*
arm64 may need to guarantee the caches are synced. Implement versions of the kernel_force_cache API for v7. Other versions are stubbed out and can be added as appropriate. Signed-off-by: Laura Abbott <labbott@redhat.com> --- v3: Switch to force implementations per CPU instead of relying on dma_map/dma_unmap. v7 is the only one implemented right now, others can be added as needed. --- arch/arm/include/asm/cacheflush.h | 11 ++++++ arch/arm/include/asm/glue-cache.h | 2 ++ arch/arm/mm/Makefile | 2 +- arch/arm/mm/cache-fa.S | 8 +++++ arch/arm/mm/cache-nop.S | 6 ++++ arch/arm/mm/cache-v4.S | 10 ++++++ arch/arm/mm/cache-v4wb.S | 8 +++++ arch/arm/mm/cache-v4wt.S | 8 +++++ arch/arm/mm/cache-v6.S | 8 +++++ arch/arm/mm/cache-v7.S | 13 +++++++ arch/arm/mm/cacheflush.c | 71 +++++++++++++++++++++++++++++++++++++++ arch/arm/mm/proc-arm920.S | 8 +++++ arch/arm/mm/proc-arm922.S | 8 +++++ arch/arm/mm/proc-arm925.S | 8 +++++ arch/arm/mm/proc-arm926.S | 8 +++++ arch/arm/mm/proc-feroceon.S | 11 ++++++ arch/arm/mm/proc-macros.S | 2 ++ arch/arm/mm/proc-xsc3.S | 9 +++++ arch/arm/mm/proc-xscale.S | 9 +++++ 19 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 arch/arm/mm/cacheflush.c