@@ -1,4 +1,5 @@
obj-y += lib/
+obj-$(CONFIG_MMU) += mmu/
obj-y += cache.o
obj-y += cpufeature.o
@@ -28,17 +28,6 @@
#include <asm/arm64/efibind.h>
#endif
-#define PT_PT 0xf7f /* nG=1 AF=1 SH=11 AP=01 NS=1 ATTR=111 T=1 P=1 */
-#define PT_MEM 0xf7d /* nG=1 AF=1 SH=11 AP=01 NS=1 ATTR=111 T=0 P=1 */
-#define PT_MEM_L3 0xf7f /* nG=1 AF=1 SH=11 AP=01 NS=1 ATTR=111 T=1 P=1 */
-#define PT_DEV 0xe71 /* nG=1 AF=1 SH=10 AP=01 NS=1 ATTR=100 T=0 P=1 */
-#define PT_DEV_L3 0xe73 /* nG=1 AF=1 SH=10 AP=01 NS=1 ATTR=100 T=1 P=1 */
-
-/* Convenience defines to get slot used by Xen mapping. */
-#define XEN_ZEROETH_SLOT zeroeth_table_offset(XEN_VIRT_START)
-#define XEN_FIRST_SLOT first_table_offset(XEN_VIRT_START)
-#define XEN_SECOND_SLOT second_table_offset(XEN_VIRT_START)
-
#define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2)
#define __HEAD_FLAG_PHYS_BASE 1
@@ -85,19 +74,7 @@
* x30 - lr
*/
-#ifdef CONFIG_EARLY_PRINTK
-/*
- * Macro to print a string to the UART, if there is one.
- *
- * Clobbers x0 - x3
- */
-#define PRINT(_s) \
- mov x3, lr ; \
- adr_l x0, 98f ; \
- bl asm_puts ; \
- mov lr, x3 ; \
- RODATA_STR(98, _s)
-
+ #ifdef CONFIG_EARLY_PRINTK
/*
* Macro to print the value of register \xb
*
@@ -111,43 +88,11 @@
.endm
#else /* CONFIG_EARLY_PRINTK */
-#define PRINT(s)
-
.macro print_reg xb
.endm
#endif /* !CONFIG_EARLY_PRINTK */
-/*
- * Pseudo-op for PC relative adr <reg>, <symbol> where <symbol> is
- * within the range +/- 4GB of the PC.
- *
- * @dst: destination register (64 bit wide)
- * @sym: name of the symbol
- */
-.macro adr_l, dst, sym
- adrp \dst, \sym
- add \dst, \dst, :lo12:\sym
-.endm
-
-/* Load the physical address of a symbol into xb */
-.macro load_paddr xb, sym
- ldr \xb, =\sym
- add \xb, \xb, x20
-.endm
-
-/*
- * Flush local TLBs
- *
- * See asm/arm64/flushtlb.h for the explanation of the sequence.
- */
-.macro flush_xen_tlb_local
- dsb nshst
- tlbi alle2
- dsb nsh
- isb
-.endm
-
.section .text.header, "ax", %progbits
/*.aarch64*/
@@ -484,402 +429,6 @@ cpu_init:
ret
ENDPROC(cpu_init)
-/*
- * Macro to find the slot number at a given page-table level
- *
- * slot: slot computed
- * virt: virtual address
- * lvl: page-table level
- */
-.macro get_table_slot, slot, virt, lvl
- ubfx \slot, \virt, #XEN_PT_LEVEL_SHIFT(\lvl), #XEN_PT_LPAE_SHIFT
-.endm
-
-/*
- * Macro to create a page table entry in \ptbl to \tbl
- * ptbl: table symbol where the entry will be created
- * tbl: physical address of the table to point to
- * virt: virtual address
- * lvl: page-table level
- * tmp1: scratch register
- * tmp2: scratch register
- *
- * Preserves \virt
- * Clobbers \tbl, \tmp1, \tmp2
- *
- * Note that all parameters using registers should be distinct.
- */
-.macro create_table_entry_from_paddr, ptbl, tbl, virt, lvl, tmp1, tmp2
- get_table_slot \tmp1, \virt, \lvl /* \tmp1 := slot in \tbl */
-
- mov \tmp2, #PT_PT /* \tmp2 := right for linear PT */
- orr \tmp2, \tmp2, \tbl /* + \tbl */
-
- adr_l \tbl, \ptbl /* \tbl := address(\ptbl) */
-
- str \tmp2, [\tbl, \tmp1, lsl #3]
-.endm
-
-/*
- * Macro to create a page table entry in \ptbl to \tbl
- *
- * ptbl: table symbol where the entry will be created
- * tbl: table symbol to point to
- * virt: virtual address
- * lvl: page-table level
- * tmp1: scratch register
- * tmp2: scratch register
- * tmp3: scratch register
- *
- * Preserves \virt
- * Clobbers \tmp1, \tmp2, \tmp3
- *
- * Also use x20 for the phys offset.
- *
- * Note that all parameters using registers should be distinct.
- */
-.macro create_table_entry, ptbl, tbl, virt, lvl, tmp1, tmp2, tmp3
- load_paddr \tmp1, \tbl
- create_table_entry_from_paddr \ptbl, \tmp1, \virt, \lvl, \tmp2, \tmp3
-.endm
-
-/*
- * Macro to create a mapping entry in \tbl to \phys. Only mapping in 3rd
- * level table (i.e page granularity) is supported.
- *
- * ptbl: table symbol where the entry will be created
- * virt: virtual address
- * phys: physical address (should be page aligned)
- * tmp1: scratch register
- * tmp2: scratch register
- * tmp3: scratch register
- * type: mapping type. If not specified it will be normal memory (PT_MEM_L3)
- *
- * Preserves \virt, \phys
- * Clobbers \tmp1, \tmp2, \tmp3
- *
- * Note that all parameters using registers should be distinct.
- */
-.macro create_mapping_entry, ptbl, virt, phys, tmp1, tmp2, tmp3, type=PT_MEM_L3
- and \tmp3, \phys, #THIRD_MASK /* \tmp3 := PAGE_ALIGNED(phys) */
-
- get_table_slot \tmp1, \virt, 3 /* \tmp1 := slot in \tlb */
-
- mov \tmp2, #\type /* \tmp2 := right for section PT */
- orr \tmp2, \tmp2, \tmp3 /* + PAGE_ALIGNED(phys) */
-
- adr_l \tmp3, \ptbl
-
- str \tmp2, [\tmp3, \tmp1, lsl #3]
-.endm
-
-/*
- * Rebuild the boot pagetable's first-level entries. The structure
- * is described in mm.c.
- *
- * After the CPU enables paging it will add the fixmap mapping
- * to these page tables, however this may clash with the 1:1
- * mapping. So each CPU must rebuild the page tables here with
- * the 1:1 in place.
- *
- * Inputs:
- * x19: paddr(start)
- * x20: phys offset
- *
- * Clobbers x0 - x4
- */
-create_page_tables:
- /* Prepare the page-tables for mapping Xen */
- ldr x0, =XEN_VIRT_START
- create_table_entry boot_pgtable, boot_first, x0, 0, x1, x2, x3
- create_table_entry boot_first, boot_second, x0, 1, x1, x2, x3
-
- /*
- * We need to use a stash register because
- * create_table_entry_paddr() will clobber the register storing
- * the physical address of the table to point to.
- */
- load_paddr x4, boot_third
- ldr x1, =XEN_VIRT_START
-.rept XEN_NR_ENTRIES(2)
- mov x0, x4 /* x0 := paddr(l3 table) */
- create_table_entry_from_paddr boot_second, x0, x1, 2, x2, x3
- add x1, x1, #XEN_PT_LEVEL_SIZE(2) /* x1 := Next vaddr */
- add x4, x4, #PAGE_SIZE /* x4 := Next table */
-.endr
-
- /*
- * Find the size of Xen in pages and multiply by the size of a
- * PTE. This will then be compared in the mapping loop below.
- *
- * Note the multiplication is just to avoid using an extra
- * register/instruction per iteration.
- */
- ldr x0, =_start /* x0 := vaddr(_start) */
- ldr x1, =_end /* x1 := vaddr(_end) */
- sub x0, x1, x0 /* x0 := effective size of Xen */
- lsr x0, x0, #PAGE_SHIFT /* x0 := Number of pages for Xen */
- lsl x0, x0, #3 /* x0 := Number of pages * PTE size */
-
- /* Map Xen */
- adr_l x4, boot_third
-
- lsr x2, x19, #THIRD_SHIFT /* Base address for 4K mapping */
- lsl x2, x2, #THIRD_SHIFT
- mov x3, #PT_MEM_L3 /* x2 := Section map */
- orr x2, x2, x3
-
- /* ... map of vaddr(start) in boot_third */
- mov x1, xzr
-1: str x2, [x4, x1] /* Map vaddr(start) */
- add x2, x2, #PAGE_SIZE /* Next page */
- add x1, x1, #8 /* Next slot */
- cmp x1, x0 /* Loop until we map all of Xen */
- b.lt 1b
-
- /*
- * If Xen is loaded at exactly XEN_VIRT_START then we don't
- * need an additional 1:1 mapping, the virtual mapping will
- * suffice.
- */
- ldr x0, =XEN_VIRT_START
- cmp x19, x0
- bne 1f
- ret
-1:
- /*
- * Setup the 1:1 mapping so we can turn the MMU on. Note that
- * only the first page of Xen will be part of the 1:1 mapping.
- */
-
- /*
- * Find the zeroeth slot used. If the slot is not
- * XEN_ZEROETH_SLOT, then the 1:1 mapping will use its own set of
- * page-tables from the first level.
- */
- get_table_slot x0, x19, 0 /* x0 := zeroeth slot */
- cmp x0, #XEN_ZEROETH_SLOT
- beq 1f
- create_table_entry boot_pgtable, boot_first_id, x19, 0, x0, x1, x2
- b link_from_first_id
-
-1:
- /*
- * Find the first slot used. If the slot is not XEN_FIRST_SLOT,
- * then the 1:1 mapping will use its own set of page-tables from
- * the second level.
- */
- get_table_slot x0, x19, 1 /* x0 := first slot */
- cmp x0, #XEN_FIRST_SLOT
- beq 1f
- create_table_entry boot_first, boot_second_id, x19, 1, x0, x1, x2
- b link_from_second_id
-
-1:
- /*
- * Find the second slot used. If the slot is XEN_SECOND_SLOT, then the
- * 1:1 mapping will use its own set of page-tables from the
- * third level. For slot XEN_SECOND_SLOT, Xen is not yet able to handle
- * it.
- */
- get_table_slot x0, x19, 2 /* x0 := second slot */
- cmp x0, #XEN_SECOND_SLOT
- beq virtphys_clash
- create_table_entry boot_second, boot_third_id, x19, 2, x0, x1, x2
- b link_from_third_id
-
-link_from_first_id:
- create_table_entry boot_first_id, boot_second_id, x19, 1, x0, x1, x2
-link_from_second_id:
- create_table_entry boot_second_id, boot_third_id, x19, 2, x0, x1, x2
-link_from_third_id:
- create_mapping_entry boot_third_id, x19, x19, x0, x1, x2
- ret
-
-virtphys_clash:
- /* Identity map clashes with boot_third, which we cannot handle yet */
- PRINT("- Unable to build boot page tables - virt and phys addresses clash. -\r\n")
- b fail
-ENDPROC(create_page_tables)
-
-/*
- * Turn on the Data Cache and the MMU. The function will return on the 1:1
- * mapping. In other word, the caller is responsible to switch to the runtime
- * mapping.
- *
- * Inputs:
- * x0 : Physical address of the page tables.
- *
- * Clobbers x0 - x4
- */
-enable_mmu:
- mov x4, x0
- PRINT("- Turning on paging -\r\n")
-
- /*
- * The state of the TLBs is unknown before turning on the MMU.
- * Flush them to avoid stale one.
- */
- flush_xen_tlb_local
-
- /* Write Xen's PT's paddr into TTBR0_EL2 */
- msr TTBR0_EL2, x4
- isb
-
- mrs x0, SCTLR_EL2
- orr x0, x0, #SCTLR_Axx_ELx_M /* Enable MMU */
- orr x0, x0, #SCTLR_Axx_ELx_C /* Enable D-cache */
- dsb sy /* Flush PTE writes and finish reads */
- msr SCTLR_EL2, x0 /* now paging is enabled */
- isb /* Now, flush the icache */
- ret
-ENDPROC(enable_mmu)
-
-/*
- * Enable mm (turn on the data cache and the MMU) for secondary CPUs.
- * The function will return to the virtual address provided in LR (e.g. the
- * runtime mapping).
- *
- * Inputs:
- * lr : Virtual address to return to.
- *
- * Clobbers x0 - x5
- */
-enable_secondary_cpu_mm:
- mov x5, lr
-
- load_paddr x0, init_ttbr
- ldr x0, [x0]
-
- bl enable_mmu
- mov lr, x5
-
- /* Return to the virtual address requested by the caller. */
- ret
-ENDPROC(enable_secondary_cpu_mm)
-
-/*
- * Enable mm (turn on the data cache and the MMU) for the boot CPU.
- * The function will return to the virtual address provided in LR (e.g. the
- * runtime mapping).
- *
- * Inputs:
- * lr : Virtual address to return to.
- *
- * Clobbers x0 - x5
- */
-enable_boot_cpu_mm:
- mov x5, lr
-
- bl create_page_tables
- load_paddr x0, boot_pgtable
-
- bl enable_mmu
-
- /*
- * The MMU is turned on and we are in the 1:1 mapping. Switch
- * to the runtime mapping.
- */
- ldr x0, =1f
- br x0
-1:
- mov lr, x5
- /*
- * The 1:1 map may clash with other parts of the Xen virtual memory
- * layout. As it is not used anymore, remove it completely to avoid
- * having to worry about replacing existing mapping afterwards.
- * Function will return to the virtual address requested by the caller.
- */
- b remove_identity_mapping
-ENDPROC(enable_boot_cpu_mm)
-
-/*
- * Remove the 1:1 map from the page-tables. It is not easy to keep track
- * where the 1:1 map was mapped, so we will look for the top-level entry
- * exclusive to the 1:1 map and remove it.
- *
- * Inputs:
- * x19: paddr(start)
- *
- * Clobbers x0 - x1
- */
-remove_identity_mapping:
- /*
- * Find the zeroeth slot used. Remove the entry from zeroeth
- * table if the slot is not XEN_ZEROETH_SLOT.
- */
- get_table_slot x1, x19, 0 /* x1 := zeroeth slot */
- cmp x1, #XEN_ZEROETH_SLOT
- beq 1f
- /* It is not in slot XEN_ZEROETH_SLOT, remove the entry. */
- ldr x0, =boot_pgtable /* x0 := root table */
- str xzr, [x0, x1, lsl #3]
- b identity_mapping_removed
-
-1:
- /*
- * Find the first slot used. Remove the entry for the first
- * table if the slot is not XEN_FIRST_SLOT.
- */
- get_table_slot x1, x19, 1 /* x1 := first slot */
- cmp x1, #XEN_FIRST_SLOT
- beq 1f
- /* It is not in slot XEN_FIRST_SLOT, remove the entry. */
- ldr x0, =boot_first /* x0 := first table */
- str xzr, [x0, x1, lsl #3]
- b identity_mapping_removed
-
-1:
- /*
- * Find the second slot used. Remove the entry for the first
- * table if the slot is not XEN_SECOND_SLOT.
- */
- get_table_slot x1, x19, 2 /* x1 := second slot */
- cmp x1, #XEN_SECOND_SLOT
- beq identity_mapping_removed
- /* It is not in slot 1, remove the entry */
- ldr x0, =boot_second /* x0 := second table */
- str xzr, [x0, x1, lsl #3]
-
-identity_mapping_removed:
- flush_xen_tlb_local
-
- ret
-ENDPROC(remove_identity_mapping)
-
-/*
- * Map the UART in the fixmap (when earlyprintk is used) and hook the
- * fixmap table in the page tables.
- *
- * The fixmap cannot be mapped in create_page_tables because it may
- * clash with the 1:1 mapping.
- *
- * Inputs:
- * x20: Physical offset
- * x23: Early UART base physical address
- *
- * Clobbers x0 - x3
- */
-setup_fixmap:
-#ifdef CONFIG_EARLY_PRINTK
- /* Add UART to the fixmap table */
- ldr x0, =EARLY_UART_VIRTUAL_ADDRESS
- create_mapping_entry xen_fixmap, x0, x23, x1, x2, x3, type=PT_DEV_L3
-#endif
- /* Map fixmap into boot_second */
- ldr x0, =FIXMAP_ADDR(0)
- create_table_entry boot_second, xen_fixmap, x0, 2, x1, x2, x3
- /* Ensure any page table updates made above have occurred. */
- dsb nshst
- /*
- * The fixmap area will be used soon after. So ensure no hardware
- * translation happens before the dsb completes.
- */
- isb
-
- ret
-ENDPROC(setup_fixmap)
-
/*
* Setup the initial stack and jump to the C world
*
@@ -908,45 +457,6 @@ fail: PRINT("- Boot failed -\r\n")
b 1b
ENDPROC(fail)
-/*
- * Switch TTBR
- *
- * x0 ttbr
- */
-ENTRY(switch_ttbr_id)
- /* 1) Ensure any previous read/write have completed */
- dsb ish
- isb
-
- /* 2) Turn off MMU */
- mrs x1, SCTLR_EL2
- bic x1, x1, #SCTLR_Axx_ELx_M
- msr SCTLR_EL2, x1
- isb
-
- /* 3) Flush the TLBs */
- flush_xen_tlb_local
-
- /* 4) Update the TTBR */
- msr TTBR0_EL2, x0
- isb
-
- /*
- * 5) Flush I-cache
- * This should not be necessary but it is kept for safety.
- */
- ic iallu
- isb
-
- /* 6) Turn on the MMU */
- mrs x1, SCTLR_EL2
- orr x1, x1, #SCTLR_Axx_ELx_M /* Enable MMU */
- msr SCTLR_EL2, x1
- isb
-
- ret
-ENDPROC(switch_ttbr_id)
-
#ifdef CONFIG_EARLY_PRINTK
/*
* Initialize the UART. Should only be called on the boot CPU.
new file mode 100644
@@ -0,0 +1 @@
+obj-y += head.o
new file mode 100644
@@ -0,0 +1,488 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * xen/arch/arm/arm64/mmu/head.S
+ *
+ * Arm64 MMU specific start-of-day code.
+ */
+
+#include <asm/page.h>
+#include <asm/early_printk.h>
+
+#define PT_PT 0xf7f /* nG=1 AF=1 SH=11 AP=01 NS=1 ATTR=111 T=1 P=1 */
+#define PT_MEM 0xf7d /* nG=1 AF=1 SH=11 AP=01 NS=1 ATTR=111 T=0 P=1 */
+#define PT_MEM_L3 0xf7f /* nG=1 AF=1 SH=11 AP=01 NS=1 ATTR=111 T=1 P=1 */
+#define PT_DEV 0xe71 /* nG=1 AF=1 SH=10 AP=01 NS=1 ATTR=100 T=0 P=1 */
+#define PT_DEV_L3 0xe73 /* nG=1 AF=1 SH=10 AP=01 NS=1 ATTR=100 T=1 P=1 */
+
+/* Convenience defines to get slot used by Xen mapping. */
+#define XEN_ZEROETH_SLOT zeroeth_table_offset(XEN_VIRT_START)
+#define XEN_FIRST_SLOT first_table_offset(XEN_VIRT_START)
+#define XEN_SECOND_SLOT second_table_offset(XEN_VIRT_START)
+
+/*
+ * Flush local TLBs
+ *
+ * See asm/arm64/flushtlb.h for the explanation of the sequence.
+ */
+.macro flush_xen_tlb_local
+ dsb nshst
+ tlbi alle2
+ dsb nsh
+ isb
+.endm
+
+/*
+ * Macro to find the slot number at a given page-table level
+ *
+ * slot: slot computed
+ * virt: virtual address
+ * lvl: page-table level
+ */
+.macro get_table_slot, slot, virt, lvl
+ ubfx \slot, \virt, #XEN_PT_LEVEL_SHIFT(\lvl), #XEN_PT_LPAE_SHIFT
+.endm
+
+/*
+ * Macro to create a page table entry in \ptbl to \tbl
+ * ptbl: table symbol where the entry will be created
+ * tbl: physical address of the table to point to
+ * virt: virtual address
+ * lvl: page-table level
+ * tmp1: scratch register
+ * tmp2: scratch register
+ *
+ * Preserves \virt
+ * Clobbers \tbl, \tmp1, \tmp2
+ *
+ * Note that all parameters using registers should be distinct.
+ */
+.macro create_table_entry_from_paddr, ptbl, tbl, virt, lvl, tmp1, tmp2
+ get_table_slot \tmp1, \virt, \lvl /* \tmp1 := slot in \tbl */
+
+ mov \tmp2, #PT_PT /* \tmp2 := right for linear PT */
+ orr \tmp2, \tmp2, \tbl /* + \tbl */
+
+ adr_l \tbl, \ptbl /* \tbl := address(\ptbl) */
+
+ str \tmp2, [\tbl, \tmp1, lsl #3]
+.endm
+
+/*
+ * Macro to create a page table entry in \ptbl to \tbl
+ *
+ * ptbl: table symbol where the entry will be created
+ * tbl: table symbol to point to
+ * virt: virtual address
+ * lvl: page-table level
+ * tmp1: scratch register
+ * tmp2: scratch register
+ * tmp3: scratch register
+ *
+ * Preserves \virt
+ * Clobbers \tmp1, \tmp2, \tmp3
+ *
+ * Also use x20 for the phys offset.
+ *
+ * Note that all parameters using registers should be distinct.
+ */
+.macro create_table_entry, ptbl, tbl, virt, lvl, tmp1, tmp2, tmp3
+ load_paddr \tmp1, \tbl
+ create_table_entry_from_paddr \ptbl, \tmp1, \virt, \lvl, \tmp2, \tmp3
+.endm
+
+/*
+ * Macro to create a mapping entry in \tbl to \phys. Only mapping in 3rd
+ * level table (i.e page granularity) is supported.
+ *
+ * ptbl: table symbol where the entry will be created
+ * virt: virtual address
+ * phys: physical address (should be page aligned)
+ * tmp1: scratch register
+ * tmp2: scratch register
+ * tmp3: scratch register
+ * type: mapping type. If not specified it will be normal memory (PT_MEM_L3)
+ *
+ * Preserves \virt, \phys
+ * Clobbers \tmp1, \tmp2, \tmp3
+ *
+ * Note that all parameters using registers should be distinct.
+ */
+.macro create_mapping_entry, ptbl, virt, phys, tmp1, tmp2, tmp3, type=PT_MEM_L3
+ and \tmp3, \phys, #THIRD_MASK /* \tmp3 := PAGE_ALIGNED(phys) */
+
+ get_table_slot \tmp1, \virt, 3 /* \tmp1 := slot in \tlb */
+
+ mov \tmp2, #\type /* \tmp2 := right for section PT */
+ orr \tmp2, \tmp2, \tmp3 /* + PAGE_ALIGNED(phys) */
+
+ adr_l \tmp3, \ptbl
+
+ str \tmp2, [\tmp3, \tmp1, lsl #3]
+.endm
+
+.section .text.idmap, "ax", %progbits
+
+/*
+ * Rebuild the boot pagetable's first-level entries. The structure
+ * is described in mm.c.
+ *
+ * After the CPU enables paging it will add the fixmap mapping
+ * to these page tables, however this may clash with the 1:1
+ * mapping. So each CPU must rebuild the page tables here with
+ * the 1:1 in place.
+ *
+ * Inputs:
+ * x19: paddr(start)
+ * x20: phys offset
+ *
+ * Clobbers x0 - x4
+ */
+create_page_tables:
+ /* Prepare the page-tables for mapping Xen */
+ ldr x0, =XEN_VIRT_START
+ create_table_entry boot_pgtable, boot_first, x0, 0, x1, x2, x3
+ create_table_entry boot_first, boot_second, x0, 1, x1, x2, x3
+
+ /*
+ * We need to use a stash register because
+ * create_table_entry_paddr() will clobber the register storing
+ * the physical address of the table to point to.
+ */
+ load_paddr x4, boot_third
+ ldr x1, =XEN_VIRT_START
+.rept XEN_NR_ENTRIES(2)
+ mov x0, x4 /* x0 := paddr(l3 table) */
+ create_table_entry_from_paddr boot_second, x0, x1, 2, x2, x3
+ add x1, x1, #XEN_PT_LEVEL_SIZE(2) /* x1 := Next vaddr */
+ add x4, x4, #PAGE_SIZE /* x4 := Next table */
+.endr
+
+ /*
+ * Find the size of Xen in pages and multiply by the size of a
+ * PTE. This will then be compared in the mapping loop below.
+ *
+ * Note the multiplication is just to avoid using an extra
+ * register/instruction per iteration.
+ */
+ ldr x0, =_start /* x0 := vaddr(_start) */
+ ldr x1, =_end /* x1 := vaddr(_end) */
+ sub x0, x1, x0 /* x0 := effective size of Xen */
+ lsr x0, x0, #PAGE_SHIFT /* x0 := Number of pages for Xen */
+ lsl x0, x0, #3 /* x0 := Number of pages * PTE size */
+
+ /* Map Xen */
+ adr_l x4, boot_third
+
+ lsr x2, x19, #THIRD_SHIFT /* Base address for 4K mapping */
+ lsl x2, x2, #THIRD_SHIFT
+ mov x3, #PT_MEM_L3 /* x2 := Section map */
+ orr x2, x2, x3
+
+ /* ... map of vaddr(start) in boot_third */
+ mov x1, xzr
+1: str x2, [x4, x1] /* Map vaddr(start) */
+ add x2, x2, #PAGE_SIZE /* Next page */
+ add x1, x1, #8 /* Next slot */
+ cmp x1, x0 /* Loop until we map all of Xen */
+ b.lt 1b
+
+ /*
+ * If Xen is loaded at exactly XEN_VIRT_START then we don't
+ * need an additional 1:1 mapping, the virtual mapping will
+ * suffice.
+ */
+ ldr x0, =XEN_VIRT_START
+ cmp x19, x0
+ bne 1f
+ ret
+1:
+ /*
+ * Setup the 1:1 mapping so we can turn the MMU on. Note that
+ * only the first page of Xen will be part of the 1:1 mapping.
+ */
+
+ /*
+ * Find the zeroeth slot used. If the slot is not
+ * XEN_ZEROETH_SLOT, then the 1:1 mapping will use its own set of
+ * page-tables from the first level.
+ */
+ get_table_slot x0, x19, 0 /* x0 := zeroeth slot */
+ cmp x0, #XEN_ZEROETH_SLOT
+ beq 1f
+ create_table_entry boot_pgtable, boot_first_id, x19, 0, x0, x1, x2
+ b link_from_first_id
+
+1:
+ /*
+ * Find the first slot used. If the slot is not XEN_FIRST_SLOT,
+ * then the 1:1 mapping will use its own set of page-tables from
+ * the second level.
+ */
+ get_table_slot x0, x19, 1 /* x0 := first slot */
+ cmp x0, #XEN_FIRST_SLOT
+ beq 1f
+ create_table_entry boot_first, boot_second_id, x19, 1, x0, x1, x2
+ b link_from_second_id
+
+1:
+ /*
+ * Find the second slot used. If the slot is XEN_SECOND_SLOT, then the
+ * 1:1 mapping will use its own set of page-tables from the
+ * third level. For slot XEN_SECOND_SLOT, Xen is not yet able to handle
+ * it.
+ */
+ get_table_slot x0, x19, 2 /* x0 := second slot */
+ cmp x0, #XEN_SECOND_SLOT
+ beq virtphys_clash
+ create_table_entry boot_second, boot_third_id, x19, 2, x0, x1, x2
+ b link_from_third_id
+
+link_from_first_id:
+ create_table_entry boot_first_id, boot_second_id, x19, 1, x0, x1, x2
+link_from_second_id:
+ create_table_entry boot_second_id, boot_third_id, x19, 2, x0, x1, x2
+link_from_third_id:
+ create_mapping_entry boot_third_id, x19, x19, x0, x1, x2
+ ret
+
+virtphys_clash:
+ /* Identity map clashes with boot_third, which we cannot handle yet */
+ PRINT("- Unable to build boot page tables - virt and phys addresses clash. -\r\n")
+ b fail
+ENDPROC(create_page_tables)
+
+/*
+ * Turn on the Data Cache and the MMU. The function will return on the 1:1
+ * mapping. In other word, the caller is responsible to switch to the runtime
+ * mapping.
+ *
+ * Inputs:
+ * x0 : Physical address of the page tables.
+ *
+ * Clobbers x0 - x4
+ */
+enable_mmu:
+ mov x4, x0
+ PRINT("- Turning on paging -\r\n")
+
+ /*
+ * The state of the TLBs is unknown before turning on the MMU.
+ * Flush them to avoid stale one.
+ */
+ flush_xen_tlb_local
+
+ /* Write Xen's PT's paddr into TTBR0_EL2 */
+ msr TTBR0_EL2, x4
+ isb
+
+ mrs x0, SCTLR_EL2
+ orr x0, x0, #SCTLR_Axx_ELx_M /* Enable MMU */
+ orr x0, x0, #SCTLR_Axx_ELx_C /* Enable D-cache */
+ dsb sy /* Flush PTE writes and finish reads */
+ msr SCTLR_EL2, x0 /* now paging is enabled */
+ isb /* Now, flush the icache */
+ ret
+ENDPROC(enable_mmu)
+
+/*
+ * Enable mm (turn on the data cache and the MMU) for secondary CPUs.
+ * The function will return to the virtual address provided in LR (e.g. the
+ * runtime mapping).
+ *
+ * Inputs:
+ * lr : Virtual address to return to.
+ *
+ * Clobbers x0 - x5
+ */
+ENTRY(enable_secondary_cpu_mm)
+ mov x5, lr
+
+ load_paddr x0, init_ttbr
+ ldr x0, [x0]
+
+ bl enable_mmu
+ mov lr, x5
+
+ /* return to secondary_switched */
+ ret
+ENDPROC(enable_secondary_cpu_mm)
+
+/*
+ * Enable mm (turn on the data cache and the MMU) for the boot CPU.
+ * The function will return to the virtual address provided in LR (e.g. the
+ * runtime mapping).
+ *
+ * Inputs:
+ * lr : Virtual address to return to.
+ *
+ * Clobbers x0 - x5
+ */
+ENTRY(enable_boot_cpu_mm)
+ mov x5, lr
+
+ bl create_page_tables
+ load_paddr x0, boot_pgtable
+
+ bl enable_mmu
+ mov lr, x5
+
+ /*
+ * The MMU is turned on and we are in the 1:1 mapping. Switch
+ * to the runtime mapping.
+ */
+ ldr x0, =1f
+ br x0
+1:
+ /*
+ * The 1:1 map may clash with other parts of the Xen virtual memory
+ * layout. As it is not used anymore, remove it completely to
+ * avoid having to worry about replacing existing mapping
+ * afterwards. Function will return to primary_switched.
+ */
+ b remove_identity_mapping
+
+ /*
+ * Below is supposed to be unreachable code, as "ret" in
+ * remove_identity_mapping will use the return address in LR in advance.
+ */
+ b fail
+ENDPROC(enable_boot_cpu_mm)
+
+/*
+ * Remove the 1:1 map from the page-tables. It is not easy to keep track
+ * where the 1:1 map was mapped, so we will look for the top-level entry
+ * exclusive to the 1:1 map and remove it.
+ *
+ * Inputs:
+ * x19: paddr(start)
+ *
+ * Clobbers x0 - x1
+ */
+remove_identity_mapping:
+ /*
+ * Find the zeroeth slot used. Remove the entry from zeroeth
+ * table if the slot is not XEN_ZEROETH_SLOT.
+ */
+ get_table_slot x1, x19, 0 /* x1 := zeroeth slot */
+ cmp x1, #XEN_ZEROETH_SLOT
+ beq 1f
+ /* It is not in slot XEN_ZEROETH_SLOT, remove the entry. */
+ ldr x0, =boot_pgtable /* x0 := root table */
+ str xzr, [x0, x1, lsl #3]
+ b identity_mapping_removed
+
+1:
+ /*
+ * Find the first slot used. Remove the entry for the first
+ * table if the slot is not XEN_FIRST_SLOT.
+ */
+ get_table_slot x1, x19, 1 /* x1 := first slot */
+ cmp x1, #XEN_FIRST_SLOT
+ beq 1f
+ /* It is not in slot XEN_FIRST_SLOT, remove the entry. */
+ ldr x0, =boot_first /* x0 := first table */
+ str xzr, [x0, x1, lsl #3]
+ b identity_mapping_removed
+
+1:
+ /*
+ * Find the second slot used. Remove the entry for the first
+ * table if the slot is not XEN_SECOND_SLOT.
+ */
+ get_table_slot x1, x19, 2 /* x1 := second slot */
+ cmp x1, #XEN_SECOND_SLOT
+ beq identity_mapping_removed
+ /* It is not in slot 1, remove the entry */
+ ldr x0, =boot_second /* x0 := second table */
+ str xzr, [x0, x1, lsl #3]
+
+identity_mapping_removed:
+ flush_xen_tlb_local
+
+ ret
+ENDPROC(remove_identity_mapping)
+
+/*
+ * Map the UART in the fixmap (when earlyprintk is used) and hook the
+ * fixmap table in the page tables.
+ *
+ * The fixmap cannot be mapped in create_page_tables because it may
+ * clash with the 1:1 mapping.
+ *
+ * Inputs:
+ * x20: Physical offset
+ * x23: Early UART base physical address
+ *
+ * Clobbers x0 - x3
+ */
+ENTRY(setup_fixmap)
+#ifdef CONFIG_EARLY_PRINTK
+ /* Add UART to the fixmap table */
+ ldr x0, =EARLY_UART_VIRTUAL_ADDRESS
+ create_mapping_entry xen_fixmap, x0, x23, x1, x2, x3, type=PT_DEV_L3
+#endif
+ /* Map fixmap into boot_second */
+ ldr x0, =FIXMAP_ADDR(0)
+ create_table_entry boot_second, xen_fixmap, x0, 2, x1, x2, x3
+ /* Ensure any page table updates made above have occurred. */
+ dsb nshst
+ /*
+ * The fixmap area will be used soon after. So ensure no hardware
+ * translation happens before the dsb completes.
+ */
+ isb
+
+ ret
+ENDPROC(setup_fixmap)
+
+/* Fail-stop */
+fail: PRINT("- Boot failed -\r\n")
+1: wfe
+ b 1b
+ENDPROC(fail)
+
+/*
+ * Switch TTBR
+ *
+ * x0 ttbr
+ */
+ENTRY(switch_ttbr_id)
+ /* 1) Ensure any previous read/write have completed */
+ dsb ish
+ isb
+
+ /* 2) Turn off MMU */
+ mrs x1, SCTLR_EL2
+ bic x1, x1, #SCTLR_Axx_ELx_M
+ msr SCTLR_EL2, x1
+ isb
+
+ /* 3) Flush the TLBs */
+ flush_xen_tlb_local
+
+ /* 4) Update the TTBR */
+ msr TTBR0_EL2, x0
+ isb
+
+ /*
+ * 5) Flush I-cache
+ * This should not be necessary but it is kept for safety.
+ */
+ ic iallu
+ isb
+
+ /* 6) Turn on the MMU */
+ mrs x1, SCTLR_EL2
+ orr x1, x1, #SCTLR_Axx_ELx_M /* Enable MMU */
+ msr SCTLR_EL2, x1
+ isb
+
+ ret
+ENDPROC(switch_ttbr_id)
+
+/*
+ * Local variables:
+ * mode: ASM
+ * indent-tabs-mode: nil
+ * End:
+ */
@@ -32,6 +32,42 @@
hint #22
.endm
+#ifdef CONFIG_EARLY_PRINTK
+/*
+ * Macro to print a string to the UART, if there is one.
+ *
+ * Clobbers x0 - x3
+ */
+#define PRINT(_s) \
+ mov x3, lr ; \
+ adr_l x0, 98f ; \
+ bl asm_puts ; \
+ mov lr, x3 ; \
+ RODATA_STR(98, _s)
+
+#else /* CONFIG_EARLY_PRINTK */
+#define PRINT(s)
+
+#endif /* !CONFIG_EARLY_PRINTK */
+
+/*
+ * Pseudo-op for PC relative adr <reg>, <symbol> where <symbol> is
+ * within the range +/- 4GB of the PC.
+ *
+ * @dst: destination register (64 bit wide)
+ * @sym: name of the symbol
+ */
+.macro adr_l, dst, sym
+ adrp \dst, \sym
+ add \dst, \dst, :lo12:\sym
+.endm
+
+/* Load the physical address of a symbol into xb */
+.macro load_paddr xb, sym
+ ldr \xb, =\sym
+ add \xb, \xb, x20
+.endm
+
/*
* Register aliases.
*/
@@ -34,6 +34,7 @@ SECTIONS
_stext = .; /* Text section */
_idmap_start = .;
*(.text.header)
+ *(.text.idmap)
_idmap_end = .;
*(.text.cold)