diff mbox series

[17/22] x86/mm: introduce support to populate a per-CPU page-table region

Message ID 20240726152206.28411-18-roger.pau@citrix.com (mailing list archive)
State New
Headers show
Series x86: adventures in Address Space Isolation | expand

Commit Message

Roger Pau Monné July 26, 2024, 3:22 p.m. UTC
Add logic in map_pages_to_xen() and modify_xen_mappings() so that TLB flushes
are only performed locally when dealing with entries in the per-CPU area of the
page-tables.

No functional change intended, as there are no callers added that create or
modify per-CPU mappings, nor is the per-CPU area still properly setup in
the page-tables yet.

Note that the removed flush_area() ended up calling flush_area_mask() through
the flush_area_all() alias.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
---
 xen/arch/x86/include/asm/config.h   |  4 ++
 xen/arch/x86/include/asm/flushtlb.h |  1 -
 xen/arch/x86/mm.c                   | 64 +++++++++++++++++++----------
 3 files changed, 47 insertions(+), 22 deletions(-)
diff mbox series

Patch

diff --git a/xen/arch/x86/include/asm/config.h b/xen/arch/x86/include/asm/config.h
index 2a260a2581fd..c24d735a0cee 100644
--- a/xen/arch/x86/include/asm/config.h
+++ b/xen/arch/x86/include/asm/config.h
@@ -204,6 +204,10 @@  extern unsigned char boot_edid_info[128];
 #define PERDOMAIN_SLOTS         3
 #define PERDOMAIN_VIRT_SLOT(s)  (PERDOMAIN_VIRT_START + (s) * \
                                  (PERDOMAIN_SLOT_MBYTES << 20))
+#define PERCPU_VIRT_START       PERDOMAIN_VIRT_SLOT(PERDOMAIN_SLOTS)
+#define PERCPU_SLOTS            1
+#define PERCPU_VIRT_SLOT(s)     (PERCPU_VIRT_START + (s) * \
+                                 (PERDOMAIN_SLOT_MBYTES << 20))
 /* Slot 4: mirror of per-domain mappings (for compat xlat area accesses). */
 #define PERDOMAIN_ALT_VIRT_START PML4_ADDR(4)
 /* Slot 261: machine-to-phys conversion table (256GB). */
diff --git a/xen/arch/x86/include/asm/flushtlb.h b/xen/arch/x86/include/asm/flushtlb.h
index 1b98d03decdc..affe944d1a5b 100644
--- a/xen/arch/x86/include/asm/flushtlb.h
+++ b/xen/arch/x86/include/asm/flushtlb.h
@@ -146,7 +146,6 @@  void flush_area_mask(const cpumask_t *mask, const void *va,
 #define flush_mask(mask, flags) flush_area_mask(mask, NULL, flags)
 
 /* Flush all CPUs' TLBs/caches */
-#define flush_area_all(va, flags) flush_area_mask(&cpu_online_map, va, flags)
 #define flush_all(flags) flush_mask(&cpu_online_map, flags)
 
 /* Flush local TLBs */
diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 1367f3361ffe..c468b46a9d1b 100644
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -5023,9 +5023,13 @@  static DEFINE_SPINLOCK(map_pgdir_lock);
  */
 static l3_pgentry_t *virt_to_xen_l3e(unsigned long v)
 {
+    unsigned int cpu = smp_processor_id();
+    /* Called before idle_vcpu is populated, fallback to idle_pg_table. */
+    root_pgentry_t *root_pgt = idle_vcpu[cpu] ?
+        maddr_to_virt(idle_vcpu[cpu]->arch.cr3) : idle_pg_table;
     l4_pgentry_t *pl4e;
 
-    pl4e = &idle_pg_table[l4_table_offset(v)];
+    pl4e = &root_pgt[l4_table_offset(v)];
     if ( !(l4e_get_flags(*pl4e) & _PAGE_PRESENT) )
     {
         bool locking = system_state > SYS_STATE_boot;
@@ -5138,8 +5142,8 @@  static l1_pgentry_t *virt_to_xen_l1e(unsigned long v)
 #define l1f_to_lNf(f) (((f) & _PAGE_PRESENT) ? ((f) |  _PAGE_PSE) : (f))
 #define lNf_to_l1f(f) (((f) & _PAGE_PRESENT) ? ((f) & ~_PAGE_PSE) : (f))
 
-/* flush_area_all() can be used prior to any other CPU being online.  */
-#define flush_area(v, f) flush_area_all((const void *)(v), f)
+/* flush_area_mask() can be used prior to any other CPU being online.  */
+#define flush_area_mask(m, v, f) flush_area_mask(m, (const void *)(v), f)
 
 #define L3T_INIT(page) (page) = ZERO_BLOCK_PTR
 
@@ -5222,7 +5226,11 @@  int map_pages_to_xen(
     unsigned long nr_mfns,
     unsigned int flags)
 {
-    bool locking = system_state > SYS_STATE_boot;
+    bool global = virt < PERCPU_VIRT_START ||
+                  virt >= PERCPU_VIRT_SLOT(PERCPU_SLOTS);
+    bool locking = system_state > SYS_STATE_boot && global;
+    const cpumask_t *flush_mask = global ? &cpu_online_map
+                                         : cpumask_of(smp_processor_id());
     l3_pgentry_t *pl3e = NULL, ol3e;
     l2_pgentry_t *pl2e = NULL, ol2e;
     l1_pgentry_t *pl1e, ol1e;
@@ -5244,6 +5252,11 @@  int map_pages_to_xen(
     }                                          \
 } while (0)
 
+    /* Ensure it's a global mapping or it's only modifying the per-CPU area. */
+    ASSERT(global ||
+           (virt + nr_mfns * PAGE_SIZE >= PERCPU_VIRT_START &&
+            virt + nr_mfns * PAGE_SIZE <  PERCPU_VIRT_SLOT(PERCPU_SLOTS)));
+
     L3T_INIT(current_l3page);
 
     while ( nr_mfns != 0 )
@@ -5278,7 +5291,7 @@  int map_pages_to_xen(
                 if ( l3e_get_flags(ol3e) & _PAGE_PSE )
                 {
                     flush_flags(lNf_to_l1f(l3e_get_flags(ol3e)));
-                    flush_area(virt, flush_flags);
+                    flush_area_mask(flush_mask, virt, flush_flags);
                 }
                 else
                 {
@@ -5301,7 +5314,7 @@  int map_pages_to_xen(
                             unmap_domain_page(l1t);
                         }
                     }
-                    flush_area(virt, flush_flags);
+                    flush_area_mask(flush_mask, virt, flush_flags);
                     for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
                     {
                         ol2e = l2t[i];
@@ -5373,7 +5386,7 @@  int map_pages_to_xen(
             }
             if ( locking )
                 spin_unlock(&map_pgdir_lock);
-            flush_area(virt, flush_flags);
+            flush_area_mask(flush_mask, virt, flush_flags);
 
             free_xen_pagetable(l2mfn);
         }
@@ -5399,7 +5412,7 @@  int map_pages_to_xen(
                 if ( l2e_get_flags(ol2e) & _PAGE_PSE )
                 {
                     flush_flags(lNf_to_l1f(l2e_get_flags(ol2e)));
-                    flush_area(virt, flush_flags);
+                    flush_area_mask(flush_mask, virt, flush_flags);
                 }
                 else
                 {
@@ -5407,7 +5420,7 @@  int map_pages_to_xen(
 
                     for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
                         flush_flags(l1e_get_flags(l1t[i]));
-                    flush_area(virt, flush_flags);
+                    flush_area_mask(flush_mask, virt, flush_flags);
                     unmap_domain_page(l1t);
                     free_xen_pagetable(l2e_get_mfn(ol2e));
                 }
@@ -5476,7 +5489,7 @@  int map_pages_to_xen(
                 }
                 if ( locking )
                     spin_unlock(&map_pgdir_lock);
-                flush_area(virt, flush_flags);
+                flush_area_mask(flush_mask, virt, flush_flags);
 
                 free_xen_pagetable(l1mfn);
             }
@@ -5491,7 +5504,7 @@  int map_pages_to_xen(
                 unsigned int flush_flags = FLUSH_TLB | FLUSH_ORDER(0);
 
                 flush_flags(l1e_get_flags(ol1e));
-                flush_area(virt, flush_flags);
+                flush_area_mask(flush_mask, virt, flush_flags);
             }
 
             virt    += 1UL << L1_PAGETABLE_SHIFT;
@@ -5540,9 +5553,9 @@  int map_pages_to_xen(
                     l2e_write(pl2e, l2e_from_pfn(base_mfn, l1f_to_lNf(flags)));
                     if ( locking )
                         spin_unlock(&map_pgdir_lock);
-                    flush_area(virt - PAGE_SIZE,
-                               FLUSH_TLB_GLOBAL |
-                               FLUSH_ORDER(PAGETABLE_ORDER));
+                    flush_area_mask(flush_mask, virt - PAGE_SIZE,
+                                    FLUSH_TLB_GLOBAL |
+                                    FLUSH_ORDER(PAGETABLE_ORDER));
                     free_xen_pagetable(l2e_get_mfn(ol2e));
                 }
                 else if ( locking )
@@ -5589,9 +5602,9 @@  int map_pages_to_xen(
                 l3e_write(pl3e, l3e_from_pfn(base_mfn, l1f_to_lNf(flags)));
                 if ( locking )
                     spin_unlock(&map_pgdir_lock);
-                flush_area(virt - PAGE_SIZE,
-                           FLUSH_TLB_GLOBAL |
-                           FLUSH_ORDER(2*PAGETABLE_ORDER));
+                flush_area_mask(flush_mask, virt - PAGE_SIZE,
+                                FLUSH_TLB_GLOBAL |
+                                FLUSH_ORDER(2*PAGETABLE_ORDER));
                 free_xen_pagetable(l3e_get_mfn(ol3e));
             }
             else if ( locking )
@@ -5629,7 +5642,11 @@  int __init populate_pt_range(unsigned long virt, unsigned long nr_mfns)
  */
 int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int nf)
 {
-    bool locking = system_state > SYS_STATE_boot;
+    bool global = s < PERCPU_VIRT_START ||
+                  s >= PERCPU_VIRT_SLOT(PERCPU_SLOTS);
+    bool locking = system_state > SYS_STATE_boot && global;
+    const cpumask_t *flush_mask = global ? &cpu_online_map
+                                         : cpumask_of(smp_processor_id());
     l3_pgentry_t *pl3e = NULL;
     l2_pgentry_t *pl2e = NULL;
     l1_pgentry_t *pl1e;
@@ -5638,6 +5655,9 @@  int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int nf)
     int rc = -ENOMEM;
     struct page_info *current_l3page;
 
+    ASSERT(global ||
+           (e >= PERCPU_VIRT_START && e < PERCPU_VIRT_SLOT(PERCPU_SLOTS)));
+
     /* Set of valid PTE bits which may be altered. */
 #define FLAGS_MASK (_PAGE_NX|_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW|_PAGE_PRESENT)
     nf &= FLAGS_MASK;
@@ -5836,7 +5856,8 @@  int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int nf)
                 l2e_write(pl2e, l2e_empty());
                 if ( locking )
                     spin_unlock(&map_pgdir_lock);
-                flush_area(NULL, FLUSH_TLB_GLOBAL); /* flush before free */
+                /* flush before free */
+                flush_area_mask(flush_mask, NULL, FLUSH_TLB_GLOBAL);
                 free_xen_pagetable(l1mfn);
             }
             else if ( locking )
@@ -5880,7 +5901,8 @@  int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int nf)
                 l3e_write(pl3e, l3e_empty());
                 if ( locking )
                     spin_unlock(&map_pgdir_lock);
-                flush_area(NULL, FLUSH_TLB_GLOBAL); /* flush before free */
+                /* flush before free */
+                flush_area_mask(flush_mask, NULL, FLUSH_TLB_GLOBAL);
                 free_xen_pagetable(l2mfn);
             }
             else if ( locking )
@@ -5888,7 +5910,7 @@  int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int nf)
         }
     }
 
-    flush_area(NULL, FLUSH_TLB_GLOBAL);
+    flush_area_mask(flush_mask, NULL, FLUSH_TLB_GLOBAL);
 
 #undef FLAGS_MASK
     rc = 0;