| Message ID | ZTLnw6cIVbPovI5l@mx3210.localdomain (mailing list archive) |
|---|---|
| State | Accepted, archived |
| Headers | show |
| Series | [v3] parisc: Add nop instructions after TLB inserts | expand |
Backport for v6.1.59.
Signed-off-by: John David Anglin <dave.anglin@bell.net>
---
diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
index 0e5ebfe8d9d2..4e4787a73b65 100644
--- a/arch/parisc/kernel/entry.S
+++ b/arch/parisc/kernel/entry.S
@@ -35,6 +35,24 @@
.level 2.0
#endif
+/*
+ * We need seven instructions after a TLB insert for it to take effect.
+ * The PA8800/PA8900 processors are an exception and need 12 instructions.
+ * The RFI changes both IAOQ_Back and IAOQ_Front, so it counts as one.
+ */
+#ifdef CONFIG_64BIT
+#define NUM_PIPELINE_INSNS 12
+#else
+#define NUM_PIPELINE_INSNS 7
+#endif
+
+ /* Insert num nops */
+ .macro insert_nops num
+ .rept \num
+ nop
+ .endr
+ .endm
+
/* Get aligned page_table_lock address for this mm from cr28/tr4 */
.macro get_ptl reg
mfctl %cr28,\reg
@@ -414,26 +432,23 @@
3:
.endm
- /* Release page_table_lock without reloading lock address.
+ /* Release page_table_lock if for user space.
Note that the values in the register spc are limited to
NR_SPACE_IDS (262144). Thus, the stw instruction always
stores a nonzero value even when register spc is 64 bits.
We use an ordered store to ensure all prior accesses are
- performed prior to releasing the lock. */
- .macro ptl_unlock0 spc,tmp
-#ifdef CONFIG_TLB_PTLOCK
-98: or,COND(=) %r0,\spc,%r0
- stw,ma \spc,0(\tmp)
-99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
-#endif
- .endm
-
- /* Release page_table_lock. */
- .macro ptl_unlock1 spc,tmp
+ performed prior to releasing the lock. Note stw may not
+ be executed, so we provide one extra nop when
+ CONFIG_TLB_PTLOCK is defined.*/
+ .macro ptl_unlock spc,tmp
#ifdef CONFIG_TLB_PTLOCK
98: get_ptl \tmp
- ptl_unlock0 \spc,\tmp
+ or,COND(=) %r0,\spc,%r0
+ stw,ma \spc,0(\tmp)
99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
+ insert_nops NUM_PIPELINE_INSNS - 3
+#else
+ insert_nops NUM_PIPELINE_INSNS - 1
#endif
.endm
@@ -1125,7 +1140,7 @@ dtlb_miss_20w:
idtlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1134,6 +1149,7 @@ dtlb_check_alias_20w:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1151,7 +1167,7 @@ nadtlb_miss_20w:
idtlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1160,6 +1176,7 @@ nadtlb_check_alias_20w:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1185,7 +1202,7 @@ dtlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1195,6 +1212,7 @@ dtlb_check_alias_11:
idtlba pte,(va)
idtlbp prot,(va)
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1218,7 +1236,7 @@ nadtlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1228,6 +1246,7 @@ nadtlb_check_alias_11:
idtlba pte,(va)
idtlbp prot,(va)
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1247,7 +1266,7 @@ dtlb_miss_20:
idtlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1256,6 +1275,7 @@ dtlb_check_alias_20:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1275,7 +1295,7 @@ nadtlb_miss_20:
idtlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1284,6 +1304,7 @@ nadtlb_check_alias_20:
idtlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1320,7 +1341,7 @@ itlb_miss_20w:
iitlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1344,7 +1365,7 @@ naitlb_miss_20w:
iitlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1353,6 +1374,7 @@ naitlb_check_alias_20w:
iitlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1378,7 +1400,7 @@ itlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1402,7 +1424,7 @@ naitlb_miss_11:
mtsp t1, %sr1 /* Restore sr1 */
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1412,6 +1434,7 @@ naitlb_check_alias_11:
iitlba pte,(%sr0, va)
iitlbp prot,(%sr0, va)
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1432,7 +1455,7 @@ itlb_miss_20:
iitlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1452,7 +1475,7 @@ naitlb_miss_20:
iitlbt pte,prot
- ptl_unlock1 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1461,6 +1484,7 @@ naitlb_check_alias_20:
iitlbt pte,prot
+ insert_nops NUM_PIPELINE_INSNS - 1
rfir
nop
@@ -1482,7 +1506,7 @@ dbit_trap_20w:
idtlbt pte,prot
- ptl_unlock0 spc,t0
+ ptl_unlock spc,t0
rfir
nop
#else
@@ -1508,7 +1532,7 @@ dbit_trap_11:
mtsp t1, %sr1 /* Restore sr1 */
- ptl_unlock0 spc,t0
+ ptl_unlock spc,t0
rfir
nop
@@ -1528,7 +1552,7 @@ dbit_trap_20:
idtlbt pte,prot
- ptl_unlock0 spc,t0
+ ptl_unlock spc,t0
rfir
nop
#endif
diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index ae03b8679696..cab1ec23e0d7 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -36,6 +36,24 @@ .level 2.0 #endif +/* + * We need seven instructions after a TLB insert for it to take effect. + * The PA8800/PA8900 processors are an exception and need 12 instructions. + * The RFI changes both IAOQ_Back and IAOQ_Front, so it counts as one. + */ +#ifdef CONFIG_64BIT +#define NUM_PIPELINE_INSNS 12 +#else +#define NUM_PIPELINE_INSNS 7 +#endif + + /* Insert num nops */ + .macro insert_nops num + .rept \num + nop + .endr + .endm + /* Get aligned page_table_lock address for this mm from cr28/tr4 */ .macro get_ptl reg mfctl %cr28,\reg @@ -415,24 +433,20 @@ 3: .endm - /* Release page_table_lock without reloading lock address. - We use an ordered store to ensure all prior accesses are - performed prior to releasing the lock. */ - .macro ptl_unlock0 spc,tmp,tmp2 + /* Release page_table_lock if for user space. We use an ordered + store to ensure all prior accesses are performed prior to + releasing the lock. Note stw may not be executed, so we + provide one extra nop when CONFIG_TLB_PTLOCK is defined. */ + .macro ptl_unlock spc,tmp,tmp2 #ifdef CONFIG_TLB_PTLOCK -98: ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmp2 +98: get_ptl \tmp + ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmp2 or,COND(=) %r0,\spc,%r0 stw,ma \tmp2,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) -#endif - .endm - - /* Release page_table_lock. */ - .macro ptl_unlock1 spc,tmp,tmp2 -#ifdef CONFIG_TLB_PTLOCK -98: get_ptl \tmp - ptl_unlock0 \spc,\tmp,\tmp2 -99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) + insert_nops NUM_PIPELINE_INSNS - 4 +#else + insert_nops NUM_PIPELINE_INSNS - 1 #endif .endm @@ -1124,7 +1138,7 @@ dtlb_miss_20w: idtlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1133,6 +1147,7 @@ dtlb_check_alias_20w: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1150,7 +1165,7 @@ nadtlb_miss_20w: idtlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1159,6 +1174,7 @@ nadtlb_check_alias_20w: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1184,7 +1200,7 @@ dtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1194,6 +1210,7 @@ dtlb_check_alias_11: idtlba pte,(va) idtlbp prot,(va) + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1217,7 +1234,7 @@ nadtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1227,6 +1244,7 @@ nadtlb_check_alias_11: idtlba pte,(va) idtlbp prot,(va) + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1246,7 +1264,7 @@ dtlb_miss_20: idtlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1255,6 +1273,7 @@ dtlb_check_alias_20: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1274,7 +1293,7 @@ nadtlb_miss_20: idtlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1283,6 +1302,7 @@ nadtlb_check_alias_20: idtlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1319,7 +1339,7 @@ itlb_miss_20w: iitlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1343,7 +1363,7 @@ naitlb_miss_20w: iitlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1352,6 +1372,7 @@ naitlb_check_alias_20w: iitlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1377,7 +1398,7 @@ itlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1401,7 +1422,7 @@ naitlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1411,6 +1432,7 @@ naitlb_check_alias_11: iitlba pte,(%sr0, va) iitlbp prot,(%sr0, va) + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1431,7 +1453,7 @@ itlb_miss_20: iitlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1451,7 +1473,7 @@ naitlb_miss_20: iitlbt pte,prot - ptl_unlock1 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1460,6 +1482,7 @@ naitlb_check_alias_20: iitlbt pte,prot + insert_nops NUM_PIPELINE_INSNS - 1 rfir nop @@ -1481,7 +1504,7 @@ dbit_trap_20w: idtlbt pte,prot - ptl_unlock0 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop #else @@ -1507,7 +1530,7 @@ dbit_trap_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock0 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop @@ -1527,7 +1550,7 @@ dbit_trap_20: idtlbt pte,prot - ptl_unlock0 spc,t0,t1 + ptl_unlock spc,t0,t1 rfir nop #endif
An excerpt from the PA8800 ERS states: o The PA8800 violates the seven instruction pipeline rule when performing TLB inserts or PxTLBE instructions with the PSW C bit on. The instruction will take effect by the 12th instruction after the insert or purge. I believe we have a problem with handling TLB misses. We don't fill the pipeline following TLB inserts. As a result, we likely fault again after returning from the interruption. The above statement indicates that we need at least seven instructions after the insert on pre PA8800 processors and we need 12 instructions on PA8800/PA8900 processors. Here we add macros and code to provide the required number instructions after a TLB insert. Tested on c8000. v2: Revise per Helge's suggestions. Applies to master 6.6.0-rc6. v3: Rename NUM_INSNS to NUM_PIPELINE_INSNS. Simply PTL unlock code. Add one extra nop when CONFIG_TLB_PTLOCK isdefined. Signed-off-by: John David Anglin <dave.anglin@bell.net> Suggested-by: Helge Deller <deller@gmx.de> ---