| Message ID | 20241216-vdso-store-rng-v1-12-f7aed1bdb3b2@linutronix.de (mailing list archive) |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | vDSO: Introduce generic data storage | expand |
Le 16/12/2024 à 15:10, Thomas Weißschuh a écrit : > The generic storage implementation provides the same features as the > custom one. However it can be shared between architectures, making > maintenance easier. > > Co-developed-by: Nam Cao <namcao@linutronix.de> > Signed-off-by: Nam Cao <namcao@linutronix.de> > Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de> > --- > arch/powerpc/Kconfig | 2 + > arch/powerpc/include/asm/vdso.h | 1 + > arch/powerpc/include/asm/vdso/arch_data.h | 37 +++++++++ > arch/powerpc/include/asm/vdso/getrandom.h | 11 +-- > arch/powerpc/include/asm/vdso/gettimeofday.h | 36 +++++---- > arch/powerpc/include/asm/vdso/vsyscall.h | 13 --- > arch/powerpc/include/asm/vdso_datapage.h | 44 +--------- > arch/powerpc/kernel/asm-offsets.c | 1 - > arch/powerpc/kernel/time.c | 2 +- > arch/powerpc/kernel/vdso.c | 115 +++------------------------ > arch/powerpc/kernel/vdso/cacheflush.S | 2 +- > arch/powerpc/kernel/vdso/datapage.S | 4 +- > arch/powerpc/kernel/vdso/gettimeofday.S | 4 +- > arch/powerpc/kernel/vdso/vdso32.lds.S | 4 +- > arch/powerpc/kernel/vdso/vdso64.lds.S | 4 +- > arch/powerpc/kernel/vdso/vgettimeofday.c | 14 ++-- > 16 files changed, 101 insertions(+), 193 deletions(-) > > diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig > index a0ce777f97063bf858942c60654d8411bcf2a3dc..600fa3b917ee902d016f2a04376950a9dc49074f 100644 > --- a/arch/powerpc/Kconfig > +++ b/arch/powerpc/Kconfig > @@ -156,6 +156,7 @@ config PPC > select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST > select ARCH_HAS_UACCESS_FLUSHCACHE > select ARCH_HAS_UBSAN > + select ARCH_HAS_VDSO_ARCH_DATA > select ARCH_HAVE_NMI_SAFE_CMPXCHG > select ARCH_HAVE_EXTRA_ELF_NOTES if SPU_BASE > select ARCH_KEEP_MEMBLOCK > @@ -206,6 +207,7 @@ config PPC > select GENERIC_PTDUMP > select GENERIC_SMP_IDLE_THREAD > select GENERIC_TIME_VSYSCALL > + select GENERIC_VDSO_DATA_STORE > select GENERIC_VDSO_TIME_NS > select HAS_IOPORT if PCI > select HAVE_ARCH_AUDITSYSCALL > diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h > index 8d972bc98b55fe916f23488ca9e2a5918046b9aa..1ca23fbfe087ae90b90c4286335f86d9f8121078 100644 > --- a/arch/powerpc/include/asm/vdso.h > +++ b/arch/powerpc/include/asm/vdso.h > @@ -3,6 +3,7 @@ > #define _ASM_POWERPC_VDSO_H > > #define VDSO_VERSION_STRING LINUX_2.6.15 > +#define __VDSO_PAGES 4 > > #ifndef __ASSEMBLY__ > > diff --git a/arch/powerpc/include/asm/vdso/arch_data.h b/arch/powerpc/include/asm/vdso/arch_data.h > new file mode 100644 > index 0000000000000000000000000000000000000000..c240a6b875181ac4159f2e80b11f9bf214e22808 > --- /dev/null > +++ b/arch/powerpc/include/asm/vdso/arch_data.h > @@ -0,0 +1,37 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM > + * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>, > + * IBM Corp. > + */ > +#ifndef _ASM_POWERPC_VDSO_ARCH_DATA_H > +#define _ASM_POWERPC_VDSO_ARCH_DATA_H > + > +#include <linux/unistd.h> > +#include <linux/types.h> > + > +#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32) > + > +#ifdef CONFIG_PPC64 > + > +struct vdso_arch_data { > + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ > + __u32 dcache_block_size; /* L1 d-cache block size */ > + __u32 icache_block_size; /* L1 i-cache block size */ > + __u32 dcache_log_block_size; /* L1 d-cache log block size */ > + __u32 icache_log_block_size; /* L1 i-cache log block size */ > + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ > + __u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */ > +}; > + > +#else /* CONFIG_PPC64 */ > + > +struct vdso_arch_data { > + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ > + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ > + __u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */ > +}; > + > +#endif /* CONFIG_PPC64 */ > + > +#endif /* _ASM_POWERPC_VDSO_ARCH_DATA_H */ > diff --git a/arch/powerpc/include/asm/vdso/getrandom.h b/arch/powerpc/include/asm/vdso/getrandom.h > index 80ce0709725eb89c1f3b69e0733038b458fbf24f..c82eb0d8237681a7396abfe7d161292636b8cce4 100644 > --- a/arch/powerpc/include/asm/vdso/getrandom.h > +++ b/arch/powerpc/include/asm/vdso/getrandom.h > @@ -43,20 +43,21 @@ static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsig > (unsigned long)len, (unsigned long)flags); > } > > -static __always_inline struct vdso_rng_data *__arch_get_vdso_rng_data(void) > +static __always_inline const struct vdso_rng_data *__ppc_get_vdso_u_rng_data(void) > { > - struct vdso_arch_data *data; > + struct vdso_rng_data *data; > > asm ( > " bcl 20, 31, .+4 ;" > "0: mflr %0 ;" > - " addis %0, %0, (_vdso_datapage - 0b)@ha ;" > - " addi %0, %0, (_vdso_datapage - 0b)@l ;" > + " addis %0, %0, (vdso_u_rng_data - 0b)@ha ;" > + " addi %0, %0, (vdso_u_rng_data - 0b)@l ;" > : "=r" (data) : : "lr" > ); > > - return &data->rng_data; > + return data; > } > +#define __arch_get_vdso_u_rng_data __ppc_get_vdso_u_rng_data > > ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, > size_t opaque_len); > diff --git a/arch/powerpc/include/asm/vdso/gettimeofday.h b/arch/powerpc/include/asm/vdso/gettimeofday.h > index c6390890a60c2fdcb608bf321b2945c3fb372f54..bddd9cde97db197d0d3daba6c2289cb29e6b5a75 100644 > --- a/arch/powerpc/include/asm/vdso/gettimeofday.h > +++ b/arch/powerpc/include/asm/vdso/gettimeofday.h > @@ -94,22 +94,29 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) > #endif > > static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, > - const struct vdso_data *vd) > + const struct vdso_time_data *vd) > { > return get_tb(); > } > > -const struct vdso_data *__arch_get_vdso_data(void); > - > #ifdef CONFIG_TIME_NS > -static __always_inline > -const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) > +static __always_inline const struct vdso_time_data *__ppc_get_vdso_u_timens_data(void) > { > - return (void *)vd + (1U << CONFIG_PAGE_SHIFT); > + struct vdso_time_data *time_data; > + > + asm( > + " bcl 20, 31, .+4\n" > + "0: mflr %0\n" > + " addis %0, %0, (vdso_u_timens_data - 0b)@ha\n" > + " addi %0, %0, (vdso_u_timens_data - 0b)@l\n" > + : "=r" (time_data) :: "lr"); > + > + return time_data; Please don't do that, it kills optimisation efforts done when implementing VDSO time. Commit ce7d8056e38b ("powerpc/vdso: Prepare for switching VDSO to generic C implementation.") explains why. For time data, the bcl/mflr dance is done by get_datapage macro called by cvdso_call macro in gettimeofday.S, and given to __cvdso_clock_gettime_data() by __c_kernel_clock_gettime() in vgettimeofday.c . Use that information and don't redo the bcl/mflr sequence. See for instance function __c_kernel_clock_getres(): Before your series it is 30 instructions. After your series it is 59 instructions. Before: 000010f8 <__c_kernel_clock_getres>: 10f8: 28 03 00 0f cmplwi r3,15 10fc: 41 81 00 60 bgt 115c <__c_kernel_clock_getres+0x64> 1100: 81 45 00 04 lwz r10,4(r5) 1104: 3d 20 7f ff lis r9,32767 1108: 61 29 ff ff ori r9,r9,65535 110c: 7c 0a 48 00 cmpw r10,r9 1110: 40 a2 00 08 bne 1118 <__c_kernel_clock_getres+0x20> 1114: 38 a5 40 00 addi r5,r5,16384 1118: 39 20 00 01 li r9,1 111c: 7d 29 18 30 slw r9,r9,r3 1120: 71 2a 08 93 andi. r10,r9,2195 1124: 40 82 00 30 bne 1154 <__c_kernel_clock_getres+0x5c> 1128: 71 29 00 60 andi. r9,r9,96 112c: 41 82 00 30 beq 115c <__c_kernel_clock_getres+0x64> 1130: 3d 20 00 98 lis r9,152 1134: 61 29 96 80 ori r9,r9,38528 1138: 2c 04 00 00 cmpwi r4,0 113c: 41 82 00 10 beq 114c <__c_kernel_clock_getres+0x54> 1140: 39 40 00 00 li r10,0 1144: 91 24 00 04 stw r9,4(r4) 1148: 91 44 00 00 stw r10,0(r4) 114c: 38 60 00 00 li r3,0 1150: 4e 80 00 20 blr 1154: 81 25 00 e8 lwz r9,232(r5) 1158: 4b ff ff e0 b 1138 <__c_kernel_clock_getres+0x40> 115c: 38 00 00 f7 li r0,247 1160: 44 00 00 02 sc 1164: 40 a3 00 08 bns 116c <__c_kernel_clock_getres+0x74> 1168: 7c 63 00 d0 neg r3,r3 116c: 4e 80 00 20 blr After: 000011ac <__c_kernel_clock_getres>: 11ac: 28 03 00 0f cmplwi r3,15 11b0: 41 81 00 c0 bgt 1270 <__c_kernel_clock_getres+0xc4> 11b4: 81 45 00 04 lwz r10,4(r5) 11b8: 3d 20 7f ff lis r9,32767 11bc: 61 29 ff ff ori r9,r9,65535 11c0: 7c 0a 48 00 cmpw r10,r9 11c4: 41 82 00 48 beq 120c <__c_kernel_clock_getres+0x60> 11c8: 39 20 00 01 li r9,1 11cc: 7d 29 18 30 slw r9,r9,r3 11d0: 71 2a 08 93 andi. r10,r9,2195 11d4: 40 82 00 30 bne 1204 <__c_kernel_clock_getres+0x58> 11d8: 71 29 00 60 andi. r9,r9,96 11dc: 41 82 00 94 beq 1270 <__c_kernel_clock_getres+0xc4> 11e0: 3d 20 00 98 lis r9,152 11e4: 61 29 96 80 ori r9,r9,38528 11e8: 2c 04 00 00 cmpwi r4,0 11ec: 41 82 00 10 beq 11fc <__c_kernel_clock_getres+0x50> 11f0: 39 40 00 00 li r10,0 11f4: 91 24 00 04 stw r9,4(r4) 11f8: 91 44 00 00 stw r10,0(r4) 11fc: 38 60 00 00 li r3,0 1200: 4e 80 00 20 blr 1204: 81 25 00 e8 lwz r9,232(r5) 1208: 4b ff ff e0 b 11e8 <__c_kernel_clock_getres+0x3c> 120c: 39 20 00 01 li r9,1 1210: 7d 29 18 30 slw r9,r9,r3 1214: 71 2a 08 93 andi. r10,r9,2195 1218: 7c 08 02 a6 mflr r0 121c: 90 01 00 04 stw r0,4(r1) 1220: 42 9f 00 05 bcl 20,4*cr7+so,1224 <__c_kernel_clock_getres+0x78> 1224: 7c a8 02 a6 mflr r5 1228: 3c a5 ff ff addis r5,r5,-1 122c: 38 a5 2d dc addi r5,r5,11740 1230: 40 82 00 38 bne 1268 <__c_kernel_clock_getres+0xbc> 1234: 71 29 00 60 andi. r9,r9,96 1238: 41 82 00 4c beq 1284 <__c_kernel_clock_getres+0xd8> 123c: 3d 20 00 98 lis r9,152 1240: 61 29 96 80 ori r9,r9,38528 1244: 2c 04 00 00 cmpwi r4,0 1248: 41 82 00 10 beq 1258 <__c_kernel_clock_getres+0xac> 124c: 39 40 00 00 li r10,0 1250: 91 24 00 04 stw r9,4(r4) 1254: 91 44 00 00 stw r10,0(r4) 1258: 38 60 00 00 li r3,0 125c: 80 01 00 04 lwz r0,4(r1) 1260: 7c 08 03 a6 mtlr r0 1264: 4e 80 00 20 blr 1268: 81 25 00 e8 lwz r9,232(r5) 126c: 4b ff ff d8 b 1244 <__c_kernel_clock_getres+0x98> 1270: 38 00 00 f7 li r0,247 1274: 44 00 00 02 sc 1278: 40 a3 00 08 bns 1280 <__c_kernel_clock_getres+0xd4> 127c: 7c 63 00 d0 neg r3,r3 1280: 4e 80 00 20 blr 1284: 38 00 00 f7 li r0,247 1288: 44 00 00 02 sc 128c: 40 a3 00 08 bns 1294 <__c_kernel_clock_getres+0xe8> 1290: 7c 63 00 d0 neg r3,r3 1294: 4b ff ff c8 b 125c <__c_kernel_clock_getres+0xb0> > } > +#define __arch_get_vdso_u_timens_data __ppc_get_vdso_u_timens_data There is not #ifdef __arch_get_vdso_u_timens_data anywhere, this #define is not needed, the function should be called __arch_get_vdso_u_timens_data() directly as before, unnecessary indirections reduce readability. > #endif > > -static inline bool vdso_clocksource_ok(const struct vdso_data *vd) > +static inline bool vdso_clocksource_ok(const struct vdso_time_data *vd) > { > return true; > }
Hi Christophe, On Wed, Dec 18, 2024 at 08:20:51AM +0100, Christophe Leroy wrote: > Le 16/12/2024 à 15:10, Thomas Weißschuh a écrit : [..] > > #ifdef CONFIG_TIME_NS > > -static __always_inline > > -const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) > > +static __always_inline const struct vdso_time_data *__ppc_get_vdso_u_timens_data(void) > > { > > - return (void *)vd + (1U << CONFIG_PAGE_SHIFT); > > + struct vdso_time_data *time_data; > > + > > + asm( > > + " bcl 20, 31, .+4\n" > > + "0: mflr %0\n" > > + " addis %0, %0, (vdso_u_timens_data - 0b)@ha\n" > > + " addi %0, %0, (vdso_u_timens_data - 0b)@l\n" > > + : "=r" (time_data) :: "lr"); > > + > > + return time_data; > > Please don't do that, it kills optimisation efforts done when implementing > VDSO time. Commit ce7d8056e38b ("powerpc/vdso: Prepare for switching VDSO to > generic C implementation.") explains why. > > For time data, the bcl/mflr dance is done by get_datapage macro called by > cvdso_call macro in gettimeofday.S, and given to > __cvdso_clock_gettime_data() by __c_kernel_clock_gettime() in > vgettimeofday.c . Use that information and don't redo the bcl/mflr sequence. So instead keeping the logic of this: static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) { return (void *)vd + (1U << CONFIG_PAGE_SHIFT); } Makes sense. Adding a constant value should be cheaper or just as cheap as a PC-relative addressing for all architectures, so it can go into the generic code, too. [..] > > } > > +#define __arch_get_vdso_u_timens_data __ppc_get_vdso_u_timens_data > > There is not #ifdef __arch_get_vdso_u_timens_data anywhere, this #define is > not needed, the function should be called __arch_get_vdso_u_timens_data() > directly as before, unnecessary indirections reduce readability. I'll see how this works out with the include order and conflicts with symbols in include/vdso/datapage.h. > > #endif > > -static inline bool vdso_clocksource_ok(const struct vdso_data *vd) > > +static inline bool vdso_clocksource_ok(const struct vdso_time_data *vd) > > { > > return true; > > } Thanks! Thomas
>> #ifdef CONFIG_TIME_NS >> -static __always_inline >> -const struct vdso_data *__arch_get_timens_vdso_data(const struct >> vdso_data *vd) >> +static __always_inline const struct vdso_time_data >> *__ppc_get_vdso_u_timens_data(void) >> { >> - return (void *)vd + (1U << CONFIG_PAGE_SHIFT); >> + struct vdso_time_data *time_data; >> + >> + asm( >> + " bcl 20, 31, .+4\n" >> + "0: mflr %0\n" >> + " addis %0, %0, (vdso_u_timens_data - 0b)@ha\n" >> + " addi %0, %0, (vdso_u_timens_data - 0b)@l\n" >> + : "=r" (time_data) :: "lr"); >> + >> + return time_data; > > Please don't do that, it kills optimisation efforts done when > implementing VDSO time. Commit ce7d8056e38b ("powerpc/vdso: Prepare for > switching VDSO to generic C implementation.") explains why. > > For time data, the bcl/mflr dance is done by get_datapage macro called > by cvdso_call macro in gettimeofday.S, and given to > __cvdso_clock_gettime_data() by __c_kernel_clock_gettime() in > vgettimeofday.c . Use that information and don't redo the bcl/mflr > sequence. > > See for instance function __c_kernel_clock_getres(): > > Before your series it is 30 instructions. > After your series it is 59 instructions. > It is even more obvious with __c_kernel_time() Before your series it has 12 instructions, After your series it has 26 instructions. Before 00001408 <__c_kernel_time>: 1408: 81 44 00 04 lwz r10,4(r4) 140c: 6d 49 80 00 xoris r9,r10,32768 1410: 2c 09 ff ff cmpwi r9,-1 1414: 40 82 00 08 bne 141c <__c_kernel_time+0x14> 1418: 38 84 40 00 addi r4,r4,16384 141c: 2c 03 00 00 cmpwi r3,0 1420: 81 44 00 20 lwz r10,32(r4) 1424: 81 64 00 24 lwz r11,36(r4) 1428: 41 82 00 08 beq 1430 <__c_kernel_time+0x28> 142c: 91 63 00 00 stw r11,0(r3) 1430: 7d 63 5b 78 mr r3,r11 1434: 4e 80 00 20 blr Versus after 00001534 <__c_kernel_time>: 1534: 81 44 00 04 lwz r10,4(r4) 1538: 6d 49 80 00 xoris r9,r10,32768 153c: 2c 09 ff ff cmpwi r9,-1 1540: 41 82 00 20 beq 1560 <__c_kernel_time+0x2c> 1544: 2c 03 00 00 cmpwi r3,0 1548: 81 44 00 20 lwz r10,32(r4) 154c: 81 64 00 24 lwz r11,36(r4) 1550: 41 82 00 08 beq 1558 <__c_kernel_time+0x24> 1554: 91 63 00 00 stw r11,0(r3) 1558: 7d 63 5b 78 mr r3,r11 155c: 4e 80 00 20 blr 1560: 7c 08 02 a6 mflr r0 1564: 2c 03 00 00 cmpwi r3,0 1568: 90 01 00 04 stw r0,4(r1) 156c: 42 9f 00 05 bcl 20,4*cr7+so,1570 <__c_kernel_time+0x3c> 1570: 7c 88 02 a6 mflr r4 1574: 3c 84 ff ff addis r4,r4,-1 1578: 38 84 2a 90 addi r4,r4,10896 157c: 81 44 00 20 lwz r10,32(r4) 1580: 81 64 00 24 lwz r11,36(r4) 1584: 41 82 00 08 beq 158c <__c_kernel_time+0x58> 1588: 91 63 00 00 stw r11,0(r3) 158c: 80 01 00 04 lwz r0,4(r1) 1590: 7d 63 5b 78 mr r3,r11 1594: 7c 08 03 a6 mtlr r0 1598: 4e 80 00 20 blr Christophe
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index a0ce777f97063bf858942c60654d8411bcf2a3dc..600fa3b917ee902d016f2a04376950a9dc49074f 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -156,6 +156,7 @@ config PPC select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE select ARCH_HAS_UBSAN + select ARCH_HAS_VDSO_ARCH_DATA select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_HAVE_EXTRA_ELF_NOTES if SPU_BASE select ARCH_KEEP_MEMBLOCK @@ -206,6 +207,7 @@ config PPC select GENERIC_PTDUMP select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_DATA_STORE select GENERIC_VDSO_TIME_NS select HAS_IOPORT if PCI select HAVE_ARCH_AUDITSYSCALL diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h index 8d972bc98b55fe916f23488ca9e2a5918046b9aa..1ca23fbfe087ae90b90c4286335f86d9f8121078 100644 --- a/arch/powerpc/include/asm/vdso.h +++ b/arch/powerpc/include/asm/vdso.h @@ -3,6 +3,7 @@ #define _ASM_POWERPC_VDSO_H #define VDSO_VERSION_STRING LINUX_2.6.15 +#define __VDSO_PAGES 4 #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/vdso/arch_data.h b/arch/powerpc/include/asm/vdso/arch_data.h new file mode 100644 index 0000000000000000000000000000000000000000..c240a6b875181ac4159f2e80b11f9bf214e22808 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/arch_data.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM + * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>, + * IBM Corp. + */ +#ifndef _ASM_POWERPC_VDSO_ARCH_DATA_H +#define _ASM_POWERPC_VDSO_ARCH_DATA_H + +#include <linux/unistd.h> +#include <linux/types.h> + +#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32) + +#ifdef CONFIG_PPC64 + +struct vdso_arch_data { + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ + __u32 dcache_block_size; /* L1 d-cache block size */ + __u32 icache_block_size; /* L1 i-cache block size */ + __u32 dcache_log_block_size; /* L1 d-cache log block size */ + __u32 icache_log_block_size; /* L1 i-cache log block size */ + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ + __u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */ +}; + +#else /* CONFIG_PPC64 */ + +struct vdso_arch_data { + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ + __u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */ +}; + +#endif /* CONFIG_PPC64 */ + +#endif /* _ASM_POWERPC_VDSO_ARCH_DATA_H */ diff --git a/arch/powerpc/include/asm/vdso/getrandom.h b/arch/powerpc/include/asm/vdso/getrandom.h index 80ce0709725eb89c1f3b69e0733038b458fbf24f..c82eb0d8237681a7396abfe7d161292636b8cce4 100644 --- a/arch/powerpc/include/asm/vdso/getrandom.h +++ b/arch/powerpc/include/asm/vdso/getrandom.h @@ -43,20 +43,21 @@ static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsig (unsigned long)len, (unsigned long)flags); } -static __always_inline struct vdso_rng_data *__arch_get_vdso_rng_data(void) +static __always_inline const struct vdso_rng_data *__ppc_get_vdso_u_rng_data(void) { - struct vdso_arch_data *data; + struct vdso_rng_data *data; asm ( " bcl 20, 31, .+4 ;" "0: mflr %0 ;" - " addis %0, %0, (_vdso_datapage - 0b)@ha ;" - " addi %0, %0, (_vdso_datapage - 0b)@l ;" + " addis %0, %0, (vdso_u_rng_data - 0b)@ha ;" + " addi %0, %0, (vdso_u_rng_data - 0b)@l ;" : "=r" (data) : : "lr" ); - return &data->rng_data; + return data; } +#define __arch_get_vdso_u_rng_data __ppc_get_vdso_u_rng_data ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len); diff --git a/arch/powerpc/include/asm/vdso/gettimeofday.h b/arch/powerpc/include/asm/vdso/gettimeofday.h index c6390890a60c2fdcb608bf321b2945c3fb372f54..bddd9cde97db197d0d3daba6c2289cb29e6b5a75 100644 --- a/arch/powerpc/include/asm/vdso/gettimeofday.h +++ b/arch/powerpc/include/asm/vdso/gettimeofday.h @@ -94,22 +94,29 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) #endif static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return get_tb(); } -const struct vdso_data *__arch_get_vdso_data(void); - #ifdef CONFIG_TIME_NS -static __always_inline -const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) +static __always_inline const struct vdso_time_data *__ppc_get_vdso_u_timens_data(void) { - return (void *)vd + (1U << CONFIG_PAGE_SHIFT); + struct vdso_time_data *time_data; + + asm( + " bcl 20, 31, .+4\n" + "0: mflr %0\n" + " addis %0, %0, (vdso_u_timens_data - 0b)@ha\n" + " addi %0, %0, (vdso_u_timens_data - 0b)@l\n" + : "=r" (time_data) :: "lr"); + + return time_data; } +#define __arch_get_vdso_u_timens_data __ppc_get_vdso_u_timens_data #endif -static inline bool vdso_clocksource_ok(const struct vdso_data *vd) +static inline bool vdso_clocksource_ok(const struct vdso_time_data *vd) { return true; } @@ -135,21 +142,22 @@ static __always_inline u64 vdso_shift_ns(u64 ns, unsigned long shift) #ifdef __powerpc64__ int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd); + const struct vdso_time_data *vd); int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res, - const struct vdso_data *vd); + const struct vdso_time_data *vd); #else int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts, - const struct vdso_data *vd); + const struct vdso_time_data *vd); int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd); + const struct vdso_time_data *vd); int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res, - const struct vdso_data *vd); + const struct vdso_time_data *vd); #endif int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz, - const struct vdso_data *vd); + const struct vdso_time_data *vd); __kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, - const struct vdso_data *vd); + const struct vdso_time_data *vd); + #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/powerpc/include/asm/vdso/vsyscall.h b/arch/powerpc/include/asm/vdso/vsyscall.h index 48560a11955956b8fbb59360334a81972723bd57..c2c9ae1b22e71a3f87e5a1a351699c7ab42b2f95 100644 --- a/arch/powerpc/include/asm/vdso/vsyscall.h +++ b/arch/powerpc/include/asm/vdso/vsyscall.h @@ -6,19 +6,6 @@ #include <asm/vdso_datapage.h> -static __always_inline -struct vdso_data *__arch_get_k_vdso_data(void) -{ - return vdso_data->data; -} -#define __arch_get_k_vdso_data __arch_get_k_vdso_data - -static __always_inline -struct vdso_rng_data *__arch_get_k_vdso_rng_data(void) -{ - return &vdso_data->rng_data; -} - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/powerpc/include/asm/vdso_datapage.h b/arch/powerpc/include/asm/vdso_datapage.h index a202f5b63479533a7f45a74df015feb59f3d7c87..95d45a50355d269454dd3e175a5b3844181536b5 100644 --- a/arch/powerpc/include/asm/vdso_datapage.h +++ b/arch/powerpc/include/asm/vdso_datapage.h @@ -11,56 +11,18 @@ #ifndef __ASSEMBLY__ -#include <linux/unistd.h> -#include <linux/time.h> #include <vdso/datapage.h> -#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32) - -#ifdef CONFIG_PPC64 - -struct vdso_arch_data { - __u64 tb_ticks_per_sec; /* Timebase tics / sec */ - __u32 dcache_block_size; /* L1 d-cache block size */ - __u32 icache_block_size; /* L1 i-cache block size */ - __u32 dcache_log_block_size; /* L1 d-cache log block size */ - __u32 icache_log_block_size; /* L1 i-cache log block size */ - __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ - __u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */ - - struct vdso_rng_data rng_data; - - struct vdso_data data[CS_BASES] __aligned(1 << CONFIG_PAGE_SHIFT); -}; - -#else /* CONFIG_PPC64 */ - -struct vdso_arch_data { - __u64 tb_ticks_per_sec; /* Timebase tics / sec */ - __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ - __u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */ - struct vdso_rng_data rng_data; - - struct vdso_data data[CS_BASES] __aligned(1 << CONFIG_PAGE_SHIFT); -}; - -#endif /* CONFIG_PPC64 */ - -extern struct vdso_arch_data *vdso_data; - #else /* __ASSEMBLY__ */ -.macro get_datapage ptr offset=0 +.macro get_datapage ptr symbol bcl 20, 31, .+4 999: mflr \ptr - addis \ptr, \ptr, (_vdso_datapage - 999b + \offset)@ha - addi \ptr, \ptr, (_vdso_datapage - 999b + \offset)@l + addis \ptr, \ptr, (\symbol - 999b)@ha + addi \ptr, \ptr, (\symbol - 999b)@l .endm -#include <asm/asm-offsets.h> -#include <asm/page.h> - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 7a390bd4f4af3c7408b3e3c5ef6d43b95b3b6463..b3048f6d3822c0c457f4aa2ccb5dc870495ba79b 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -334,7 +334,6 @@ int main(void) #endif /* ! CONFIG_PPC64 */ /* datapage offsets for use by vdso */ - OFFSET(VDSO_DATA_OFFSET, vdso_arch_data, data); OFFSET(CFG_TB_TICKS_PER_SEC, vdso_arch_data, tb_ticks_per_sec); #ifdef CONFIG_PPC64 OFFSET(CFG_ICACHE_BLOCKSZ, vdso_arch_data, icache_block_size); diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 0727332ad86fbcfcf8ca18b344ba04381e827c79..15784c5c95c77f1eccfa948a36ba69386a2c175b 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -950,7 +950,7 @@ void __init time_init(void) sys_tz.tz_dsttime = 0; } - vdso_data->tb_ticks_per_sec = tb_ticks_per_sec; + vdso_k_arch_data->tb_ticks_per_sec = tb_ticks_per_sec; #ifdef CONFIG_PPC64_PROC_SYSTEMCFG systemcfg->tb_ticks_per_sec = tb_ticks_per_sec; #endif diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index 43379365ce1b37cfba662ea58feca5e73dd5f700..219d67bcf747e79f48d09a50f5cb9624bcc0f7b1 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -17,7 +17,7 @@ #include <linux/elf.h> #include <linux/security.h> #include <linux/syscalls.h> -#include <linux/time_namespace.h> +#include <linux/vdso_datastore.h> #include <vdso/datapage.h> #include <asm/syscall.h> @@ -32,6 +32,8 @@ #include <asm/vdso_datapage.h> #include <asm/setup.h> +static_assert(__VDSO_PAGES == VDSO_NR_PAGES); + /* The alignment of the vDSO */ #define VDSO_ALIGNMENT (1 << 16) @@ -40,24 +42,6 @@ extern char vdso64_start, vdso64_end; long sys_ni_syscall(void); -/* - * The vdso data page (aka. systemcfg for old ppc64 fans) is here. - * Once the early boot kernel code no longer needs to muck around - * with it, it will become dynamically allocated - */ -static union { - struct vdso_arch_data data; - u8 page[2 * PAGE_SIZE]; -} vdso_data_store __page_aligned_data; -struct vdso_arch_data *vdso_data = &vdso_data_store.data; - -enum vvar_pages { - VVAR_BASE_PAGE_OFFSET, - VVAR_TIME_PAGE_OFFSET, - VVAR_TIMENS_PAGE_OFFSET, - VVAR_NR_PAGES, -}; - static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma, unsigned long text_size) { @@ -96,14 +80,6 @@ static void vdso_close(const struct vm_special_mapping *sm, struct vm_area_struc mm->context.vdso = NULL; } -static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, - struct vm_area_struct *vma, struct vm_fault *vmf); - -static struct vm_special_mapping vvar_spec __ro_after_init = { - .name = "[vvar]", - .fault = vvar_fault, -}; - static struct vm_special_mapping vdso32_spec __ro_after_init = { .name = "[vdso]", .mremap = vdso32_mremap, @@ -116,73 +92,6 @@ static struct vm_special_mapping vdso64_spec __ro_after_init = { .close = vdso_close, }; -#ifdef CONFIG_TIME_NS -struct vdso_data *arch_get_vdso_data(void *vvar_page) -{ - return vvar_page; -} - -/* - * The vvar mapping contains data for a specific time namespace, so when a task - * changes namespace we must unmap its vvar data for the old namespace. - * Subsequent faults will map in data for the new namespace. - * - * For more details see timens_setup_vdso_data(). - */ -int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) -{ - struct mm_struct *mm = task->mm; - VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - - mmap_read_lock(mm); - for_each_vma(vmi, vma) { - if (vma_is_special_mapping(vma, &vvar_spec)) - zap_vma_pages(vma); - } - mmap_read_unlock(mm); - - return 0; -} -#endif - -static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, - struct vm_area_struct *vma, struct vm_fault *vmf) -{ - struct page *timens_page = find_timens_vvar_page(vma); - unsigned long pfn; - - switch (vmf->pgoff) { - case VVAR_BASE_PAGE_OFFSET: - pfn = virt_to_pfn(vdso_data); - break; - case VVAR_TIME_PAGE_OFFSET: - if (timens_page) - pfn = page_to_pfn(timens_page); - else - pfn = virt_to_pfn(vdso_data->data); - break; -#ifdef CONFIG_TIME_NS - case VVAR_TIMENS_PAGE_OFFSET: - /* - * If a task belongs to a time namespace then a namespace - * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and - * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET - * offset. - * See also the comment near timens_setup_vdso_data(). - */ - if (!timens_page) - return VM_FAULT_SIGBUS; - pfn = virt_to_pfn(vdso_data->data); - break; -#endif /* CONFIG_TIME_NS */ - default: - return VM_FAULT_SIGBUS; - } - - return vmf_insert_pfn(vma, vmf->address, pfn); -} - /* * This is called from binfmt_elf, we create the special vma for the * vDSO and insert it into the mm struct tree @@ -191,7 +100,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int { unsigned long vdso_size, vdso_base, mappings_size; struct vm_special_mapping *vdso_spec; - unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE; + unsigned long vvar_size = VDSO_NR_PAGES * PAGE_SIZE; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; @@ -217,9 +126,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int /* Add required alignment. */ vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT); - vma = _install_special_mapping(mm, vdso_base, vvar_size, - VM_READ | VM_MAYREAD | VM_IO | - VM_DONTDUMP | VM_PFNMAP, &vvar_spec); + vma = vdso_install_vvar_mapping(mm, vdso_base); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -299,10 +206,10 @@ static void __init vdso_setup_syscall_map(void) for (i = 0; i < NR_syscalls; i++) { if (sys_call_table[i] != (void *)&sys_ni_syscall) - vdso_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); + vdso_k_arch_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); if (IS_ENABLED(CONFIG_COMPAT) && compat_sys_call_table[i] != (void *)&sys_ni_syscall) - vdso_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); + vdso_k_arch_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); } } @@ -352,10 +259,10 @@ static struct page ** __init vdso_setup_pages(void *start, void *end) static int __init vdso_init(void) { #ifdef CONFIG_PPC64 - vdso_data->dcache_block_size = ppc64_caches.l1d.block_size; - vdso_data->icache_block_size = ppc64_caches.l1i.block_size; - vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size; - vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size; + vdso_k_arch_data->dcache_block_size = ppc64_caches.l1d.block_size; + vdso_k_arch_data->icache_block_size = ppc64_caches.l1i.block_size; + vdso_k_arch_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size; + vdso_k_arch_data->icache_log_block_size = ppc64_caches.l1i.log_block_size; #endif /* CONFIG_PPC64 */ vdso_setup_syscall_map(); diff --git a/arch/powerpc/kernel/vdso/cacheflush.S b/arch/powerpc/kernel/vdso/cacheflush.S index 0085ae464dac9c32381625a6969a4e422ad34eb7..488d3ade11e64996b30f42777251df8499eda92c 100644 --- a/arch/powerpc/kernel/vdso/cacheflush.S +++ b/arch/powerpc/kernel/vdso/cacheflush.S @@ -30,7 +30,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) #ifdef CONFIG_PPC64 mflr r12 .cfi_register lr,r12 - get_datapage r10 + get_datapage r10 vdso_u_arch_data mtlr r12 .cfi_restore lr #endif diff --git a/arch/powerpc/kernel/vdso/datapage.S b/arch/powerpc/kernel/vdso/datapage.S index db8e167f01667eb95b3dc74f6771e610411bba90..d23b2e8e2a34ca9b142231eb3a492716a49b2248 100644 --- a/arch/powerpc/kernel/vdso/datapage.S +++ b/arch/powerpc/kernel/vdso/datapage.S @@ -28,7 +28,7 @@ V_FUNCTION_BEGIN(__kernel_get_syscall_map) mflr r12 .cfi_register lr,r12 mr. r4,r3 - get_datapage r3 + get_datapage r3 vdso_u_arch_data mtlr r12 #ifdef __powerpc64__ addi r3,r3,CFG_SYSCALL_MAP64 @@ -52,7 +52,7 @@ V_FUNCTION_BEGIN(__kernel_get_tbfreq) .cfi_startproc mflr r12 .cfi_register lr,r12 - get_datapage r3 + get_datapage r3 vdso_u_arch_data #ifndef __powerpc64__ lwz r4,(CFG_TB_TICKS_PER_SEC + 4)(r3) #endif diff --git a/arch/powerpc/kernel/vdso/gettimeofday.S b/arch/powerpc/kernel/vdso/gettimeofday.S index 5333848322ca6105018d501952e3bf42475f49df..79c967212444732da50805fd086c6f2a3c75b0cc 100644 --- a/arch/powerpc/kernel/vdso/gettimeofday.S +++ b/arch/powerpc/kernel/vdso/gettimeofday.S @@ -33,9 +33,9 @@ .cfi_rel_offset r2, PPC_MIN_STKFRM + STK_GOT #endif .ifeq \call_time - get_datapage r5 VDSO_DATA_OFFSET + get_datapage r5 vdso_u_time_data .else - get_datapage r4 VDSO_DATA_OFFSET + get_datapage r4 vdso_u_time_data .endif bl CFUNC(DOTSYM(\funct)) PPC_LL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1) diff --git a/arch/powerpc/kernel/vdso/vdso32.lds.S b/arch/powerpc/kernel/vdso/vdso32.lds.S index 1a1b0b6d681a9977e4ef8042e52d8d33da61887e..72a1012b8a205c6357cecb4b53d2d8e1ff59b051 100644 --- a/arch/powerpc/kernel/vdso/vdso32.lds.S +++ b/arch/powerpc/kernel/vdso/vdso32.lds.S @@ -6,6 +6,7 @@ #include <asm/vdso.h> #include <asm/page.h> #include <asm-generic/vmlinux.lds.h> +#include <vdso/datapage.h> #ifdef __LITTLE_ENDIAN__ OUTPUT_FORMAT("elf32-powerpcle", "elf32-powerpcle", "elf32-powerpcle") @@ -16,7 +17,8 @@ OUTPUT_ARCH(powerpc:common) SECTIONS { - PROVIDE(_vdso_datapage = . - 3 * PAGE_SIZE); + VDSO_VVAR_SYMS + . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/powerpc/kernel/vdso/vdso64.lds.S b/arch/powerpc/kernel/vdso/vdso64.lds.S index e21b5506cad62b16e677be74fda7921ec917141a..32102a05eaa7e015e0f89e4a94a3c5e31da7d460 100644 --- a/arch/powerpc/kernel/vdso/vdso64.lds.S +++ b/arch/powerpc/kernel/vdso/vdso64.lds.S @@ -6,6 +6,7 @@ #include <asm/vdso.h> #include <asm/page.h> #include <asm-generic/vmlinux.lds.h> +#include <vdso/datapage.h> #ifdef __LITTLE_ENDIAN__ OUTPUT_FORMAT("elf64-powerpcle", "elf64-powerpcle", "elf64-powerpcle") @@ -16,7 +17,8 @@ OUTPUT_ARCH(powerpc:common64) SECTIONS { - PROVIDE(_vdso_datapage = . - 3 * PAGE_SIZE); + VDSO_VVAR_SYMS + . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/powerpc/kernel/vdso/vgettimeofday.c b/arch/powerpc/kernel/vdso/vgettimeofday.c index 55a287c9a7366aa59ab4af1e760a8995f588a4d5..6f5167d81af5f3e6e755dbda4307769e45a28421 100644 --- a/arch/powerpc/kernel/vdso/vgettimeofday.c +++ b/arch/powerpc/kernel/vdso/vgettimeofday.c @@ -7,43 +7,43 @@ #ifdef __powerpc64__ int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_gettime_data(vd, clock, ts); } int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_getres_data(vd, clock_id, res); } #else int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_gettime32_data(vd, clock, ts); } int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_gettime_data(vd, clock, ts); } int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_getres_time32_data(vd, clock_id, res); } #endif int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_gettimeofday_data(vd, tv, tz); } -__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso_data *vd) +__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso_time_data *vd) { return __cvdso_time_data(vd, time); }