@@ -83,7 +83,7 @@ endif
# Currently we save and restore fpregs on all kernel entry/interruption paths.
# If that gets optimized, we might need to disable the use of fpregs in the
# kernel.
-cflags-y += -mdisable-fpregs
+# cflags-y += -mdisable-fpregs
# Use long jumps instead of long branches (needed if your linker fails to
# link a too big vmlinux executable). Not enabled for building modules.
@@ -18,7 +18,8 @@ KBUILD_CFLAGS := -D__KERNEL__ -O2 -DBOOTLOADER
KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
KBUILD_CFLAGS += -fno-strict-aliasing
KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks -fno-builtin-printf
-KBUILD_CFLAGS += -fno-PIE -mno-space-regs -mdisable-fpregs -Os
+# KBUILD_CFLAGS += -fno-PIE -mno-space-regs -mdisable-fpregs -Os
+KBUILD_CFLAGS += -fno-PIE -mno-space-regs -Os
ifndef CONFIG_64BIT
KBUILD_CFLAGS += -mfast-indirect-calls
endif
@@ -135,6 +135,16 @@
extrd,u \r, 63-(\sa), 64-(\sa), \t
.endm
+ /* The extru instruction leaves the most significant 32 bits of the
+ * target register in an undefined state on PA 2.0 systems. */
+ .macro extru_safe r, p, len, t
+#ifdef CONFIG_64BIT
+ extrd,u \r, 32+(\p), \len, \t
+#else
+ extru \r, \p, \len, \t
+#endif
+ .endm
+
/* load 32-bit 'value' into 'reg' compensating for the ldil
* sign-extension when running in wide mode.
* WARNING!! neither 'value' nor 'reg' can be expressions
@@ -10,23 +10,28 @@
#include <asm/errno.h>
/* The following has to match the LWS code in syscall.S. We have
- sixteen four-word locks. */
+ 256 four-word locks. We xor three 8-bit hunks of the futex physical
+ address to create the hash index. */
+
+static unsigned long
+_futex_hash_index(unsigned long pa)
+{
+ return ((pa >> 6) ^ (pa >> 14) ^ (pa >> 22)) & 0x3fc;
+}
static inline void
-_futex_spin_lock_irqsave(u32 __user *uaddr, unsigned long int *flags)
+_futex_spin_lock_irqsave(unsigned long index, unsigned long int *flags)
{
extern u32 lws_lock_start[];
- long index = ((long)uaddr & 0x3f8) >> 1;
arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index];
local_irq_save(*flags);
arch_spin_lock(s);
}
static inline void
-_futex_spin_unlock_irqrestore(u32 __user *uaddr, unsigned long int *flags)
+_futex_spin_unlock_irqrestore(unsigned long index, unsigned long int *flags)
{
extern u32 lws_lock_start[];
- long index = ((long)uaddr & 0x3f8) >> 1;
arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index];
arch_spin_unlock(s);
local_irq_restore(*flags);
@@ -35,11 +40,14 @@ _futex_spin_unlock_irqrestore(u32 __user *uaddr, unsigned long int *flags)
static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
{
+ unsigned long ua = (unsigned long)uaddr;
+ unsigned long index;
unsigned long int flags;
int oldval, ret;
u32 tmp;
- _futex_spin_lock_irqsave(uaddr, &flags);
+ index = _futex_hash_index(ua);
+ _futex_spin_lock_irqsave(index, &flags);
ret = -EFAULT;
if (unlikely(get_user(oldval, uaddr) != 0))
@@ -72,7 +80,7 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
ret = -EFAULT;
out_pagefault_enable:
- _futex_spin_unlock_irqrestore(uaddr, &flags);
+ _futex_spin_unlock_irqrestore(index, &flags);
if (!ret)
*oval = oldval;
@@ -84,10 +92,12 @@ static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
u32 oldval, u32 newval)
{
+ unsigned long ua = (unsigned long)uaddr;
+ unsigned long index;
u32 val;
unsigned long flags;
- /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is
+ /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is
* our gateway page, and causes no end of trouble...
*/
if (uaccess_kernel() && !uaddr)
@@ -98,23 +108,25 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
/* HPPA has no cmpxchg in hardware and therefore the
* best we can do here is use an array of locks. The
- * lock selected is based on a hash of the userspace
- * address. This should scale to a couple of CPUs.
+ * lock selected is based on a hash of the virtual
+ * address of the futex. This should scale to a couple
+ * of CPUs.
*/
- _futex_spin_lock_irqsave(uaddr, &flags);
+ index = _futex_hash_index(ua);
+ _futex_spin_lock_irqsave(index, &flags);
if (unlikely(get_user(val, uaddr) != 0)) {
- _futex_spin_unlock_irqrestore(uaddr, &flags);
+ _futex_spin_unlock_irqrestore(index, &flags);
return -EFAULT;
}
if (val == oldval && unlikely(put_user(newval, uaddr) != 0)) {
- _futex_spin_unlock_irqrestore(uaddr, &flags);
+ _futex_spin_unlock_irqrestore(index, &flags);
return -EFAULT;
}
*uval = val;
- _futex_spin_unlock_irqrestore(uaddr, &flags);
+ _futex_spin_unlock_irqrestore(index, &flags);
return 0;
}
@@ -65,6 +65,8 @@ extern int pa_serialize_tlb_flushes;
* are slow on SMP machines since the purge must be broadcast to all CPUs.
*/
+extern void __update_cache(pte_t pte);
+
static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
{
unsigned long flags;
@@ -2,26 +2,30 @@
#ifndef __PARISC_SPECIAL_INSNS_H
#define __PARISC_SPECIAL_INSNS_H
-#define lpa(va) ({ \
- unsigned long pa; \
- __asm__ __volatile__( \
- "copy %%r0,%0\n\t" \
- "lpa %%r0(%1),%0" \
- : "=r" (pa) \
- : "r" (va) \
- : "memory" \
- ); \
- pa; \
+#define lpa(va) ({ \
+ unsigned long pa; \
+ __asm__ __volatile__( \
+ "copy %%r0,%0\n" \
+ "9998:\tlpa %%r0(%1),%0\n" \
+ "9999:\n" \
+ ASM_EXCEPTIONTABLE_ENTRY(9998b, 9999b) \
+ : "=&r" (pa) \
+ : "r" (va) \
+ : "memory" \
+ ); \
+ pa; \
})
-#define lpa_user(va) ({ \
- unsigned long pa; \
- __asm__ __volatile__( \
- "copy %%r0,%0\n\t" \
- "lpa %%r0(%%sr3,%1),%0" \
- : "=r" (pa) \
- : "r" (va) \
- : "memory" \
+#define lpa_user(va) ({ \
+ unsigned long pa; \
+ __asm__ __volatile__( \
+ "copy %%r0,%0\n" \
+ "9998:\tlpa %%r0(%%sr3,%1),%0\n" \
+ "9999:\n" \
+ ASM_EXCEPTIONTABLE_ENTRY(9998b, 9999b) \
+ : "=&r" (pa) \
+ : "r" (va) \
+ : "memory" \
); \
pa; \
})
@@ -39,6 +39,7 @@ verify "$3"
if [ -n "${INSTALLKERNEL}" ]; then
if [ -x ~/bin/${INSTALLKERNEL} ]; then exec ~/bin/${INSTALLKERNEL} "$@"; fi
if [ -x /sbin/${INSTALLKERNEL} ]; then exec /sbin/${INSTALLKERNEL} "$@"; fi
+ if [ -x /usr/sbin/${INSTALLKERNEL} ]; then exec /usr/sbin/${INSTALLKERNEL} "$@"; fi
fi
# Default install
@@ -559,6 +559,7 @@ void flush_cache_mm(struct mm_struct *mm)
return;
}
+ preempt_disable();
if (mm->context == mfsp(3)) {
for (vma = mm->mmap; vma; vma = vma->vm_next) {
flush_user_dcache_range_asm(vma->vm_start, vma->vm_end);
@@ -566,8 +567,10 @@ void flush_cache_mm(struct mm_struct *mm)
flush_user_icache_range_asm(vma->vm_start, vma->vm_end);
flush_tlb_range(vma, vma->vm_start, vma->vm_end);
}
+ preempt_enable();
return;
}
+ preempt_enable();
pgd = mm->pgd;
for (vma = mm->mmap; vma; vma = vma->vm_next) {
@@ -606,13 +609,16 @@ void flush_cache_range(struct vm_area_struct *vma,
return;
}
+ preempt_disable();
if (vma->vm_mm->context == mfsp(3)) {
flush_user_dcache_range_asm(start, end);
if (vma->vm_flags & VM_EXEC)
flush_user_icache_range_asm(start, end);
flush_tlb_range(vma, start, end);
+ preempt_enable();
return;
}
+ preempt_enable();
pgd = vma->vm_mm->pgd;
for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
@@ -393,17 +393,9 @@
*/
.macro L2_ptep pmd,pte,index,va,fault
#if CONFIG_PGTABLE_LEVELS == 3
- extru \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index
+ extru_safe \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index
#else
-# if defined(CONFIG_64BIT)
- extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
- #else
- # if PAGE_SIZE > 4096
- extru \va,31-ASM_PGDIR_SHIFT,32-ASM_PGDIR_SHIFT,\index
- # else
- extru \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
- # endif
-# endif
+ extru_safe \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index
#endif
dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */
#if CONFIG_PGTABLE_LEVELS < 3
@@ -413,7 +405,7 @@
bb,>=,n \pmd,_PxD_PRESENT_BIT,\fault
dep %r0,31,PxD_FLAG_SHIFT,\pmd /* clear flags */
SHLREG \pmd,PxD_VALUE_SHIFT,\pmd
- extru \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index
+ extru_safe \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index
dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */
shladd \index,BITS_PER_PTE_ENTRY,\pmd,\pmd /* pmd is now pte */
.endm
@@ -974,8 +966,8 @@ intr_do_preempt:
/* current_thread_info()->preempt_count */
mfctl %cr30, %r1
- LDREG TI_PRE_COUNT(%r1), %r19
- cmpib,COND(<>) 0, %r19, intr_restore /* if preempt_count > 0 */
+ ldw TI_PRE_COUNT(%r1), %r19
+ cmpib,<> 0, %r19, intr_restore /* if preempt_count > 0 */
nop /* prev insn branched backwards */
/* check if we interrupted a critical path */
@@ -67,8 +67,8 @@ void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
int mapped;
/* Make sure we don't have any aliases in cache */
- flush_kernel_vmap_range(addr, len);
flush_icache_range(start, end);
+ flush_tlb_kernel_range(start, end);
p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags, &mapped);
@@ -93,7 +93,6 @@ void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len)
flush_kernel_vmap_range((void *)fixmap, (p-fixmap) * sizeof(*p));
if (mapped)
patch_unmap(FIX_TEXT_POKE0, &flags);
- flush_icache_range(start, end);
}
void __kprobes __patch_text(void *addr, u32 insn)
@@ -228,11 +228,13 @@ static inline void
send_IPI_allbutself(enum ipi_message_type op)
{
int i;
-
+
+ preempt_disable();
for_each_online_cpu(i) {
if (i != smp_processor_id())
send_IPI_single(i, op);
}
+ preempt_enable();
}
#ifdef CONFIG_KGDB
@@ -74,7 +74,7 @@ ENTRY(linux_gateway_page)
/* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */
/* Light-weight-syscall entry must always be located at 0xb0 */
/* WARNING: Keep this number updated with table size changes */
-#define __NR_lws_entries (3)
+#define __NR_lws_entries (5)
lws_entry:
gate lws_start, %r0 /* increase privilege */
@@ -496,6 +496,11 @@ lws_start:
/* Jump to lws, lws table pointers already relocated */
be,n 0(%sr2,%r21)
+lws_stby_fault:
+ ldo 1(%r0),%r28
+ b lws_exit
+ ldo -EFAULT(%r0),%r21 /* set errno */
+
lws_exit_nosys:
ldo -ENOSYS(%r0),%r21 /* set errno */
/* Fall through: Return to userspace */
@@ -567,15 +572,25 @@ lws_compare_and_swap32:
#endif
lws_compare_and_swap:
- /* Load start of lock table */
- ldil L%lws_lock_start, %r20
- ldo R%lws_lock_start(%r20), %r28
+ /* Dirty cache line at r26 to avoid COW break in critical region */
+ b 3f
+1: stbys,e %r0, 0(%r26)
+2: b,n lws_stby_fault
+ ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 2b-linux_gateway_page)
+
+ /* Calculate 8-bit hash index from virtual address */
+3: extru_safe %r26, 23, 8, %r28
+ extru_safe %r26, 15, 8, %r1
+ xor %r1, %r28, %r28
+ extru_safe %r26, 7, 8, %r1
+ xor %r1, %r28, %r20
- /* Extract eight bits from r26 and hash lock (Bits 3-11) */
- extru %r26, 28, 8, %r20
+ /* Load start of lock table */
+ ldil L%lws_lock_start, %r28
+ ldo R%lws_lock_start(%r28), %r28
- /* Find lock to use, the hash is either one of 0 to
- 15, multiplied by 16 (keep it 16-byte aligned)
+ /* Find lock to use, the hash index is one of 0 to
+ 255, multiplied by 16 (keep it 16-byte aligned)
and add to the lock table offset. */
shlw %r20, 4, %r20
add %r20, %r28, %r20
@@ -603,10 +618,20 @@ cas_nocontend:
# endif
/* ENABLE_LWS_DEBUG */
+ /* We disable interrupts around the critical region. However,
+ the store instruction in the critical region might cause a
+ COW break. The trap handler needs to re-enable interrupts
+ when page faults are enabled as the process might sleep.
+ When this happens, the lock may be held by a process for
+ an extended period and affect unrelated processes because
+ locks are shared. */
rsm PSW_SM_I, %r0 /* Disable interrupts */
- /* COW breaks can cause contention on UP systems */
+
+ /* Check lock first to minimize cache line bounce */
+ ldw 0(%sr2,%r20), %r28 /* Load the lock */
+ cmpb,=,n %r0, %r28, cas_wouldblock /* Spin if locked */
LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */
- cmpb,<>,n %r0, %r28, cas_action /* Did we get it? */
+ cmpb,<>,n %r28, %r0, cas_action /* Did we get it? */
cas_wouldblock:
ldo 2(%r0), %r28 /* 2nd case */
ssm PSW_SM_I, %r0
@@ -621,7 +646,7 @@ cas_wouldblock:
*/
/* NOTES:
- This all works becuse intr_do_signal
+ This all works because intr_do_signal
and schedule both check the return iasq
and see that we are on the kernel page
so this process is never scheduled off
@@ -629,6 +654,7 @@ cas_wouldblock:
thus it is wholly atomic from usrspaces
perspective
*/
+
cas_action:
#if defined CONFIG_SMP && ENABLE_LWS_DEBUG
/* DEBUG */
@@ -757,23 +783,49 @@ lws_compare_and_swap_2:
#endif
cas2_lock_start:
- /* Load start of lock table */
- ldil L%lws_lock_start, %r20
- ldo R%lws_lock_start(%r20), %r28
+ /* Dirty cache line at r26 to avoid COW break in critical region */
+ copy %r26, %r28
+#ifdef CONFIG_64BIT
+ depdi 0, 63, 2, %r28
+#else
+ depi 0, 31, 2, %r28
+#endif
+ b 3f
+1: stbys,e %r0, 0(%r28)
+2: b,n lws_stby_fault
+ ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 2b-linux_gateway_page)
+
+ /* Calculate 8-bit hash index from virtual address */
+3: extru_safe %r26, 23, 8, %r28
+ extru_safe %r26, 15, 8, %r1
+ xor %r1, %r28, %r28
+ extru_safe %r26, 7, 8, %r1
+ xor %r1, %r28, %r20
- /* Extract eight bits from r26 and hash lock (Bits 3-11) */
- extru %r26, 28, 8, %r20
+ /* Load start of lock table */
+ ldil L%lws_lock_start, %r28
+ ldo R%lws_lock_start(%r28), %r28
- /* Find lock to use, the hash is either one of 0 to
- 15, multiplied by 16 (keep it 16-byte aligned)
+ /* Find lock to use, the hash index is one of 0 to
+ 255, multiplied by 16 (keep it 16-byte aligned)
and add to the lock table offset. */
shlw %r20, 4, %r20
add %r20, %r28, %r20
+ /* We disable interrupts around the critical region. However,
+ the store instruction in the critical region might cause a
+ COW break. The trap handler needs to re-enable interrupts
+ when page faults are enabled as the process might sleep.
+ When this happens, the lock may be held by a process for
+ an extended period and affect unrelated processes because
+ locks are shared. */
rsm PSW_SM_I, %r0 /* Disable interrupts */
- /* COW breaks can cause contention on UP systems */
+
+ /* Check lock first to minimize cache line bounce */
+ ldw 0(%sr2,%r20), %r28 /* Load the lock */
+ cmpb,=,n %r0, %r28, cas2_wouldblock /* Spin if locked */
LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */
- cmpb,<>,n %r0, %r28, cas2_action /* Did we get it? */
+ cmpb,<>,n %r28, %r0, cas2_action /* Did we get it? */
cas2_wouldblock:
ldo 2(%r0), %r28 /* 2nd case */
ssm PSW_SM_I, %r0
@@ -788,7 +840,7 @@ cas2_wouldblock:
*/
/* NOTES:
- This all works becuse intr_do_signal
+ This all works because intr_do_signal
and schedule both check the return iasq
and see that we are on the kernel page
so this process is never scheduled off
@@ -796,6 +848,7 @@ cas2_wouldblock:
thus it is wholly atomic from usrspaces
perspective
*/
+
cas2_action:
/* Jump to the correct function */
blr %r29, %r0
@@ -897,6 +950,342 @@ cas2_end:
ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
#endif
+ /***************************************************
+ LWS atomic exchange. When using using a synthesized CAS
+ operation, we need to perform atomic loads using the CAS
+ lock to ensure sequential consistency.
+
+ %r26 - Exchange address
+ %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit)
+ %r24 - Address of new value
+ %r23 - Address of old value
+ %r28 - Return non-zero on failure
+ %r21 - Kernel error code
+
+ %r21 has the following meanings:
+
+ EAGAIN - Lock is busy, ldcw failed, try again.
+ EFAULT - Read failed.
+
+ Scratch: r20, r1
+
+ ****************************************************/
+
+ /* ELF32 Process entry path */
+lws_atomic_xchg:
+#ifdef CONFIG_64BIT
+ /* Clip the input registers. We don't need to clip %r25 as we
+ only use it for word operations */
+ depdi 0, 31, 32, %r26
+ depdi 0, 31, 32, %r25
+ depdi 0, 31, 32, %r24
+ depdi 0, 31, 32, %r23
+#endif
+
+ /* Check the validity of the size pointer */
+ subi,>>= 3, %r25, %r0
+ b,n lws_exit_nosys
+
+ /* Dirty cache line at r26 to avoid COW break in critical region */
+ copy %r26, %r28
+#ifdef CONFIG_64BIT
+ depdi 0, 63, 2, %r28
+#else
+ depi 0, 31, 2, %r28
+#endif
+ b 3f
+1: stbys,e %r0, 0(%r28)
+2: b,n lws_stby_fault
+ ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 2b-linux_gateway_page)
+
+ /* Calculate 8-bit hash index from virtual address */
+3: extru_safe %r26, 23, 8, %r28
+ extru_safe %r26, 15, 8, %r1
+ xor %r1, %r28, %r28
+ extru_safe %r26, 7, 8, %r1
+ xor %r1, %r28, %r20
+
+ /* Load start of lock table */
+ ldil L%lws_lock_start, %r28
+ ldo R%lws_lock_start(%r28), %r28
+
+ /* Find lock to use, the hash index is one of 0 to
+ 255, multiplied by 16 (keep it 16-byte aligned)
+ and add to the lock table offset. */
+ shlw %r20, 4, %r20
+ add %r20, %r28, %r20
+
+ shlw %r25, 2, %r1
+
+ /* We disable interrupts around the critical region. However,
+ the store instruction in the critical region might cause a
+ COW break. The trap handler needs to re-enable interrupts
+ when page faults are enabled as the process might sleep.
+ When this happens, the lock may be held by a process for
+ an extended period and affect unrelated processes because
+ locks are shared. */
+ rsm PSW_SM_I, %r0 /* Disable interrupts */
+
+ /* Check lock first to minimize cache line bounce */
+ ldw 0(%sr2,%r20), %r28 /* Load the lock */
+ cmpb,=,n %r0, %r28, 1f /* Spin if locked */
+ LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */
+ cmpb,<>,n %r28, %r0, atomic_xchg_action /* Did we get it? */
+1: ldo 2(%r0), %r28 /* 2nd case */
+ ssm PSW_SM_I, %r0
+ b lws_exit /* Contended... */
+ ldo -EAGAIN(%r0), %r21 /* Spin in userspace */
+
+ /* NOTES:
+ This all works because intr_do_signal
+ and schedule both check the return iasq
+ and see that we are on the kernel page
+ so this process is never scheduled off
+ or is ever sent any signal of any sort,
+ thus it is wholly atomic from usrspaces
+ perspective
+ */
+
+atomic_xchg_action:
+ /* Jump to the correct function */
+ blr %r1, %r0
+ /* Set %r28 as non-zero for now */
+ ldo 1(%r0),%r28
+
+ /* 8bit exchange */
+1: ldb 0(%r26), %r1
+2: stb %r1, 0(%r23)
+3: ldb 0(%r24), %r1
+4: stb %r1, 0(%r26)
+ b atomic_xchg_end
+ copy %r0, %r28
+ nop
+ nop
+
+ /* 16bit exchange */
+5: ldh 0(%r26), %r1
+6: sth %r1, 0(%r23)
+7: ldh 0(%r24), %r1
+8: sth %r1, 0(%r26)
+ b atomic_xchg_end
+ copy %r0, %r28
+ nop
+ nop
+
+ /* 32bit exchange */
+9: ldw 0(%r26), %r1
+10: stw %r1, 0(%r23)
+11: ldw 0(%r24), %r1
+12: stw %r1, 0(%r26)
+ b atomic_xchg_end
+ copy %r0, %r28
+ nop
+ nop
+
+ /* 64bit exchange */
+#ifdef CONFIG_64BIT
+13: ldd 0(%r26), %r1
+14: std %r1, 0(%r23)
+15: ldd 0(%r24), %r1
+16: std %r1, 0(%r26)
+#else
+13: flddx 0(%r26), %fr4
+14: fstdx %fr4, 0(%r23)
+15: flddx 0(%r24), %fr4
+16: fstdx %fr4, 0(%r26)
+#endif
+ copy %r0, %r28
+
+atomic_xchg_end:
+ /* Free lock */
+ stw,ma %r20, 0(%sr2,%r20)
+ /* Return to userspace, set no error */
+ b lws_exit
+ copy %r0, %r21
+
+17:
+ /* Error occurred on load or store */
+ /* Free lock */
+ stw,ma %r20, 0(%sr2,%r20)
+ ldo 1(%r0),%r28
+ b lws_exit
+ ldo -EFAULT(%r0),%r21 /* set errno */
+
+ /* Exception table entries, for the load and store, return EFAULT.
+ Each of the entries must be relocated. */
+ ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 17b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 17b-linux_gateway_page)
+
+ /***************************************************
+ LWS atomic store. When using using a synthesized CAS
+ operation, we need to perform atomic loads using the CAS
+ lock to ensure sequential consistency.
+
+ %r26 - Address to store
+ %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit)
+ %r24 - Address of value to store
+ %r28 - Return non-zero on failure
+ %r21 - Kernel error code
+
+ %r21 has the following meanings:
+
+ EAGAIN - Lock is busy, ldcw failed, try again.
+ EFAULT - Read failed.
+
+ Scratch: r20, r1
+
+ ****************************************************/
+
+ /* ELF32 Process entry path */
+lws_atomic_store:
+#ifdef CONFIG_64BIT
+ /* Clip the input registers. We don't need to clip %r25 as we
+ only use it for word operations */
+ depdi 0, 31, 32, %r26
+ depdi 0, 31, 32, %r25
+ depdi 0, 31, 32, %r24
+#endif
+
+ /* Check the validity of the size pointer */
+ subi,>>= 3, %r25, %r0
+ b,n lws_exit_nosys
+
+ shlw %r25, 1, %r1
+
+ /* Dirty cache line at r26 to avoid COW break in critical region */
+ copy %r26, %r28
+#ifdef CONFIG_64BIT
+ depdi 0, 63, 2, %r28
+#else
+ depi 0, 31, 2, %r28
+#endif
+ b 3f
+1: stbys,e %r0, 0(%r28)
+2: b,n lws_stby_fault
+ ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 2b-linux_gateway_page)
+
+ /* Calculate 8-bit hash index from virtual address */
+3: extru_safe %r26, 23, 8, %r28
+ extru_safe %r26, 15, 8, %r1
+ xor %r1, %r28, %r28
+ extru_safe %r26, 7, 8, %r1
+ xor %r1, %r28, %r20
+
+ /* Load start of lock table */
+ ldil L%lws_lock_start, %r28
+ ldo R%lws_lock_start(%r28), %r28
+
+ /* Find lock to use, the hash index is one of 0 to
+ 255, multiplied by 16 (keep it 16-byte aligned)
+ and add to the lock table offset. */
+ shlw %r20, 4, %r20
+ add %r20, %r28, %r20
+
+ shlw %r25, 1, %r1
+
+ /* We disable interrupts around the critical region. However,
+ the store instruction in the critical region might cause a
+ COW break. The trap handler needs to re-enable interrupts
+ when page faults are enabled as the process might sleep.
+ When this happens, the lock may be held by a process for
+ an extended period and affect unrelated processes because
+ locks are shared. */
+ rsm PSW_SM_I, %r0 /* Disable interrupts */
+
+ /* Check lock first to minimize cache line bounce */
+ ldw 0(%sr2,%r20), %r28 /* Load the lock */
+ cmpb,=,n %r0, %r28, 1f /* Spin if locked */
+ LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */
+ cmpb,<>,n %r0, %r28, atomic_store_action /* Did we get it? */
+1: ldo 2(%r0), %r28 /* 2nd case */
+ ssm PSW_SM_I, %r0
+ b lws_exit /* Contended... */
+ ldo -EAGAIN(%r0), %r21 /* Spin in userspace */
+
+ /* NOTES:
+ This all works because intr_do_signal
+ and schedule both check the return iasq
+ and see that we are on the kernel page
+ so this process is never scheduled off
+ or is ever sent any signal of any sort,
+ thus it is wholly atomic from usrspaces
+ perspective
+ */
+
+atomic_store_action:
+ /* Jump to the correct function */
+ blr %r1, %r0
+ /* Set %r28 as non-zero for now */
+ ldo 1(%r0),%r28
+
+ /* 8bit store */
+1: ldb 0(%r24), %r1
+2: stb %r1, 0(%r26)
+ b atomic_store_end
+ copy %r0, %r28
+
+ /* 16bit store */
+3: ldh 0(%r24), %r1
+4: sth %r1, 0(%r26)
+ b atomic_store_end
+ copy %r0, %r28
+
+ /* 32bit store */
+5: ldw 0(%r24), %r1
+6: stw %r1, 0(%r26)
+ b atomic_store_end
+ copy %r0, %r28
+
+ /* 64bit store */
+#ifdef CONFIG_64BIT
+7: ldd 0(%r24), %r1
+8: std %r1, 0(%r26)
+#else
+7: flddx 0(%r24), %fr4
+8: fstdx %fr4, 0(%r26)
+#endif
+ copy %r0, %r28
+
+atomic_store_end:
+ /* Free lock */
+ stw,ma %r20, 0(%sr2,%r20)
+ /* Return to userspace, set no error */
+ b lws_exit
+ copy %r0, %r21
+
+9:
+ /* Error occurred on store */
+ /* Free lock */
+ stw,ma %r20, 0(%sr2,%r20)
+ ldo 1(%r0),%r28
+ b lws_exit
+ ldo -EFAULT(%r0),%r21 /* set errno */
+
+ /* Exception table entries, return EFAULT.
+ Each of the entries must be relocated. */
+ ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 9b-linux_gateway_page)
+ ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 9b-linux_gateway_page)
+
/* Make sure nothing else is placed on this page */
.align PAGE_SIZE
END(linux_gateway_page)
@@ -916,6 +1305,8 @@ ENTRY(lws_table)
LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic 32bit CAS */
LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic 32bit CAS */
LWS_ENTRY(compare_and_swap_2) /* 2 - ELF32 Atomic 64bit CAS */
+ LWS_ENTRY(atomic_xchg) /* 3 - ELF32 Atomic Exchange */
+ LWS_ENTRY(atomic_store) /* 4 - ELF32 Atomic Store */
END(lws_table)
/* End of lws table */
@@ -249,27 +249,16 @@ void __init time_init(void)
static int __init init_cr16_clocksource(void)
{
/*
- * The cr16 interval timers are not syncronized across CPUs on
- * different sockets, so mark them unstable and lower rating on
- * multi-socket SMP systems.
+ * The cr16 interval timers are not syncronized across CPUs, even if
+ * they share the same socket.
*/
if (num_online_cpus() > 1 && !running_on_qemu) {
- int cpu;
- unsigned long cpu0_loc;
- cpu0_loc = per_cpu(cpu_data, 0).cpu_loc;
-
- for_each_online_cpu(cpu) {
- if (cpu == 0)
- continue;
- if ((cpu0_loc != 0) &&
- (cpu0_loc == per_cpu(cpu_data, cpu).cpu_loc))
- continue;
-
- clocksource_cr16.name = "cr16_unstable";
- clocksource_cr16.flags = CLOCK_SOURCE_UNSTABLE;
- clocksource_cr16.rating = 0;
- break;
- }
+ /* mark sched_clock unstable */
+ clear_sched_clock_stable();
+
+ clocksource_cr16.name = "cr16_unstable";
+ clocksource_cr16.flags = CLOCK_SOURCE_UNSTABLE;
+ clocksource_cr16.rating = 0;
}
/* XXX: We may want to mark sched_clock stable here if cr16 clocks are
@@ -479,9 +479,9 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
int si_code;
if (code == 1)
- pdc_console_restart(); /* switch back to pdc if HPMC */
- else
- local_irq_enable();
+ pdc_console_restart(); /* switch back to pdc if HPMC */
+ else if (!irqs_disabled_flags(regs->gr[0]))
+ local_irq_enable();
/* Security check:
* If the priority level is still user, and the
@@ -729,6 +729,8 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
}
mmap_read_unlock(current->mm);
}
+ /* CPU could not fetch instruction, so clear IIR stale value. */
+ regs->iir = 0xbaadf00d;
fallthrough;
case 27:
/* Data memory protection ID trap */
@@ -782,13 +784,13 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
* unless pagefault_disable() was called before.
*/
- if (fault_space == 0 && !faulthandler_disabled())
+ if (faulthandler_disabled() || fault_space == 0)
{
/* Clean up and return if in exception table. */
if (fixup_exception(regs))
return;
pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC);
- parisc_terminate("Kernel Fault", regs, code, fault_address);
+ parisc_terminate("Kernel Fault", regs, code, 0);
}
}
@@ -309,7 +309,8 @@ static void unwind_frame_regs(struct unwind_frame_info *info)
break;
}
- if (get_user(tmp, (unsigned long *)(info->prev_sp - RP_OFFSET)))
+ if (copy_from_kernel_nofault(&tmp,
+ (void *)info->prev_sp - RP_OFFSET, sizeof(tmp)))
break;
info->prev_ip = tmp;
sp = info->prev_sp;
@@ -57,8 +57,6 @@ SECTIONS
{
. = KERNEL_BINARY_TEXT_START;
- _stext = .; /* start of kernel text, includes init code & data */
-
__init_begin = .;
HEAD_TEXT_SECTION
MLONGCALL_DISCARD(INIT_TEXT_SECTION(8))
@@ -82,6 +80,7 @@ SECTIONS
/* freed after init ends here */
_text = .; /* Text and read-only data */
+ _stext = .;
MLONGCALL_KEEP(INIT_TEXT_SECTION(8))
.text ALIGN(PAGE_SIZE) : {
TEXT_TEXT
@@ -341,7 +341,7 @@ static void __init setup_bootmem(void)
static bool kernel_set_to_readonly;
-static void __init map_pages(unsigned long start_vaddr,
+static void map_pages(unsigned long start_vaddr,
unsigned long start_paddr, unsigned long size,
pgprot_t pgprot, int force)
{
@@ -281,7 +281,7 @@ agp_ioc_init(void __iomem *ioc_regs)
return 0;
}
-static int
+static int __init
lba_find_capability(int cap)
{
struct _parisc_agp_info *info = &parisc_agp_info;
@@ -366,7 +366,7 @@ parisc_agp_setup(void __iomem *ioc_hpa, void __iomem *lba_hpa)
return error;
}
-static int
+static int __init
find_quicksilver(struct device *dev, void *data)
{
struct parisc_device **lba = data;
@@ -378,7 +378,7 @@ find_quicksilver(struct device *dev, void *data)
return 0;
}
-static int
+static int __init
parisc_agp_init(void)
{
extern struct sba_device *sba_list;