@@ -24,29 +24,46 @@
* Hash function to index into a different SPINLOCK.
* Since "a" is usually an address, use one spinlock per cacheline.
*/
-# define ATOMIC_HASH_SIZE 4
-# define ATOMIC_HASH(a) (&(__atomic_hash[ (((unsigned long) (a))/L1_CACHE_BYTES) & (ATOMIC_HASH_SIZE-1) ]))
+# define ATOMIC_HASH_SIZE (4096/L1_CACHE_BYTES) /* 4 */
+# define ATOMIC_HASH(a) (&(__atomic_hash[ (((unsigned long) (a))/L1_CACHE_BYTES) & (ATOMIC_HASH_SIZE-1) ]))
+# define ATOMIC_USER_HASH(a) (&(__atomic_user_hash[ (((unsigned long) (a))/L1_CACHE_BYTES) & (ATOMIC_HASH_SIZE-1) ]))
extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned;
+extern arch_spinlock_t __atomic_user_hash[ATOMIC_HASH_SIZE] __lock_aligned;
/* Can't use raw_spin_lock_irq because of #include problems, so
* this is the substitute */
-#define _atomic_spin_lock_irqsave(l,f) do { \
- arch_spinlock_t *s = ATOMIC_HASH(l); \
+#define _atomic_spin_lock_irqsave_template(l,f,hash_func) do { \
+ arch_spinlock_t *s = hash_func; \
local_irq_save(f); \
arch_spin_lock(s); \
} while(0)
-#define _atomic_spin_unlock_irqrestore(l,f) do { \
- arch_spinlock_t *s = ATOMIC_HASH(l); \
+#define _atomic_spin_unlock_irqrestore_template(l,f,hash_func) do { \
+ arch_spinlock_t *s = hash_func; \
arch_spin_unlock(s); \
local_irq_restore(f); \
} while(0)
+/* kernel memory locks */
+#define _atomic_spin_lock_irqsave(l,f) \
+ _atomic_spin_lock_irqsave_template(l,f,ATOMIC_HASH(l))
+
+#define _atomic_spin_unlock_irqrestore(l,f) \
+ _atomic_spin_unlock_irqrestore_template(l,f,ATOMIC_HASH(l))
+
+/* userspace memory locks */
+#define _atomic_spin_lock_irqsave_user(l,f) \
+ _atomic_spin_lock_irqsave_template(l,f,ATOMIC_USER_HASH(l))
+
+#define _atomic_spin_unlock_irqrestore_user(l,f) \
+ _atomic_spin_unlock_irqrestore_template(l,f,ATOMIC_USER_HASH(l))
#else
# define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0)
# define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0)
+# define _atomic_spin_lock_irqsave_user(l,f) _atomic_spin_lock_irqsave(l,f)
+# define _atomic_spin_unlock_irqrestore_user(l,f) _atomic_spin_lock_irqsave_user(l,f)
#endif
/* This should get optimized out since it's never called.
@@ -55,6 +55,7 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
{
int err = 0;
int uval;
+ unsigned long flags;
/* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is
* our gateway page, and causes no end of trouble...
@@ -65,10 +66,15 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
+ _atomic_spin_lock_irqsave_user(uaddr, flags);
+
err = get_user(uval, uaddr);
- if (err) return -EFAULT;
- if (uval == oldval)
- err = put_user(newval, uaddr);
+ if (!err)
+ if (uval == oldval)
+ err = put_user(newval, uaddr);
+
+ _atomic_spin_unlock_irqrestore_user(uaddr, flags);
+
if (err) return -EFAULT;
return uval;
}
@@ -160,7 +160,7 @@ static inline void set_eiem(unsigned long val)
ldcd). */
#define __PA_LDCW_ALIGNMENT 4
-#define __ldcw_align(a) ((volatile unsigned int *)a)
+#define __ldcw_align(a) (&(a)->slock)
#define __LDCW "ldcw,co"
#endif /*!CONFIG_PA20*/
@@ -290,5 +290,11 @@ int main(void)
BLANK();
DEFINE(ASM_PDC_RESULT_SIZE, NUM_PDC_RESULT * sizeof(unsigned long));
BLANK();
+
+#ifdef CONFIG_SMP
+ DEFINE(ASM_ATOMIC_HASH_SIZE_SHIFT, __builtin_ffs(ATOMIC_HASH_SIZE)-1);
+ DEFINE(ASM_ATOMIC_HASH_ENTRY_SHIFT, __builtin_ffs(sizeof(__atomic_hash[0]))-1);
+#endif
+
return 0;
}
@@ -128,6 +131,14 @@ void __init setup_arch(char **cmdline_p)
printk(KERN_INFO "The 32-bit Kernel has started...\n");
#endif
+ /* Consistency check on the size and alignments of our spinlocks */
+#ifdef CONFIG_SMP
+ BUILD_BUG_ON(sizeof(arch_spinlock_t) != __PA_LDCW_ALIGNMENT);
+ BUG_ON((unsigned long)&__atomic_hash[0] & (__PA_LDCW_ALIGNMENT-1));
+ BUG_ON((unsigned long)&__atomic_hash[1] & (__PA_LDCW_ALIGNMENT-1));
+#endif
+ BUILD_BUG_ON((1<<L1_CACHE_SHIFT) != L1_CACHE_BYTES);
+
pdc_console_init();
#ifdef CONFIG_64BIT
@@ -11,6 +11,7 @@
#include <asm/unistd.h>
#include <asm/errno.h>
#include <asm/page.h>
+#include <asm/cache.h>
#include <asm/psw.h>
#include <asm/thread_info.h>
#include <asm/assembly.h>
@@ -47,18 +48,17 @@ ENTRY(linux_gateway_page)
KILL_INSN
.endr
- /* ADDRESS 0xb0 to 0xb4, lws uses 1 insns for entry */
+ /* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */
/* Light-weight-syscall entry must always be located at 0xb0 */
/* WARNING: Keep this number updated with table size changes */
#define __NR_lws_entries (2)
lws_entry:
- /* Unconditional branch to lws_start, located on the
- same gateway page */
- b,n lws_start
+ gate lws_start, %r0 /* increase privilege */
+ depi 3, 31, 2, %r31 /* Ensure we return into user mode. */
- /* Fill from 0xb4 to 0xe0 */
- .rept 11
+ /* Fill from 0xb8 to 0xe0 */
+ .rept 10
KILL_INSN
.endr
@@ -423,9 +423,6 @@ tracesys_sigexit:
*********************************************************/
lws_start:
- /* Gate and ensure we return to userspace */
- gate .+8, %r0
- depi 3, 31, 2, %r31 /* Ensure we return to userspace */
#ifdef CONFIG_64BIT
/* FIXME: If we are a 64-bit kernel just
@@ -473,7 +470,7 @@ lws_exit:
/* now reset the lowest bit of sp if it was set */
xor %r30,%r1,%r30
#endif
- be,n 0(%sr3, %r31)
+ be,n 0(%sr7, %r31)
@@ -530,18 +527,17 @@ lws_compare_and_swap32:
lws_compare_and_swap:
#ifdef CONFIG_SMP
- /* Load start of lock table */
- ldil L%lws_lock_start, %r20
- ldo R%lws_lock_start(%r20), %r28
+ /* Calculate lock table entry via ATOMIC_HASH(%r26) */
+ ldil L%__atomic_user_hash, %r20
+ ldo R%__atomic_user_hash(%r20), %r28
- /* Extract four bits from r26 and hash lock (Bits 4-7) */
- extru %r26, 27, 4, %r20
+#ifdef CONFIG_64BIT
+ extrd,u %r26, 63-L1_CACHE_SHIFT, ASM_ATOMIC_HASH_SIZE_SHIFT, %r20
+#else
+ extru %r26, 31-L1_CACHE_SHIFT, ASM_ATOMIC_HASH_SIZE_SHIFT, %r20
+#endif
+ shladd,l %r20, ASM_ATOMIC_HASH_ENTRY_SHIFT, %r28, %r20
- /* Find lock to use, the hash is either one of 0 to
- 15, multiplied by 16 (keep it 16-byte aligned)
- and add to the lock table offset. */
- shlw %r20, 4, %r20
- add %r20, %r28, %r20
# if ENABLE_LWS_DEBUG
/*
@@ -672,31 +668,6 @@ ENTRY(sys_call_table64)
END(sys_call_table64)
#endif
-#ifdef CONFIG_SMP
- /*
- All light-weight-syscall atomic operations
- will use this set of locks
-
- NOTE: The lws_lock_start symbol must be
- at least 16-byte aligned for safe use
- with ldcw.
- */
- .section .data
- .align PAGE_SIZE
-ENTRY(lws_lock_start)
- /* lws locks */
- .rept 16
- /* Keep locks aligned at 16-bytes */
- .word 1
- .word 0
- .word 0
- .word 0
- .endr
-END(lws_lock_start)
- .previous
-#endif
-/* CONFIG_SMP for lws_lock_start */
-
.end
@@ -15,6 +15,9 @@
arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned = {
[0 ... (ATOMIC_HASH_SIZE-1)] = __ARCH_SPIN_LOCK_UNLOCKED
};
+arch_spinlock_t __atomic_user_hash[ATOMIC_HASH_SIZE] __lock_aligned = {
+ [0 ... (ATOMIC_HASH_SIZE-1)] = __ARCH_SPIN_LOCK_UNLOCKED
+};
#endif
#ifdef CONFIG_64BIT
@@ -608,7 +608,10 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm)
* We don't check the error code - if userspace has
* not set up a proper pointer then tough luck.
*/
+ unsigned long flags;
+ _atomic_spin_lock_irqsave_user(tsk->clear_child_tid, flags);
put_user(0, tsk->clear_child_tid);
+ _atomic_spin_unlock_irqrestore_user(tsk->clear_child_tid, flags);
sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
1, NULL, NULL, 0);
}
@@ -1432,8 +1435,12 @@ long do_fork(unsigned long clone_flags,
nr = task_pid_vnr(p);
- if (clone_flags & CLONE_PARENT_SETTID)
+ if (clone_flags & CLONE_PARENT_SETTID) {
+ unsigned long flags;
+ _atomic_spin_lock_irqsave_user(parent_tidptr, flags);
put_user(nr, parent_tidptr);
+ _atomic_spin_unlock_irqrestore_user(parent_tidptr, flags);
+ }
if (clone_flags & CLONE_VFORK) {
p->vfork_done = &vfork;