| 1 | /* $NetBSD: lock.h,v 1.27 2013/01/22 22:09:44 christos Exp $ */ |
| 2 | |
| 3 | /*- |
| 4 | * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc. |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * This code is derived from software contributed to The NetBSD Foundation |
| 8 | * by Jason R. Thorpe and Andrew Doran. |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * 1. Redistributions of source code must retain the above copyright |
| 14 | * notice, this list of conditions and the following disclaimer. |
| 15 | * 2. Redistributions in binary form must reproduce the above copyright |
| 16 | * notice, this list of conditions and the following disclaimer in the |
| 17 | * documentation and/or other materials provided with the distribution. |
| 18 | * |
| 19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 29 | * POSSIBILITY OF SUCH DAMAGE. |
| 30 | */ |
| 31 | |
| 32 | /* |
| 33 | * Machine-dependent spin lock operations. |
| 34 | */ |
| 35 | |
| 36 | #ifndef _X86_LOCK_H_ |
| 37 | #define _X86_LOCK_H_ |
| 38 | |
| 39 | #include <sys/param.h> |
| 40 | |
| 41 | static __inline int |
| 42 | __SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t *__ptr) |
| 43 | { |
| 44 | return *__ptr == __SIMPLELOCK_LOCKED; |
| 45 | } |
| 46 | |
| 47 | static __inline int |
| 48 | __SIMPLELOCK_UNLOCKED_P(__cpu_simple_lock_t *__ptr) |
| 49 | { |
| 50 | return *__ptr == __SIMPLELOCK_UNLOCKED; |
| 51 | } |
| 52 | |
| 53 | static __inline void |
| 54 | __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr) |
| 55 | { |
| 56 | |
| 57 | *__ptr = __SIMPLELOCK_LOCKED; |
| 58 | } |
| 59 | |
| 60 | static __inline void |
| 61 | __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr) |
| 62 | { |
| 63 | |
| 64 | *__ptr = __SIMPLELOCK_UNLOCKED; |
| 65 | } |
| 66 | |
| 67 | #ifdef _HARDKERNEL |
| 68 | # include <machine/cpufunc.h> |
| 69 | # define SPINLOCK_SPIN_HOOK /* nothing */ |
| 70 | # ifdef SPINLOCK_BACKOFF_HOOK |
| 71 | # undef SPINLOCK_BACKOFF_HOOK |
| 72 | # endif |
| 73 | # define SPINLOCK_BACKOFF_HOOK x86_pause() |
| 74 | # define SPINLOCK_INLINE |
| 75 | #else /* !_HARDKERNEL */ |
| 76 | # define SPINLOCK_BODY |
| 77 | # define SPINLOCK_INLINE static __inline __unused |
| 78 | #endif /* _HARDKERNEL */ |
| 79 | |
| 80 | SPINLOCK_INLINE void __cpu_simple_lock_init(__cpu_simple_lock_t *); |
| 81 | SPINLOCK_INLINE void __cpu_simple_lock(__cpu_simple_lock_t *); |
| 82 | SPINLOCK_INLINE int __cpu_simple_lock_try(__cpu_simple_lock_t *); |
| 83 | SPINLOCK_INLINE void __cpu_simple_unlock(__cpu_simple_lock_t *); |
| 84 | |
| 85 | #ifdef SPINLOCK_BODY |
| 86 | SPINLOCK_INLINE void |
| 87 | __cpu_simple_lock_init(__cpu_simple_lock_t *lockp) |
| 88 | { |
| 89 | |
| 90 | *lockp = __SIMPLELOCK_UNLOCKED; |
| 91 | __insn_barrier(); |
| 92 | } |
| 93 | |
| 94 | SPINLOCK_INLINE int |
| 95 | __cpu_simple_lock_try(__cpu_simple_lock_t *lockp) |
| 96 | { |
| 97 | uint8_t val; |
| 98 | |
| 99 | val = __SIMPLELOCK_LOCKED; |
| 100 | __asm volatile ("xchgb %0,(%2)" : |
| 101 | "=qQ" (val) |
| 102 | :"0" (val), "r" (lockp)); |
| 103 | __insn_barrier(); |
| 104 | return val == __SIMPLELOCK_UNLOCKED; |
| 105 | } |
| 106 | |
| 107 | SPINLOCK_INLINE void |
| 108 | __cpu_simple_lock(__cpu_simple_lock_t *lockp) |
| 109 | { |
| 110 | |
| 111 | while (!__cpu_simple_lock_try(lockp)) |
| 112 | /* nothing */; |
| 113 | __insn_barrier(); |
| 114 | } |
| 115 | |
| 116 | /* |
| 117 | * Note on x86 memory ordering |
| 118 | * |
| 119 | * When releasing a lock we must ensure that no stores or loads from within |
| 120 | * the critical section are re-ordered by the CPU to occur outside of it: |
| 121 | * they must have completed and be visible to other processors once the lock |
| 122 | * has been released. |
| 123 | * |
| 124 | * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write |
| 125 | * back) memory region. In that case, memory ordering on x86 platforms |
| 126 | * looks like this: |
| 127 | * |
| 128 | * i386 All loads/stores occur in instruction sequence. |
| 129 | * |
| 130 | * i486 All loads/stores occur in instruction sequence. In |
| 131 | * Pentium exceptional circumstances, loads can be re-ordered around |
| 132 | * stores, but for the purposes of releasing a lock it does |
| 133 | * not matter. Stores may not be immediately visible to other |
| 134 | * processors as they can be buffered. However, since the |
| 135 | * stores are buffered in order the lock release will always be |
| 136 | * the last operation in the critical section that becomes |
| 137 | * visible to other CPUs. |
| 138 | * |
| 139 | * Pentium Pro The "Intel 64 and IA-32 Architectures Software Developer's |
| 140 | * onwards Manual" volume 3A (order number 248966) says that (1) "Reads |
| 141 | * can be carried out speculatively and in any order" and (2) |
| 142 | * "Reads can pass buffered stores, but the processor is |
| 143 | * self-consistent.". This would be a problem for the below, |
| 144 | * and would mandate a locked instruction cycle or load fence |
| 145 | * before releasing the simple lock. |
| 146 | * |
| 147 | * The "Intel Pentium 4 Processor Optimization" guide (order |
| 148 | * number 253668-022US) says: "Loads can be moved before stores |
| 149 | * that occurred earlier in the program if they are not |
| 150 | * predicted to load from the same linear address.". This is |
| 151 | * not a problem since the only loads that can be re-ordered |
| 152 | * take place once the lock has been released via a store. |
| 153 | * |
| 154 | * The above two documents seem to contradict each other, |
| 155 | * however with the exception of early steppings of the Pentium |
| 156 | * Pro, the second document is closer to the truth: a store |
| 157 | * will always act as a load fence for all loads that precede |
| 158 | * the store in instruction order. |
| 159 | * |
| 160 | * Again, note that stores can be buffered and will not always |
| 161 | * become immediately visible to other CPUs: they are however |
| 162 | * buffered in order. |
| 163 | * |
| 164 | * AMD64 Stores occur in order and are buffered. Loads can be |
| 165 | * reordered, however stores act as load fences, meaning that |
| 166 | * loads can not be reordered around stores. |
| 167 | */ |
| 168 | SPINLOCK_INLINE void |
| 169 | __cpu_simple_unlock(__cpu_simple_lock_t *lockp) |
| 170 | { |
| 171 | |
| 172 | __insn_barrier(); |
| 173 | *lockp = __SIMPLELOCK_UNLOCKED; |
| 174 | } |
| 175 | |
| 176 | #endif /* SPINLOCK_BODY */ |
| 177 | |
| 178 | #endif /* _X86_LOCK_H_ */ |
| 179 | |