| 1 | /* $NetBSD: patch.c,v 1.22 2013/11/15 08:47:55 msaitoh Exp $ */ |
| 2 | |
| 3 | /*- |
| 4 | * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * This code is derived from software contributed to The NetBSD Foundation |
| 8 | * by Andrew Doran. |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * 1. Redistributions of source code must retain the above copyright |
| 14 | * notice, this list of conditions and the following disclaimer. |
| 15 | * 2. Redistributions in binary form must reproduce the above copyright |
| 16 | * notice, this list of conditions and the following disclaimer in the |
| 17 | * documentation and/or other materials provided with the distribution. |
| 18 | * |
| 19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 29 | * POSSIBILITY OF SUCH DAMAGE. |
| 30 | */ |
| 31 | |
| 32 | /* |
| 33 | * Patch kernel code at boot time, depending on available CPU features. |
| 34 | */ |
| 35 | |
| 36 | #include <sys/cdefs.h> |
| 37 | __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.22 2013/11/15 08:47:55 msaitoh Exp $" ); |
| 38 | |
| 39 | #include "opt_lockdebug.h" |
| 40 | #ifdef i386 |
| 41 | #include "opt_spldebug.h" |
| 42 | #endif |
| 43 | |
| 44 | #include <sys/types.h> |
| 45 | #include <sys/systm.h> |
| 46 | |
| 47 | #include <machine/cpu.h> |
| 48 | #include <machine/cpufunc.h> |
| 49 | #include <machine/specialreg.h> |
| 50 | |
| 51 | #include <x86/cpuvar.h> |
| 52 | #include <x86/cputypes.h> |
| 53 | |
| 54 | void spllower(int); |
| 55 | void spllower_end(void); |
| 56 | void cx8_spllower(int); |
| 57 | void cx8_spllower_end(void); |
| 58 | void cx8_spllower_patch(void); |
| 59 | |
| 60 | void mutex_spin_exit_end(void); |
| 61 | void i686_mutex_spin_exit(int); |
| 62 | void i686_mutex_spin_exit_end(void); |
| 63 | void i686_mutex_spin_exit_patch(void); |
| 64 | |
| 65 | void membar_consumer(void); |
| 66 | void membar_consumer_end(void); |
| 67 | void membar_sync(void); |
| 68 | void membar_sync_end(void); |
| 69 | void sse2_lfence(void); |
| 70 | void sse2_lfence_end(void); |
| 71 | void sse2_mfence(void); |
| 72 | void sse2_mfence_end(void); |
| 73 | |
| 74 | void _atomic_cas_64(void); |
| 75 | void _atomic_cas_64_end(void); |
| 76 | void _atomic_cas_cx8(void); |
| 77 | void _atomic_cas_cx8_end(void); |
| 78 | |
| 79 | extern void *x86_lockpatch[]; |
| 80 | extern void *x86_retpatch[]; |
| 81 | extern void *atomic_lockpatch[]; |
| 82 | |
| 83 | #define X86_NOP 0x90 |
| 84 | #define X86_REP 0xf3 |
| 85 | #define X86_RET 0xc3 |
| 86 | #define X86_CS 0x2e |
| 87 | #define X86_DS 0x3e |
| 88 | #define X86_GROUP_0F 0x0f |
| 89 | |
| 90 | static void |
| 91 | adjust_jumpoff(uint8_t *ptr, void *from_s, void *to_s) |
| 92 | { |
| 93 | |
| 94 | /* Branch hints */ |
| 95 | if (ptr[0] == X86_CS || ptr[0] == X86_DS) |
| 96 | ptr++; |
| 97 | /* Conditional jumps */ |
| 98 | if (ptr[0] == X86_GROUP_0F) |
| 99 | ptr++; |
| 100 | /* 4-byte relative jump or call */ |
| 101 | *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) += |
| 102 | ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s); |
| 103 | } |
| 104 | |
| 105 | static void __unused |
| 106 | patchfunc(void *from_s, void *from_e, void *to_s, void *to_e, |
| 107 | void *pcrel) |
| 108 | { |
| 109 | |
| 110 | if ((uintptr_t)from_e - (uintptr_t)from_s != |
| 111 | (uintptr_t)to_e - (uintptr_t)to_s) |
| 112 | panic("patchfunc: sizes do not match (from=%p)" , from_s); |
| 113 | |
| 114 | memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s); |
| 115 | if (pcrel != NULL) |
| 116 | adjust_jumpoff(pcrel, from_s, to_s); |
| 117 | |
| 118 | #ifdef GPROF |
| 119 | #ifdef i386 |
| 120 | #define MCOUNT_CALL_OFFSET 3 |
| 121 | #endif |
| 122 | #ifdef __x86_64__ |
| 123 | #define MCOUNT_CALL_OFFSET 5 |
| 124 | #endif |
| 125 | /* Patch mcount call offset */ |
| 126 | adjust_jumpoff((uint8_t *)from_s + MCOUNT_CALL_OFFSET, from_s, to_s); |
| 127 | #endif |
| 128 | } |
| 129 | |
| 130 | static inline void __unused |
| 131 | patchbytes(void *addr, const int byte1, const int byte2, const int byte3) |
| 132 | { |
| 133 | |
| 134 | ((uint8_t *)addr)[0] = (uint8_t)byte1; |
| 135 | if (byte2 != -1) |
| 136 | ((uint8_t *)addr)[1] = (uint8_t)byte2; |
| 137 | if (byte3 != -1) |
| 138 | ((uint8_t *)addr)[2] = (uint8_t)byte3; |
| 139 | } |
| 140 | |
| 141 | void |
| 142 | x86_patch(bool early) |
| 143 | { |
| 144 | static bool first, second; |
| 145 | u_long psl; |
| 146 | u_long cr0; |
| 147 | int i; |
| 148 | |
| 149 | if (early) { |
| 150 | if (first) |
| 151 | return; |
| 152 | first = true; |
| 153 | } else { |
| 154 | if (second) |
| 155 | return; |
| 156 | second = true; |
| 157 | } |
| 158 | |
| 159 | /* Disable interrupts. */ |
| 160 | psl = x86_read_psl(); |
| 161 | x86_disable_intr(); |
| 162 | |
| 163 | /* Disable write protection in supervisor mode. */ |
| 164 | cr0 = rcr0(); |
| 165 | lcr0(cr0 & ~CR0_WP); |
| 166 | |
| 167 | #if !defined(GPROF) |
| 168 | if (!early && ncpu == 1) { |
| 169 | #ifndef LOCKDEBUG |
| 170 | /* Uniprocessor: kill LOCK prefixes. */ |
| 171 | for (i = 0; x86_lockpatch[i] != 0; i++) |
| 172 | patchbytes(x86_lockpatch[i], X86_NOP, -1, -1); |
| 173 | for (i = 0; atomic_lockpatch[i] != 0; i++) |
| 174 | patchbytes(atomic_lockpatch[i], X86_NOP, -1, -1); |
| 175 | #endif /* !LOCKDEBUG */ |
| 176 | } |
| 177 | if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) { |
| 178 | /* Faster memory barriers. */ |
| 179 | patchfunc( |
| 180 | sse2_lfence, sse2_lfence_end, |
| 181 | membar_consumer, membar_consumer_end, |
| 182 | NULL |
| 183 | ); |
| 184 | patchfunc( |
| 185 | sse2_mfence, sse2_mfence_end, |
| 186 | membar_sync, membar_sync_end, |
| 187 | NULL |
| 188 | ); |
| 189 | } |
| 190 | #endif /* GPROF */ |
| 191 | |
| 192 | #ifdef i386 |
| 193 | /* |
| 194 | * Patch early and late. Second time around the 'lock' prefix |
| 195 | * may be gone. |
| 196 | */ |
| 197 | if ((cpu_feature[0] & CPUID_CX8) != 0) { |
| 198 | patchfunc( |
| 199 | _atomic_cas_cx8, _atomic_cas_cx8_end, |
| 200 | _atomic_cas_64, _atomic_cas_64_end, |
| 201 | NULL |
| 202 | ); |
| 203 | } |
| 204 | #endif /* i386 */ |
| 205 | |
| 206 | #if !defined(SPLDEBUG) |
| 207 | if (!early && (cpu_feature[0] & CPUID_CX8) != 0) { |
| 208 | /* Faster splx(), mutex_spin_exit(). */ |
| 209 | patchfunc( |
| 210 | cx8_spllower, cx8_spllower_end, |
| 211 | spllower, spllower_end, |
| 212 | cx8_spllower_patch |
| 213 | ); |
| 214 | #if defined(i386) && !defined(LOCKDEBUG) |
| 215 | patchfunc( |
| 216 | i686_mutex_spin_exit, i686_mutex_spin_exit_end, |
| 217 | mutex_spin_exit, mutex_spin_exit_end, |
| 218 | i686_mutex_spin_exit_patch |
| 219 | ); |
| 220 | #endif /* i386 && !LOCKDEBUG */ |
| 221 | } |
| 222 | #endif /* !SPLDEBUG */ |
| 223 | |
| 224 | /* |
| 225 | * On some Opteron revisions, locked operations erroneously |
| 226 | * allow memory references to be `bled' outside of critical |
| 227 | * sections. Apply workaround. |
| 228 | */ |
| 229 | if (cpu_vendor == CPUVENDOR_AMD && |
| 230 | (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe || |
| 231 | (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf && |
| 232 | CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) { |
| 233 | for (i = 0; x86_retpatch[i] != 0; i++) { |
| 234 | /* ret,nop,nop,ret -> lfence,ret */ |
| 235 | patchbytes(x86_retpatch[i], 0x0f, 0xae, 0xe8); |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | /* Write back and invalidate cache, flush pipelines. */ |
| 240 | wbinvd(); |
| 241 | x86_flush(); |
| 242 | x86_write_psl(psl); |
| 243 | |
| 244 | /* Re-enable write protection. */ |
| 245 | lcr0(cr0); |
| 246 | } |
| 247 | |