| 1 | /* $NetBSD: xen_pmap.c,v 1.23 2016/11/21 04:10:05 ozaki-r Exp $ */ |
| 2 | |
| 3 | /* |
| 4 | * Copyright (c) 2007 Manuel Bouyer. |
| 5 | * |
| 6 | * Redistribution and use in source and binary forms, with or without |
| 7 | * modification, are permitted provided that the following conditions |
| 8 | * are met: |
| 9 | * 1. Redistributions of source code must retain the above copyright |
| 10 | * notice, this list of conditions and the following disclaimer. |
| 11 | * 2. Redistributions in binary form must reproduce the above copyright |
| 12 | * notice, this list of conditions and the following disclaimer in the |
| 13 | * documentation and/or other materials provided with the distribution. |
| 14 | * |
| 15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| 16 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| 17 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| 18 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| 19 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| 20 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 21 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 22 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| 24 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 25 | * |
| 26 | */ |
| 27 | |
| 28 | /* |
| 29 | * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr> |
| 30 | * |
| 31 | * Permission to use, copy, modify, and distribute this software for any |
| 32 | * purpose with or without fee is hereby granted, provided that the above |
| 33 | * copyright notice and this permission notice appear in all copies. |
| 34 | * |
| 35 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 36 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 37 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 38 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 39 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 40 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 41 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 42 | */ |
| 43 | |
| 44 | /* |
| 45 | * Copyright (c) 1997 Charles D. Cranor and Washington University. |
| 46 | * All rights reserved. |
| 47 | * |
| 48 | * Redistribution and use in source and binary forms, with or without |
| 49 | * modification, are permitted provided that the following conditions |
| 50 | * are met: |
| 51 | * 1. Redistributions of source code must retain the above copyright |
| 52 | * notice, this list of conditions and the following disclaimer. |
| 53 | * 2. Redistributions in binary form must reproduce the above copyright |
| 54 | * notice, this list of conditions and the following disclaimer in the |
| 55 | * documentation and/or other materials provided with the distribution. |
| 56 | * |
| 57 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| 58 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| 59 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| 60 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| 61 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| 62 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 63 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 64 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 65 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| 66 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 67 | */ |
| 68 | |
| 69 | /* |
| 70 | * Copyright 2001 (c) Wasabi Systems, Inc. |
| 71 | * All rights reserved. |
| 72 | * |
| 73 | * Written by Frank van der Linden for Wasabi Systems, Inc. |
| 74 | * |
| 75 | * Redistribution and use in source and binary forms, with or without |
| 76 | * modification, are permitted provided that the following conditions |
| 77 | * are met: |
| 78 | * 1. Redistributions of source code must retain the above copyright |
| 79 | * notice, this list of conditions and the following disclaimer. |
| 80 | * 2. Redistributions in binary form must reproduce the above copyright |
| 81 | * notice, this list of conditions and the following disclaimer in the |
| 82 | * documentation and/or other materials provided with the distribution. |
| 83 | * 3. All advertising materials mentioning features or use of this software |
| 84 | * must display the following acknowledgement: |
| 85 | * This product includes software developed for the NetBSD Project by |
| 86 | * Wasabi Systems, Inc. |
| 87 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse |
| 88 | * or promote products derived from this software without specific prior |
| 89 | * written permission. |
| 90 | * |
| 91 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND |
| 92 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 93 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 94 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC |
| 95 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 96 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 97 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 98 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 99 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 100 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 101 | * POSSIBILITY OF SUCH DAMAGE. |
| 102 | */ |
| 103 | |
| 104 | #include <sys/cdefs.h> |
| 105 | __KERNEL_RCSID(0, "$NetBSD: xen_pmap.c,v 1.23 2016/11/21 04:10:05 ozaki-r Exp $" ); |
| 106 | |
| 107 | #include "opt_user_ldt.h" |
| 108 | #include "opt_lockdebug.h" |
| 109 | #include "opt_multiprocessor.h" |
| 110 | #include "opt_xen.h" |
| 111 | #if !defined(__x86_64__) |
| 112 | #include "opt_kstack_dr0.h" |
| 113 | #endif /* !defined(__x86_64__) */ |
| 114 | |
| 115 | #include <sys/param.h> |
| 116 | #include <sys/systm.h> |
| 117 | #include <sys/proc.h> |
| 118 | #include <sys/pool.h> |
| 119 | #include <sys/kernel.h> |
| 120 | #include <sys/atomic.h> |
| 121 | #include <sys/cpu.h> |
| 122 | #include <sys/intr.h> |
| 123 | |
| 124 | #include <uvm/uvm.h> |
| 125 | |
| 126 | #include <dev/isa/isareg.h> |
| 127 | |
| 128 | #include <machine/specialreg.h> |
| 129 | #include <machine/gdt.h> |
| 130 | #include <machine/isa_machdep.h> |
| 131 | #include <machine/cpuvar.h> |
| 132 | |
| 133 | #include <x86/pmap.h> |
| 134 | #include <x86/pmap_pv.h> |
| 135 | |
| 136 | #include <x86/i82489reg.h> |
| 137 | #include <x86/i82489var.h> |
| 138 | |
| 139 | #include <xen/xen-public/xen.h> |
| 140 | #include <xen/hypervisor.h> |
| 141 | #include <xen/xenpmap.h> |
| 142 | |
| 143 | #define COUNT(x) /* nothing */ |
| 144 | |
| 145 | extern pd_entry_t * const normal_pdes[]; |
| 146 | |
| 147 | extern paddr_t pmap_pa_start; /* PA of first physical page for this domain */ |
| 148 | extern paddr_t pmap_pa_end; /* PA of last physical page for this domain */ |
| 149 | |
| 150 | int |
| 151 | pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) |
| 152 | { |
| 153 | paddr_t ma; |
| 154 | |
| 155 | if (__predict_false(pa < pmap_pa_start || pmap_pa_end <= pa)) { |
| 156 | ma = pa; /* XXX hack */ |
| 157 | } else { |
| 158 | ma = xpmap_ptom(pa); |
| 159 | } |
| 160 | |
| 161 | return pmap_enter_ma(pmap, va, ma, pa, prot, flags, DOMID_SELF); |
| 162 | } |
| 163 | |
| 164 | /* |
| 165 | * pmap_kenter_ma: enter a kernel mapping without R/M (pv_entry) tracking |
| 166 | * |
| 167 | * => no need to lock anything, assume va is already allocated |
| 168 | * => should be faster than normal pmap enter function |
| 169 | * => we expect a MACHINE address |
| 170 | */ |
| 171 | |
| 172 | void |
| 173 | pmap_kenter_ma(vaddr_t va, paddr_t ma, vm_prot_t prot, u_int flags) |
| 174 | { |
| 175 | pt_entry_t *pte, opte, npte; |
| 176 | |
| 177 | if (va < VM_MIN_KERNEL_ADDRESS) |
| 178 | pte = vtopte(va); |
| 179 | else |
| 180 | pte = kvtopte(va); |
| 181 | |
| 182 | npte = ma | ((prot & VM_PROT_WRITE) ? PG_RW : PG_RO) | |
| 183 | PG_V | PG_k; |
| 184 | if (flags & PMAP_NOCACHE) |
| 185 | npte |= PG_N; |
| 186 | |
| 187 | if ((cpu_feature[2] & CPUID_NOX) && !(prot & VM_PROT_EXECUTE)) |
| 188 | npte |= PG_NX; |
| 189 | |
| 190 | opte = pmap_pte_testset (pte, npte); /* zap! */ |
| 191 | |
| 192 | if (pmap_valid_entry(opte)) { |
| 193 | #if defined(MULTIPROCESSOR) |
| 194 | kpreempt_disable(); |
| 195 | pmap_tlb_shootdown(pmap_kernel(), va, opte, TLBSHOOT_KENTER); |
| 196 | kpreempt_enable(); |
| 197 | #else |
| 198 | /* Don't bother deferring in the single CPU case. */ |
| 199 | pmap_update_pg(va); |
| 200 | #endif |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | /* |
| 205 | * pmap_extract_ma: extract a MA for the given VA |
| 206 | */ |
| 207 | |
| 208 | bool |
| 209 | (struct pmap *pmap, vaddr_t va, paddr_t *pap) |
| 210 | { |
| 211 | pt_entry_t *ptes, pte; |
| 212 | pd_entry_t pde; |
| 213 | pd_entry_t * const *pdes; |
| 214 | struct pmap *pmap2; |
| 215 | |
| 216 | kpreempt_disable(); |
| 217 | pmap_map_ptes(pmap, &pmap2, &ptes, &pdes); |
| 218 | if (!pmap_pdes_valid(va, pdes, &pde)) { |
| 219 | pmap_unmap_ptes(pmap, pmap2); |
| 220 | kpreempt_enable(); |
| 221 | return false; |
| 222 | } |
| 223 | |
| 224 | pte = ptes[pl1_i(va)]; |
| 225 | pmap_unmap_ptes(pmap, pmap2); |
| 226 | kpreempt_enable(); |
| 227 | |
| 228 | if (__predict_true((pte & PG_V) != 0)) { |
| 229 | if (pap != NULL) |
| 230 | *pap = (pte & PG_FRAME) | (va & (NBPD_L1 - 1)); |
| 231 | return true; |
| 232 | } |
| 233 | |
| 234 | return false; |
| 235 | } |
| 236 | |
| 237 | /* |
| 238 | * Xen pmap's handlers for save/restore |
| 239 | */ |
| 240 | void |
| 241 | pmap_xen_suspend(void) |
| 242 | { |
| 243 | pmap_unmap_recursive_entries(); |
| 244 | |
| 245 | xpq_flush_queue(); |
| 246 | } |
| 247 | |
| 248 | void |
| 249 | pmap_xen_resume(void) |
| 250 | { |
| 251 | pmap_map_recursive_entries(); |
| 252 | |
| 253 | xpq_flush_queue(); |
| 254 | } |
| 255 | |
| 256 | /* |
| 257 | * NetBSD uses L2 shadow pages to support PAE with Xen. However, Xen does not |
| 258 | * handle them correctly during save/restore, leading to incorrect page |
| 259 | * tracking and pinning during restore. |
| 260 | * For save/restore to succeed, two functions are introduced: |
| 261 | * - pmap_map_recursive_entries(), used by resume code to set the recursive |
| 262 | * mapping entries to their correct value |
| 263 | * - pmap_unmap_recursive_entries(), used by suspend code to clear all |
| 264 | * PDIR_SLOT_PTE entries |
| 265 | */ |
| 266 | void |
| 267 | pmap_map_recursive_entries(void) |
| 268 | { |
| 269 | int i; |
| 270 | struct pmap *pm; |
| 271 | |
| 272 | mutex_enter(&pmaps_lock); |
| 273 | LIST_FOREACH(pm, &pmaps, pm_list) { |
| 274 | for (i = 0; i < PDP_SIZE; i++) { |
| 275 | xpq_queue_pte_update( |
| 276 | xpmap_ptom(pmap_pdirpa(pm, PDIR_SLOT_PTE + i)), |
| 277 | xpmap_ptom((pm)->pm_pdirpa[i]) | PG_V); |
| 278 | } |
| 279 | } |
| 280 | mutex_exit(&pmaps_lock); |
| 281 | |
| 282 | for (i = 0; i < PDP_SIZE; i++) { |
| 283 | xpq_queue_pte_update( |
| 284 | xpmap_ptom(pmap_pdirpa(pmap_kernel(), PDIR_SLOT_PTE + i)), |
| 285 | xpmap_ptom(pmap_kernel()->pm_pdirpa[i]) | PG_V); |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | /* |
| 290 | * Unmap recursive entries found in pmaps. Required during Xen |
| 291 | * save/restore operations, as Xen does not handle recursive mappings |
| 292 | * properly. |
| 293 | */ |
| 294 | void |
| 295 | pmap_unmap_recursive_entries(void) |
| 296 | { |
| 297 | int i; |
| 298 | struct pmap *pm; |
| 299 | |
| 300 | /* |
| 301 | * Invalidate pmap_pdp_cache as it contains L2-pinned objects with |
| 302 | * recursive entries. |
| 303 | * XXX jym@ : find a way to drain per-CPU caches to. pool_cache_inv |
| 304 | * does not do that. |
| 305 | */ |
| 306 | pool_cache_invalidate(&pmap_pdp_cache); |
| 307 | |
| 308 | mutex_enter(&pmaps_lock); |
| 309 | LIST_FOREACH(pm, &pmaps, pm_list) { |
| 310 | for (i = 0; i < PDP_SIZE; i++) { |
| 311 | xpq_queue_pte_update( |
| 312 | xpmap_ptom(pmap_pdirpa(pm, PDIR_SLOT_PTE + i)), 0); |
| 313 | } |
| 314 | } |
| 315 | mutex_exit(&pmaps_lock); |
| 316 | |
| 317 | /* do it for pmap_kernel() too! */ |
| 318 | for (i = 0; i < PDP_SIZE; i++) { |
| 319 | xpq_queue_pte_update( |
| 320 | xpmap_ptom(pmap_pdirpa(pmap_kernel(), PDIR_SLOT_PTE + i)), |
| 321 | 0); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | #if defined(PAE) || defined(__x86_64__) |
| 326 | |
| 327 | static __inline void |
| 328 | pmap_kpm_setpte(struct cpu_info *ci, struct pmap *pmap, int index) |
| 329 | { |
| 330 | KASSERT(mutex_owned(pmap->pm_lock)); |
| 331 | KASSERT(mutex_owned(&ci->ci_kpm_mtx)); |
| 332 | if (pmap == pmap_kernel()) { |
| 333 | KASSERT(index >= PDIR_SLOT_KERN); |
| 334 | } |
| 335 | #ifdef PAE |
| 336 | xpq_queue_pte_update( |
| 337 | xpmap_ptetomach(&ci->ci_kpm_pdir[l2tol2(index)]), |
| 338 | pmap->pm_pdir[index]); |
| 339 | #elif defined(__x86_64__) |
| 340 | xpq_queue_pte_update( |
| 341 | xpmap_ptetomach(&ci->ci_kpm_pdir[index]), |
| 342 | pmap->pm_pdir[index]); |
| 343 | #endif /* PAE */ |
| 344 | xpq_flush_queue(); |
| 345 | } |
| 346 | |
| 347 | /* |
| 348 | * Synchronise shadow pdir with the pmap on all cpus on which it is |
| 349 | * loaded. |
| 350 | */ |
| 351 | void |
| 352 | xen_kpm_sync(struct pmap *pmap, int index) |
| 353 | { |
| 354 | CPU_INFO_ITERATOR cii; |
| 355 | struct cpu_info *ci; |
| 356 | |
| 357 | KASSERT(pmap != NULL); |
| 358 | KASSERT(kpreempt_disabled()); |
| 359 | |
| 360 | pmap_pte_flush(); |
| 361 | |
| 362 | for (CPU_INFO_FOREACH(cii, ci)) { |
| 363 | if (ci == NULL) { |
| 364 | continue; |
| 365 | } |
| 366 | cpuid_t cid = cpu_index(ci); |
| 367 | if (pmap != pmap_kernel() && |
| 368 | !kcpuset_isset(pmap->pm_xen_ptp_cpus, cid)) |
| 369 | continue; |
| 370 | |
| 371 | /* take the lock and check again */ |
| 372 | mutex_enter(&ci->ci_kpm_mtx); |
| 373 | if (pmap == pmap_kernel() || |
| 374 | kcpuset_isset(pmap->pm_xen_ptp_cpus, cid)) { |
| 375 | pmap_kpm_setpte(ci, pmap, index); |
| 376 | } |
| 377 | mutex_exit(&ci->ci_kpm_mtx); |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | #endif /* PAE || __x86_64__ */ |
| 382 | |