| 1 | /* $NetBSD: xen_bus_dma.c,v 1.26 2012/06/30 23:36:20 jym Exp $ */ |
| 2 | /* NetBSD bus_dma.c,v 1.21 2005/04/16 07:53:35 yamt Exp */ |
| 3 | |
| 4 | /*- |
| 5 | * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. |
| 6 | * All rights reserved. |
| 7 | * |
| 8 | * This code is derived from software contributed to The NetBSD Foundation |
| 9 | * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace |
| 10 | * Simulation Facility, NASA Ames Research Center. |
| 11 | * |
| 12 | * Redistribution and use in source and binary forms, with or without |
| 13 | * modification, are permitted provided that the following conditions |
| 14 | * are met: |
| 15 | * 1. Redistributions of source code must retain the above copyright |
| 16 | * notice, this list of conditions and the following disclaimer. |
| 17 | * 2. Redistributions in binary form must reproduce the above copyright |
| 18 | * notice, this list of conditions and the following disclaimer in the |
| 19 | * documentation and/or other materials provided with the distribution. |
| 20 | * |
| 21 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 23 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 24 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 25 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 26 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 27 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 28 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 29 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 30 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 31 | * POSSIBILITY OF SUCH DAMAGE. |
| 32 | */ |
| 33 | |
| 34 | #include <sys/cdefs.h> |
| 35 | __KERNEL_RCSID(0, "$NetBSD: xen_bus_dma.c,v 1.26 2012/06/30 23:36:20 jym Exp $" ); |
| 36 | |
| 37 | #include <sys/param.h> |
| 38 | #include <sys/systm.h> |
| 39 | #include <sys/kernel.h> |
| 40 | #include <sys/mbuf.h> |
| 41 | #include <sys/proc.h> |
| 42 | |
| 43 | #include <sys/bus.h> |
| 44 | #include <machine/bus_private.h> |
| 45 | |
| 46 | #include <uvm/uvm.h> |
| 47 | |
| 48 | extern paddr_t avail_end; |
| 49 | |
| 50 | /* Pure 2^n version of get_order */ |
| 51 | static inline int get_order(unsigned long size) |
| 52 | { |
| 53 | int order = -1; |
| 54 | size = (size - 1) >> (PAGE_SHIFT - 1); |
| 55 | do { |
| 56 | size >>= 1; |
| 57 | order++; |
| 58 | } while (size); |
| 59 | return order; |
| 60 | } |
| 61 | |
| 62 | static int |
| 63 | _xen_alloc_contig(bus_size_t size, bus_size_t alignment, |
| 64 | struct pglist *mlistp, int flags, bus_addr_t low, bus_addr_t high) |
| 65 | { |
| 66 | int order, i; |
| 67 | unsigned long npagesreq, npages, mfn; |
| 68 | bus_addr_t pa; |
| 69 | struct vm_page *pg, *pgnext; |
| 70 | int s, error; |
| 71 | struct xen_memory_reservation res; |
| 72 | |
| 73 | /* |
| 74 | * When requesting a contigous memory region, the hypervisor will |
| 75 | * return a memory range aligned on size. |
| 76 | * The only way to enforce alignment is to request a memory region |
| 77 | * of size max(alignment, size). |
| 78 | */ |
| 79 | order = max(get_order(size), get_order(alignment)); |
| 80 | npages = (1 << order); |
| 81 | npagesreq = (size >> PAGE_SHIFT); |
| 82 | KASSERT(npages >= npagesreq); |
| 83 | |
| 84 | /* get npages from UVM, and give them back to the hypervisor */ |
| 85 | error = uvm_pglistalloc(((psize_t)npages) << PAGE_SHIFT, |
| 86 | 0, avail_end, 0, 0, mlistp, npages, (flags & BUS_DMA_NOWAIT) == 0); |
| 87 | if (error) |
| 88 | return (error); |
| 89 | |
| 90 | for (pg = mlistp->tqh_first; pg != NULL; pg = pg->pageq.queue.tqe_next) { |
| 91 | pa = VM_PAGE_TO_PHYS(pg); |
| 92 | mfn = xpmap_ptom(pa) >> PAGE_SHIFT; |
| 93 | xpmap_ptom_unmap(pa); |
| 94 | set_xen_guest_handle(res.extent_start, &mfn); |
| 95 | res.nr_extents = 1; |
| 96 | res.extent_order = 0; |
| 97 | res.address_bits = 0; |
| 98 | res.domid = DOMID_SELF; |
| 99 | error = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &res); |
| 100 | if (error != 1) { |
| 101 | #ifdef DEBUG |
| 102 | printf("xen_alloc_contig: XENMEM_decrease_reservation " |
| 103 | "failed: err %d (pa %#" PRIxPADDR " mfn %#lx)\n" , |
| 104 | error, pa, mfn); |
| 105 | #endif |
| 106 | xpmap_ptom_map(pa, ptoa(mfn)); |
| 107 | |
| 108 | error = ENOMEM; |
| 109 | goto failed; |
| 110 | } |
| 111 | } |
| 112 | /* Get the new contiguous memory extent */ |
| 113 | set_xen_guest_handle(res.extent_start, &mfn); |
| 114 | res.nr_extents = 1; |
| 115 | res.extent_order = order; |
| 116 | res.address_bits = get_order(high) + PAGE_SHIFT; |
| 117 | res.domid = DOMID_SELF; |
| 118 | error = HYPERVISOR_memory_op(XENMEM_increase_reservation, &res); |
| 119 | if (error != 1) { |
| 120 | #ifdef DEBUG |
| 121 | printf("xen_alloc_contig: XENMEM_increase_reservation " |
| 122 | "failed: %d (order %d address_bits %d)\n" , |
| 123 | error, order, res.address_bits); |
| 124 | #endif |
| 125 | error = ENOMEM; |
| 126 | pg = NULL; |
| 127 | goto failed; |
| 128 | } |
| 129 | s = splvm(); |
| 130 | /* Map the new extent in place of the old pages */ |
| 131 | for (pg = mlistp->tqh_first, i = 0; pg != NULL; pg = pgnext, i++) { |
| 132 | pgnext = pg->pageq.queue.tqe_next; |
| 133 | pa = VM_PAGE_TO_PHYS(pg); |
| 134 | xpmap_ptom_map(pa, ptoa(mfn+i)); |
| 135 | xpq_queue_machphys_update(((paddr_t)(mfn+i)) << PAGE_SHIFT, pa); |
| 136 | /* while here, give extra pages back to UVM */ |
| 137 | if (i >= npagesreq) { |
| 138 | TAILQ_REMOVE(mlistp, pg, pageq.queue); |
| 139 | uvm_pagefree(pg); |
| 140 | } |
| 141 | } |
| 142 | /* Flush updates through and flush the TLB */ |
| 143 | xpq_queue_tlb_flush(); |
| 144 | splx(s); |
| 145 | return 0; |
| 146 | |
| 147 | failed: |
| 148 | /* |
| 149 | * Attempt to recover from a failed decrease or increase reservation: |
| 150 | * if decrease_reservation failed, we don't have given all pages |
| 151 | * back to Xen; give them back to UVM, and get the missing pages |
| 152 | * from Xen. |
| 153 | * if increase_reservation failed, we expect pg to be NULL and we just |
| 154 | * get back the missing pages from Xen one by one. |
| 155 | */ |
| 156 | /* give back remaining pages to UVM */ |
| 157 | for (; pg != NULL; pg = pgnext) { |
| 158 | pgnext = pg->pageq.queue.tqe_next; |
| 159 | TAILQ_REMOVE(mlistp, pg, pageq.queue); |
| 160 | uvm_pagefree(pg); |
| 161 | } |
| 162 | /* remplace the pages that we already gave to Xen */ |
| 163 | s = splvm(); |
| 164 | for (pg = mlistp->tqh_first; pg != NULL; pg = pgnext) { |
| 165 | pgnext = pg->pageq.queue.tqe_next; |
| 166 | set_xen_guest_handle(res.extent_start, &mfn); |
| 167 | res.nr_extents = 1; |
| 168 | res.extent_order = 0; |
| 169 | res.address_bits = 32; |
| 170 | res.domid = DOMID_SELF; |
| 171 | if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &res) |
| 172 | < 0) { |
| 173 | printf("xen_alloc_contig: recovery " |
| 174 | "XENMEM_increase_reservation failed!\n" ); |
| 175 | break; |
| 176 | } |
| 177 | pa = VM_PAGE_TO_PHYS(pg); |
| 178 | xpmap_ptom_map(pa, ptoa(mfn)); |
| 179 | xpq_queue_machphys_update(((paddr_t)mfn) << PAGE_SHIFT, pa); |
| 180 | TAILQ_REMOVE(mlistp, pg, pageq.queue); |
| 181 | uvm_pagefree(pg); |
| 182 | } |
| 183 | /* Flush updates through and flush the TLB */ |
| 184 | xpq_queue_tlb_flush(); |
| 185 | splx(s); |
| 186 | return error; |
| 187 | } |
| 188 | |
| 189 | |
| 190 | /* |
| 191 | * Allocate physical memory from the given physical address range. |
| 192 | * Called by DMA-safe memory allocation methods. |
| 193 | * We need our own version to deal with physical vs machine addresses. |
| 194 | */ |
| 195 | int |
| 196 | _xen_bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, |
| 197 | bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, |
| 198 | int nsegs, int *rsegs, int flags, bus_addr_t low, bus_addr_t high) |
| 199 | { |
| 200 | bus_addr_t curaddr, lastaddr; |
| 201 | struct vm_page *m; |
| 202 | struct pglist mlist; |
| 203 | int curseg, error; |
| 204 | int doingrealloc = 0; |
| 205 | bus_size_t uboundary; |
| 206 | |
| 207 | /* Always round the size. */ |
| 208 | size = round_page(size); |
| 209 | |
| 210 | KASSERT((alignment & (alignment - 1)) == 0); |
| 211 | KASSERT((boundary & (boundary - 1)) == 0); |
| 212 | KASSERT(boundary >= PAGE_SIZE || boundary == 0); |
| 213 | |
| 214 | if (alignment < PAGE_SIZE) |
| 215 | alignment = PAGE_SIZE; |
| 216 | |
| 217 | /* |
| 218 | * Allocate pages from the VM system. |
| 219 | * We accept boundaries < size, splitting in multiple segments |
| 220 | * if needed. uvm_pglistalloc does not, so compute an appropriate |
| 221 | * boundary: next power of 2 >= size |
| 222 | */ |
| 223 | if (boundary == 0) |
| 224 | uboundary = 0; |
| 225 | else { |
| 226 | uboundary = boundary; |
| 227 | while (uboundary < size) |
| 228 | uboundary = uboundary << 1; |
| 229 | } |
| 230 | error = uvm_pglistalloc(size, 0, avail_end, alignment, uboundary, |
| 231 | &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); |
| 232 | if (error) |
| 233 | return (error); |
| 234 | again: |
| 235 | |
| 236 | /* |
| 237 | * Compute the location, size, and number of segments actually |
| 238 | * returned by the VM code. |
| 239 | */ |
| 240 | m = mlist.tqh_first; |
| 241 | curseg = 0; |
| 242 | curaddr = lastaddr = segs[curseg].ds_addr = _BUS_VM_PAGE_TO_BUS(m); |
| 243 | if (curaddr < low || curaddr >= high) |
| 244 | goto badaddr; |
| 245 | segs[curseg].ds_len = PAGE_SIZE; |
| 246 | m = m->pageq.queue.tqe_next; |
| 247 | if ((segs[curseg].ds_addr & (alignment - 1)) != 0) |
| 248 | goto dorealloc; |
| 249 | |
| 250 | for (; m != NULL; m = m->pageq.queue.tqe_next) { |
| 251 | curaddr = _BUS_VM_PAGE_TO_BUS(m); |
| 252 | if (curaddr < low || curaddr >= high) |
| 253 | goto badaddr; |
| 254 | if (curaddr == (lastaddr + PAGE_SIZE) && |
| 255 | (lastaddr & boundary) == (curaddr & boundary)) { |
| 256 | segs[curseg].ds_len += PAGE_SIZE; |
| 257 | } else { |
| 258 | curseg++; |
| 259 | if (curseg >= nsegs || |
| 260 | (curaddr & (alignment - 1)) != 0) { |
| 261 | if (doingrealloc) |
| 262 | return EFBIG; |
| 263 | else |
| 264 | goto dorealloc; |
| 265 | } |
| 266 | segs[curseg].ds_addr = curaddr; |
| 267 | segs[curseg].ds_len = PAGE_SIZE; |
| 268 | } |
| 269 | lastaddr = curaddr; |
| 270 | } |
| 271 | |
| 272 | *rsegs = curseg + 1; |
| 273 | return (0); |
| 274 | |
| 275 | badaddr: |
| 276 | if (doingrealloc == 0) |
| 277 | goto dorealloc; |
| 278 | if (curaddr < low) { |
| 279 | /* no way to enforce this */ |
| 280 | printf("_xen_bus_dmamem_alloc_range: no way to " |
| 281 | "enforce address range (0x%" PRIx64 " - 0x%" PRIx64 ")\n" , |
| 282 | (uint64_t)low, (uint64_t)high); |
| 283 | uvm_pglistfree(&mlist); |
| 284 | return EINVAL; |
| 285 | } |
| 286 | printf("xen_bus_dmamem_alloc_range: " |
| 287 | "curraddr=0x%lx > high=0x%lx\n" , |
| 288 | (u_long)curaddr, (u_long)high); |
| 289 | panic("xen_bus_dmamem_alloc_range 1" ); |
| 290 | dorealloc: |
| 291 | if (doingrealloc == 1) |
| 292 | panic("_xen_bus_dmamem_alloc_range: " |
| 293 | "xen_alloc_contig returned " |
| 294 | "too much segments" ); |
| 295 | doingrealloc = 1; |
| 296 | /* |
| 297 | * Too much segments, or memory doesn't fit |
| 298 | * constraints. Free this memory and |
| 299 | * get a contigous segment from the hypervisor. |
| 300 | */ |
| 301 | uvm_pglistfree(&mlist); |
| 302 | for (curseg = 0; curseg < nsegs; curseg++) { |
| 303 | segs[curseg].ds_addr = 0; |
| 304 | segs[curseg].ds_len = 0; |
| 305 | } |
| 306 | error = _xen_alloc_contig(size, alignment, |
| 307 | &mlist, flags, low, high); |
| 308 | if (error) |
| 309 | return error; |
| 310 | goto again; |
| 311 | } |
| 312 | |