| 1 | /* $NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $ */ |
| 2 | |
| 3 | /* |
| 4 | * Copyright (c) 1998 The NetBSD Foundation, Inc. |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * This code is derived from software contributed to The NetBSD Foundation |
| 8 | * by Lennart Augustsson (lennart@augustsson.net) at |
| 9 | * Carlstedt Research & Technology. |
| 10 | * |
| 11 | * Redistribution and use in source and binary forms, with or without |
| 12 | * modification, are permitted provided that the following conditions |
| 13 | * are met: |
| 14 | * 1. Redistributions of source code must retain the above copyright |
| 15 | * notice, this list of conditions and the following disclaimer. |
| 16 | * 2. Redistributions in binary form must reproduce the above copyright |
| 17 | * notice, this list of conditions and the following disclaimer in the |
| 18 | * documentation and/or other materials provided with the distribution. |
| 19 | * |
| 20 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 21 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 22 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 23 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 24 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 25 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 26 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 27 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 28 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 29 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 30 | * POSSIBILITY OF SUCH DAMAGE. |
| 31 | */ |
| 32 | |
| 33 | /* |
| 34 | * USB DMA memory allocation. |
| 35 | * We need to allocate a lot of small (many 8 byte, some larger) |
| 36 | * memory blocks that can be used for DMA. Using the bus_dma |
| 37 | * routines directly would incur large overheads in space and time. |
| 38 | */ |
| 39 | |
| 40 | #include <sys/cdefs.h> |
| 41 | __KERNEL_RCSID(0, "$NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $" ); |
| 42 | |
| 43 | #ifdef _KERNEL_OPT |
| 44 | #include "opt_usb.h" |
| 45 | #endif |
| 46 | |
| 47 | #include <sys/param.h> |
| 48 | #include <sys/systm.h> |
| 49 | #include <sys/kernel.h> |
| 50 | #include <sys/kmem.h> |
| 51 | #include <sys/queue.h> |
| 52 | #include <sys/device.h> /* for usbdivar.h */ |
| 53 | #include <sys/bus.h> |
| 54 | #include <sys/cpu.h> |
| 55 | #include <sys/once.h> |
| 56 | |
| 57 | #ifdef DIAGNOSTIC |
| 58 | #include <sys/proc.h> |
| 59 | #endif |
| 60 | |
| 61 | #include <dev/usb/usb.h> |
| 62 | #include <dev/usb/usbdi.h> |
| 63 | #include <dev/usb/usbdivar.h> /* just for usb_dma_t */ |
| 64 | #include <dev/usb/usb_mem.h> |
| 65 | #include <dev/usb/usbhist.h> |
| 66 | |
| 67 | #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D) |
| 68 | #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D) |
| 69 | |
| 70 | #define USB_MEM_SMALL roundup(64, CACHE_LINE_SIZE) |
| 71 | #define USB_MEM_CHUNKS 64 |
| 72 | #define USB_MEM_BLOCK (USB_MEM_SMALL * USB_MEM_CHUNKS) |
| 73 | |
| 74 | /* This struct is overlayed on free fragments. */ |
| 75 | struct usb_frag_dma { |
| 76 | usb_dma_block_t *ufd_block; |
| 77 | u_int ufd_offs; |
| 78 | LIST_ENTRY(usb_frag_dma) ufd_next; |
| 79 | }; |
| 80 | |
| 81 | Static usbd_status usb_block_allocmem(bus_dma_tag_t, size_t, size_t, |
| 82 | usb_dma_block_t **, bool); |
| 83 | Static void usb_block_freemem(usb_dma_block_t *); |
| 84 | |
| 85 | LIST_HEAD(usb_dma_block_qh, usb_dma_block); |
| 86 | Static struct usb_dma_block_qh usb_blk_freelist = |
| 87 | LIST_HEAD_INITIALIZER(usb_blk_freelist); |
| 88 | kmutex_t usb_blk_lock; |
| 89 | |
| 90 | #ifdef DEBUG |
| 91 | Static struct usb_dma_block_qh usb_blk_fraglist = |
| 92 | LIST_HEAD_INITIALIZER(usb_blk_fraglist); |
| 93 | Static struct usb_dma_block_qh usb_blk_fulllist = |
| 94 | LIST_HEAD_INITIALIZER(usb_blk_fulllist); |
| 95 | #endif |
| 96 | Static u_int usb_blk_nfree = 0; |
| 97 | /* XXX should have different free list for different tags (for speed) */ |
| 98 | Static LIST_HEAD(, usb_frag_dma) usb_frag_freelist = |
| 99 | LIST_HEAD_INITIALIZER(usb_frag_freelist); |
| 100 | |
| 101 | Static int usb_mem_init(void); |
| 102 | |
| 103 | Static int |
| 104 | usb_mem_init(void) |
| 105 | { |
| 106 | |
| 107 | mutex_init(&usb_blk_lock, MUTEX_DEFAULT, IPL_NONE); |
| 108 | return 0; |
| 109 | } |
| 110 | |
| 111 | Static usbd_status |
| 112 | usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align, |
| 113 | usb_dma_block_t **dmap, bool multiseg) |
| 114 | { |
| 115 | usb_dma_block_t *b; |
| 116 | int error; |
| 117 | |
| 118 | USBHIST_FUNC(); USBHIST_CALLED(usbdebug); |
| 119 | DPRINTFN(5, "size=%zu align=%zu" , size, align, 0, 0); |
| 120 | |
| 121 | ASSERT_SLEEPABLE(); |
| 122 | KASSERT(size != 0); |
| 123 | KASSERT(mutex_owned(&usb_blk_lock)); |
| 124 | |
| 125 | /* First check the free list. */ |
| 126 | LIST_FOREACH(b, &usb_blk_freelist, next) { |
| 127 | /* Don't allocate multiple segments to unwilling callers */ |
| 128 | if (b->nsegs != 1 && !multiseg) |
| 129 | continue; |
| 130 | if (b->tag == tag && b->size >= size && b->align >= align) { |
| 131 | LIST_REMOVE(b, next); |
| 132 | usb_blk_nfree--; |
| 133 | *dmap = b; |
| 134 | DPRINTFN(6, "free list size=%zu" , b->size, 0, 0, 0); |
| 135 | return USBD_NORMAL_COMPLETION; |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | DPRINTFN(6, "no free" , 0, 0, 0, 0); |
| 140 | mutex_exit(&usb_blk_lock); |
| 141 | |
| 142 | b = kmem_zalloc(sizeof(*b), KM_SLEEP); |
| 143 | if (b == NULL) { |
| 144 | goto fail; |
| 145 | } |
| 146 | |
| 147 | b->tag = tag; |
| 148 | b->size = size; |
| 149 | b->align = align; |
| 150 | |
| 151 | if (!multiseg) |
| 152 | /* Caller wants one segment */ |
| 153 | b->nsegs = 1; |
| 154 | else |
| 155 | b->nsegs = (size + (PAGE_SIZE-1)) / PAGE_SIZE; |
| 156 | |
| 157 | b->segs = kmem_alloc(b->nsegs * sizeof(*b->segs), KM_SLEEP); |
| 158 | if (b->segs == NULL) { |
| 159 | kmem_free(b, sizeof(*b)); |
| 160 | goto fail; |
| 161 | } |
| 162 | b->nsegs_alloc = b->nsegs; |
| 163 | |
| 164 | error = bus_dmamem_alloc(tag, b->size, align, 0, |
| 165 | b->segs, b->nsegs, |
| 166 | &b->nsegs, BUS_DMA_WAITOK); |
| 167 | if (error) |
| 168 | goto free0; |
| 169 | |
| 170 | error = bus_dmamem_map(tag, b->segs, b->nsegs, b->size, |
| 171 | &b->kaddr, BUS_DMA_WAITOK|BUS_DMA_COHERENT); |
| 172 | if (error) |
| 173 | goto free1; |
| 174 | |
| 175 | error = bus_dmamap_create(tag, b->size, b->nsegs, b->size, |
| 176 | 0, BUS_DMA_WAITOK, &b->map); |
| 177 | if (error) |
| 178 | goto unmap; |
| 179 | |
| 180 | error = bus_dmamap_load(tag, b->map, b->kaddr, b->size, NULL, |
| 181 | BUS_DMA_WAITOK); |
| 182 | if (error) |
| 183 | goto destroy; |
| 184 | |
| 185 | *dmap = b; |
| 186 | #ifdef USB_FRAG_DMA_WORKAROUND |
| 187 | memset(b->kaddr, 0, b->size); |
| 188 | #endif |
| 189 | mutex_enter(&usb_blk_lock); |
| 190 | |
| 191 | return USBD_NORMAL_COMPLETION; |
| 192 | |
| 193 | destroy: |
| 194 | bus_dmamap_destroy(tag, b->map); |
| 195 | unmap: |
| 196 | bus_dmamem_unmap(tag, b->kaddr, b->size); |
| 197 | free1: |
| 198 | bus_dmamem_free(tag, b->segs, b->nsegs); |
| 199 | free0: |
| 200 | kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs)); |
| 201 | kmem_free(b, sizeof(*b)); |
| 202 | fail: |
| 203 | mutex_enter(&usb_blk_lock); |
| 204 | |
| 205 | return USBD_NOMEM; |
| 206 | } |
| 207 | |
| 208 | #if 0 |
| 209 | void |
| 210 | usb_block_real_freemem(usb_dma_block_t *b) |
| 211 | { |
| 212 | #ifdef DIAGNOSTIC |
| 213 | if (cpu_softintr_p() || cpu_intr_p()) { |
| 214 | printf("usb_block_real_freemem: in interrupt context\n" ); |
| 215 | return; |
| 216 | } |
| 217 | #endif |
| 218 | bus_dmamap_unload(b->tag, b->map); |
| 219 | bus_dmamap_destroy(b->tag, b->map); |
| 220 | bus_dmamem_unmap(b->tag, b->kaddr, b->size); |
| 221 | bus_dmamem_free(b->tag, b->segs, b->nsegs); |
| 222 | kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs)); |
| 223 | kmem_free(b, sizeof(*b)); |
| 224 | } |
| 225 | #endif |
| 226 | |
| 227 | #ifdef DEBUG |
| 228 | static bool |
| 229 | usb_valid_block_p(usb_dma_block_t *b, struct usb_dma_block_qh *qh) |
| 230 | { |
| 231 | usb_dma_block_t *xb; |
| 232 | LIST_FOREACH(xb, qh, next) { |
| 233 | if (xb == b) |
| 234 | return true; |
| 235 | } |
| 236 | return false; |
| 237 | } |
| 238 | #endif |
| 239 | |
| 240 | /* |
| 241 | * Do not free the memory unconditionally since we might be called |
| 242 | * from an interrupt context and that is BAD. |
| 243 | * XXX when should we really free? |
| 244 | */ |
| 245 | Static void |
| 246 | usb_block_freemem(usb_dma_block_t *b) |
| 247 | { |
| 248 | |
| 249 | KASSERT(mutex_owned(&usb_blk_lock)); |
| 250 | |
| 251 | USBHIST_FUNC(); USBHIST_CALLED(usbdebug); |
| 252 | DPRINTFN(6, "size=%zu" , b->size, 0, 0, 0); |
| 253 | #ifdef DEBUG |
| 254 | LIST_REMOVE(b, next); |
| 255 | #endif |
| 256 | LIST_INSERT_HEAD(&usb_blk_freelist, b, next); |
| 257 | usb_blk_nfree++; |
| 258 | } |
| 259 | |
| 260 | usbd_status |
| 261 | usb_allocmem(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p) |
| 262 | { |
| 263 | |
| 264 | return usb_allocmem_flags(bus, size, align, p, 0); |
| 265 | } |
| 266 | |
| 267 | usbd_status |
| 268 | usb_allocmem_flags(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p, |
| 269 | int flags) |
| 270 | { |
| 271 | bus_dma_tag_t tag = bus->ub_dmatag; |
| 272 | usbd_status err; |
| 273 | struct usb_frag_dma *f; |
| 274 | usb_dma_block_t *b; |
| 275 | int i; |
| 276 | static ONCE_DECL(init_control); |
| 277 | bool frag; |
| 278 | |
| 279 | USBHIST_FUNC(); USBHIST_CALLED(usbdebug); |
| 280 | |
| 281 | ASSERT_SLEEPABLE(); |
| 282 | |
| 283 | RUN_ONCE(&init_control, usb_mem_init); |
| 284 | |
| 285 | frag = (flags & USBMALLOC_MULTISEG); |
| 286 | |
| 287 | /* If the request is large then just use a full block. */ |
| 288 | if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) { |
| 289 | DPRINTFN(1, "large alloc %d" , size, 0, 0, 0); |
| 290 | size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1); |
| 291 | mutex_enter(&usb_blk_lock); |
| 292 | err = usb_block_allocmem(tag, size, align, &p->udma_block, frag); |
| 293 | if (!err) { |
| 294 | #ifdef DEBUG |
| 295 | LIST_INSERT_HEAD(&usb_blk_fulllist, p->udma_block, next); |
| 296 | #endif |
| 297 | p->udma_block->flags = USB_DMA_FULLBLOCK; |
| 298 | p->udma_offs = 0; |
| 299 | } |
| 300 | mutex_exit(&usb_blk_lock); |
| 301 | return err; |
| 302 | } |
| 303 | |
| 304 | mutex_enter(&usb_blk_lock); |
| 305 | /* Check for free fragments. */ |
| 306 | LIST_FOREACH(f, &usb_frag_freelist, ufd_next) { |
| 307 | KDASSERTMSG(usb_valid_block_p(f->ufd_block, &usb_blk_fraglist), |
| 308 | "%s: usb frag %p: unknown block pointer %p" , |
| 309 | __func__, f, f->ufd_block); |
| 310 | if (f->ufd_block->tag == tag) |
| 311 | break; |
| 312 | } |
| 313 | if (f == NULL) { |
| 314 | DPRINTFN(1, "adding fragments" , 0, 0, 0, 0); |
| 315 | err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL, &b, |
| 316 | false); |
| 317 | if (err) { |
| 318 | mutex_exit(&usb_blk_lock); |
| 319 | return err; |
| 320 | } |
| 321 | #ifdef DEBUG |
| 322 | LIST_INSERT_HEAD(&usb_blk_fraglist, b, next); |
| 323 | #endif |
| 324 | b->flags = 0; |
| 325 | for (i = 0; i < USB_MEM_BLOCK; i += USB_MEM_SMALL) { |
| 326 | f = (struct usb_frag_dma *)((char *)b->kaddr + i); |
| 327 | f->ufd_block = b; |
| 328 | f->ufd_offs = i; |
| 329 | LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next); |
| 330 | #ifdef USB_FRAG_DMA_WORKAROUND |
| 331 | i += 1 * USB_MEM_SMALL; |
| 332 | #endif |
| 333 | } |
| 334 | f = LIST_FIRST(&usb_frag_freelist); |
| 335 | } |
| 336 | p->udma_block = f->ufd_block; |
| 337 | p->udma_offs = f->ufd_offs; |
| 338 | #ifdef USB_FRAG_DMA_WORKAROUND |
| 339 | p->udma_offs += USB_MEM_SMALL; |
| 340 | #endif |
| 341 | LIST_REMOVE(f, ufd_next); |
| 342 | mutex_exit(&usb_blk_lock); |
| 343 | DPRINTFN(5, "use frag=%p size=%d" , f, size, 0, 0); |
| 344 | |
| 345 | return USBD_NORMAL_COMPLETION; |
| 346 | } |
| 347 | |
| 348 | void |
| 349 | usb_freemem(struct usbd_bus *bus, usb_dma_t *p) |
| 350 | { |
| 351 | struct usb_frag_dma *f; |
| 352 | |
| 353 | USBHIST_FUNC(); USBHIST_CALLED(usbdebug); |
| 354 | |
| 355 | mutex_enter(&usb_blk_lock); |
| 356 | if (p->udma_block->flags & USB_DMA_FULLBLOCK) { |
| 357 | KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fulllist), |
| 358 | "%s: dma %p: invalid block pointer %p" , |
| 359 | __func__, p, p->udma_block); |
| 360 | DPRINTFN(1, "large free" , 0, 0, 0, 0); |
| 361 | usb_block_freemem(p->udma_block); |
| 362 | mutex_exit(&usb_blk_lock); |
| 363 | return; |
| 364 | } |
| 365 | KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fraglist), |
| 366 | "%s: dma %p: invalid block pointer %p" , |
| 367 | __func__, p, p->udma_block); |
| 368 | //usb_syncmem(p, 0, USB_MEM_SMALL, BUS_DMASYNC_POSTREAD); |
| 369 | f = KERNADDR(p, 0); |
| 370 | #ifdef USB_FRAG_DMA_WORKAROUND |
| 371 | f = (void *)((uintptr_t)f - USB_MEM_SMALL); |
| 372 | #endif |
| 373 | f->ufd_block = p->udma_block; |
| 374 | f->ufd_offs = p->udma_offs; |
| 375 | #ifdef USB_FRAG_DMA_WORKAROUND |
| 376 | f->ufd_offs -= USB_MEM_SMALL; |
| 377 | #endif |
| 378 | LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next); |
| 379 | mutex_exit(&usb_blk_lock); |
| 380 | DPRINTFN(5, "frag=%p" , f, 0, 0, 0); |
| 381 | } |
| 382 | |
| 383 | bus_addr_t |
| 384 | usb_dmaaddr(usb_dma_t *dma, unsigned int offset) |
| 385 | { |
| 386 | unsigned int i; |
| 387 | bus_size_t seg_offs; |
| 388 | |
| 389 | offset += dma->udma_offs; |
| 390 | |
| 391 | KASSERTMSG(offset < dma->udma_block->size, "offset %d vs %zu" , offset, |
| 392 | dma->udma_block->size); |
| 393 | |
| 394 | if (dma->udma_block->nsegs == 1) { |
| 395 | KASSERT(dma->udma_block->map->dm_segs[0].ds_len > offset); |
| 396 | return dma->udma_block->map->dm_segs[0].ds_addr + offset; |
| 397 | } |
| 398 | |
| 399 | /* |
| 400 | * Search for a bus_segment_t corresponding to this offset. With no |
| 401 | * record of the offset in the map to a particular dma_segment_t, we |
| 402 | * have to iterate from the start of the list each time. Could be |
| 403 | * improved |
| 404 | */ |
| 405 | seg_offs = 0; |
| 406 | for (i = 0; i < dma->udma_block->nsegs; i++) { |
| 407 | if (seg_offs + dma->udma_block->map->dm_segs[i].ds_len > offset) |
| 408 | break; |
| 409 | |
| 410 | seg_offs += dma->udma_block->map->dm_segs[i].ds_len; |
| 411 | } |
| 412 | |
| 413 | KASSERT(i != dma->udma_block->nsegs); |
| 414 | offset -= seg_offs; |
| 415 | return dma->udma_block->map->dm_segs[i].ds_addr + offset; |
| 416 | } |
| 417 | |
| 418 | void |
| 419 | usb_syncmem(usb_dma_t *p, bus_addr_t offset, bus_size_t len, int ops) |
| 420 | { |
| 421 | |
| 422 | bus_dmamap_sync(p->udma_block->tag, p->udma_block->map, p->udma_offs + offset, |
| 423 | len, ops); |
| 424 | } |
| 425 | |