| 1 | /* $NetBSD: subr_pool.c,v 1.206 2016/02/05 03:04:52 knakahara Exp $ */ |
| 2 | |
| 3 | /*- |
| 4 | * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010, 2014, 2015 |
| 5 | * The NetBSD Foundation, Inc. |
| 6 | * All rights reserved. |
| 7 | * |
| 8 | * This code is derived from software contributed to The NetBSD Foundation |
| 9 | * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace |
| 10 | * Simulation Facility, NASA Ames Research Center; by Andrew Doran, and by |
| 11 | * Maxime Villard. |
| 12 | * |
| 13 | * Redistribution and use in source and binary forms, with or without |
| 14 | * modification, are permitted provided that the following conditions |
| 15 | * are met: |
| 16 | * 1. Redistributions of source code must retain the above copyright |
| 17 | * notice, this list of conditions and the following disclaimer. |
| 18 | * 2. Redistributions in binary form must reproduce the above copyright |
| 19 | * notice, this list of conditions and the following disclaimer in the |
| 20 | * documentation and/or other materials provided with the distribution. |
| 21 | * |
| 22 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 23 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 24 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 25 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 26 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 27 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 28 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 29 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 30 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 31 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 32 | * POSSIBILITY OF SUCH DAMAGE. |
| 33 | */ |
| 34 | |
| 35 | #include <sys/cdefs.h> |
| 36 | __KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.206 2016/02/05 03:04:52 knakahara Exp $" ); |
| 37 | |
| 38 | #ifdef _KERNEL_OPT |
| 39 | #include "opt_ddb.h" |
| 40 | #include "opt_lockdebug.h" |
| 41 | #endif |
| 42 | |
| 43 | #include <sys/param.h> |
| 44 | #include <sys/systm.h> |
| 45 | #include <sys/sysctl.h> |
| 46 | #include <sys/bitops.h> |
| 47 | #include <sys/proc.h> |
| 48 | #include <sys/errno.h> |
| 49 | #include <sys/kernel.h> |
| 50 | #include <sys/vmem.h> |
| 51 | #include <sys/pool.h> |
| 52 | #include <sys/syslog.h> |
| 53 | #include <sys/debug.h> |
| 54 | #include <sys/lockdebug.h> |
| 55 | #include <sys/xcall.h> |
| 56 | #include <sys/cpu.h> |
| 57 | #include <sys/atomic.h> |
| 58 | |
| 59 | #include <uvm/uvm_extern.h> |
| 60 | |
| 61 | /* |
| 62 | * Pool resource management utility. |
| 63 | * |
| 64 | * Memory is allocated in pages which are split into pieces according to |
| 65 | * the pool item size. Each page is kept on one of three lists in the |
| 66 | * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages', |
| 67 | * for empty, full and partially-full pages respectively. The individual |
| 68 | * pool items are on a linked list headed by `ph_itemlist' in each page |
| 69 | * header. The memory for building the page list is either taken from |
| 70 | * the allocated pages themselves (for small pool items) or taken from |
| 71 | * an internal pool of page headers (`phpool'). |
| 72 | */ |
| 73 | |
| 74 | /* List of all pools. Non static as needed by 'vmstat -i' */ |
| 75 | TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head); |
| 76 | |
| 77 | /* Private pool for page header structures */ |
| 78 | #define PHPOOL_MAX 8 |
| 79 | static struct pool phpool[PHPOOL_MAX]; |
| 80 | #define PHPOOL_FREELIST_NELEM(idx) \ |
| 81 | (((idx) == 0) ? 0 : BITMAP_SIZE * (1 << (idx))) |
| 82 | |
| 83 | #ifdef POOL_SUBPAGE |
| 84 | /* Pool of subpages for use by normal pools. */ |
| 85 | static struct pool psppool; |
| 86 | #endif |
| 87 | |
| 88 | #ifdef POOL_REDZONE |
| 89 | # define POOL_REDZONE_SIZE 2 |
| 90 | static void pool_redzone_init(struct pool *, size_t); |
| 91 | static void pool_redzone_fill(struct pool *, void *); |
| 92 | static void pool_redzone_check(struct pool *, void *); |
| 93 | #else |
| 94 | # define pool_redzone_init(pp, sz) /* NOTHING */ |
| 95 | # define pool_redzone_fill(pp, ptr) /* NOTHING */ |
| 96 | # define pool_redzone_check(pp, ptr) /* NOTHING */ |
| 97 | #endif |
| 98 | |
| 99 | static void *pool_page_alloc_meta(struct pool *, int); |
| 100 | static void pool_page_free_meta(struct pool *, void *); |
| 101 | |
| 102 | /* allocator for pool metadata */ |
| 103 | struct pool_allocator pool_allocator_meta = { |
| 104 | .pa_alloc = pool_page_alloc_meta, |
| 105 | .pa_free = pool_page_free_meta, |
| 106 | .pa_pagesz = 0 |
| 107 | }; |
| 108 | |
| 109 | /* # of seconds to retain page after last use */ |
| 110 | int pool_inactive_time = 10; |
| 111 | |
| 112 | /* Next candidate for drainage (see pool_drain()) */ |
| 113 | static struct pool *drainpp; |
| 114 | |
| 115 | /* This lock protects both pool_head and drainpp. */ |
| 116 | static kmutex_t pool_head_lock; |
| 117 | static kcondvar_t pool_busy; |
| 118 | |
| 119 | /* This lock protects initialization of a potentially shared pool allocator */ |
| 120 | static kmutex_t pool_allocator_lock; |
| 121 | |
| 122 | typedef uint32_t pool_item_bitmap_t; |
| 123 | #define BITMAP_SIZE (CHAR_BIT * sizeof(pool_item_bitmap_t)) |
| 124 | #define BITMAP_MASK (BITMAP_SIZE - 1) |
| 125 | |
| 126 | struct { |
| 127 | /* Page headers */ |
| 128 | LIST_ENTRY(pool_item_header) |
| 129 | ; /* pool page list */ |
| 130 | SPLAY_ENTRY(pool_item_header) |
| 131 | ; /* Off-page page headers */ |
| 132 | void * ; /* this page's address */ |
| 133 | uint32_t ; /* last referenced */ |
| 134 | uint16_t ; /* # of chunks in use */ |
| 135 | uint16_t ; /* start offset in page */ |
| 136 | union { |
| 137 | /* !PR_NOTOUCH */ |
| 138 | struct { |
| 139 | LIST_HEAD(, pool_item) |
| 140 | ; /* chunk list for this page */ |
| 141 | } ; |
| 142 | /* PR_NOTOUCH */ |
| 143 | struct { |
| 144 | pool_item_bitmap_t [1]; |
| 145 | } ; |
| 146 | } ; |
| 147 | }; |
| 148 | #define ph_itemlist ph_u.phu_normal.phu_itemlist |
| 149 | #define ph_bitmap ph_u.phu_notouch.phu_bitmap |
| 150 | |
| 151 | struct pool_item { |
| 152 | #ifdef DIAGNOSTIC |
| 153 | u_int pi_magic; |
| 154 | #endif |
| 155 | #define PI_MAGIC 0xdeaddeadU |
| 156 | /* Other entries use only this list entry */ |
| 157 | LIST_ENTRY(pool_item) pi_list; |
| 158 | }; |
| 159 | |
| 160 | #define POOL_NEEDS_CATCHUP(pp) \ |
| 161 | ((pp)->pr_nitems < (pp)->pr_minitems) |
| 162 | |
| 163 | /* |
| 164 | * Pool cache management. |
| 165 | * |
| 166 | * Pool caches provide a way for constructed objects to be cached by the |
| 167 | * pool subsystem. This can lead to performance improvements by avoiding |
| 168 | * needless object construction/destruction; it is deferred until absolutely |
| 169 | * necessary. |
| 170 | * |
| 171 | * Caches are grouped into cache groups. Each cache group references up |
| 172 | * to PCG_NUMOBJECTS constructed objects. When a cache allocates an |
| 173 | * object from the pool, it calls the object's constructor and places it |
| 174 | * into a cache group. When a cache group frees an object back to the |
| 175 | * pool, it first calls the object's destructor. This allows the object |
| 176 | * to persist in constructed form while freed to the cache. |
| 177 | * |
| 178 | * The pool references each cache, so that when a pool is drained by the |
| 179 | * pagedaemon, it can drain each individual cache as well. Each time a |
| 180 | * cache is drained, the most idle cache group is freed to the pool in |
| 181 | * its entirety. |
| 182 | * |
| 183 | * Pool caches are layed on top of pools. By layering them, we can avoid |
| 184 | * the complexity of cache management for pools which would not benefit |
| 185 | * from it. |
| 186 | */ |
| 187 | |
| 188 | static struct pool pcg_normal_pool; |
| 189 | static struct pool pcg_large_pool; |
| 190 | static struct pool cache_pool; |
| 191 | static struct pool cache_cpu_pool; |
| 192 | |
| 193 | pool_cache_t pnbuf_cache; /* pathname buffer cache */ |
| 194 | |
| 195 | /* List of all caches. */ |
| 196 | TAILQ_HEAD(,pool_cache) pool_cache_head = |
| 197 | TAILQ_HEAD_INITIALIZER(pool_cache_head); |
| 198 | |
| 199 | int pool_cache_disable; /* global disable for caching */ |
| 200 | static const pcg_t pcg_dummy; /* zero sized: always empty, yet always full */ |
| 201 | |
| 202 | static bool pool_cache_put_slow(pool_cache_cpu_t *, int, |
| 203 | void *); |
| 204 | static bool pool_cache_get_slow(pool_cache_cpu_t *, int, |
| 205 | void **, paddr_t *, int); |
| 206 | static void pool_cache_cpu_init1(struct cpu_info *, pool_cache_t); |
| 207 | static void pool_cache_invalidate_groups(pool_cache_t, pcg_t *); |
| 208 | static void pool_cache_invalidate_cpu(pool_cache_t, u_int); |
| 209 | static void pool_cache_transfer(pool_cache_t); |
| 210 | |
| 211 | static int pool_catchup(struct pool *); |
| 212 | static void pool_prime_page(struct pool *, void *, |
| 213 | struct pool_item_header *); |
| 214 | static void pool_update_curpage(struct pool *); |
| 215 | |
| 216 | static int pool_grow(struct pool *, int); |
| 217 | static void *pool_allocator_alloc(struct pool *, int); |
| 218 | static void pool_allocator_free(struct pool *, void *); |
| 219 | |
| 220 | static void pool_print_pagelist(struct pool *, struct pool_pagelist *, |
| 221 | void (*)(const char *, ...) __printflike(1, 2)); |
| 222 | static void pool_print1(struct pool *, const char *, |
| 223 | void (*)(const char *, ...) __printflike(1, 2)); |
| 224 | |
| 225 | static int pool_chk_page(struct pool *, const char *, |
| 226 | struct pool_item_header *); |
| 227 | |
| 228 | static inline unsigned int |
| 229 | pr_item_notouch_index(const struct pool *pp, const struct pool_item_header *ph, |
| 230 | const void *v) |
| 231 | { |
| 232 | const char *cp = v; |
| 233 | unsigned int idx; |
| 234 | |
| 235 | KASSERT(pp->pr_roflags & PR_NOTOUCH); |
| 236 | idx = (cp - (char *)ph->ph_page - ph->ph_off) / pp->pr_size; |
| 237 | KASSERT(idx < pp->pr_itemsperpage); |
| 238 | return idx; |
| 239 | } |
| 240 | |
| 241 | static inline void |
| 242 | pr_item_notouch_put(const struct pool *pp, struct pool_item_header *ph, |
| 243 | void *obj) |
| 244 | { |
| 245 | unsigned int idx = pr_item_notouch_index(pp, ph, obj); |
| 246 | pool_item_bitmap_t *bitmap = ph->ph_bitmap + (idx / BITMAP_SIZE); |
| 247 | pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); |
| 248 | |
| 249 | KASSERT((*bitmap & mask) == 0); |
| 250 | *bitmap |= mask; |
| 251 | } |
| 252 | |
| 253 | static inline void * |
| 254 | pr_item_notouch_get(const struct pool *pp, struct pool_item_header *ph) |
| 255 | { |
| 256 | pool_item_bitmap_t *bitmap = ph->ph_bitmap; |
| 257 | unsigned int idx; |
| 258 | int i; |
| 259 | |
| 260 | for (i = 0; ; i++) { |
| 261 | int bit; |
| 262 | |
| 263 | KASSERT((i * BITMAP_SIZE) < pp->pr_itemsperpage); |
| 264 | bit = ffs32(bitmap[i]); |
| 265 | if (bit) { |
| 266 | pool_item_bitmap_t mask; |
| 267 | |
| 268 | bit--; |
| 269 | idx = (i * BITMAP_SIZE) + bit; |
| 270 | mask = 1 << bit; |
| 271 | KASSERT((bitmap[i] & mask) != 0); |
| 272 | bitmap[i] &= ~mask; |
| 273 | break; |
| 274 | } |
| 275 | } |
| 276 | KASSERT(idx < pp->pr_itemsperpage); |
| 277 | return (char *)ph->ph_page + ph->ph_off + idx * pp->pr_size; |
| 278 | } |
| 279 | |
| 280 | static inline void |
| 281 | pr_item_notouch_init(const struct pool *pp, struct pool_item_header *ph) |
| 282 | { |
| 283 | pool_item_bitmap_t *bitmap = ph->ph_bitmap; |
| 284 | const int n = howmany(pp->pr_itemsperpage, BITMAP_SIZE); |
| 285 | int i; |
| 286 | |
| 287 | for (i = 0; i < n; i++) { |
| 288 | bitmap[i] = (pool_item_bitmap_t)-1; |
| 289 | } |
| 290 | } |
| 291 | |
| 292 | static inline int |
| 293 | phtree_compare(struct pool_item_header *a, struct pool_item_header *b) |
| 294 | { |
| 295 | |
| 296 | /* |
| 297 | * we consider pool_item_header with smaller ph_page bigger. |
| 298 | * (this unnatural ordering is for the benefit of pr_find_pagehead.) |
| 299 | */ |
| 300 | |
| 301 | if (a->ph_page < b->ph_page) |
| 302 | return (1); |
| 303 | else if (a->ph_page > b->ph_page) |
| 304 | return (-1); |
| 305 | else |
| 306 | return (0); |
| 307 | } |
| 308 | |
| 309 | SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare); |
| 310 | SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare); |
| 311 | |
| 312 | static inline struct pool_item_header * |
| 313 | pr_find_pagehead_noalign(struct pool *pp, void *v) |
| 314 | { |
| 315 | struct pool_item_header *ph, tmp; |
| 316 | |
| 317 | tmp.ph_page = (void *)(uintptr_t)v; |
| 318 | ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
| 319 | if (ph == NULL) { |
| 320 | ph = SPLAY_ROOT(&pp->pr_phtree); |
| 321 | if (ph != NULL && phtree_compare(&tmp, ph) >= 0) { |
| 322 | ph = SPLAY_NEXT(phtree, &pp->pr_phtree, ph); |
| 323 | } |
| 324 | KASSERT(ph == NULL || phtree_compare(&tmp, ph) < 0); |
| 325 | } |
| 326 | |
| 327 | return ph; |
| 328 | } |
| 329 | |
| 330 | /* |
| 331 | * Return the pool page header based on item address. |
| 332 | */ |
| 333 | static inline struct pool_item_header * |
| 334 | pr_find_pagehead(struct pool *pp, void *v) |
| 335 | { |
| 336 | struct pool_item_header *ph, tmp; |
| 337 | |
| 338 | if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
| 339 | ph = pr_find_pagehead_noalign(pp, v); |
| 340 | } else { |
| 341 | void *page = |
| 342 | (void *)((uintptr_t)v & pp->pr_alloc->pa_pagemask); |
| 343 | |
| 344 | if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
| 345 | ph = (struct pool_item_header *)((char *)page + pp->pr_phoffset); |
| 346 | } else { |
| 347 | tmp.ph_page = page; |
| 348 | ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp); |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | KASSERT(ph == NULL || ((pp->pr_roflags & PR_PHINPAGE) != 0) || |
| 353 | ((char *)ph->ph_page <= (char *)v && |
| 354 | (char *)v < (char *)ph->ph_page + pp->pr_alloc->pa_pagesz)); |
| 355 | return ph; |
| 356 | } |
| 357 | |
| 358 | static void |
| 359 | pr_pagelist_free(struct pool *pp, struct pool_pagelist *pq) |
| 360 | { |
| 361 | struct pool_item_header *ph; |
| 362 | |
| 363 | while ((ph = LIST_FIRST(pq)) != NULL) { |
| 364 | LIST_REMOVE(ph, ph_pagelist); |
| 365 | pool_allocator_free(pp, ph->ph_page); |
| 366 | if ((pp->pr_roflags & PR_PHINPAGE) == 0) |
| 367 | pool_put(pp->pr_phpool, ph); |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | /* |
| 372 | * Remove a page from the pool. |
| 373 | */ |
| 374 | static inline void |
| 375 | pr_rmpage(struct pool *pp, struct pool_item_header *ph, |
| 376 | struct pool_pagelist *pq) |
| 377 | { |
| 378 | |
| 379 | KASSERT(mutex_owned(&pp->pr_lock)); |
| 380 | |
| 381 | /* |
| 382 | * If the page was idle, decrement the idle page count. |
| 383 | */ |
| 384 | if (ph->ph_nmissing == 0) { |
| 385 | #ifdef DIAGNOSTIC |
| 386 | if (pp->pr_nidle == 0) |
| 387 | panic("pr_rmpage: nidle inconsistent" ); |
| 388 | if (pp->pr_nitems < pp->pr_itemsperpage) |
| 389 | panic("pr_rmpage: nitems inconsistent" ); |
| 390 | #endif |
| 391 | pp->pr_nidle--; |
| 392 | } |
| 393 | |
| 394 | pp->pr_nitems -= pp->pr_itemsperpage; |
| 395 | |
| 396 | /* |
| 397 | * Unlink the page from the pool and queue it for release. |
| 398 | */ |
| 399 | LIST_REMOVE(ph, ph_pagelist); |
| 400 | if ((pp->pr_roflags & PR_PHINPAGE) == 0) |
| 401 | SPLAY_REMOVE(phtree, &pp->pr_phtree, ph); |
| 402 | LIST_INSERT_HEAD(pq, ph, ph_pagelist); |
| 403 | |
| 404 | pp->pr_npages--; |
| 405 | pp->pr_npagefree++; |
| 406 | |
| 407 | pool_update_curpage(pp); |
| 408 | } |
| 409 | |
| 410 | /* |
| 411 | * Initialize all the pools listed in the "pools" link set. |
| 412 | */ |
| 413 | void |
| 414 | pool_subsystem_init(void) |
| 415 | { |
| 416 | size_t size; |
| 417 | int idx; |
| 418 | |
| 419 | mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE); |
| 420 | mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE); |
| 421 | cv_init(&pool_busy, "poolbusy" ); |
| 422 | |
| 423 | /* |
| 424 | * Initialize private page header pool and cache magazine pool if we |
| 425 | * haven't done so yet. |
| 426 | */ |
| 427 | for (idx = 0; idx < PHPOOL_MAX; idx++) { |
| 428 | static char phpool_names[PHPOOL_MAX][6+1+6+1]; |
| 429 | int nelem; |
| 430 | size_t sz; |
| 431 | |
| 432 | nelem = PHPOOL_FREELIST_NELEM(idx); |
| 433 | snprintf(phpool_names[idx], sizeof(phpool_names[idx]), |
| 434 | "phpool-%d" , nelem); |
| 435 | sz = sizeof(struct pool_item_header); |
| 436 | if (nelem) { |
| 437 | sz = offsetof(struct pool_item_header, |
| 438 | ph_bitmap[howmany(nelem, BITMAP_SIZE)]); |
| 439 | } |
| 440 | pool_init(&phpool[idx], sz, 0, 0, 0, |
| 441 | phpool_names[idx], &pool_allocator_meta, IPL_VM); |
| 442 | } |
| 443 | #ifdef POOL_SUBPAGE |
| 444 | pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0, |
| 445 | PR_RECURSIVE, "psppool" , &pool_allocator_meta, IPL_VM); |
| 446 | #endif |
| 447 | |
| 448 | size = sizeof(pcg_t) + |
| 449 | (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t); |
| 450 | pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0, |
| 451 | "pcgnormal" , &pool_allocator_meta, IPL_VM); |
| 452 | |
| 453 | size = sizeof(pcg_t) + |
| 454 | (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t); |
| 455 | pool_init(&pcg_large_pool, size, coherency_unit, 0, 0, |
| 456 | "pcglarge" , &pool_allocator_meta, IPL_VM); |
| 457 | |
| 458 | pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit, |
| 459 | 0, 0, "pcache" , &pool_allocator_meta, IPL_NONE); |
| 460 | |
| 461 | pool_init(&cache_cpu_pool, sizeof(pool_cache_cpu_t), coherency_unit, |
| 462 | 0, 0, "pcachecpu" , &pool_allocator_meta, IPL_NONE); |
| 463 | } |
| 464 | |
| 465 | /* |
| 466 | * Initialize the given pool resource structure. |
| 467 | * |
| 468 | * We export this routine to allow other kernel parts to declare |
| 469 | * static pools that must be initialized before kmem(9) is available. |
| 470 | */ |
| 471 | void |
| 472 | pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags, |
| 473 | const char *wchan, struct pool_allocator *palloc, int ipl) |
| 474 | { |
| 475 | struct pool *pp1; |
| 476 | size_t trysize, phsize, prsize; |
| 477 | int off, slack; |
| 478 | |
| 479 | #ifdef DEBUG |
| 480 | if (__predict_true(!cold)) |
| 481 | mutex_enter(&pool_head_lock); |
| 482 | /* |
| 483 | * Check that the pool hasn't already been initialised and |
| 484 | * added to the list of all pools. |
| 485 | */ |
| 486 | TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
| 487 | if (pp == pp1) |
| 488 | panic("pool_init: pool %s already initialised" , |
| 489 | wchan); |
| 490 | } |
| 491 | if (__predict_true(!cold)) |
| 492 | mutex_exit(&pool_head_lock); |
| 493 | #endif |
| 494 | |
| 495 | if (palloc == NULL) |
| 496 | palloc = &pool_allocator_kmem; |
| 497 | #ifdef POOL_SUBPAGE |
| 498 | if (size > palloc->pa_pagesz) { |
| 499 | if (palloc == &pool_allocator_kmem) |
| 500 | palloc = &pool_allocator_kmem_fullpage; |
| 501 | else if (palloc == &pool_allocator_nointr) |
| 502 | palloc = &pool_allocator_nointr_fullpage; |
| 503 | } |
| 504 | #endif /* POOL_SUBPAGE */ |
| 505 | if (!cold) |
| 506 | mutex_enter(&pool_allocator_lock); |
| 507 | if (palloc->pa_refcnt++ == 0) { |
| 508 | if (palloc->pa_pagesz == 0) |
| 509 | palloc->pa_pagesz = PAGE_SIZE; |
| 510 | |
| 511 | TAILQ_INIT(&palloc->pa_list); |
| 512 | |
| 513 | mutex_init(&palloc->pa_lock, MUTEX_DEFAULT, IPL_VM); |
| 514 | palloc->pa_pagemask = ~(palloc->pa_pagesz - 1); |
| 515 | palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1; |
| 516 | } |
| 517 | if (!cold) |
| 518 | mutex_exit(&pool_allocator_lock); |
| 519 | |
| 520 | if (align == 0) |
| 521 | align = ALIGN(1); |
| 522 | |
| 523 | prsize = size; |
| 524 | if ((flags & PR_NOTOUCH) == 0 && prsize < sizeof(struct pool_item)) |
| 525 | prsize = sizeof(struct pool_item); |
| 526 | |
| 527 | prsize = roundup(prsize, align); |
| 528 | #ifdef DIAGNOSTIC |
| 529 | if (prsize > palloc->pa_pagesz) |
| 530 | panic("pool_init: pool item size (%zu) too large" , prsize); |
| 531 | #endif |
| 532 | |
| 533 | /* |
| 534 | * Initialize the pool structure. |
| 535 | */ |
| 536 | LIST_INIT(&pp->pr_emptypages); |
| 537 | LIST_INIT(&pp->pr_fullpages); |
| 538 | LIST_INIT(&pp->pr_partpages); |
| 539 | pp->pr_cache = NULL; |
| 540 | pp->pr_curpage = NULL; |
| 541 | pp->pr_npages = 0; |
| 542 | pp->pr_minitems = 0; |
| 543 | pp->pr_minpages = 0; |
| 544 | pp->pr_maxpages = UINT_MAX; |
| 545 | pp->pr_roflags = flags; |
| 546 | pp->pr_flags = 0; |
| 547 | pp->pr_size = prsize; |
| 548 | pp->pr_align = align; |
| 549 | pp->pr_wchan = wchan; |
| 550 | pp->pr_alloc = palloc; |
| 551 | pp->pr_nitems = 0; |
| 552 | pp->pr_nout = 0; |
| 553 | pp->pr_hardlimit = UINT_MAX; |
| 554 | pp->pr_hardlimit_warning = NULL; |
| 555 | pp->pr_hardlimit_ratecap.tv_sec = 0; |
| 556 | pp->pr_hardlimit_ratecap.tv_usec = 0; |
| 557 | pp->pr_hardlimit_warning_last.tv_sec = 0; |
| 558 | pp->pr_hardlimit_warning_last.tv_usec = 0; |
| 559 | pp->pr_drain_hook = NULL; |
| 560 | pp->pr_drain_hook_arg = NULL; |
| 561 | pp->pr_freecheck = NULL; |
| 562 | pool_redzone_init(pp, size); |
| 563 | |
| 564 | /* |
| 565 | * Decide whether to put the page header off page to avoid |
| 566 | * wasting too large a part of the page or too big item. |
| 567 | * Off-page page headers go on a hash table, so we can match |
| 568 | * a returned item with its header based on the page address. |
| 569 | * We use 1/16 of the page size and about 8 times of the item |
| 570 | * size as the threshold (XXX: tune) |
| 571 | * |
| 572 | * However, we'll put the header into the page if we can put |
| 573 | * it without wasting any items. |
| 574 | * |
| 575 | * Silently enforce `0 <= ioff < align'. |
| 576 | */ |
| 577 | pp->pr_itemoffset = ioff %= align; |
| 578 | /* See the comment below about reserved bytes. */ |
| 579 | trysize = palloc->pa_pagesz - ((align - ioff) % align); |
| 580 | phsize = ALIGN(sizeof(struct pool_item_header)); |
| 581 | if (pp->pr_roflags & PR_PHINPAGE || |
| 582 | ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 && |
| 583 | (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) || |
| 584 | trysize / pp->pr_size == (trysize - phsize) / pp->pr_size))) { |
| 585 | /* Use the end of the page for the page header */ |
| 586 | pp->pr_roflags |= PR_PHINPAGE; |
| 587 | pp->pr_phoffset = off = palloc->pa_pagesz - phsize; |
| 588 | } else { |
| 589 | /* The page header will be taken from our page header pool */ |
| 590 | pp->pr_phoffset = 0; |
| 591 | off = palloc->pa_pagesz; |
| 592 | SPLAY_INIT(&pp->pr_phtree); |
| 593 | } |
| 594 | |
| 595 | /* |
| 596 | * Alignment is to take place at `ioff' within the item. This means |
| 597 | * we must reserve up to `align - 1' bytes on the page to allow |
| 598 | * appropriate positioning of each item. |
| 599 | */ |
| 600 | pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size; |
| 601 | KASSERT(pp->pr_itemsperpage != 0); |
| 602 | if ((pp->pr_roflags & PR_NOTOUCH)) { |
| 603 | int idx; |
| 604 | |
| 605 | for (idx = 0; pp->pr_itemsperpage > PHPOOL_FREELIST_NELEM(idx); |
| 606 | idx++) { |
| 607 | /* nothing */ |
| 608 | } |
| 609 | if (idx >= PHPOOL_MAX) { |
| 610 | /* |
| 611 | * if you see this panic, consider to tweak |
| 612 | * PHPOOL_MAX and PHPOOL_FREELIST_NELEM. |
| 613 | */ |
| 614 | panic("%s: too large itemsperpage(%d) for PR_NOTOUCH" , |
| 615 | pp->pr_wchan, pp->pr_itemsperpage); |
| 616 | } |
| 617 | pp->pr_phpool = &phpool[idx]; |
| 618 | } else if ((pp->pr_roflags & PR_PHINPAGE) == 0) { |
| 619 | pp->pr_phpool = &phpool[0]; |
| 620 | } |
| 621 | #if defined(DIAGNOSTIC) |
| 622 | else { |
| 623 | pp->pr_phpool = NULL; |
| 624 | } |
| 625 | #endif |
| 626 | |
| 627 | /* |
| 628 | * Use the slack between the chunks and the page header |
| 629 | * for "cache coloring". |
| 630 | */ |
| 631 | slack = off - pp->pr_itemsperpage * pp->pr_size; |
| 632 | pp->pr_maxcolor = (slack / align) * align; |
| 633 | pp->pr_curcolor = 0; |
| 634 | |
| 635 | pp->pr_nget = 0; |
| 636 | pp->pr_nfail = 0; |
| 637 | pp->pr_nput = 0; |
| 638 | pp->pr_npagealloc = 0; |
| 639 | pp->pr_npagefree = 0; |
| 640 | pp->pr_hiwat = 0; |
| 641 | pp->pr_nidle = 0; |
| 642 | pp->pr_refcnt = 0; |
| 643 | |
| 644 | mutex_init(&pp->pr_lock, MUTEX_DEFAULT, ipl); |
| 645 | cv_init(&pp->pr_cv, wchan); |
| 646 | pp->pr_ipl = ipl; |
| 647 | |
| 648 | /* Insert into the list of all pools. */ |
| 649 | if (!cold) |
| 650 | mutex_enter(&pool_head_lock); |
| 651 | TAILQ_FOREACH(pp1, &pool_head, pr_poollist) { |
| 652 | if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0) |
| 653 | break; |
| 654 | } |
| 655 | if (pp1 == NULL) |
| 656 | TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist); |
| 657 | else |
| 658 | TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist); |
| 659 | if (!cold) |
| 660 | mutex_exit(&pool_head_lock); |
| 661 | |
| 662 | /* Insert this into the list of pools using this allocator. */ |
| 663 | if (!cold) |
| 664 | mutex_enter(&palloc->pa_lock); |
| 665 | TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list); |
| 666 | if (!cold) |
| 667 | mutex_exit(&palloc->pa_lock); |
| 668 | } |
| 669 | |
| 670 | /* |
| 671 | * De-commision a pool resource. |
| 672 | */ |
| 673 | void |
| 674 | pool_destroy(struct pool *pp) |
| 675 | { |
| 676 | struct pool_pagelist pq; |
| 677 | struct pool_item_header *ph; |
| 678 | |
| 679 | /* Remove from global pool list */ |
| 680 | mutex_enter(&pool_head_lock); |
| 681 | while (pp->pr_refcnt != 0) |
| 682 | cv_wait(&pool_busy, &pool_head_lock); |
| 683 | TAILQ_REMOVE(&pool_head, pp, pr_poollist); |
| 684 | if (drainpp == pp) |
| 685 | drainpp = NULL; |
| 686 | mutex_exit(&pool_head_lock); |
| 687 | |
| 688 | /* Remove this pool from its allocator's list of pools. */ |
| 689 | mutex_enter(&pp->pr_alloc->pa_lock); |
| 690 | TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list); |
| 691 | mutex_exit(&pp->pr_alloc->pa_lock); |
| 692 | |
| 693 | mutex_enter(&pool_allocator_lock); |
| 694 | if (--pp->pr_alloc->pa_refcnt == 0) |
| 695 | mutex_destroy(&pp->pr_alloc->pa_lock); |
| 696 | mutex_exit(&pool_allocator_lock); |
| 697 | |
| 698 | mutex_enter(&pp->pr_lock); |
| 699 | |
| 700 | KASSERT(pp->pr_cache == NULL); |
| 701 | |
| 702 | #ifdef DIAGNOSTIC |
| 703 | if (pp->pr_nout != 0) { |
| 704 | panic("pool_destroy: pool busy: still out: %u" , |
| 705 | pp->pr_nout); |
| 706 | } |
| 707 | #endif |
| 708 | |
| 709 | KASSERT(LIST_EMPTY(&pp->pr_fullpages)); |
| 710 | KASSERT(LIST_EMPTY(&pp->pr_partpages)); |
| 711 | |
| 712 | /* Remove all pages */ |
| 713 | LIST_INIT(&pq); |
| 714 | while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) |
| 715 | pr_rmpage(pp, ph, &pq); |
| 716 | |
| 717 | mutex_exit(&pp->pr_lock); |
| 718 | |
| 719 | pr_pagelist_free(pp, &pq); |
| 720 | cv_destroy(&pp->pr_cv); |
| 721 | mutex_destroy(&pp->pr_lock); |
| 722 | } |
| 723 | |
| 724 | void |
| 725 | pool_set_drain_hook(struct pool *pp, void (*fn)(void *, int), void *arg) |
| 726 | { |
| 727 | |
| 728 | /* XXX no locking -- must be used just after pool_init() */ |
| 729 | #ifdef DIAGNOSTIC |
| 730 | if (pp->pr_drain_hook != NULL) |
| 731 | panic("pool_set_drain_hook(%s): already set" , pp->pr_wchan); |
| 732 | #endif |
| 733 | pp->pr_drain_hook = fn; |
| 734 | pp->pr_drain_hook_arg = arg; |
| 735 | } |
| 736 | |
| 737 | static struct pool_item_header * |
| 738 | (struct pool *pp, void *storage, int flags) |
| 739 | { |
| 740 | struct pool_item_header *ph; |
| 741 | |
| 742 | if ((pp->pr_roflags & PR_PHINPAGE) != 0) |
| 743 | ph = (struct pool_item_header *) ((char *)storage + pp->pr_phoffset); |
| 744 | else |
| 745 | ph = pool_get(pp->pr_phpool, flags); |
| 746 | |
| 747 | return (ph); |
| 748 | } |
| 749 | |
| 750 | /* |
| 751 | * Grab an item from the pool. |
| 752 | */ |
| 753 | void * |
| 754 | pool_get(struct pool *pp, int flags) |
| 755 | { |
| 756 | struct pool_item *pi; |
| 757 | struct pool_item_header *ph; |
| 758 | void *v; |
| 759 | |
| 760 | #ifdef DIAGNOSTIC |
| 761 | if (pp->pr_itemsperpage == 0) |
| 762 | panic("pool_get: pool '%s': pr_itemsperpage is zero, " |
| 763 | "pool not initialized?" , pp->pr_wchan); |
| 764 | if ((cpu_intr_p() || cpu_softintr_p()) && pp->pr_ipl == IPL_NONE && |
| 765 | !cold && panicstr == NULL) |
| 766 | panic("pool '%s' is IPL_NONE, but called from " |
| 767 | "interrupt context\n" , pp->pr_wchan); |
| 768 | #endif |
| 769 | if (flags & PR_WAITOK) { |
| 770 | ASSERT_SLEEPABLE(); |
| 771 | } |
| 772 | |
| 773 | mutex_enter(&pp->pr_lock); |
| 774 | startover: |
| 775 | /* |
| 776 | * Check to see if we've reached the hard limit. If we have, |
| 777 | * and we can wait, then wait until an item has been returned to |
| 778 | * the pool. |
| 779 | */ |
| 780 | #ifdef DIAGNOSTIC |
| 781 | if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) { |
| 782 | mutex_exit(&pp->pr_lock); |
| 783 | panic("pool_get: %s: crossed hard limit" , pp->pr_wchan); |
| 784 | } |
| 785 | #endif |
| 786 | if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) { |
| 787 | if (pp->pr_drain_hook != NULL) { |
| 788 | /* |
| 789 | * Since the drain hook is going to free things |
| 790 | * back to the pool, unlock, call the hook, re-lock, |
| 791 | * and check the hardlimit condition again. |
| 792 | */ |
| 793 | mutex_exit(&pp->pr_lock); |
| 794 | (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); |
| 795 | mutex_enter(&pp->pr_lock); |
| 796 | if (pp->pr_nout < pp->pr_hardlimit) |
| 797 | goto startover; |
| 798 | } |
| 799 | |
| 800 | if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) { |
| 801 | /* |
| 802 | * XXX: A warning isn't logged in this case. Should |
| 803 | * it be? |
| 804 | */ |
| 805 | pp->pr_flags |= PR_WANTED; |
| 806 | cv_wait(&pp->pr_cv, &pp->pr_lock); |
| 807 | goto startover; |
| 808 | } |
| 809 | |
| 810 | /* |
| 811 | * Log a message that the hard limit has been hit. |
| 812 | */ |
| 813 | if (pp->pr_hardlimit_warning != NULL && |
| 814 | ratecheck(&pp->pr_hardlimit_warning_last, |
| 815 | &pp->pr_hardlimit_ratecap)) |
| 816 | log(LOG_ERR, "%s\n" , pp->pr_hardlimit_warning); |
| 817 | |
| 818 | pp->pr_nfail++; |
| 819 | |
| 820 | mutex_exit(&pp->pr_lock); |
| 821 | return (NULL); |
| 822 | } |
| 823 | |
| 824 | /* |
| 825 | * The convention we use is that if `curpage' is not NULL, then |
| 826 | * it points at a non-empty bucket. In particular, `curpage' |
| 827 | * never points at a page header which has PR_PHINPAGE set and |
| 828 | * has no items in its bucket. |
| 829 | */ |
| 830 | if ((ph = pp->pr_curpage) == NULL) { |
| 831 | int error; |
| 832 | |
| 833 | #ifdef DIAGNOSTIC |
| 834 | if (pp->pr_nitems != 0) { |
| 835 | mutex_exit(&pp->pr_lock); |
| 836 | printf("pool_get: %s: curpage NULL, nitems %u\n" , |
| 837 | pp->pr_wchan, pp->pr_nitems); |
| 838 | panic("pool_get: nitems inconsistent" ); |
| 839 | } |
| 840 | #endif |
| 841 | |
| 842 | /* |
| 843 | * Call the back-end page allocator for more memory. |
| 844 | * Release the pool lock, as the back-end page allocator |
| 845 | * may block. |
| 846 | */ |
| 847 | error = pool_grow(pp, flags); |
| 848 | if (error != 0) { |
| 849 | /* |
| 850 | * We were unable to allocate a page or item |
| 851 | * header, but we released the lock during |
| 852 | * allocation, so perhaps items were freed |
| 853 | * back to the pool. Check for this case. |
| 854 | */ |
| 855 | if (pp->pr_curpage != NULL) |
| 856 | goto startover; |
| 857 | |
| 858 | pp->pr_nfail++; |
| 859 | mutex_exit(&pp->pr_lock); |
| 860 | return (NULL); |
| 861 | } |
| 862 | |
| 863 | /* Start the allocation process over. */ |
| 864 | goto startover; |
| 865 | } |
| 866 | if (pp->pr_roflags & PR_NOTOUCH) { |
| 867 | #ifdef DIAGNOSTIC |
| 868 | if (__predict_false(ph->ph_nmissing == pp->pr_itemsperpage)) { |
| 869 | mutex_exit(&pp->pr_lock); |
| 870 | panic("pool_get: %s: page empty" , pp->pr_wchan); |
| 871 | } |
| 872 | #endif |
| 873 | v = pr_item_notouch_get(pp, ph); |
| 874 | } else { |
| 875 | v = pi = LIST_FIRST(&ph->ph_itemlist); |
| 876 | if (__predict_false(v == NULL)) { |
| 877 | mutex_exit(&pp->pr_lock); |
| 878 | panic("pool_get: %s: page empty" , pp->pr_wchan); |
| 879 | } |
| 880 | #ifdef DIAGNOSTIC |
| 881 | if (__predict_false(pp->pr_nitems == 0)) { |
| 882 | mutex_exit(&pp->pr_lock); |
| 883 | printf("pool_get: %s: items on itemlist, nitems %u\n" , |
| 884 | pp->pr_wchan, pp->pr_nitems); |
| 885 | panic("pool_get: nitems inconsistent" ); |
| 886 | } |
| 887 | #endif |
| 888 | |
| 889 | #ifdef DIAGNOSTIC |
| 890 | if (__predict_false(pi->pi_magic != PI_MAGIC)) { |
| 891 | panic("pool_get(%s): free list modified: " |
| 892 | "magic=%x; page %p; item addr %p\n" , |
| 893 | pp->pr_wchan, pi->pi_magic, ph->ph_page, pi); |
| 894 | } |
| 895 | #endif |
| 896 | |
| 897 | /* |
| 898 | * Remove from item list. |
| 899 | */ |
| 900 | LIST_REMOVE(pi, pi_list); |
| 901 | } |
| 902 | pp->pr_nitems--; |
| 903 | pp->pr_nout++; |
| 904 | if (ph->ph_nmissing == 0) { |
| 905 | #ifdef DIAGNOSTIC |
| 906 | if (__predict_false(pp->pr_nidle == 0)) |
| 907 | panic("pool_get: nidle inconsistent" ); |
| 908 | #endif |
| 909 | pp->pr_nidle--; |
| 910 | |
| 911 | /* |
| 912 | * This page was previously empty. Move it to the list of |
| 913 | * partially-full pages. This page is already curpage. |
| 914 | */ |
| 915 | LIST_REMOVE(ph, ph_pagelist); |
| 916 | LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); |
| 917 | } |
| 918 | ph->ph_nmissing++; |
| 919 | if (ph->ph_nmissing == pp->pr_itemsperpage) { |
| 920 | #ifdef DIAGNOSTIC |
| 921 | if (__predict_false((pp->pr_roflags & PR_NOTOUCH) == 0 && |
| 922 | !LIST_EMPTY(&ph->ph_itemlist))) { |
| 923 | mutex_exit(&pp->pr_lock); |
| 924 | panic("pool_get: %s: nmissing inconsistent" , |
| 925 | pp->pr_wchan); |
| 926 | } |
| 927 | #endif |
| 928 | /* |
| 929 | * This page is now full. Move it to the full list |
| 930 | * and select a new current page. |
| 931 | */ |
| 932 | LIST_REMOVE(ph, ph_pagelist); |
| 933 | LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist); |
| 934 | pool_update_curpage(pp); |
| 935 | } |
| 936 | |
| 937 | pp->pr_nget++; |
| 938 | |
| 939 | /* |
| 940 | * If we have a low water mark and we are now below that low |
| 941 | * water mark, add more items to the pool. |
| 942 | */ |
| 943 | if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { |
| 944 | /* |
| 945 | * XXX: Should we log a warning? Should we set up a timeout |
| 946 | * to try again in a second or so? The latter could break |
| 947 | * a caller's assumptions about interrupt protection, etc. |
| 948 | */ |
| 949 | } |
| 950 | |
| 951 | mutex_exit(&pp->pr_lock); |
| 952 | KASSERT((((vaddr_t)v + pp->pr_itemoffset) & (pp->pr_align - 1)) == 0); |
| 953 | FREECHECK_OUT(&pp->pr_freecheck, v); |
| 954 | pool_redzone_fill(pp, v); |
| 955 | return (v); |
| 956 | } |
| 957 | |
| 958 | /* |
| 959 | * Internal version of pool_put(). Pool is already locked/entered. |
| 960 | */ |
| 961 | static void |
| 962 | pool_do_put(struct pool *pp, void *v, struct pool_pagelist *pq) |
| 963 | { |
| 964 | struct pool_item *pi = v; |
| 965 | struct pool_item_header *ph; |
| 966 | |
| 967 | KASSERT(mutex_owned(&pp->pr_lock)); |
| 968 | pool_redzone_check(pp, v); |
| 969 | FREECHECK_IN(&pp->pr_freecheck, v); |
| 970 | LOCKDEBUG_MEM_CHECK(v, pp->pr_size); |
| 971 | |
| 972 | #ifdef DIAGNOSTIC |
| 973 | if (__predict_false(pp->pr_nout == 0)) { |
| 974 | printf("pool %s: putting with none out\n" , |
| 975 | pp->pr_wchan); |
| 976 | panic("pool_put" ); |
| 977 | } |
| 978 | #endif |
| 979 | |
| 980 | if (__predict_false((ph = pr_find_pagehead(pp, v)) == NULL)) { |
| 981 | panic("pool_put: %s: page header missing" , pp->pr_wchan); |
| 982 | } |
| 983 | |
| 984 | /* |
| 985 | * Return to item list. |
| 986 | */ |
| 987 | if (pp->pr_roflags & PR_NOTOUCH) { |
| 988 | pr_item_notouch_put(pp, ph, v); |
| 989 | } else { |
| 990 | #ifdef DIAGNOSTIC |
| 991 | pi->pi_magic = PI_MAGIC; |
| 992 | #endif |
| 993 | #ifdef DEBUG |
| 994 | { |
| 995 | int i, *ip = v; |
| 996 | |
| 997 | for (i = 0; i < pp->pr_size / sizeof(int); i++) { |
| 998 | *ip++ = PI_MAGIC; |
| 999 | } |
| 1000 | } |
| 1001 | #endif |
| 1002 | |
| 1003 | LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); |
| 1004 | } |
| 1005 | KDASSERT(ph->ph_nmissing != 0); |
| 1006 | ph->ph_nmissing--; |
| 1007 | pp->pr_nput++; |
| 1008 | pp->pr_nitems++; |
| 1009 | pp->pr_nout--; |
| 1010 | |
| 1011 | /* Cancel "pool empty" condition if it exists */ |
| 1012 | if (pp->pr_curpage == NULL) |
| 1013 | pp->pr_curpage = ph; |
| 1014 | |
| 1015 | if (pp->pr_flags & PR_WANTED) { |
| 1016 | pp->pr_flags &= ~PR_WANTED; |
| 1017 | cv_broadcast(&pp->pr_cv); |
| 1018 | } |
| 1019 | |
| 1020 | /* |
| 1021 | * If this page is now empty, do one of two things: |
| 1022 | * |
| 1023 | * (1) If we have more pages than the page high water mark, |
| 1024 | * free the page back to the system. ONLY CONSIDER |
| 1025 | * FREEING BACK A PAGE IF WE HAVE MORE THAN OUR MINIMUM PAGE |
| 1026 | * CLAIM. |
| 1027 | * |
| 1028 | * (2) Otherwise, move the page to the empty page list. |
| 1029 | * |
| 1030 | * Either way, select a new current page (so we use a partially-full |
| 1031 | * page if one is available). |
| 1032 | */ |
| 1033 | if (ph->ph_nmissing == 0) { |
| 1034 | pp->pr_nidle++; |
| 1035 | if (pp->pr_npages > pp->pr_minpages && |
| 1036 | pp->pr_npages > pp->pr_maxpages) { |
| 1037 | pr_rmpage(pp, ph, pq); |
| 1038 | } else { |
| 1039 | LIST_REMOVE(ph, ph_pagelist); |
| 1040 | LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); |
| 1041 | |
| 1042 | /* |
| 1043 | * Update the timestamp on the page. A page must |
| 1044 | * be idle for some period of time before it can |
| 1045 | * be reclaimed by the pagedaemon. This minimizes |
| 1046 | * ping-pong'ing for memory. |
| 1047 | * |
| 1048 | * note for 64-bit time_t: truncating to 32-bit is not |
| 1049 | * a problem for our usage. |
| 1050 | */ |
| 1051 | ph->ph_time = time_uptime; |
| 1052 | } |
| 1053 | pool_update_curpage(pp); |
| 1054 | } |
| 1055 | |
| 1056 | /* |
| 1057 | * If the page was previously completely full, move it to the |
| 1058 | * partially-full list and make it the current page. The next |
| 1059 | * allocation will get the item from this page, instead of |
| 1060 | * further fragmenting the pool. |
| 1061 | */ |
| 1062 | else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) { |
| 1063 | LIST_REMOVE(ph, ph_pagelist); |
| 1064 | LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist); |
| 1065 | pp->pr_curpage = ph; |
| 1066 | } |
| 1067 | } |
| 1068 | |
| 1069 | void |
| 1070 | pool_put(struct pool *pp, void *v) |
| 1071 | { |
| 1072 | struct pool_pagelist pq; |
| 1073 | |
| 1074 | LIST_INIT(&pq); |
| 1075 | |
| 1076 | mutex_enter(&pp->pr_lock); |
| 1077 | pool_do_put(pp, v, &pq); |
| 1078 | mutex_exit(&pp->pr_lock); |
| 1079 | |
| 1080 | pr_pagelist_free(pp, &pq); |
| 1081 | } |
| 1082 | |
| 1083 | /* |
| 1084 | * pool_grow: grow a pool by a page. |
| 1085 | * |
| 1086 | * => called with pool locked. |
| 1087 | * => unlock and relock the pool. |
| 1088 | * => return with pool locked. |
| 1089 | */ |
| 1090 | |
| 1091 | static int |
| 1092 | pool_grow(struct pool *pp, int flags) |
| 1093 | { |
| 1094 | struct pool_item_header *ph = NULL; |
| 1095 | char *cp; |
| 1096 | |
| 1097 | mutex_exit(&pp->pr_lock); |
| 1098 | cp = pool_allocator_alloc(pp, flags); |
| 1099 | if (__predict_true(cp != NULL)) { |
| 1100 | ph = pool_alloc_item_header(pp, cp, flags); |
| 1101 | } |
| 1102 | if (__predict_false(cp == NULL || ph == NULL)) { |
| 1103 | if (cp != NULL) { |
| 1104 | pool_allocator_free(pp, cp); |
| 1105 | } |
| 1106 | mutex_enter(&pp->pr_lock); |
| 1107 | return ENOMEM; |
| 1108 | } |
| 1109 | |
| 1110 | mutex_enter(&pp->pr_lock); |
| 1111 | pool_prime_page(pp, cp, ph); |
| 1112 | pp->pr_npagealloc++; |
| 1113 | return 0; |
| 1114 | } |
| 1115 | |
| 1116 | /* |
| 1117 | * Add N items to the pool. |
| 1118 | */ |
| 1119 | int |
| 1120 | pool_prime(struct pool *pp, int n) |
| 1121 | { |
| 1122 | int newpages; |
| 1123 | int error = 0; |
| 1124 | |
| 1125 | mutex_enter(&pp->pr_lock); |
| 1126 | |
| 1127 | newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
| 1128 | |
| 1129 | while (newpages-- > 0) { |
| 1130 | error = pool_grow(pp, PR_NOWAIT); |
| 1131 | if (error) { |
| 1132 | break; |
| 1133 | } |
| 1134 | pp->pr_minpages++; |
| 1135 | } |
| 1136 | |
| 1137 | if (pp->pr_minpages >= pp->pr_maxpages) |
| 1138 | pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */ |
| 1139 | |
| 1140 | mutex_exit(&pp->pr_lock); |
| 1141 | return error; |
| 1142 | } |
| 1143 | |
| 1144 | /* |
| 1145 | * Add a page worth of items to the pool. |
| 1146 | * |
| 1147 | * Note, we must be called with the pool descriptor LOCKED. |
| 1148 | */ |
| 1149 | static void |
| 1150 | pool_prime_page(struct pool *pp, void *storage, struct pool_item_header *ph) |
| 1151 | { |
| 1152 | struct pool_item *pi; |
| 1153 | void *cp = storage; |
| 1154 | const unsigned int align = pp->pr_align; |
| 1155 | const unsigned int ioff = pp->pr_itemoffset; |
| 1156 | int n; |
| 1157 | |
| 1158 | KASSERT(mutex_owned(&pp->pr_lock)); |
| 1159 | |
| 1160 | #ifdef DIAGNOSTIC |
| 1161 | if ((pp->pr_roflags & PR_NOALIGN) == 0 && |
| 1162 | ((uintptr_t)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0) |
| 1163 | panic("pool_prime_page: %s: unaligned page" , pp->pr_wchan); |
| 1164 | #endif |
| 1165 | |
| 1166 | /* |
| 1167 | * Insert page header. |
| 1168 | */ |
| 1169 | LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist); |
| 1170 | LIST_INIT(&ph->ph_itemlist); |
| 1171 | ph->ph_page = storage; |
| 1172 | ph->ph_nmissing = 0; |
| 1173 | ph->ph_time = time_uptime; |
| 1174 | if ((pp->pr_roflags & PR_PHINPAGE) == 0) |
| 1175 | SPLAY_INSERT(phtree, &pp->pr_phtree, ph); |
| 1176 | |
| 1177 | pp->pr_nidle++; |
| 1178 | |
| 1179 | /* |
| 1180 | * Color this page. |
| 1181 | */ |
| 1182 | ph->ph_off = pp->pr_curcolor; |
| 1183 | cp = (char *)cp + ph->ph_off; |
| 1184 | if ((pp->pr_curcolor += align) > pp->pr_maxcolor) |
| 1185 | pp->pr_curcolor = 0; |
| 1186 | |
| 1187 | /* |
| 1188 | * Adjust storage to apply aligment to `pr_itemoffset' in each item. |
| 1189 | */ |
| 1190 | if (ioff != 0) |
| 1191 | cp = (char *)cp + align - ioff; |
| 1192 | |
| 1193 | KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); |
| 1194 | |
| 1195 | /* |
| 1196 | * Insert remaining chunks on the bucket list. |
| 1197 | */ |
| 1198 | n = pp->pr_itemsperpage; |
| 1199 | pp->pr_nitems += n; |
| 1200 | |
| 1201 | if (pp->pr_roflags & PR_NOTOUCH) { |
| 1202 | pr_item_notouch_init(pp, ph); |
| 1203 | } else { |
| 1204 | while (n--) { |
| 1205 | pi = (struct pool_item *)cp; |
| 1206 | |
| 1207 | KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0); |
| 1208 | |
| 1209 | /* Insert on page list */ |
| 1210 | LIST_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list); |
| 1211 | #ifdef DIAGNOSTIC |
| 1212 | pi->pi_magic = PI_MAGIC; |
| 1213 | #endif |
| 1214 | cp = (char *)cp + pp->pr_size; |
| 1215 | |
| 1216 | KASSERT((((vaddr_t)cp + ioff) & (align - 1)) == 0); |
| 1217 | } |
| 1218 | } |
| 1219 | |
| 1220 | /* |
| 1221 | * If the pool was depleted, point at the new page. |
| 1222 | */ |
| 1223 | if (pp->pr_curpage == NULL) |
| 1224 | pp->pr_curpage = ph; |
| 1225 | |
| 1226 | if (++pp->pr_npages > pp->pr_hiwat) |
| 1227 | pp->pr_hiwat = pp->pr_npages; |
| 1228 | } |
| 1229 | |
| 1230 | /* |
| 1231 | * Used by pool_get() when nitems drops below the low water mark. This |
| 1232 | * is used to catch up pr_nitems with the low water mark. |
| 1233 | * |
| 1234 | * Note 1, we never wait for memory here, we let the caller decide what to do. |
| 1235 | * |
| 1236 | * Note 2, we must be called with the pool already locked, and we return |
| 1237 | * with it locked. |
| 1238 | */ |
| 1239 | static int |
| 1240 | pool_catchup(struct pool *pp) |
| 1241 | { |
| 1242 | int error = 0; |
| 1243 | |
| 1244 | while (POOL_NEEDS_CATCHUP(pp)) { |
| 1245 | error = pool_grow(pp, PR_NOWAIT); |
| 1246 | if (error) { |
| 1247 | break; |
| 1248 | } |
| 1249 | } |
| 1250 | return error; |
| 1251 | } |
| 1252 | |
| 1253 | static void |
| 1254 | pool_update_curpage(struct pool *pp) |
| 1255 | { |
| 1256 | |
| 1257 | pp->pr_curpage = LIST_FIRST(&pp->pr_partpages); |
| 1258 | if (pp->pr_curpage == NULL) { |
| 1259 | pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages); |
| 1260 | } |
| 1261 | KASSERT((pp->pr_curpage == NULL && pp->pr_nitems == 0) || |
| 1262 | (pp->pr_curpage != NULL && pp->pr_nitems > 0)); |
| 1263 | } |
| 1264 | |
| 1265 | void |
| 1266 | pool_setlowat(struct pool *pp, int n) |
| 1267 | { |
| 1268 | |
| 1269 | mutex_enter(&pp->pr_lock); |
| 1270 | |
| 1271 | pp->pr_minitems = n; |
| 1272 | pp->pr_minpages = (n == 0) |
| 1273 | ? 0 |
| 1274 | : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
| 1275 | |
| 1276 | /* Make sure we're caught up with the newly-set low water mark. */ |
| 1277 | if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) { |
| 1278 | /* |
| 1279 | * XXX: Should we log a warning? Should we set up a timeout |
| 1280 | * to try again in a second or so? The latter could break |
| 1281 | * a caller's assumptions about interrupt protection, etc. |
| 1282 | */ |
| 1283 | } |
| 1284 | |
| 1285 | mutex_exit(&pp->pr_lock); |
| 1286 | } |
| 1287 | |
| 1288 | void |
| 1289 | pool_sethiwat(struct pool *pp, int n) |
| 1290 | { |
| 1291 | |
| 1292 | mutex_enter(&pp->pr_lock); |
| 1293 | |
| 1294 | pp->pr_maxpages = (n == 0) |
| 1295 | ? 0 |
| 1296 | : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
| 1297 | |
| 1298 | mutex_exit(&pp->pr_lock); |
| 1299 | } |
| 1300 | |
| 1301 | void |
| 1302 | pool_sethardlimit(struct pool *pp, int n, const char *warnmess, int ratecap) |
| 1303 | { |
| 1304 | |
| 1305 | mutex_enter(&pp->pr_lock); |
| 1306 | |
| 1307 | pp->pr_hardlimit = n; |
| 1308 | pp->pr_hardlimit_warning = warnmess; |
| 1309 | pp->pr_hardlimit_ratecap.tv_sec = ratecap; |
| 1310 | pp->pr_hardlimit_warning_last.tv_sec = 0; |
| 1311 | pp->pr_hardlimit_warning_last.tv_usec = 0; |
| 1312 | |
| 1313 | /* |
| 1314 | * In-line version of pool_sethiwat(), because we don't want to |
| 1315 | * release the lock. |
| 1316 | */ |
| 1317 | pp->pr_maxpages = (n == 0) |
| 1318 | ? 0 |
| 1319 | : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage; |
| 1320 | |
| 1321 | mutex_exit(&pp->pr_lock); |
| 1322 | } |
| 1323 | |
| 1324 | /* |
| 1325 | * Release all complete pages that have not been used recently. |
| 1326 | * |
| 1327 | * Must not be called from interrupt context. |
| 1328 | */ |
| 1329 | int |
| 1330 | pool_reclaim(struct pool *pp) |
| 1331 | { |
| 1332 | struct pool_item_header *ph, *phnext; |
| 1333 | struct pool_pagelist pq; |
| 1334 | uint32_t curtime; |
| 1335 | bool klock; |
| 1336 | int rv; |
| 1337 | |
| 1338 | KASSERT(!cpu_intr_p() && !cpu_softintr_p()); |
| 1339 | |
| 1340 | if (pp->pr_drain_hook != NULL) { |
| 1341 | /* |
| 1342 | * The drain hook must be called with the pool unlocked. |
| 1343 | */ |
| 1344 | (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, PR_NOWAIT); |
| 1345 | } |
| 1346 | |
| 1347 | /* |
| 1348 | * XXXSMP Because we do not want to cause non-MPSAFE code |
| 1349 | * to block. |
| 1350 | */ |
| 1351 | if (pp->pr_ipl == IPL_SOFTNET || pp->pr_ipl == IPL_SOFTCLOCK || |
| 1352 | pp->pr_ipl == IPL_SOFTSERIAL) { |
| 1353 | KERNEL_LOCK(1, NULL); |
| 1354 | klock = true; |
| 1355 | } else |
| 1356 | klock = false; |
| 1357 | |
| 1358 | /* Reclaim items from the pool's cache (if any). */ |
| 1359 | if (pp->pr_cache != NULL) |
| 1360 | pool_cache_invalidate(pp->pr_cache); |
| 1361 | |
| 1362 | if (mutex_tryenter(&pp->pr_lock) == 0) { |
| 1363 | if (klock) { |
| 1364 | KERNEL_UNLOCK_ONE(NULL); |
| 1365 | } |
| 1366 | return (0); |
| 1367 | } |
| 1368 | |
| 1369 | LIST_INIT(&pq); |
| 1370 | |
| 1371 | curtime = time_uptime; |
| 1372 | |
| 1373 | for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) { |
| 1374 | phnext = LIST_NEXT(ph, ph_pagelist); |
| 1375 | |
| 1376 | /* Check our minimum page claim */ |
| 1377 | if (pp->pr_npages <= pp->pr_minpages) |
| 1378 | break; |
| 1379 | |
| 1380 | KASSERT(ph->ph_nmissing == 0); |
| 1381 | if (curtime - ph->ph_time < pool_inactive_time) |
| 1382 | continue; |
| 1383 | |
| 1384 | /* |
| 1385 | * If freeing this page would put us below |
| 1386 | * the low water mark, stop now. |
| 1387 | */ |
| 1388 | if ((pp->pr_nitems - pp->pr_itemsperpage) < |
| 1389 | pp->pr_minitems) |
| 1390 | break; |
| 1391 | |
| 1392 | pr_rmpage(pp, ph, &pq); |
| 1393 | } |
| 1394 | |
| 1395 | mutex_exit(&pp->pr_lock); |
| 1396 | |
| 1397 | if (LIST_EMPTY(&pq)) |
| 1398 | rv = 0; |
| 1399 | else { |
| 1400 | pr_pagelist_free(pp, &pq); |
| 1401 | rv = 1; |
| 1402 | } |
| 1403 | |
| 1404 | if (klock) { |
| 1405 | KERNEL_UNLOCK_ONE(NULL); |
| 1406 | } |
| 1407 | |
| 1408 | return (rv); |
| 1409 | } |
| 1410 | |
| 1411 | /* |
| 1412 | * Drain pools, one at a time. The drained pool is returned within ppp. |
| 1413 | * |
| 1414 | * Note, must never be called from interrupt context. |
| 1415 | */ |
| 1416 | bool |
| 1417 | pool_drain(struct pool **ppp) |
| 1418 | { |
| 1419 | bool reclaimed; |
| 1420 | struct pool *pp; |
| 1421 | |
| 1422 | KASSERT(!TAILQ_EMPTY(&pool_head)); |
| 1423 | |
| 1424 | pp = NULL; |
| 1425 | |
| 1426 | /* Find next pool to drain, and add a reference. */ |
| 1427 | mutex_enter(&pool_head_lock); |
| 1428 | do { |
| 1429 | if (drainpp == NULL) { |
| 1430 | drainpp = TAILQ_FIRST(&pool_head); |
| 1431 | } |
| 1432 | if (drainpp != NULL) { |
| 1433 | pp = drainpp; |
| 1434 | drainpp = TAILQ_NEXT(pp, pr_poollist); |
| 1435 | } |
| 1436 | /* |
| 1437 | * Skip completely idle pools. We depend on at least |
| 1438 | * one pool in the system being active. |
| 1439 | */ |
| 1440 | } while (pp == NULL || pp->pr_npages == 0); |
| 1441 | pp->pr_refcnt++; |
| 1442 | mutex_exit(&pool_head_lock); |
| 1443 | |
| 1444 | /* Drain the cache (if any) and pool.. */ |
| 1445 | reclaimed = pool_reclaim(pp); |
| 1446 | |
| 1447 | /* Finally, unlock the pool. */ |
| 1448 | mutex_enter(&pool_head_lock); |
| 1449 | pp->pr_refcnt--; |
| 1450 | cv_broadcast(&pool_busy); |
| 1451 | mutex_exit(&pool_head_lock); |
| 1452 | |
| 1453 | if (ppp != NULL) |
| 1454 | *ppp = pp; |
| 1455 | |
| 1456 | return reclaimed; |
| 1457 | } |
| 1458 | |
| 1459 | /* |
| 1460 | * Diagnostic helpers. |
| 1461 | */ |
| 1462 | |
| 1463 | void |
| 1464 | pool_printall(const char *modif, void (*pr)(const char *, ...)) |
| 1465 | { |
| 1466 | struct pool *pp; |
| 1467 | |
| 1468 | TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
| 1469 | pool_printit(pp, modif, pr); |
| 1470 | } |
| 1471 | } |
| 1472 | |
| 1473 | void |
| 1474 | pool_printit(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) |
| 1475 | { |
| 1476 | |
| 1477 | if (pp == NULL) { |
| 1478 | (*pr)("Must specify a pool to print.\n" ); |
| 1479 | return; |
| 1480 | } |
| 1481 | |
| 1482 | pool_print1(pp, modif, pr); |
| 1483 | } |
| 1484 | |
| 1485 | static void |
| 1486 | pool_print_pagelist(struct pool *pp, struct pool_pagelist *pl, |
| 1487 | void (*pr)(const char *, ...)) |
| 1488 | { |
| 1489 | struct pool_item_header *ph; |
| 1490 | #ifdef DIAGNOSTIC |
| 1491 | struct pool_item *pi; |
| 1492 | #endif |
| 1493 | |
| 1494 | LIST_FOREACH(ph, pl, ph_pagelist) { |
| 1495 | (*pr)("\t\tpage %p, nmissing %d, time %" PRIu32 "\n" , |
| 1496 | ph->ph_page, ph->ph_nmissing, ph->ph_time); |
| 1497 | #ifdef DIAGNOSTIC |
| 1498 | if (!(pp->pr_roflags & PR_NOTOUCH)) { |
| 1499 | LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { |
| 1500 | if (pi->pi_magic != PI_MAGIC) { |
| 1501 | (*pr)("\t\t\titem %p, magic 0x%x\n" , |
| 1502 | pi, pi->pi_magic); |
| 1503 | } |
| 1504 | } |
| 1505 | } |
| 1506 | #endif |
| 1507 | } |
| 1508 | } |
| 1509 | |
| 1510 | static void |
| 1511 | pool_print1(struct pool *pp, const char *modif, void (*pr)(const char *, ...)) |
| 1512 | { |
| 1513 | struct pool_item_header *ph; |
| 1514 | pool_cache_t pc; |
| 1515 | pcg_t *pcg; |
| 1516 | pool_cache_cpu_t *cc; |
| 1517 | uint64_t cpuhit, cpumiss; |
| 1518 | int i, print_log = 0, print_pagelist = 0, print_cache = 0; |
| 1519 | char c; |
| 1520 | |
| 1521 | while ((c = *modif++) != '\0') { |
| 1522 | if (c == 'l') |
| 1523 | print_log = 1; |
| 1524 | if (c == 'p') |
| 1525 | print_pagelist = 1; |
| 1526 | if (c == 'c') |
| 1527 | print_cache = 1; |
| 1528 | } |
| 1529 | |
| 1530 | if ((pc = pp->pr_cache) != NULL) { |
| 1531 | (*pr)("POOL CACHE" ); |
| 1532 | } else { |
| 1533 | (*pr)("POOL" ); |
| 1534 | } |
| 1535 | |
| 1536 | (*pr)(" %s: size %u, align %u, ioff %u, roflags 0x%08x\n" , |
| 1537 | pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset, |
| 1538 | pp->pr_roflags); |
| 1539 | (*pr)("\talloc %p\n" , pp->pr_alloc); |
| 1540 | (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n" , |
| 1541 | pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages); |
| 1542 | (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n" , |
| 1543 | pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit); |
| 1544 | |
| 1545 | (*pr)("\tnget %lu, nfail %lu, nput %lu\n" , |
| 1546 | pp->pr_nget, pp->pr_nfail, pp->pr_nput); |
| 1547 | (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n" , |
| 1548 | pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle); |
| 1549 | |
| 1550 | if (print_pagelist == 0) |
| 1551 | goto skip_pagelist; |
| 1552 | |
| 1553 | if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL) |
| 1554 | (*pr)("\n\tempty page list:\n" ); |
| 1555 | pool_print_pagelist(pp, &pp->pr_emptypages, pr); |
| 1556 | if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL) |
| 1557 | (*pr)("\n\tfull page list:\n" ); |
| 1558 | pool_print_pagelist(pp, &pp->pr_fullpages, pr); |
| 1559 | if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL) |
| 1560 | (*pr)("\n\tpartial-page list:\n" ); |
| 1561 | pool_print_pagelist(pp, &pp->pr_partpages, pr); |
| 1562 | |
| 1563 | if (pp->pr_curpage == NULL) |
| 1564 | (*pr)("\tno current page\n" ); |
| 1565 | else |
| 1566 | (*pr)("\tcurpage %p\n" , pp->pr_curpage->ph_page); |
| 1567 | |
| 1568 | skip_pagelist: |
| 1569 | if (print_log == 0) |
| 1570 | goto skip_log; |
| 1571 | |
| 1572 | (*pr)("\n" ); |
| 1573 | |
| 1574 | skip_log: |
| 1575 | |
| 1576 | #define PR_GROUPLIST(pcg) \ |
| 1577 | (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail); \ |
| 1578 | for (i = 0; i < pcg->pcg_size; i++) { \ |
| 1579 | if (pcg->pcg_objects[i].pcgo_pa != \ |
| 1580 | POOL_PADDR_INVALID) { \ |
| 1581 | (*pr)("\t\t\t%p, 0x%llx\n", \ |
| 1582 | pcg->pcg_objects[i].pcgo_va, \ |
| 1583 | (unsigned long long) \ |
| 1584 | pcg->pcg_objects[i].pcgo_pa); \ |
| 1585 | } else { \ |
| 1586 | (*pr)("\t\t\t%p\n", \ |
| 1587 | pcg->pcg_objects[i].pcgo_va); \ |
| 1588 | } \ |
| 1589 | } |
| 1590 | |
| 1591 | if (pc != NULL) { |
| 1592 | cpuhit = 0; |
| 1593 | cpumiss = 0; |
| 1594 | for (i = 0; i < __arraycount(pc->pc_cpus); i++) { |
| 1595 | if ((cc = pc->pc_cpus[i]) == NULL) |
| 1596 | continue; |
| 1597 | cpuhit += cc->cc_hits; |
| 1598 | cpumiss += cc->cc_misses; |
| 1599 | } |
| 1600 | (*pr)("\tcpu layer hits %llu misses %llu\n" , cpuhit, cpumiss); |
| 1601 | (*pr)("\tcache layer hits %llu misses %llu\n" , |
| 1602 | pc->pc_hits, pc->pc_misses); |
| 1603 | (*pr)("\tcache layer entry uncontended %llu contended %llu\n" , |
| 1604 | pc->pc_hits + pc->pc_misses - pc->pc_contended, |
| 1605 | pc->pc_contended); |
| 1606 | (*pr)("\tcache layer empty groups %u full groups %u\n" , |
| 1607 | pc->pc_nempty, pc->pc_nfull); |
| 1608 | if (print_cache) { |
| 1609 | (*pr)("\tfull cache groups:\n" ); |
| 1610 | for (pcg = pc->pc_fullgroups; pcg != NULL; |
| 1611 | pcg = pcg->pcg_next) { |
| 1612 | PR_GROUPLIST(pcg); |
| 1613 | } |
| 1614 | (*pr)("\tempty cache groups:\n" ); |
| 1615 | for (pcg = pc->pc_emptygroups; pcg != NULL; |
| 1616 | pcg = pcg->pcg_next) { |
| 1617 | PR_GROUPLIST(pcg); |
| 1618 | } |
| 1619 | } |
| 1620 | } |
| 1621 | #undef PR_GROUPLIST |
| 1622 | } |
| 1623 | |
| 1624 | static int |
| 1625 | pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph) |
| 1626 | { |
| 1627 | struct pool_item *pi; |
| 1628 | void *page; |
| 1629 | int n; |
| 1630 | |
| 1631 | if ((pp->pr_roflags & PR_NOALIGN) == 0) { |
| 1632 | page = (void *)((uintptr_t)ph & pp->pr_alloc->pa_pagemask); |
| 1633 | if (page != ph->ph_page && |
| 1634 | (pp->pr_roflags & PR_PHINPAGE) != 0) { |
| 1635 | if (label != NULL) |
| 1636 | printf("%s: " , label); |
| 1637 | printf("pool(%p:%s): page inconsistency: page %p;" |
| 1638 | " at page head addr %p (p %p)\n" , pp, |
| 1639 | pp->pr_wchan, ph->ph_page, |
| 1640 | ph, page); |
| 1641 | return 1; |
| 1642 | } |
| 1643 | } |
| 1644 | |
| 1645 | if ((pp->pr_roflags & PR_NOTOUCH) != 0) |
| 1646 | return 0; |
| 1647 | |
| 1648 | for (pi = LIST_FIRST(&ph->ph_itemlist), n = 0; |
| 1649 | pi != NULL; |
| 1650 | pi = LIST_NEXT(pi,pi_list), n++) { |
| 1651 | |
| 1652 | #ifdef DIAGNOSTIC |
| 1653 | if (pi->pi_magic != PI_MAGIC) { |
| 1654 | if (label != NULL) |
| 1655 | printf("%s: " , label); |
| 1656 | printf("pool(%s): free list modified: magic=%x;" |
| 1657 | " page %p; item ordinal %d; addr %p\n" , |
| 1658 | pp->pr_wchan, pi->pi_magic, ph->ph_page, |
| 1659 | n, pi); |
| 1660 | panic("pool" ); |
| 1661 | } |
| 1662 | #endif |
| 1663 | if ((pp->pr_roflags & PR_NOALIGN) != 0) { |
| 1664 | continue; |
| 1665 | } |
| 1666 | page = (void *)((uintptr_t)pi & pp->pr_alloc->pa_pagemask); |
| 1667 | if (page == ph->ph_page) |
| 1668 | continue; |
| 1669 | |
| 1670 | if (label != NULL) |
| 1671 | printf("%s: " , label); |
| 1672 | printf("pool(%p:%s): page inconsistency: page %p;" |
| 1673 | " item ordinal %d; addr %p (p %p)\n" , pp, |
| 1674 | pp->pr_wchan, ph->ph_page, |
| 1675 | n, pi, page); |
| 1676 | return 1; |
| 1677 | } |
| 1678 | return 0; |
| 1679 | } |
| 1680 | |
| 1681 | |
| 1682 | int |
| 1683 | pool_chk(struct pool *pp, const char *label) |
| 1684 | { |
| 1685 | struct pool_item_header *ph; |
| 1686 | int r = 0; |
| 1687 | |
| 1688 | mutex_enter(&pp->pr_lock); |
| 1689 | LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { |
| 1690 | r = pool_chk_page(pp, label, ph); |
| 1691 | if (r) { |
| 1692 | goto out; |
| 1693 | } |
| 1694 | } |
| 1695 | LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
| 1696 | r = pool_chk_page(pp, label, ph); |
| 1697 | if (r) { |
| 1698 | goto out; |
| 1699 | } |
| 1700 | } |
| 1701 | LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
| 1702 | r = pool_chk_page(pp, label, ph); |
| 1703 | if (r) { |
| 1704 | goto out; |
| 1705 | } |
| 1706 | } |
| 1707 | |
| 1708 | out: |
| 1709 | mutex_exit(&pp->pr_lock); |
| 1710 | return (r); |
| 1711 | } |
| 1712 | |
| 1713 | /* |
| 1714 | * pool_cache_init: |
| 1715 | * |
| 1716 | * Initialize a pool cache. |
| 1717 | */ |
| 1718 | pool_cache_t |
| 1719 | pool_cache_init(size_t size, u_int align, u_int align_offset, u_int flags, |
| 1720 | const char *wchan, struct pool_allocator *palloc, int ipl, |
| 1721 | int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), void *arg) |
| 1722 | { |
| 1723 | pool_cache_t pc; |
| 1724 | |
| 1725 | pc = pool_get(&cache_pool, PR_WAITOK); |
| 1726 | if (pc == NULL) |
| 1727 | return NULL; |
| 1728 | |
| 1729 | pool_cache_bootstrap(pc, size, align, align_offset, flags, wchan, |
| 1730 | palloc, ipl, ctor, dtor, arg); |
| 1731 | |
| 1732 | return pc; |
| 1733 | } |
| 1734 | |
| 1735 | /* |
| 1736 | * pool_cache_bootstrap: |
| 1737 | * |
| 1738 | * Kernel-private version of pool_cache_init(). The caller |
| 1739 | * provides initial storage. |
| 1740 | */ |
| 1741 | void |
| 1742 | pool_cache_bootstrap(pool_cache_t pc, size_t size, u_int align, |
| 1743 | u_int align_offset, u_int flags, const char *wchan, |
| 1744 | struct pool_allocator *palloc, int ipl, |
| 1745 | int (*ctor)(void *, void *, int), void (*dtor)(void *, void *), |
| 1746 | void *arg) |
| 1747 | { |
| 1748 | CPU_INFO_ITERATOR cii; |
| 1749 | pool_cache_t pc1; |
| 1750 | struct cpu_info *ci; |
| 1751 | struct pool *pp; |
| 1752 | |
| 1753 | pp = &pc->pc_pool; |
| 1754 | if (palloc == NULL && ipl == IPL_NONE) |
| 1755 | palloc = &pool_allocator_nointr; |
| 1756 | pool_init(pp, size, align, align_offset, flags, wchan, palloc, ipl); |
| 1757 | mutex_init(&pc->pc_lock, MUTEX_DEFAULT, ipl); |
| 1758 | |
| 1759 | if (ctor == NULL) { |
| 1760 | ctor = (int (*)(void *, void *, int))nullop; |
| 1761 | } |
| 1762 | if (dtor == NULL) { |
| 1763 | dtor = (void (*)(void *, void *))nullop; |
| 1764 | } |
| 1765 | |
| 1766 | pc->pc_emptygroups = NULL; |
| 1767 | pc->pc_fullgroups = NULL; |
| 1768 | pc->pc_partgroups = NULL; |
| 1769 | pc->pc_ctor = ctor; |
| 1770 | pc->pc_dtor = dtor; |
| 1771 | pc->pc_arg = arg; |
| 1772 | pc->pc_hits = 0; |
| 1773 | pc->pc_misses = 0; |
| 1774 | pc->pc_nempty = 0; |
| 1775 | pc->pc_npart = 0; |
| 1776 | pc->pc_nfull = 0; |
| 1777 | pc->pc_contended = 0; |
| 1778 | pc->pc_refcnt = 0; |
| 1779 | pc->pc_freecheck = NULL; |
| 1780 | |
| 1781 | if ((flags & PR_LARGECACHE) != 0) { |
| 1782 | pc->pc_pcgsize = PCG_NOBJECTS_LARGE; |
| 1783 | pc->pc_pcgpool = &pcg_large_pool; |
| 1784 | } else { |
| 1785 | pc->pc_pcgsize = PCG_NOBJECTS_NORMAL; |
| 1786 | pc->pc_pcgpool = &pcg_normal_pool; |
| 1787 | } |
| 1788 | |
| 1789 | /* Allocate per-CPU caches. */ |
| 1790 | memset(pc->pc_cpus, 0, sizeof(pc->pc_cpus)); |
| 1791 | pc->pc_ncpu = 0; |
| 1792 | if (ncpu < 2) { |
| 1793 | /* XXX For sparc: boot CPU is not attached yet. */ |
| 1794 | pool_cache_cpu_init1(curcpu(), pc); |
| 1795 | } else { |
| 1796 | for (CPU_INFO_FOREACH(cii, ci)) { |
| 1797 | pool_cache_cpu_init1(ci, pc); |
| 1798 | } |
| 1799 | } |
| 1800 | |
| 1801 | /* Add to list of all pools. */ |
| 1802 | if (__predict_true(!cold)) |
| 1803 | mutex_enter(&pool_head_lock); |
| 1804 | TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) { |
| 1805 | if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0) |
| 1806 | break; |
| 1807 | } |
| 1808 | if (pc1 == NULL) |
| 1809 | TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist); |
| 1810 | else |
| 1811 | TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist); |
| 1812 | if (__predict_true(!cold)) |
| 1813 | mutex_exit(&pool_head_lock); |
| 1814 | |
| 1815 | membar_sync(); |
| 1816 | pp->pr_cache = pc; |
| 1817 | } |
| 1818 | |
| 1819 | /* |
| 1820 | * pool_cache_destroy: |
| 1821 | * |
| 1822 | * Destroy a pool cache. |
| 1823 | */ |
| 1824 | void |
| 1825 | pool_cache_destroy(pool_cache_t pc) |
| 1826 | { |
| 1827 | |
| 1828 | pool_cache_bootstrap_destroy(pc); |
| 1829 | pool_put(&cache_pool, pc); |
| 1830 | } |
| 1831 | |
| 1832 | /* |
| 1833 | * pool_cache_bootstrap_destroy: |
| 1834 | * |
| 1835 | * Destroy a pool cache. |
| 1836 | */ |
| 1837 | void |
| 1838 | pool_cache_bootstrap_destroy(pool_cache_t pc) |
| 1839 | { |
| 1840 | struct pool *pp = &pc->pc_pool; |
| 1841 | u_int i; |
| 1842 | |
| 1843 | /* Remove it from the global list. */ |
| 1844 | mutex_enter(&pool_head_lock); |
| 1845 | while (pc->pc_refcnt != 0) |
| 1846 | cv_wait(&pool_busy, &pool_head_lock); |
| 1847 | TAILQ_REMOVE(&pool_cache_head, pc, pc_cachelist); |
| 1848 | mutex_exit(&pool_head_lock); |
| 1849 | |
| 1850 | /* First, invalidate the entire cache. */ |
| 1851 | pool_cache_invalidate(pc); |
| 1852 | |
| 1853 | /* Disassociate it from the pool. */ |
| 1854 | mutex_enter(&pp->pr_lock); |
| 1855 | pp->pr_cache = NULL; |
| 1856 | mutex_exit(&pp->pr_lock); |
| 1857 | |
| 1858 | /* Destroy per-CPU data */ |
| 1859 | for (i = 0; i < __arraycount(pc->pc_cpus); i++) |
| 1860 | pool_cache_invalidate_cpu(pc, i); |
| 1861 | |
| 1862 | /* Finally, destroy it. */ |
| 1863 | mutex_destroy(&pc->pc_lock); |
| 1864 | pool_destroy(pp); |
| 1865 | } |
| 1866 | |
| 1867 | /* |
| 1868 | * pool_cache_cpu_init1: |
| 1869 | * |
| 1870 | * Called for each pool_cache whenever a new CPU is attached. |
| 1871 | */ |
| 1872 | static void |
| 1873 | pool_cache_cpu_init1(struct cpu_info *ci, pool_cache_t pc) |
| 1874 | { |
| 1875 | pool_cache_cpu_t *cc; |
| 1876 | int index; |
| 1877 | |
| 1878 | index = ci->ci_index; |
| 1879 | |
| 1880 | KASSERT(index < __arraycount(pc->pc_cpus)); |
| 1881 | |
| 1882 | if ((cc = pc->pc_cpus[index]) != NULL) { |
| 1883 | KASSERT(cc->cc_cpuindex == index); |
| 1884 | return; |
| 1885 | } |
| 1886 | |
| 1887 | /* |
| 1888 | * The first CPU is 'free'. This needs to be the case for |
| 1889 | * bootstrap - we may not be able to allocate yet. |
| 1890 | */ |
| 1891 | if (pc->pc_ncpu == 0) { |
| 1892 | cc = &pc->pc_cpu0; |
| 1893 | pc->pc_ncpu = 1; |
| 1894 | } else { |
| 1895 | mutex_enter(&pc->pc_lock); |
| 1896 | pc->pc_ncpu++; |
| 1897 | mutex_exit(&pc->pc_lock); |
| 1898 | cc = pool_get(&cache_cpu_pool, PR_WAITOK); |
| 1899 | } |
| 1900 | |
| 1901 | cc->cc_ipl = pc->pc_pool.pr_ipl; |
| 1902 | cc->cc_iplcookie = makeiplcookie(cc->cc_ipl); |
| 1903 | cc->cc_cache = pc; |
| 1904 | cc->cc_cpuindex = index; |
| 1905 | cc->cc_hits = 0; |
| 1906 | cc->cc_misses = 0; |
| 1907 | cc->cc_current = __UNCONST(&pcg_dummy); |
| 1908 | cc->cc_previous = __UNCONST(&pcg_dummy); |
| 1909 | |
| 1910 | pc->pc_cpus[index] = cc; |
| 1911 | } |
| 1912 | |
| 1913 | /* |
| 1914 | * pool_cache_cpu_init: |
| 1915 | * |
| 1916 | * Called whenever a new CPU is attached. |
| 1917 | */ |
| 1918 | void |
| 1919 | pool_cache_cpu_init(struct cpu_info *ci) |
| 1920 | { |
| 1921 | pool_cache_t pc; |
| 1922 | |
| 1923 | mutex_enter(&pool_head_lock); |
| 1924 | TAILQ_FOREACH(pc, &pool_cache_head, pc_cachelist) { |
| 1925 | pc->pc_refcnt++; |
| 1926 | mutex_exit(&pool_head_lock); |
| 1927 | |
| 1928 | pool_cache_cpu_init1(ci, pc); |
| 1929 | |
| 1930 | mutex_enter(&pool_head_lock); |
| 1931 | pc->pc_refcnt--; |
| 1932 | cv_broadcast(&pool_busy); |
| 1933 | } |
| 1934 | mutex_exit(&pool_head_lock); |
| 1935 | } |
| 1936 | |
| 1937 | /* |
| 1938 | * pool_cache_reclaim: |
| 1939 | * |
| 1940 | * Reclaim memory from a pool cache. |
| 1941 | */ |
| 1942 | bool |
| 1943 | pool_cache_reclaim(pool_cache_t pc) |
| 1944 | { |
| 1945 | |
| 1946 | return pool_reclaim(&pc->pc_pool); |
| 1947 | } |
| 1948 | |
| 1949 | static void |
| 1950 | pool_cache_destruct_object1(pool_cache_t pc, void *object) |
| 1951 | { |
| 1952 | |
| 1953 | (*pc->pc_dtor)(pc->pc_arg, object); |
| 1954 | pool_put(&pc->pc_pool, object); |
| 1955 | } |
| 1956 | |
| 1957 | /* |
| 1958 | * pool_cache_destruct_object: |
| 1959 | * |
| 1960 | * Force destruction of an object and its release back into |
| 1961 | * the pool. |
| 1962 | */ |
| 1963 | void |
| 1964 | pool_cache_destruct_object(pool_cache_t pc, void *object) |
| 1965 | { |
| 1966 | |
| 1967 | FREECHECK_IN(&pc->pc_freecheck, object); |
| 1968 | |
| 1969 | pool_cache_destruct_object1(pc, object); |
| 1970 | } |
| 1971 | |
| 1972 | /* |
| 1973 | * pool_cache_invalidate_groups: |
| 1974 | * |
| 1975 | * Invalidate a chain of groups and destruct all objects. |
| 1976 | */ |
| 1977 | static void |
| 1978 | pool_cache_invalidate_groups(pool_cache_t pc, pcg_t *pcg) |
| 1979 | { |
| 1980 | void *object; |
| 1981 | pcg_t *next; |
| 1982 | int i; |
| 1983 | |
| 1984 | for (; pcg != NULL; pcg = next) { |
| 1985 | next = pcg->pcg_next; |
| 1986 | |
| 1987 | for (i = 0; i < pcg->pcg_avail; i++) { |
| 1988 | object = pcg->pcg_objects[i].pcgo_va; |
| 1989 | pool_cache_destruct_object1(pc, object); |
| 1990 | } |
| 1991 | |
| 1992 | if (pcg->pcg_size == PCG_NOBJECTS_LARGE) { |
| 1993 | pool_put(&pcg_large_pool, pcg); |
| 1994 | } else { |
| 1995 | KASSERT(pcg->pcg_size == PCG_NOBJECTS_NORMAL); |
| 1996 | pool_put(&pcg_normal_pool, pcg); |
| 1997 | } |
| 1998 | } |
| 1999 | } |
| 2000 | |
| 2001 | /* |
| 2002 | * pool_cache_invalidate: |
| 2003 | * |
| 2004 | * Invalidate a pool cache (destruct and release all of the |
| 2005 | * cached objects). Does not reclaim objects from the pool. |
| 2006 | * |
| 2007 | * Note: For pool caches that provide constructed objects, there |
| 2008 | * is an assumption that another level of synchronization is occurring |
| 2009 | * between the input to the constructor and the cache invalidation. |
| 2010 | * |
| 2011 | * Invalidation is a costly process and should not be called from |
| 2012 | * interrupt context. |
| 2013 | */ |
| 2014 | void |
| 2015 | pool_cache_invalidate(pool_cache_t pc) |
| 2016 | { |
| 2017 | uint64_t where; |
| 2018 | pcg_t *full, *empty, *part; |
| 2019 | |
| 2020 | KASSERT(!cpu_intr_p() && !cpu_softintr_p()); |
| 2021 | |
| 2022 | if (ncpu < 2 || !mp_online) { |
| 2023 | /* |
| 2024 | * We might be called early enough in the boot process |
| 2025 | * for the CPU data structures to not be fully initialized. |
| 2026 | * In this case, transfer the content of the local CPU's |
| 2027 | * cache back into global cache as only this CPU is currently |
| 2028 | * running. |
| 2029 | */ |
| 2030 | pool_cache_transfer(pc); |
| 2031 | } else { |
| 2032 | /* |
| 2033 | * Signal all CPUs that they must transfer their local |
| 2034 | * cache back to the global pool then wait for the xcall to |
| 2035 | * complete. |
| 2036 | */ |
| 2037 | where = xc_broadcast(0, (xcfunc_t)pool_cache_transfer, |
| 2038 | pc, NULL); |
| 2039 | xc_wait(where); |
| 2040 | } |
| 2041 | |
| 2042 | /* Empty pool caches, then invalidate objects */ |
| 2043 | mutex_enter(&pc->pc_lock); |
| 2044 | full = pc->pc_fullgroups; |
| 2045 | empty = pc->pc_emptygroups; |
| 2046 | part = pc->pc_partgroups; |
| 2047 | pc->pc_fullgroups = NULL; |
| 2048 | pc->pc_emptygroups = NULL; |
| 2049 | pc->pc_partgroups = NULL; |
| 2050 | pc->pc_nfull = 0; |
| 2051 | pc->pc_nempty = 0; |
| 2052 | pc->pc_npart = 0; |
| 2053 | mutex_exit(&pc->pc_lock); |
| 2054 | |
| 2055 | pool_cache_invalidate_groups(pc, full); |
| 2056 | pool_cache_invalidate_groups(pc, empty); |
| 2057 | pool_cache_invalidate_groups(pc, part); |
| 2058 | } |
| 2059 | |
| 2060 | /* |
| 2061 | * pool_cache_invalidate_cpu: |
| 2062 | * |
| 2063 | * Invalidate all CPU-bound cached objects in pool cache, the CPU being |
| 2064 | * identified by its associated index. |
| 2065 | * It is caller's responsibility to ensure that no operation is |
| 2066 | * taking place on this pool cache while doing this invalidation. |
| 2067 | * WARNING: as no inter-CPU locking is enforced, trying to invalidate |
| 2068 | * pool cached objects from a CPU different from the one currently running |
| 2069 | * may result in an undefined behaviour. |
| 2070 | */ |
| 2071 | static void |
| 2072 | pool_cache_invalidate_cpu(pool_cache_t pc, u_int index) |
| 2073 | { |
| 2074 | pool_cache_cpu_t *cc; |
| 2075 | pcg_t *pcg; |
| 2076 | |
| 2077 | if ((cc = pc->pc_cpus[index]) == NULL) |
| 2078 | return; |
| 2079 | |
| 2080 | if ((pcg = cc->cc_current) != &pcg_dummy) { |
| 2081 | pcg->pcg_next = NULL; |
| 2082 | pool_cache_invalidate_groups(pc, pcg); |
| 2083 | } |
| 2084 | if ((pcg = cc->cc_previous) != &pcg_dummy) { |
| 2085 | pcg->pcg_next = NULL; |
| 2086 | pool_cache_invalidate_groups(pc, pcg); |
| 2087 | } |
| 2088 | if (cc != &pc->pc_cpu0) |
| 2089 | pool_put(&cache_cpu_pool, cc); |
| 2090 | |
| 2091 | } |
| 2092 | |
| 2093 | void |
| 2094 | pool_cache_set_drain_hook(pool_cache_t pc, void (*fn)(void *, int), void *arg) |
| 2095 | { |
| 2096 | |
| 2097 | pool_set_drain_hook(&pc->pc_pool, fn, arg); |
| 2098 | } |
| 2099 | |
| 2100 | void |
| 2101 | pool_cache_setlowat(pool_cache_t pc, int n) |
| 2102 | { |
| 2103 | |
| 2104 | pool_setlowat(&pc->pc_pool, n); |
| 2105 | } |
| 2106 | |
| 2107 | void |
| 2108 | pool_cache_sethiwat(pool_cache_t pc, int n) |
| 2109 | { |
| 2110 | |
| 2111 | pool_sethiwat(&pc->pc_pool, n); |
| 2112 | } |
| 2113 | |
| 2114 | void |
| 2115 | pool_cache_sethardlimit(pool_cache_t pc, int n, const char *warnmess, int ratecap) |
| 2116 | { |
| 2117 | |
| 2118 | pool_sethardlimit(&pc->pc_pool, n, warnmess, ratecap); |
| 2119 | } |
| 2120 | |
| 2121 | static bool __noinline |
| 2122 | pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp, |
| 2123 | paddr_t *pap, int flags) |
| 2124 | { |
| 2125 | pcg_t *pcg, *cur; |
| 2126 | uint64_t ncsw; |
| 2127 | pool_cache_t pc; |
| 2128 | void *object; |
| 2129 | |
| 2130 | KASSERT(cc->cc_current->pcg_avail == 0); |
| 2131 | KASSERT(cc->cc_previous->pcg_avail == 0); |
| 2132 | |
| 2133 | pc = cc->cc_cache; |
| 2134 | cc->cc_misses++; |
| 2135 | |
| 2136 | /* |
| 2137 | * Nothing was available locally. Try and grab a group |
| 2138 | * from the cache. |
| 2139 | */ |
| 2140 | if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
| 2141 | ncsw = curlwp->l_ncsw; |
| 2142 | mutex_enter(&pc->pc_lock); |
| 2143 | pc->pc_contended++; |
| 2144 | |
| 2145 | /* |
| 2146 | * If we context switched while locking, then |
| 2147 | * our view of the per-CPU data is invalid: |
| 2148 | * retry. |
| 2149 | */ |
| 2150 | if (curlwp->l_ncsw != ncsw) { |
| 2151 | mutex_exit(&pc->pc_lock); |
| 2152 | return true; |
| 2153 | } |
| 2154 | } |
| 2155 | |
| 2156 | if (__predict_true((pcg = pc->pc_fullgroups) != NULL)) { |
| 2157 | /* |
| 2158 | * If there's a full group, release our empty |
| 2159 | * group back to the cache. Install the full |
| 2160 | * group as cc_current and return. |
| 2161 | */ |
| 2162 | if (__predict_true((cur = cc->cc_current) != &pcg_dummy)) { |
| 2163 | KASSERT(cur->pcg_avail == 0); |
| 2164 | cur->pcg_next = pc->pc_emptygroups; |
| 2165 | pc->pc_emptygroups = cur; |
| 2166 | pc->pc_nempty++; |
| 2167 | } |
| 2168 | KASSERT(pcg->pcg_avail == pcg->pcg_size); |
| 2169 | cc->cc_current = pcg; |
| 2170 | pc->pc_fullgroups = pcg->pcg_next; |
| 2171 | pc->pc_hits++; |
| 2172 | pc->pc_nfull--; |
| 2173 | mutex_exit(&pc->pc_lock); |
| 2174 | return true; |
| 2175 | } |
| 2176 | |
| 2177 | /* |
| 2178 | * Nothing available locally or in cache. Take the slow |
| 2179 | * path: fetch a new object from the pool and construct |
| 2180 | * it. |
| 2181 | */ |
| 2182 | pc->pc_misses++; |
| 2183 | mutex_exit(&pc->pc_lock); |
| 2184 | splx(s); |
| 2185 | |
| 2186 | object = pool_get(&pc->pc_pool, flags); |
| 2187 | *objectp = object; |
| 2188 | if (__predict_false(object == NULL)) |
| 2189 | return false; |
| 2190 | |
| 2191 | if (__predict_false((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0)) { |
| 2192 | pool_put(&pc->pc_pool, object); |
| 2193 | *objectp = NULL; |
| 2194 | return false; |
| 2195 | } |
| 2196 | |
| 2197 | KASSERT((((vaddr_t)object + pc->pc_pool.pr_itemoffset) & |
| 2198 | (pc->pc_pool.pr_align - 1)) == 0); |
| 2199 | |
| 2200 | if (pap != NULL) { |
| 2201 | #ifdef POOL_VTOPHYS |
| 2202 | *pap = POOL_VTOPHYS(object); |
| 2203 | #else |
| 2204 | *pap = POOL_PADDR_INVALID; |
| 2205 | #endif |
| 2206 | } |
| 2207 | |
| 2208 | FREECHECK_OUT(&pc->pc_freecheck, object); |
| 2209 | pool_redzone_fill(&pc->pc_pool, object); |
| 2210 | return false; |
| 2211 | } |
| 2212 | |
| 2213 | /* |
| 2214 | * pool_cache_get{,_paddr}: |
| 2215 | * |
| 2216 | * Get an object from a pool cache (optionally returning |
| 2217 | * the physical address of the object). |
| 2218 | */ |
| 2219 | void * |
| 2220 | pool_cache_get_paddr(pool_cache_t pc, int flags, paddr_t *pap) |
| 2221 | { |
| 2222 | pool_cache_cpu_t *cc; |
| 2223 | pcg_t *pcg; |
| 2224 | void *object; |
| 2225 | int s; |
| 2226 | |
| 2227 | KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()) || |
| 2228 | (pc->pc_pool.pr_ipl != IPL_NONE || cold || panicstr != NULL), |
| 2229 | "pool '%s' is IPL_NONE, but called from interrupt context\n" , |
| 2230 | pc->pc_pool.pr_wchan); |
| 2231 | |
| 2232 | if (flags & PR_WAITOK) { |
| 2233 | ASSERT_SLEEPABLE(); |
| 2234 | } |
| 2235 | |
| 2236 | /* Lock out interrupts and disable preemption. */ |
| 2237 | s = splvm(); |
| 2238 | while (/* CONSTCOND */ true) { |
| 2239 | /* Try and allocate an object from the current group. */ |
| 2240 | cc = pc->pc_cpus[curcpu()->ci_index]; |
| 2241 | KASSERT(cc->cc_cache == pc); |
| 2242 | pcg = cc->cc_current; |
| 2243 | if (__predict_true(pcg->pcg_avail > 0)) { |
| 2244 | object = pcg->pcg_objects[--pcg->pcg_avail].pcgo_va; |
| 2245 | if (__predict_false(pap != NULL)) |
| 2246 | *pap = pcg->pcg_objects[pcg->pcg_avail].pcgo_pa; |
| 2247 | #if defined(DIAGNOSTIC) |
| 2248 | pcg->pcg_objects[pcg->pcg_avail].pcgo_va = NULL; |
| 2249 | KASSERT(pcg->pcg_avail < pcg->pcg_size); |
| 2250 | KASSERT(object != NULL); |
| 2251 | #endif |
| 2252 | cc->cc_hits++; |
| 2253 | splx(s); |
| 2254 | FREECHECK_OUT(&pc->pc_freecheck, object); |
| 2255 | pool_redzone_fill(&pc->pc_pool, object); |
| 2256 | return object; |
| 2257 | } |
| 2258 | |
| 2259 | /* |
| 2260 | * That failed. If the previous group isn't empty, swap |
| 2261 | * it with the current group and allocate from there. |
| 2262 | */ |
| 2263 | pcg = cc->cc_previous; |
| 2264 | if (__predict_true(pcg->pcg_avail > 0)) { |
| 2265 | cc->cc_previous = cc->cc_current; |
| 2266 | cc->cc_current = pcg; |
| 2267 | continue; |
| 2268 | } |
| 2269 | |
| 2270 | /* |
| 2271 | * Can't allocate from either group: try the slow path. |
| 2272 | * If get_slow() allocated an object for us, or if |
| 2273 | * no more objects are available, it will return false. |
| 2274 | * Otherwise, we need to retry. |
| 2275 | */ |
| 2276 | if (!pool_cache_get_slow(cc, s, &object, pap, flags)) |
| 2277 | break; |
| 2278 | } |
| 2279 | |
| 2280 | return object; |
| 2281 | } |
| 2282 | |
| 2283 | static bool __noinline |
| 2284 | pool_cache_put_slow(pool_cache_cpu_t *cc, int s, void *object) |
| 2285 | { |
| 2286 | struct lwp *l = curlwp; |
| 2287 | pcg_t *pcg, *cur; |
| 2288 | uint64_t ncsw; |
| 2289 | pool_cache_t pc; |
| 2290 | |
| 2291 | KASSERT(cc->cc_current->pcg_avail == cc->cc_current->pcg_size); |
| 2292 | KASSERT(cc->cc_previous->pcg_avail == cc->cc_previous->pcg_size); |
| 2293 | |
| 2294 | pc = cc->cc_cache; |
| 2295 | pcg = NULL; |
| 2296 | cc->cc_misses++; |
| 2297 | ncsw = l->l_ncsw; |
| 2298 | |
| 2299 | /* |
| 2300 | * If there are no empty groups in the cache then allocate one |
| 2301 | * while still unlocked. |
| 2302 | */ |
| 2303 | if (__predict_false(pc->pc_emptygroups == NULL)) { |
| 2304 | if (__predict_true(!pool_cache_disable)) { |
| 2305 | pcg = pool_get(pc->pc_pcgpool, PR_NOWAIT); |
| 2306 | } |
| 2307 | /* |
| 2308 | * If pool_get() blocked, then our view of |
| 2309 | * the per-CPU data is invalid: retry. |
| 2310 | */ |
| 2311 | if (__predict_false(l->l_ncsw != ncsw)) { |
| 2312 | if (pcg != NULL) { |
| 2313 | pool_put(pc->pc_pcgpool, pcg); |
| 2314 | } |
| 2315 | return true; |
| 2316 | } |
| 2317 | if (__predict_true(pcg != NULL)) { |
| 2318 | pcg->pcg_avail = 0; |
| 2319 | pcg->pcg_size = pc->pc_pcgsize; |
| 2320 | } |
| 2321 | } |
| 2322 | |
| 2323 | /* Lock the cache. */ |
| 2324 | if (__predict_false(!mutex_tryenter(&pc->pc_lock))) { |
| 2325 | mutex_enter(&pc->pc_lock); |
| 2326 | pc->pc_contended++; |
| 2327 | |
| 2328 | /* |
| 2329 | * If we context switched while locking, then our view of |
| 2330 | * the per-CPU data is invalid: retry. |
| 2331 | */ |
| 2332 | if (__predict_false(l->l_ncsw != ncsw)) { |
| 2333 | mutex_exit(&pc->pc_lock); |
| 2334 | if (pcg != NULL) { |
| 2335 | pool_put(pc->pc_pcgpool, pcg); |
| 2336 | } |
| 2337 | return true; |
| 2338 | } |
| 2339 | } |
| 2340 | |
| 2341 | /* If there are no empty groups in the cache then allocate one. */ |
| 2342 | if (pcg == NULL && pc->pc_emptygroups != NULL) { |
| 2343 | pcg = pc->pc_emptygroups; |
| 2344 | pc->pc_emptygroups = pcg->pcg_next; |
| 2345 | pc->pc_nempty--; |
| 2346 | } |
| 2347 | |
| 2348 | /* |
| 2349 | * If there's a empty group, release our full group back |
| 2350 | * to the cache. Install the empty group to the local CPU |
| 2351 | * and return. |
| 2352 | */ |
| 2353 | if (pcg != NULL) { |
| 2354 | KASSERT(pcg->pcg_avail == 0); |
| 2355 | if (__predict_false(cc->cc_previous == &pcg_dummy)) { |
| 2356 | cc->cc_previous = pcg; |
| 2357 | } else { |
| 2358 | cur = cc->cc_current; |
| 2359 | if (__predict_true(cur != &pcg_dummy)) { |
| 2360 | KASSERT(cur->pcg_avail == cur->pcg_size); |
| 2361 | cur->pcg_next = pc->pc_fullgroups; |
| 2362 | pc->pc_fullgroups = cur; |
| 2363 | pc->pc_nfull++; |
| 2364 | } |
| 2365 | cc->cc_current = pcg; |
| 2366 | } |
| 2367 | pc->pc_hits++; |
| 2368 | mutex_exit(&pc->pc_lock); |
| 2369 | return true; |
| 2370 | } |
| 2371 | |
| 2372 | /* |
| 2373 | * Nothing available locally or in cache, and we didn't |
| 2374 | * allocate an empty group. Take the slow path and destroy |
| 2375 | * the object here and now. |
| 2376 | */ |
| 2377 | pc->pc_misses++; |
| 2378 | mutex_exit(&pc->pc_lock); |
| 2379 | splx(s); |
| 2380 | pool_cache_destruct_object(pc, object); |
| 2381 | |
| 2382 | return false; |
| 2383 | } |
| 2384 | |
| 2385 | /* |
| 2386 | * pool_cache_put{,_paddr}: |
| 2387 | * |
| 2388 | * Put an object back to the pool cache (optionally caching the |
| 2389 | * physical address of the object). |
| 2390 | */ |
| 2391 | void |
| 2392 | pool_cache_put_paddr(pool_cache_t pc, void *object, paddr_t pa) |
| 2393 | { |
| 2394 | pool_cache_cpu_t *cc; |
| 2395 | pcg_t *pcg; |
| 2396 | int s; |
| 2397 | |
| 2398 | KASSERT(object != NULL); |
| 2399 | pool_redzone_check(&pc->pc_pool, object); |
| 2400 | FREECHECK_IN(&pc->pc_freecheck, object); |
| 2401 | |
| 2402 | /* Lock out interrupts and disable preemption. */ |
| 2403 | s = splvm(); |
| 2404 | while (/* CONSTCOND */ true) { |
| 2405 | /* If the current group isn't full, release it there. */ |
| 2406 | cc = pc->pc_cpus[curcpu()->ci_index]; |
| 2407 | KASSERT(cc->cc_cache == pc); |
| 2408 | pcg = cc->cc_current; |
| 2409 | if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { |
| 2410 | pcg->pcg_objects[pcg->pcg_avail].pcgo_va = object; |
| 2411 | pcg->pcg_objects[pcg->pcg_avail].pcgo_pa = pa; |
| 2412 | pcg->pcg_avail++; |
| 2413 | cc->cc_hits++; |
| 2414 | splx(s); |
| 2415 | return; |
| 2416 | } |
| 2417 | |
| 2418 | /* |
| 2419 | * That failed. If the previous group isn't full, swap |
| 2420 | * it with the current group and try again. |
| 2421 | */ |
| 2422 | pcg = cc->cc_previous; |
| 2423 | if (__predict_true(pcg->pcg_avail < pcg->pcg_size)) { |
| 2424 | cc->cc_previous = cc->cc_current; |
| 2425 | cc->cc_current = pcg; |
| 2426 | continue; |
| 2427 | } |
| 2428 | |
| 2429 | /* |
| 2430 | * Can't free to either group: try the slow path. |
| 2431 | * If put_slow() releases the object for us, it |
| 2432 | * will return false. Otherwise we need to retry. |
| 2433 | */ |
| 2434 | if (!pool_cache_put_slow(cc, s, object)) |
| 2435 | break; |
| 2436 | } |
| 2437 | } |
| 2438 | |
| 2439 | /* |
| 2440 | * pool_cache_transfer: |
| 2441 | * |
| 2442 | * Transfer objects from the per-CPU cache to the global cache. |
| 2443 | * Run within a cross-call thread. |
| 2444 | */ |
| 2445 | static void |
| 2446 | pool_cache_transfer(pool_cache_t pc) |
| 2447 | { |
| 2448 | pool_cache_cpu_t *cc; |
| 2449 | pcg_t *prev, *cur, **list; |
| 2450 | int s; |
| 2451 | |
| 2452 | s = splvm(); |
| 2453 | mutex_enter(&pc->pc_lock); |
| 2454 | cc = pc->pc_cpus[curcpu()->ci_index]; |
| 2455 | cur = cc->cc_current; |
| 2456 | cc->cc_current = __UNCONST(&pcg_dummy); |
| 2457 | prev = cc->cc_previous; |
| 2458 | cc->cc_previous = __UNCONST(&pcg_dummy); |
| 2459 | if (cur != &pcg_dummy) { |
| 2460 | if (cur->pcg_avail == cur->pcg_size) { |
| 2461 | list = &pc->pc_fullgroups; |
| 2462 | pc->pc_nfull++; |
| 2463 | } else if (cur->pcg_avail == 0) { |
| 2464 | list = &pc->pc_emptygroups; |
| 2465 | pc->pc_nempty++; |
| 2466 | } else { |
| 2467 | list = &pc->pc_partgroups; |
| 2468 | pc->pc_npart++; |
| 2469 | } |
| 2470 | cur->pcg_next = *list; |
| 2471 | *list = cur; |
| 2472 | } |
| 2473 | if (prev != &pcg_dummy) { |
| 2474 | if (prev->pcg_avail == prev->pcg_size) { |
| 2475 | list = &pc->pc_fullgroups; |
| 2476 | pc->pc_nfull++; |
| 2477 | } else if (prev->pcg_avail == 0) { |
| 2478 | list = &pc->pc_emptygroups; |
| 2479 | pc->pc_nempty++; |
| 2480 | } else { |
| 2481 | list = &pc->pc_partgroups; |
| 2482 | pc->pc_npart++; |
| 2483 | } |
| 2484 | prev->pcg_next = *list; |
| 2485 | *list = prev; |
| 2486 | } |
| 2487 | mutex_exit(&pc->pc_lock); |
| 2488 | splx(s); |
| 2489 | } |
| 2490 | |
| 2491 | /* |
| 2492 | * Pool backend allocators. |
| 2493 | * |
| 2494 | * Each pool has a backend allocator that handles allocation, deallocation, |
| 2495 | * and any additional draining that might be needed. |
| 2496 | * |
| 2497 | * We provide two standard allocators: |
| 2498 | * |
| 2499 | * pool_allocator_kmem - the default when no allocator is specified |
| 2500 | * |
| 2501 | * pool_allocator_nointr - used for pools that will not be accessed |
| 2502 | * in interrupt context. |
| 2503 | */ |
| 2504 | void *pool_page_alloc(struct pool *, int); |
| 2505 | void pool_page_free(struct pool *, void *); |
| 2506 | |
| 2507 | #ifdef POOL_SUBPAGE |
| 2508 | struct pool_allocator pool_allocator_kmem_fullpage = { |
| 2509 | .pa_alloc = pool_page_alloc, |
| 2510 | .pa_free = pool_page_free, |
| 2511 | .pa_pagesz = 0 |
| 2512 | }; |
| 2513 | #else |
| 2514 | struct pool_allocator pool_allocator_kmem = { |
| 2515 | .pa_alloc = pool_page_alloc, |
| 2516 | .pa_free = pool_page_free, |
| 2517 | .pa_pagesz = 0 |
| 2518 | }; |
| 2519 | #endif |
| 2520 | |
| 2521 | #ifdef POOL_SUBPAGE |
| 2522 | struct pool_allocator pool_allocator_nointr_fullpage = { |
| 2523 | .pa_alloc = pool_page_alloc, |
| 2524 | .pa_free = pool_page_free, |
| 2525 | .pa_pagesz = 0 |
| 2526 | }; |
| 2527 | #else |
| 2528 | struct pool_allocator pool_allocator_nointr = { |
| 2529 | .pa_alloc = pool_page_alloc, |
| 2530 | .pa_free = pool_page_free, |
| 2531 | .pa_pagesz = 0 |
| 2532 | }; |
| 2533 | #endif |
| 2534 | |
| 2535 | #ifdef POOL_SUBPAGE |
| 2536 | void *pool_subpage_alloc(struct pool *, int); |
| 2537 | void pool_subpage_free(struct pool *, void *); |
| 2538 | |
| 2539 | struct pool_allocator pool_allocator_kmem = { |
| 2540 | .pa_alloc = pool_subpage_alloc, |
| 2541 | .pa_free = pool_subpage_free, |
| 2542 | .pa_pagesz = POOL_SUBPAGE |
| 2543 | }; |
| 2544 | |
| 2545 | struct pool_allocator pool_allocator_nointr = { |
| 2546 | .pa_alloc = pool_subpage_alloc, |
| 2547 | .pa_free = pool_subpage_free, |
| 2548 | .pa_pagesz = POOL_SUBPAGE |
| 2549 | }; |
| 2550 | #endif /* POOL_SUBPAGE */ |
| 2551 | |
| 2552 | static void * |
| 2553 | pool_allocator_alloc(struct pool *pp, int flags) |
| 2554 | { |
| 2555 | struct pool_allocator *pa = pp->pr_alloc; |
| 2556 | void *res; |
| 2557 | |
| 2558 | res = (*pa->pa_alloc)(pp, flags); |
| 2559 | if (res == NULL && (flags & PR_WAITOK) == 0) { |
| 2560 | /* |
| 2561 | * We only run the drain hook here if PR_NOWAIT. |
| 2562 | * In other cases, the hook will be run in |
| 2563 | * pool_reclaim(). |
| 2564 | */ |
| 2565 | if (pp->pr_drain_hook != NULL) { |
| 2566 | (*pp->pr_drain_hook)(pp->pr_drain_hook_arg, flags); |
| 2567 | res = (*pa->pa_alloc)(pp, flags); |
| 2568 | } |
| 2569 | } |
| 2570 | return res; |
| 2571 | } |
| 2572 | |
| 2573 | static void |
| 2574 | pool_allocator_free(struct pool *pp, void *v) |
| 2575 | { |
| 2576 | struct pool_allocator *pa = pp->pr_alloc; |
| 2577 | |
| 2578 | (*pa->pa_free)(pp, v); |
| 2579 | } |
| 2580 | |
| 2581 | void * |
| 2582 | pool_page_alloc(struct pool *pp, int flags) |
| 2583 | { |
| 2584 | const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; |
| 2585 | vmem_addr_t va; |
| 2586 | int ret; |
| 2587 | |
| 2588 | ret = uvm_km_kmem_alloc(kmem_va_arena, pp->pr_alloc->pa_pagesz, |
| 2589 | vflags | VM_INSTANTFIT, &va); |
| 2590 | |
| 2591 | return ret ? NULL : (void *)va; |
| 2592 | } |
| 2593 | |
| 2594 | void |
| 2595 | pool_page_free(struct pool *pp, void *v) |
| 2596 | { |
| 2597 | |
| 2598 | uvm_km_kmem_free(kmem_va_arena, (vaddr_t)v, pp->pr_alloc->pa_pagesz); |
| 2599 | } |
| 2600 | |
| 2601 | static void * |
| 2602 | pool_page_alloc_meta(struct pool *pp, int flags) |
| 2603 | { |
| 2604 | const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP; |
| 2605 | vmem_addr_t va; |
| 2606 | int ret; |
| 2607 | |
| 2608 | ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz, |
| 2609 | vflags | VM_INSTANTFIT, &va); |
| 2610 | |
| 2611 | return ret ? NULL : (void *)va; |
| 2612 | } |
| 2613 | |
| 2614 | static void |
| 2615 | pool_page_free_meta(struct pool *pp, void *v) |
| 2616 | { |
| 2617 | |
| 2618 | vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz); |
| 2619 | } |
| 2620 | |
| 2621 | #ifdef POOL_REDZONE |
| 2622 | #if defined(_LP64) |
| 2623 | # define PRIME 0x9e37fffffffc0000UL |
| 2624 | #else /* defined(_LP64) */ |
| 2625 | # define PRIME 0x9e3779b1 |
| 2626 | #endif /* defined(_LP64) */ |
| 2627 | #define STATIC_BYTE 0xFE |
| 2628 | CTASSERT(POOL_REDZONE_SIZE > 1); |
| 2629 | |
| 2630 | static inline uint8_t |
| 2631 | pool_pattern_generate(const void *p) |
| 2632 | { |
| 2633 | return (uint8_t)(((uintptr_t)p) * PRIME |
| 2634 | >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT); |
| 2635 | } |
| 2636 | |
| 2637 | static void |
| 2638 | pool_redzone_init(struct pool *pp, size_t requested_size) |
| 2639 | { |
| 2640 | size_t nsz; |
| 2641 | |
| 2642 | if (pp->pr_roflags & PR_NOTOUCH) { |
| 2643 | pp->pr_reqsize = 0; |
| 2644 | pp->pr_redzone = false; |
| 2645 | return; |
| 2646 | } |
| 2647 | |
| 2648 | /* |
| 2649 | * We may have extended the requested size earlier; check if |
| 2650 | * there's naturally space in the padding for a red zone. |
| 2651 | */ |
| 2652 | if (pp->pr_size - requested_size >= POOL_REDZONE_SIZE) { |
| 2653 | pp->pr_reqsize = requested_size; |
| 2654 | pp->pr_redzone = true; |
| 2655 | return; |
| 2656 | } |
| 2657 | |
| 2658 | /* |
| 2659 | * No space in the natural padding; check if we can extend a |
| 2660 | * bit the size of the pool. |
| 2661 | */ |
| 2662 | nsz = roundup(pp->pr_size + POOL_REDZONE_SIZE, pp->pr_align); |
| 2663 | if (nsz <= pp->pr_alloc->pa_pagesz) { |
| 2664 | /* Ok, we can */ |
| 2665 | pp->pr_size = nsz; |
| 2666 | pp->pr_reqsize = requested_size; |
| 2667 | pp->pr_redzone = true; |
| 2668 | } else { |
| 2669 | /* No space for a red zone... snif :'( */ |
| 2670 | pp->pr_reqsize = 0; |
| 2671 | pp->pr_redzone = false; |
| 2672 | printf("pool redzone disabled for '%s'\n" , pp->pr_wchan); |
| 2673 | } |
| 2674 | } |
| 2675 | |
| 2676 | static void |
| 2677 | pool_redzone_fill(struct pool *pp, void *p) |
| 2678 | { |
| 2679 | uint8_t *cp, pat; |
| 2680 | const uint8_t *ep; |
| 2681 | |
| 2682 | if (!pp->pr_redzone) |
| 2683 | return; |
| 2684 | |
| 2685 | cp = (uint8_t *)p + pp->pr_reqsize; |
| 2686 | ep = cp + POOL_REDZONE_SIZE; |
| 2687 | |
| 2688 | /* |
| 2689 | * We really don't want the first byte of the red zone to be '\0'; |
| 2690 | * an off-by-one in a string may not be properly detected. |
| 2691 | */ |
| 2692 | pat = pool_pattern_generate(cp); |
| 2693 | *cp = (pat == '\0') ? STATIC_BYTE: pat; |
| 2694 | cp++; |
| 2695 | |
| 2696 | while (cp < ep) { |
| 2697 | *cp = pool_pattern_generate(cp); |
| 2698 | cp++; |
| 2699 | } |
| 2700 | } |
| 2701 | |
| 2702 | static void |
| 2703 | pool_redzone_check(struct pool *pp, void *p) |
| 2704 | { |
| 2705 | uint8_t *cp, pat, expected; |
| 2706 | const uint8_t *ep; |
| 2707 | |
| 2708 | if (!pp->pr_redzone) |
| 2709 | return; |
| 2710 | |
| 2711 | cp = (uint8_t *)p + pp->pr_reqsize; |
| 2712 | ep = cp + POOL_REDZONE_SIZE; |
| 2713 | |
| 2714 | pat = pool_pattern_generate(cp); |
| 2715 | expected = (pat == '\0') ? STATIC_BYTE: pat; |
| 2716 | if (expected != *cp) { |
| 2717 | panic("%s: %p: 0x%02x != 0x%02x\n" , |
| 2718 | __func__, cp, *cp, expected); |
| 2719 | } |
| 2720 | cp++; |
| 2721 | |
| 2722 | while (cp < ep) { |
| 2723 | expected = pool_pattern_generate(cp); |
| 2724 | if (*cp != expected) { |
| 2725 | panic("%s: %p: 0x%02x != 0x%02x\n" , |
| 2726 | __func__, cp, *cp, expected); |
| 2727 | } |
| 2728 | cp++; |
| 2729 | } |
| 2730 | } |
| 2731 | |
| 2732 | #endif /* POOL_REDZONE */ |
| 2733 | |
| 2734 | |
| 2735 | #ifdef POOL_SUBPAGE |
| 2736 | /* Sub-page allocator, for machines with large hardware pages. */ |
| 2737 | void * |
| 2738 | pool_subpage_alloc(struct pool *pp, int flags) |
| 2739 | { |
| 2740 | return pool_get(&psppool, flags); |
| 2741 | } |
| 2742 | |
| 2743 | void |
| 2744 | pool_subpage_free(struct pool *pp, void *v) |
| 2745 | { |
| 2746 | pool_put(&psppool, v); |
| 2747 | } |
| 2748 | |
| 2749 | #endif /* POOL_SUBPAGE */ |
| 2750 | |
| 2751 | #if defined(DDB) |
| 2752 | static bool |
| 2753 | pool_in_page(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
| 2754 | { |
| 2755 | |
| 2756 | return (uintptr_t)ph->ph_page <= addr && |
| 2757 | addr < (uintptr_t)ph->ph_page + pp->pr_alloc->pa_pagesz; |
| 2758 | } |
| 2759 | |
| 2760 | static bool |
| 2761 | pool_in_item(struct pool *pp, void *item, uintptr_t addr) |
| 2762 | { |
| 2763 | |
| 2764 | return (uintptr_t)item <= addr && addr < (uintptr_t)item + pp->pr_size; |
| 2765 | } |
| 2766 | |
| 2767 | static bool |
| 2768 | pool_in_cg(struct pool *pp, struct pool_cache_group *pcg, uintptr_t addr) |
| 2769 | { |
| 2770 | int i; |
| 2771 | |
| 2772 | if (pcg == NULL) { |
| 2773 | return false; |
| 2774 | } |
| 2775 | for (i = 0; i < pcg->pcg_avail; i++) { |
| 2776 | if (pool_in_item(pp, pcg->pcg_objects[i].pcgo_va, addr)) { |
| 2777 | return true; |
| 2778 | } |
| 2779 | } |
| 2780 | return false; |
| 2781 | } |
| 2782 | |
| 2783 | static bool |
| 2784 | pool_allocated(struct pool *pp, struct pool_item_header *ph, uintptr_t addr) |
| 2785 | { |
| 2786 | |
| 2787 | if ((pp->pr_roflags & PR_NOTOUCH) != 0) { |
| 2788 | unsigned int idx = pr_item_notouch_index(pp, ph, (void *)addr); |
| 2789 | pool_item_bitmap_t *bitmap = |
| 2790 | ph->ph_bitmap + (idx / BITMAP_SIZE); |
| 2791 | pool_item_bitmap_t mask = 1 << (idx & BITMAP_MASK); |
| 2792 | |
| 2793 | return (*bitmap & mask) == 0; |
| 2794 | } else { |
| 2795 | struct pool_item *pi; |
| 2796 | |
| 2797 | LIST_FOREACH(pi, &ph->ph_itemlist, pi_list) { |
| 2798 | if (pool_in_item(pp, pi, addr)) { |
| 2799 | return false; |
| 2800 | } |
| 2801 | } |
| 2802 | return true; |
| 2803 | } |
| 2804 | } |
| 2805 | |
| 2806 | void |
| 2807 | pool_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
| 2808 | { |
| 2809 | struct pool *pp; |
| 2810 | |
| 2811 | TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
| 2812 | struct pool_item_header *ph; |
| 2813 | uintptr_t item; |
| 2814 | bool allocated = true; |
| 2815 | bool incache = false; |
| 2816 | bool incpucache = false; |
| 2817 | char cpucachestr[32]; |
| 2818 | |
| 2819 | if ((pp->pr_roflags & PR_PHINPAGE) != 0) { |
| 2820 | LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) { |
| 2821 | if (pool_in_page(pp, ph, addr)) { |
| 2822 | goto found; |
| 2823 | } |
| 2824 | } |
| 2825 | LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) { |
| 2826 | if (pool_in_page(pp, ph, addr)) { |
| 2827 | allocated = |
| 2828 | pool_allocated(pp, ph, addr); |
| 2829 | goto found; |
| 2830 | } |
| 2831 | } |
| 2832 | LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) { |
| 2833 | if (pool_in_page(pp, ph, addr)) { |
| 2834 | allocated = false; |
| 2835 | goto found; |
| 2836 | } |
| 2837 | } |
| 2838 | continue; |
| 2839 | } else { |
| 2840 | ph = pr_find_pagehead_noalign(pp, (void *)addr); |
| 2841 | if (ph == NULL || !pool_in_page(pp, ph, addr)) { |
| 2842 | continue; |
| 2843 | } |
| 2844 | allocated = pool_allocated(pp, ph, addr); |
| 2845 | } |
| 2846 | found: |
| 2847 | if (allocated && pp->pr_cache) { |
| 2848 | pool_cache_t pc = pp->pr_cache; |
| 2849 | struct pool_cache_group *pcg; |
| 2850 | int i; |
| 2851 | |
| 2852 | for (pcg = pc->pc_fullgroups; pcg != NULL; |
| 2853 | pcg = pcg->pcg_next) { |
| 2854 | if (pool_in_cg(pp, pcg, addr)) { |
| 2855 | incache = true; |
| 2856 | goto print; |
| 2857 | } |
| 2858 | } |
| 2859 | for (i = 0; i < __arraycount(pc->pc_cpus); i++) { |
| 2860 | pool_cache_cpu_t *cc; |
| 2861 | |
| 2862 | if ((cc = pc->pc_cpus[i]) == NULL) { |
| 2863 | continue; |
| 2864 | } |
| 2865 | if (pool_in_cg(pp, cc->cc_current, addr) || |
| 2866 | pool_in_cg(pp, cc->cc_previous, addr)) { |
| 2867 | struct cpu_info *ci = |
| 2868 | cpu_lookup(i); |
| 2869 | |
| 2870 | incpucache = true; |
| 2871 | snprintf(cpucachestr, |
| 2872 | sizeof(cpucachestr), |
| 2873 | "cached by CPU %u" , |
| 2874 | ci->ci_index); |
| 2875 | goto print; |
| 2876 | } |
| 2877 | } |
| 2878 | } |
| 2879 | print: |
| 2880 | item = (uintptr_t)ph->ph_page + ph->ph_off; |
| 2881 | item = item + rounddown(addr - item, pp->pr_size); |
| 2882 | (*pr)("%p is %p+%zu in POOL '%s' (%s)\n" , |
| 2883 | (void *)addr, item, (size_t)(addr - item), |
| 2884 | pp->pr_wchan, |
| 2885 | incpucache ? cpucachestr : |
| 2886 | incache ? "cached" : allocated ? "allocated" : "free" ); |
| 2887 | } |
| 2888 | } |
| 2889 | #endif /* defined(DDB) */ |
| 2890 | |
| 2891 | static int |
| 2892 | pool_sysctl(SYSCTLFN_ARGS) |
| 2893 | { |
| 2894 | struct pool_sysctl data; |
| 2895 | struct pool *pp; |
| 2896 | struct pool_cache *pc; |
| 2897 | pool_cache_cpu_t *cc; |
| 2898 | int error; |
| 2899 | size_t i, written; |
| 2900 | |
| 2901 | if (oldp == NULL) { |
| 2902 | *oldlenp = 0; |
| 2903 | TAILQ_FOREACH(pp, &pool_head, pr_poollist) |
| 2904 | *oldlenp += sizeof(data); |
| 2905 | return 0; |
| 2906 | } |
| 2907 | |
| 2908 | memset(&data, 0, sizeof(data)); |
| 2909 | error = 0; |
| 2910 | written = 0; |
| 2911 | TAILQ_FOREACH(pp, &pool_head, pr_poollist) { |
| 2912 | if (written + sizeof(data) > *oldlenp) |
| 2913 | break; |
| 2914 | strlcpy(data.pr_wchan, pp->pr_wchan, sizeof(data.pr_wchan)); |
| 2915 | data.pr_pagesize = pp->pr_alloc->pa_pagesz; |
| 2916 | data.pr_flags = pp->pr_roflags | pp->pr_flags; |
| 2917 | #define COPY(field) data.field = pp->field |
| 2918 | COPY(pr_size); |
| 2919 | |
| 2920 | COPY(pr_itemsperpage); |
| 2921 | COPY(pr_nitems); |
| 2922 | COPY(pr_nout); |
| 2923 | COPY(pr_hardlimit); |
| 2924 | COPY(pr_npages); |
| 2925 | COPY(pr_minpages); |
| 2926 | COPY(pr_maxpages); |
| 2927 | |
| 2928 | COPY(pr_nget); |
| 2929 | COPY(pr_nfail); |
| 2930 | COPY(pr_nput); |
| 2931 | COPY(pr_npagealloc); |
| 2932 | COPY(pr_npagefree); |
| 2933 | COPY(pr_hiwat); |
| 2934 | COPY(pr_nidle); |
| 2935 | #undef COPY |
| 2936 | |
| 2937 | data.pr_cache_nmiss_pcpu = 0; |
| 2938 | data.pr_cache_nhit_pcpu = 0; |
| 2939 | if (pp->pr_cache) { |
| 2940 | pc = pp->pr_cache; |
| 2941 | data.pr_cache_meta_size = pc->pc_pcgsize; |
| 2942 | data.pr_cache_nfull = pc->pc_nfull; |
| 2943 | data.pr_cache_npartial = pc->pc_npart; |
| 2944 | data.pr_cache_nempty = pc->pc_nempty; |
| 2945 | data.pr_cache_ncontended = pc->pc_contended; |
| 2946 | data.pr_cache_nmiss_global = pc->pc_misses; |
| 2947 | data.pr_cache_nhit_global = pc->pc_hits; |
| 2948 | for (i = 0; i < pc->pc_ncpu; ++i) { |
| 2949 | cc = pc->pc_cpus[i]; |
| 2950 | if (cc == NULL) |
| 2951 | continue; |
| 2952 | data.pr_cache_nmiss_pcpu += cc->cc_misses; |
| 2953 | data.pr_cache_nhit_pcpu += cc->cc_hits; |
| 2954 | } |
| 2955 | } else { |
| 2956 | data.pr_cache_meta_size = 0; |
| 2957 | data.pr_cache_nfull = 0; |
| 2958 | data.pr_cache_npartial = 0; |
| 2959 | data.pr_cache_nempty = 0; |
| 2960 | data.pr_cache_ncontended = 0; |
| 2961 | data.pr_cache_nmiss_global = 0; |
| 2962 | data.pr_cache_nhit_global = 0; |
| 2963 | } |
| 2964 | |
| 2965 | error = sysctl_copyout(l, &data, oldp, sizeof(data)); |
| 2966 | if (error) |
| 2967 | break; |
| 2968 | written += sizeof(data); |
| 2969 | oldp = (char *)oldp + sizeof(data); |
| 2970 | } |
| 2971 | |
| 2972 | *oldlenp = written; |
| 2973 | return error; |
| 2974 | } |
| 2975 | |
| 2976 | SYSCTL_SETUP(sysctl_pool_setup, "sysctl kern.pool setup" ) |
| 2977 | { |
| 2978 | const struct sysctlnode *rnode = NULL; |
| 2979 | |
| 2980 | sysctl_createv(clog, 0, NULL, &rnode, |
| 2981 | CTLFLAG_PERMANENT, |
| 2982 | CTLTYPE_STRUCT, "pool" , |
| 2983 | SYSCTL_DESCR("Get pool statistics" ), |
| 2984 | pool_sysctl, 0, NULL, 0, |
| 2985 | CTL_KERN, CTL_CREATE, CTL_EOL); |
| 2986 | } |
| 2987 | |