| 1 | /* $NetBSD: lockstat.c,v 1.24 2015/08/20 14:40:17 christos Exp $ */ |
| 2 | |
| 3 | /*- |
| 4 | * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc. |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * This code is derived from software contributed to The NetBSD Foundation |
| 8 | * by Andrew Doran. |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * 1. Redistributions of source code must retain the above copyright |
| 14 | * notice, this list of conditions and the following disclaimer. |
| 15 | * 2. Redistributions in binary form must reproduce the above copyright |
| 16 | * notice, this list of conditions and the following disclaimer in the |
| 17 | * documentation and/or other materials provided with the distribution. |
| 18 | * |
| 19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 29 | * POSSIBILITY OF SUCH DAMAGE. |
| 30 | */ |
| 31 | |
| 32 | /* |
| 33 | * Lock statistics driver, providing kernel support for the lockstat(8) |
| 34 | * command. |
| 35 | * |
| 36 | * We use a global lock word (lockstat_lock) to track device opens. |
| 37 | * Only one thread can hold the device at a time, providing a global lock. |
| 38 | * |
| 39 | * XXX Timings for contention on sleep locks are currently incorrect. |
| 40 | */ |
| 41 | |
| 42 | #include <sys/cdefs.h> |
| 43 | __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.24 2015/08/20 14:40:17 christos Exp $" ); |
| 44 | |
| 45 | #include <sys/types.h> |
| 46 | #include <sys/param.h> |
| 47 | #include <sys/proc.h> |
| 48 | #include <sys/resourcevar.h> |
| 49 | #include <sys/systm.h> |
| 50 | #include <sys/kernel.h> |
| 51 | #include <sys/kmem.h> |
| 52 | #include <sys/conf.h> |
| 53 | #include <sys/cpu.h> |
| 54 | #include <sys/syslog.h> |
| 55 | #include <sys/atomic.h> |
| 56 | |
| 57 | #include <dev/lockstat.h> |
| 58 | |
| 59 | #include <machine/lock.h> |
| 60 | |
| 61 | #include "ioconf.h" |
| 62 | |
| 63 | #ifndef __HAVE_CPU_COUNTER |
| 64 | #error CPU counters not available |
| 65 | #endif |
| 66 | |
| 67 | #if LONG_BIT == 64 |
| 68 | #define LOCKSTAT_HASH_SHIFT 3 |
| 69 | #elif LONG_BIT == 32 |
| 70 | #define LOCKSTAT_HASH_SHIFT 2 |
| 71 | #endif |
| 72 | |
| 73 | #define LOCKSTAT_MINBUFS 1000 |
| 74 | #define LOCKSTAT_DEFBUFS 10000 |
| 75 | #define LOCKSTAT_MAXBUFS 1000000 |
| 76 | |
| 77 | #define LOCKSTAT_HASH_SIZE 128 |
| 78 | #define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1) |
| 79 | #define LOCKSTAT_HASH(key) \ |
| 80 | ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK) |
| 81 | |
| 82 | typedef struct lscpu { |
| 83 | SLIST_HEAD(, lsbuf) lc_free; |
| 84 | u_int lc_overflow; |
| 85 | LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE]; |
| 86 | } lscpu_t; |
| 87 | |
| 88 | typedef struct lslist lslist_t; |
| 89 | |
| 90 | void lockstat_start(lsenable_t *); |
| 91 | int lockstat_alloc(lsenable_t *); |
| 92 | void lockstat_init_tables(lsenable_t *); |
| 93 | int lockstat_stop(lsdisable_t *); |
| 94 | void lockstat_free(void); |
| 95 | |
| 96 | dev_type_open(lockstat_open); |
| 97 | dev_type_close(lockstat_close); |
| 98 | dev_type_read(lockstat_read); |
| 99 | dev_type_ioctl(lockstat_ioctl); |
| 100 | |
| 101 | volatile u_int lockstat_enabled; |
| 102 | volatile u_int lockstat_dev_enabled; |
| 103 | uintptr_t lockstat_csstart; |
| 104 | uintptr_t lockstat_csend; |
| 105 | uintptr_t lockstat_csmask; |
| 106 | uintptr_t lockstat_lamask; |
| 107 | uintptr_t lockstat_lockstart; |
| 108 | uintptr_t lockstat_lockend; |
| 109 | __cpu_simple_lock_t lockstat_lock; |
| 110 | lwp_t *lockstat_lwp; |
| 111 | lsbuf_t *lockstat_baseb; |
| 112 | size_t lockstat_sizeb; |
| 113 | int lockstat_busy; |
| 114 | struct timespec lockstat_stime; |
| 115 | |
| 116 | #ifdef KDTRACE_HOOKS |
| 117 | volatile u_int lockstat_dtrace_enabled; |
| 118 | CTASSERT(LB_NEVENT <= 3); |
| 119 | CTASSERT(LB_NLOCK <= (7 << LB_LOCK_SHIFT)); |
| 120 | void |
| 121 | lockstat_probe_stub(uint32_t id, uintptr_t lock, uintptr_t callsite, |
| 122 | uintptr_t flags, uintptr_t count, uintptr_t cycles) |
| 123 | { |
| 124 | } |
| 125 | |
| 126 | uint32_t lockstat_probemap[LS_NPROBES]; |
| 127 | void (*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t, |
| 128 | uintptr_t, uintptr_t, uintptr_t) = &lockstat_probe_stub; |
| 129 | #endif |
| 130 | |
| 131 | const struct cdevsw lockstat_cdevsw = { |
| 132 | .d_open = lockstat_open, |
| 133 | .d_close = lockstat_close, |
| 134 | .d_read = lockstat_read, |
| 135 | .d_write = nowrite, |
| 136 | .d_ioctl = lockstat_ioctl, |
| 137 | .d_stop = nostop, |
| 138 | .d_tty = notty, |
| 139 | .d_poll = nopoll, |
| 140 | .d_mmap = nommap, |
| 141 | .d_kqfilter = nokqfilter, |
| 142 | .d_discard = nodiscard, |
| 143 | .d_flag = D_OTHER | D_MPSAFE |
| 144 | }; |
| 145 | |
| 146 | /* |
| 147 | * Called when the pseudo-driver is attached. |
| 148 | */ |
| 149 | void |
| 150 | lockstatattach(int nunits) |
| 151 | { |
| 152 | |
| 153 | (void)nunits; |
| 154 | |
| 155 | __cpu_simple_lock_init(&lockstat_lock); |
| 156 | } |
| 157 | |
| 158 | /* |
| 159 | * Prepare the per-CPU tables for use, or clear down tables when tracing is |
| 160 | * stopped. |
| 161 | */ |
| 162 | void |
| 163 | lockstat_init_tables(lsenable_t *le) |
| 164 | { |
| 165 | int i, per, slop, cpuno; |
| 166 | CPU_INFO_ITERATOR cii; |
| 167 | struct cpu_info *ci; |
| 168 | lscpu_t *lc; |
| 169 | lsbuf_t *lb; |
| 170 | |
| 171 | /* coverity[assert_side_effect] */ |
| 172 | KASSERT(!lockstat_dev_enabled); |
| 173 | |
| 174 | for (CPU_INFO_FOREACH(cii, ci)) { |
| 175 | if (ci->ci_lockstat != NULL) { |
| 176 | kmem_free(ci->ci_lockstat, sizeof(lscpu_t)); |
| 177 | ci->ci_lockstat = NULL; |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | if (le == NULL) |
| 182 | return; |
| 183 | |
| 184 | lb = lockstat_baseb; |
| 185 | per = le->le_nbufs / ncpu; |
| 186 | slop = le->le_nbufs - (per * ncpu); |
| 187 | cpuno = 0; |
| 188 | for (CPU_INFO_FOREACH(cii, ci)) { |
| 189 | lc = kmem_alloc(sizeof(*lc), KM_SLEEP); |
| 190 | lc->lc_overflow = 0; |
| 191 | ci->ci_lockstat = lc; |
| 192 | |
| 193 | SLIST_INIT(&lc->lc_free); |
| 194 | for (i = 0; i < LOCKSTAT_HASH_SIZE; i++) |
| 195 | LIST_INIT(&lc->lc_hash[i]); |
| 196 | |
| 197 | for (i = per; i != 0; i--, lb++) { |
| 198 | lb->lb_cpu = (uint16_t)cpuno; |
| 199 | SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); |
| 200 | } |
| 201 | if (--slop > 0) { |
| 202 | lb->lb_cpu = (uint16_t)cpuno; |
| 203 | SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); |
| 204 | lb++; |
| 205 | } |
| 206 | cpuno++; |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | /* |
| 211 | * Start collecting lock statistics. |
| 212 | */ |
| 213 | void |
| 214 | lockstat_start(lsenable_t *le) |
| 215 | { |
| 216 | |
| 217 | /* coverity[assert_side_effect] */ |
| 218 | KASSERT(!lockstat_dev_enabled); |
| 219 | |
| 220 | lockstat_init_tables(le); |
| 221 | |
| 222 | if ((le->le_flags & LE_CALLSITE) != 0) |
| 223 | lockstat_csmask = (uintptr_t)-1LL; |
| 224 | else |
| 225 | lockstat_csmask = 0; |
| 226 | |
| 227 | if ((le->le_flags & LE_LOCK) != 0) |
| 228 | lockstat_lamask = (uintptr_t)-1LL; |
| 229 | else |
| 230 | lockstat_lamask = 0; |
| 231 | |
| 232 | lockstat_csstart = le->le_csstart; |
| 233 | lockstat_csend = le->le_csend; |
| 234 | lockstat_lockstart = le->le_lockstart; |
| 235 | lockstat_lockstart = le->le_lockstart; |
| 236 | lockstat_lockend = le->le_lockend; |
| 237 | membar_sync(); |
| 238 | getnanotime(&lockstat_stime); |
| 239 | lockstat_dev_enabled = le->le_mask; |
| 240 | LOCKSTAT_ENABLED_UPDATE(); |
| 241 | } |
| 242 | |
| 243 | /* |
| 244 | * Stop collecting lock statistics. |
| 245 | */ |
| 246 | int |
| 247 | lockstat_stop(lsdisable_t *ld) |
| 248 | { |
| 249 | CPU_INFO_ITERATOR cii; |
| 250 | struct cpu_info *ci; |
| 251 | u_int cpuno, overflow; |
| 252 | struct timespec ts; |
| 253 | int error; |
| 254 | lwp_t *l; |
| 255 | |
| 256 | /* coverity[assert_side_effect] */ |
| 257 | KASSERT(lockstat_dev_enabled); |
| 258 | |
| 259 | /* |
| 260 | * Set enabled false, force a write barrier, and wait for other CPUs |
| 261 | * to exit lockstat_event(). |
| 262 | */ |
| 263 | lockstat_dev_enabled = 0; |
| 264 | LOCKSTAT_ENABLED_UPDATE(); |
| 265 | getnanotime(&ts); |
| 266 | tsleep(&lockstat_stop, PPAUSE, "lockstat" , mstohz(10)); |
| 267 | |
| 268 | /* |
| 269 | * Did we run out of buffers while tracing? |
| 270 | */ |
| 271 | overflow = 0; |
| 272 | for (CPU_INFO_FOREACH(cii, ci)) |
| 273 | overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow; |
| 274 | |
| 275 | if (overflow != 0) { |
| 276 | error = EOVERFLOW; |
| 277 | log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n" , |
| 278 | overflow); |
| 279 | } else |
| 280 | error = 0; |
| 281 | |
| 282 | lockstat_init_tables(NULL); |
| 283 | |
| 284 | /* Run through all LWPs and clear the slate for the next run. */ |
| 285 | mutex_enter(proc_lock); |
| 286 | LIST_FOREACH(l, &alllwp, l_list) { |
| 287 | l->l_pfailaddr = 0; |
| 288 | l->l_pfailtime = 0; |
| 289 | l->l_pfaillock = 0; |
| 290 | } |
| 291 | mutex_exit(proc_lock); |
| 292 | |
| 293 | if (ld == NULL) |
| 294 | return error; |
| 295 | |
| 296 | /* |
| 297 | * Fill out the disable struct for the caller. |
| 298 | */ |
| 299 | timespecsub(&ts, &lockstat_stime, &ld->ld_time); |
| 300 | ld->ld_size = lockstat_sizeb; |
| 301 | |
| 302 | cpuno = 0; |
| 303 | for (CPU_INFO_FOREACH(cii, ci)) { |
| 304 | if (cpuno >= sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) { |
| 305 | log(LOG_WARNING, "lockstat: too many CPUs\n" ); |
| 306 | break; |
| 307 | } |
| 308 | ld->ld_freq[cpuno++] = cpu_frequency(ci); |
| 309 | } |
| 310 | |
| 311 | return error; |
| 312 | } |
| 313 | |
| 314 | /* |
| 315 | * Allocate buffers for lockstat_start(). |
| 316 | */ |
| 317 | int |
| 318 | lockstat_alloc(lsenable_t *le) |
| 319 | { |
| 320 | lsbuf_t *lb; |
| 321 | size_t sz; |
| 322 | |
| 323 | /* coverity[assert_side_effect] */ |
| 324 | KASSERT(!lockstat_dev_enabled); |
| 325 | lockstat_free(); |
| 326 | |
| 327 | sz = sizeof(*lb) * le->le_nbufs; |
| 328 | |
| 329 | lb = kmem_zalloc(sz, KM_SLEEP); |
| 330 | if (lb == NULL) |
| 331 | return (ENOMEM); |
| 332 | |
| 333 | /* coverity[assert_side_effect] */ |
| 334 | KASSERT(!lockstat_dev_enabled); |
| 335 | KASSERT(lockstat_baseb == NULL); |
| 336 | lockstat_sizeb = sz; |
| 337 | lockstat_baseb = lb; |
| 338 | |
| 339 | return (0); |
| 340 | } |
| 341 | |
| 342 | /* |
| 343 | * Free allocated buffers after tracing has stopped. |
| 344 | */ |
| 345 | void |
| 346 | lockstat_free(void) |
| 347 | { |
| 348 | |
| 349 | /* coverity[assert_side_effect] */ |
| 350 | KASSERT(!lockstat_dev_enabled); |
| 351 | |
| 352 | if (lockstat_baseb != NULL) { |
| 353 | kmem_free(lockstat_baseb, lockstat_sizeb); |
| 354 | lockstat_baseb = NULL; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | /* |
| 359 | * Main entry point from lock primatives. |
| 360 | */ |
| 361 | void |
| 362 | lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count, |
| 363 | uint64_t cycles) |
| 364 | { |
| 365 | lslist_t *ll; |
| 366 | lscpu_t *lc; |
| 367 | lsbuf_t *lb; |
| 368 | u_int event; |
| 369 | int s; |
| 370 | |
| 371 | #ifdef KDTRACE_HOOKS |
| 372 | uint32_t id; |
| 373 | CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0); |
| 374 | if ((id = lockstat_probemap[LS_COMPRESS(flags)]) != 0) |
| 375 | (*lockstat_probe_func)(id, lock, callsite, flags, count, |
| 376 | cycles); |
| 377 | #endif |
| 378 | |
| 379 | if ((flags & lockstat_dev_enabled) != flags || count == 0) |
| 380 | return; |
| 381 | if (lock < lockstat_lockstart || lock > lockstat_lockend) |
| 382 | return; |
| 383 | if (callsite < lockstat_csstart || callsite > lockstat_csend) |
| 384 | return; |
| 385 | |
| 386 | callsite &= lockstat_csmask; |
| 387 | lock &= lockstat_lamask; |
| 388 | |
| 389 | /* |
| 390 | * Find the table for this lock+callsite pair, and try to locate a |
| 391 | * buffer with the same key. |
| 392 | */ |
| 393 | s = splhigh(); |
| 394 | lc = curcpu()->ci_lockstat; |
| 395 | ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)]; |
| 396 | event = (flags & LB_EVENT_MASK) - 1; |
| 397 | |
| 398 | LIST_FOREACH(lb, ll, lb_chain.list) { |
| 399 | if (lb->lb_lock == lock && lb->lb_callsite == callsite) |
| 400 | break; |
| 401 | } |
| 402 | |
| 403 | if (lb != NULL) { |
| 404 | /* |
| 405 | * We found a record. Move it to the front of the list, as |
| 406 | * we're likely to hit it again soon. |
| 407 | */ |
| 408 | if (lb != LIST_FIRST(ll)) { |
| 409 | LIST_REMOVE(lb, lb_chain.list); |
| 410 | LIST_INSERT_HEAD(ll, lb, lb_chain.list); |
| 411 | } |
| 412 | lb->lb_counts[event] += count; |
| 413 | lb->lb_times[event] += cycles; |
| 414 | } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) { |
| 415 | /* |
| 416 | * Pinch a new buffer and fill it out. |
| 417 | */ |
| 418 | SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist); |
| 419 | LIST_INSERT_HEAD(ll, lb, lb_chain.list); |
| 420 | lb->lb_flags = (uint16_t)flags; |
| 421 | lb->lb_lock = lock; |
| 422 | lb->lb_callsite = callsite; |
| 423 | lb->lb_counts[event] = count; |
| 424 | lb->lb_times[event] = cycles; |
| 425 | } else { |
| 426 | /* |
| 427 | * We didn't find a buffer and there were none free. |
| 428 | * lockstat_stop() will notice later on and report the |
| 429 | * error. |
| 430 | */ |
| 431 | lc->lc_overflow++; |
| 432 | } |
| 433 | |
| 434 | splx(s); |
| 435 | } |
| 436 | |
| 437 | /* |
| 438 | * Accept an open() on /dev/lockstat. |
| 439 | */ |
| 440 | int |
| 441 | lockstat_open(dev_t dev, int flag, int mode, lwp_t *l) |
| 442 | { |
| 443 | |
| 444 | if (!__cpu_simple_lock_try(&lockstat_lock)) |
| 445 | return EBUSY; |
| 446 | lockstat_lwp = curlwp; |
| 447 | return 0; |
| 448 | } |
| 449 | |
| 450 | /* |
| 451 | * Accept the last close() on /dev/lockstat. |
| 452 | */ |
| 453 | int |
| 454 | lockstat_close(dev_t dev, int flag, int mode, lwp_t *l) |
| 455 | { |
| 456 | |
| 457 | lockstat_lwp = NULL; |
| 458 | __cpu_simple_unlock(&lockstat_lock); |
| 459 | return 0; |
| 460 | } |
| 461 | |
| 462 | /* |
| 463 | * Handle control operations. |
| 464 | */ |
| 465 | int |
| 466 | lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) |
| 467 | { |
| 468 | lsenable_t *le; |
| 469 | int error; |
| 470 | |
| 471 | if (lockstat_lwp != curlwp) |
| 472 | return EBUSY; |
| 473 | |
| 474 | switch (cmd) { |
| 475 | case IOC_LOCKSTAT_GVERSION: |
| 476 | *(int *)data = LS_VERSION; |
| 477 | error = 0; |
| 478 | break; |
| 479 | |
| 480 | case IOC_LOCKSTAT_ENABLE: |
| 481 | le = (lsenable_t *)data; |
| 482 | |
| 483 | if (!cpu_hascounter()) { |
| 484 | error = ENODEV; |
| 485 | break; |
| 486 | } |
| 487 | if (lockstat_dev_enabled) { |
| 488 | error = EBUSY; |
| 489 | break; |
| 490 | } |
| 491 | |
| 492 | /* |
| 493 | * Sanitize the arguments passed in and set up filtering. |
| 494 | */ |
| 495 | if (le->le_nbufs == 0) |
| 496 | le->le_nbufs = LOCKSTAT_DEFBUFS; |
| 497 | else if (le->le_nbufs > LOCKSTAT_MAXBUFS || |
| 498 | le->le_nbufs < LOCKSTAT_MINBUFS) { |
| 499 | error = EINVAL; |
| 500 | break; |
| 501 | } |
| 502 | if ((le->le_flags & LE_ONE_CALLSITE) == 0) { |
| 503 | le->le_csstart = 0; |
| 504 | le->le_csend = le->le_csstart - 1; |
| 505 | } |
| 506 | if ((le->le_flags & LE_ONE_LOCK) == 0) { |
| 507 | le->le_lockstart = 0; |
| 508 | le->le_lockend = le->le_lockstart - 1; |
| 509 | } |
| 510 | if ((le->le_mask & LB_EVENT_MASK) == 0) |
| 511 | return EINVAL; |
| 512 | if ((le->le_mask & LB_LOCK_MASK) == 0) |
| 513 | return EINVAL; |
| 514 | |
| 515 | /* |
| 516 | * Start tracing. |
| 517 | */ |
| 518 | if ((error = lockstat_alloc(le)) == 0) |
| 519 | lockstat_start(le); |
| 520 | break; |
| 521 | |
| 522 | case IOC_LOCKSTAT_DISABLE: |
| 523 | if (!lockstat_dev_enabled) |
| 524 | error = EINVAL; |
| 525 | else |
| 526 | error = lockstat_stop((lsdisable_t *)data); |
| 527 | break; |
| 528 | |
| 529 | default: |
| 530 | error = ENOTTY; |
| 531 | break; |
| 532 | } |
| 533 | |
| 534 | return error; |
| 535 | } |
| 536 | |
| 537 | /* |
| 538 | * Copy buffers out to user-space. |
| 539 | */ |
| 540 | int |
| 541 | lockstat_read(dev_t dev, struct uio *uio, int flag) |
| 542 | { |
| 543 | |
| 544 | if (curlwp != lockstat_lwp || lockstat_dev_enabled) |
| 545 | return EBUSY; |
| 546 | return uiomove(lockstat_baseb, lockstat_sizeb, uio); |
| 547 | } |
| 548 | |