| 1 | /* $NetBSD: lockstat.h,v 1.14 2016/01/24 01:01:11 christos Exp $ */ |
| 2 | |
| 3 | /*- |
| 4 | * Copyright (c) 2006 The NetBSD Foundation, Inc. |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * This code is derived from software contributed to The NetBSD Foundation |
| 8 | * by Andrew Doran. |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * 1. Redistributions of source code must retain the above copyright |
| 14 | * notice, this list of conditions and the following disclaimer. |
| 15 | * 2. Redistributions in binary form must reproduce the above copyright |
| 16 | * notice, this list of conditions and the following disclaimer in the |
| 17 | * documentation and/or other materials provided with the distribution. |
| 18 | * |
| 19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
| 20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
| 23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 29 | * POSSIBILITY OF SUCH DAMAGE. |
| 30 | */ |
| 31 | |
| 32 | #ifndef _SYS_LOCKSTAT_H_ |
| 33 | #define _SYS_LOCKSTAT_H_ |
| 34 | |
| 35 | #ifdef _KERNEL_OPT |
| 36 | #include "opt_dtrace.h" |
| 37 | #include <lockstat.h> |
| 38 | #endif |
| 39 | |
| 40 | #include <sys/types.h> |
| 41 | #include <sys/ioccom.h> |
| 42 | #include <sys/queue.h> |
| 43 | #include <sys/time.h> |
| 44 | |
| 45 | #if defined(_KERNEL) && defined(__HAVE_CPU_COUNTER) |
| 46 | #include <machine/cpu_counter.h> |
| 47 | #endif |
| 48 | |
| 49 | /* |
| 50 | * Interface version. The interface is not designed to provide |
| 51 | * compatibility across NetBSD releases. |
| 52 | */ |
| 53 | |
| 54 | #define IOC_LOCKSTAT_GVERSION _IOR('L', 0, int) |
| 55 | |
| 56 | #define LS_VERSION 5 |
| 57 | |
| 58 | /* |
| 59 | * Enable request. We can limit tracing by the call site and by |
| 60 | * the lock. We also specify the number of event buffers to |
| 61 | * allocate up front, and what kind of events to track. |
| 62 | */ |
| 63 | |
| 64 | #define IOC_LOCKSTAT_ENABLE _IOW('L', 1, lsenable_t) |
| 65 | |
| 66 | #define LE_CALLSITE 0x01 /* track call sites */ |
| 67 | #define LE_ONE_CALLSITE 0x02 /* specific call site */ |
| 68 | #define LE_ONE_LOCK 0x04 /* specific lock */ |
| 69 | #define LE_LOCK 0x08 /* track locks */ |
| 70 | |
| 71 | typedef struct lsenable { |
| 72 | uintptr_t le_csstart; /* callsite start */ |
| 73 | uintptr_t le_csend; /* callsite end */ |
| 74 | uintptr_t le_lockstart; /* lock address start */ |
| 75 | uintptr_t le_lockend; /* lock address end */ |
| 76 | uintptr_t le_nbufs; /* buffers to allocate, 0 = default */ |
| 77 | u_int le_flags; /* request flags */ |
| 78 | u_int le_mask; /* event mask (LB_*) */ |
| 79 | } lsenable_t; |
| 80 | |
| 81 | /* |
| 82 | * Disable request. |
| 83 | */ |
| 84 | |
| 85 | #define IOC_LOCKSTAT_DISABLE _IOR('L', 2, lsdisable_t) |
| 86 | |
| 87 | typedef struct lsdisable { |
| 88 | size_t ld_size; /* buffer space allocated */ |
| 89 | struct timespec ld_time; /* time spent enabled */ |
| 90 | uint64_t ld_freq[64]; /* counter HZ by CPU number */ |
| 91 | } lsdisable_t; |
| 92 | |
| 93 | /* |
| 94 | * Event buffers returned from reading from the devices. |
| 95 | */ |
| 96 | |
| 97 | /* |
| 98 | * Event types, for lockstat_event(). Stored in lb_flags but should be |
| 99 | * meaningless to the consumer, also provided with the enable request |
| 100 | * in le_mask. |
| 101 | */ |
| 102 | #define LB_SPIN 0x00000001 |
| 103 | #define LB_SLEEP1 0x00000002 |
| 104 | #define LB_SLEEP2 0x00000003 |
| 105 | #define LB_NEVENT 0x00000003 |
| 106 | #define LB_EVENT_MASK 0x000000ff |
| 107 | |
| 108 | /* |
| 109 | * Lock types, the only part of lb_flags that should be inspected. Also |
| 110 | * provided with the enable request in le_mask. |
| 111 | */ |
| 112 | #define LB_ADAPTIVE_MUTEX 0x00000100 |
| 113 | #define LB_SPIN_MUTEX 0x00000200 |
| 114 | #define LB_RWLOCK 0x00000300 |
| 115 | #define LB_NOPREEMPT 0x00000400 |
| 116 | #define LB_KERNEL_LOCK 0x00000500 |
| 117 | #define LB_MISC 0x00000600 |
| 118 | #define LB_NLOCK 0x00000600 |
| 119 | #define LB_LOCK_MASK 0x0000ff00 |
| 120 | #define LB_LOCK_SHIFT 8 |
| 121 | |
| 122 | #define LB_DTRACE 0x00010000 |
| 123 | |
| 124 | typedef struct lsbuf { |
| 125 | union { |
| 126 | LIST_ENTRY(lsbuf) list; |
| 127 | SLIST_ENTRY(lsbuf) slist; |
| 128 | TAILQ_ENTRY(lsbuf) tailq; |
| 129 | } lb_chain; |
| 130 | uintptr_t lb_lock; /* lock address */ |
| 131 | uintptr_t lb_callsite; /* call site */ |
| 132 | uint64_t lb_times[LB_NEVENT]; /* cumulative times */ |
| 133 | uint32_t lb_counts[LB_NEVENT]; /* count of events */ |
| 134 | uint16_t lb_flags; /* lock type */ |
| 135 | uint16_t lb_cpu; /* CPU number */ |
| 136 | } lsbuf_t; |
| 137 | |
| 138 | /* |
| 139 | * Tracing stubs used by lock providers. |
| 140 | */ |
| 141 | |
| 142 | #if defined(_KERNEL) && defined(__HAVE_CPU_COUNTER) && NLOCKSTAT > 0 |
| 143 | |
| 144 | #define LOCKSTAT_EVENT(flag, lock, type, count, time) \ |
| 145 | do { \ |
| 146 | if (__predict_false(flag)) \ |
| 147 | lockstat_event((uintptr_t)(lock), \ |
| 148 | (uintptr_t)__builtin_return_address(0), \ |
| 149 | (type), (count), (time)); \ |
| 150 | } while (/* CONSTCOND */ 0); |
| 151 | |
| 152 | #define LOCKSTAT_EVENT_RA(flag, lock, type, count, time, ra) \ |
| 153 | do { \ |
| 154 | if (__predict_false(flag)) \ |
| 155 | lockstat_event((uintptr_t)(lock), (uintptr_t)ra, \ |
| 156 | (type), (count), (time)); \ |
| 157 | } while (/* CONSTCOND */ 0); |
| 158 | |
| 159 | #define LOCKSTAT_TIMER(name) uint64_t name = 0 |
| 160 | #define LOCKSTAT_COUNTER(name) uint64_t name = 0 |
| 161 | #define LOCKSTAT_FLAG(name) int name |
| 162 | #define LOCKSTAT_ENTER(name) name = lockstat_enabled |
| 163 | #define LOCKSTAT_EXIT(name) |
| 164 | |
| 165 | #define LOCKSTAT_START_TIMER(flag, name) \ |
| 166 | do { \ |
| 167 | if (__predict_false(flag)) \ |
| 168 | (name) -= cpu_counter(); \ |
| 169 | } while (/* CONSTCOND */ 0) |
| 170 | |
| 171 | #define LOCKSTAT_STOP_TIMER(flag, name) \ |
| 172 | do { \ |
| 173 | if (__predict_false(flag)) \ |
| 174 | (name) += cpu_counter(); \ |
| 175 | } while (/* CONSTCOND */ 0) |
| 176 | |
| 177 | #define LOCKSTAT_COUNT(name, inc) \ |
| 178 | do { \ |
| 179 | (name) += (inc); \ |
| 180 | } while (/* CONSTCOND */ 0) |
| 181 | |
| 182 | void lockstat_event(uintptr_t, uintptr_t, u_int, u_int, uint64_t); |
| 183 | |
| 184 | #else |
| 185 | |
| 186 | #define LOCKSTAT_FLAG(name) /* nothing */ |
| 187 | #define LOCKSTAT_ENTER(name) /* nothing */ |
| 188 | #define LOCKSTAT_EXIT(name) /* nothing */ |
| 189 | #define LOCKSTAT_EVENT(flag, lock, type, count, time) /* nothing */ |
| 190 | #define LOCKSTAT_EVENT_RA(flag, lock, type, count, time, ra) /* nothing */ |
| 191 | #define LOCKSTAT_TIMER(void) /* nothing */ |
| 192 | #define LOCKSTAT_COUNTER(void) /* nothing */ |
| 193 | #define LOCKSTAT_START_TIMER(flag, void) /* nothing */ |
| 194 | #define LOCKSTAT_STOP_TIMER(flag, void) /* nothing */ |
| 195 | #define LOCKSTAT_COUNT(name, int) /* nothing */ |
| 196 | |
| 197 | #endif |
| 198 | |
| 199 | #ifdef KDTRACE_HOOKS |
| 200 | extern volatile u_int lockstat_dtrace_enabled; |
| 201 | #define KDTRACE_LOCKSTAT_ENABLED lockstat_dtrace_enabled |
| 202 | #define LS_COMPRESS(f) \ |
| 203 | ((((f) & 0x3) | (((f) & 0x700) >> 6)) & (LS_NPROBES - 1)) |
| 204 | #define LS_NPROBES 0x20 /* 5 bits */ |
| 205 | |
| 206 | extern uint32_t lockstat_probemap[]; |
| 207 | extern void (*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t, |
| 208 | uintptr_t, uintptr_t, uintptr_t); |
| 209 | |
| 210 | void lockstat_probe_stub(uint32_t, uintptr_t, uintptr_t, |
| 211 | uintptr_t, uintptr_t, uintptr_t); |
| 212 | #else |
| 213 | #define KDTRACE_LOCKSTAT_ENABLED 0 |
| 214 | #endif |
| 215 | |
| 216 | #if defined(_KERNEL) && NLOCKSTAT > 0 |
| 217 | extern volatile u_int lockstat_enabled; |
| 218 | extern volatile u_int lockstat_dev_enabled; |
| 219 | |
| 220 | #define LOCKSTAT_ENABLED_UPDATE() do { \ |
| 221 | lockstat_enabled = lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED; \ |
| 222 | membar_producer(); \ |
| 223 | } while (/*CONSTCOND*/0) |
| 224 | #endif |
| 225 | |
| 226 | #endif /* _SYS_LOCKSTAT_H_ */ |
| 227 | |