| 1 | /* $NetBSD: nouveau_dma.c,v 1.2 2014/08/06 15:01:33 riastradh Exp $ */ |
| 2 | |
| 3 | /* |
| 4 | * Copyright (C) 2007 Ben Skeggs. |
| 5 | * All Rights Reserved. |
| 6 | * |
| 7 | * Permission is hereby granted, free of charge, to any person obtaining |
| 8 | * a copy of this software and associated documentation files (the |
| 9 | * "Software"), to deal in the Software without restriction, including |
| 10 | * without limitation the rights to use, copy, modify, merge, publish, |
| 11 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 12 | * permit persons to whom the Software is furnished to do so, subject to |
| 13 | * the following conditions: |
| 14 | * |
| 15 | * The above copyright notice and this permission notice (including the |
| 16 | * next paragraph) shall be included in all copies or substantial |
| 17 | * portions of the Software. |
| 18 | * |
| 19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 20 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 21 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
| 22 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
| 23 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 24 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 25 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 26 | * |
| 27 | */ |
| 28 | |
| 29 | #include <sys/cdefs.h> |
| 30 | __KERNEL_RCSID(0, "$NetBSD: nouveau_dma.c,v 1.2 2014/08/06 15:01:33 riastradh Exp $" ); |
| 31 | |
| 32 | #include <core/client.h> |
| 33 | |
| 34 | #include "nouveau_drm.h" |
| 35 | #include "nouveau_dma.h" |
| 36 | |
| 37 | #ifdef __NetBSD__ |
| 38 | # define __iomem |
| 39 | # define __force |
| 40 | #endif |
| 41 | |
| 42 | void |
| 43 | OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords) |
| 44 | { |
| 45 | bool is_iomem; |
| 46 | u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem); |
| 47 | mem = &mem[chan->dma.cur]; |
| 48 | if (is_iomem) |
| 49 | memcpy_toio((void __force __iomem *)mem, data, nr_dwords * 4); |
| 50 | else |
| 51 | memcpy(mem, data, nr_dwords * 4); |
| 52 | chan->dma.cur += nr_dwords; |
| 53 | } |
| 54 | |
| 55 | #ifdef __NetBSD__ |
| 56 | # undef __force |
| 57 | # undef __iomem |
| 58 | #endif |
| 59 | |
| 60 | /* Fetch and adjust GPU GET pointer |
| 61 | * |
| 62 | * Returns: |
| 63 | * value >= 0, the adjusted GET pointer |
| 64 | * -EINVAL if GET pointer currently outside main push buffer |
| 65 | * -EBUSY if timeout exceeded |
| 66 | */ |
| 67 | static inline int |
| 68 | READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout) |
| 69 | { |
| 70 | uint64_t val; |
| 71 | |
| 72 | val = nv_ro32(chan->object, chan->user_get); |
| 73 | if (chan->user_get_hi) |
| 74 | val |= (uint64_t)nv_ro32(chan->object, chan->user_get_hi) << 32; |
| 75 | |
| 76 | /* reset counter as long as GET is still advancing, this is |
| 77 | * to avoid misdetecting a GPU lockup if the GPU happens to |
| 78 | * just be processing an operation that takes a long time |
| 79 | */ |
| 80 | if (val != *prev_get) { |
| 81 | *prev_get = val; |
| 82 | *timeout = 0; |
| 83 | } |
| 84 | |
| 85 | if ((++*timeout & 0xff) == 0) { |
| 86 | udelay(1); |
| 87 | if (*timeout > 100000) |
| 88 | return -EBUSY; |
| 89 | } |
| 90 | |
| 91 | if (val < chan->push.vma.offset || |
| 92 | val > chan->push.vma.offset + (chan->dma.max << 2)) |
| 93 | return -EINVAL; |
| 94 | |
| 95 | return (val - chan->push.vma.offset) >> 2; |
| 96 | } |
| 97 | |
| 98 | void |
| 99 | nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo, |
| 100 | int delta, int length) |
| 101 | { |
| 102 | struct nouveau_bo *pb = chan->push.buffer; |
| 103 | struct nouveau_vma *vma; |
| 104 | int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base; |
| 105 | u64 offset; |
| 106 | |
| 107 | vma = nouveau_bo_vma_find(bo, nv_client(chan->cli)->vm); |
| 108 | BUG_ON(!vma); |
| 109 | offset = vma->offset + delta; |
| 110 | |
| 111 | BUG_ON(chan->dma.ib_free < 1); |
| 112 | |
| 113 | nouveau_bo_wr32(pb, ip++, lower_32_bits(offset)); |
| 114 | nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8); |
| 115 | |
| 116 | chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max; |
| 117 | |
| 118 | mb(); |
| 119 | /* Flush writes. */ |
| 120 | nouveau_bo_rd32(pb, 0); |
| 121 | |
| 122 | nv_wo32(chan->object, 0x8c, chan->dma.ib_put); |
| 123 | chan->dma.ib_free--; |
| 124 | } |
| 125 | |
| 126 | static int |
| 127 | nv50_dma_push_wait(struct nouveau_channel *chan, int count) |
| 128 | { |
| 129 | uint32_t cnt = 0, prev_get = 0; |
| 130 | |
| 131 | while (chan->dma.ib_free < count) { |
| 132 | uint32_t get = nv_ro32(chan->object, 0x88); |
| 133 | if (get != prev_get) { |
| 134 | prev_get = get; |
| 135 | cnt = 0; |
| 136 | } |
| 137 | |
| 138 | if ((++cnt & 0xff) == 0) { |
| 139 | DRM_UDELAY(1); |
| 140 | if (cnt > 100000) |
| 141 | return -EBUSY; |
| 142 | } |
| 143 | |
| 144 | chan->dma.ib_free = get - chan->dma.ib_put; |
| 145 | if (chan->dma.ib_free <= 0) |
| 146 | chan->dma.ib_free += chan->dma.ib_max; |
| 147 | } |
| 148 | |
| 149 | return 0; |
| 150 | } |
| 151 | |
| 152 | static int |
| 153 | nv50_dma_wait(struct nouveau_channel *chan, int slots, int count) |
| 154 | { |
| 155 | uint64_t prev_get = 0; |
| 156 | int ret, cnt = 0; |
| 157 | |
| 158 | ret = nv50_dma_push_wait(chan, slots + 1); |
| 159 | if (unlikely(ret)) |
| 160 | return ret; |
| 161 | |
| 162 | while (chan->dma.free < count) { |
| 163 | int get = READ_GET(chan, &prev_get, &cnt); |
| 164 | if (unlikely(get < 0)) { |
| 165 | if (get == -EINVAL) |
| 166 | continue; |
| 167 | |
| 168 | return get; |
| 169 | } |
| 170 | |
| 171 | if (get <= chan->dma.cur) { |
| 172 | chan->dma.free = chan->dma.max - chan->dma.cur; |
| 173 | if (chan->dma.free >= count) |
| 174 | break; |
| 175 | |
| 176 | FIRE_RING(chan); |
| 177 | do { |
| 178 | get = READ_GET(chan, &prev_get, &cnt); |
| 179 | if (unlikely(get < 0)) { |
| 180 | if (get == -EINVAL) |
| 181 | continue; |
| 182 | return get; |
| 183 | } |
| 184 | } while (get == 0); |
| 185 | chan->dma.cur = 0; |
| 186 | chan->dma.put = 0; |
| 187 | } |
| 188 | |
| 189 | chan->dma.free = get - chan->dma.cur - 1; |
| 190 | } |
| 191 | |
| 192 | return 0; |
| 193 | } |
| 194 | |
| 195 | int |
| 196 | nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size) |
| 197 | { |
| 198 | uint64_t prev_get = 0; |
| 199 | int cnt = 0, get; |
| 200 | |
| 201 | if (chan->dma.ib_max) |
| 202 | return nv50_dma_wait(chan, slots, size); |
| 203 | |
| 204 | while (chan->dma.free < size) { |
| 205 | get = READ_GET(chan, &prev_get, &cnt); |
| 206 | if (unlikely(get == -EBUSY)) |
| 207 | return -EBUSY; |
| 208 | |
| 209 | /* loop until we have a usable GET pointer. the value |
| 210 | * we read from the GPU may be outside the main ring if |
| 211 | * PFIFO is processing a buffer called from the main ring, |
| 212 | * discard these values until something sensible is seen. |
| 213 | * |
| 214 | * the other case we discard GET is while the GPU is fetching |
| 215 | * from the SKIPS area, so the code below doesn't have to deal |
| 216 | * with some fun corner cases. |
| 217 | */ |
| 218 | if (unlikely(get == -EINVAL) || get < NOUVEAU_DMA_SKIPS) |
| 219 | continue; |
| 220 | |
| 221 | if (get <= chan->dma.cur) { |
| 222 | /* engine is fetching behind us, or is completely |
| 223 | * idle (GET == PUT) so we have free space up until |
| 224 | * the end of the push buffer |
| 225 | * |
| 226 | * we can only hit that path once per call due to |
| 227 | * looping back to the beginning of the push buffer, |
| 228 | * we'll hit the fetching-ahead-of-us path from that |
| 229 | * point on. |
| 230 | * |
| 231 | * the *one* exception to that rule is if we read |
| 232 | * GET==PUT, in which case the below conditional will |
| 233 | * always succeed and break us out of the wait loop. |
| 234 | */ |
| 235 | chan->dma.free = chan->dma.max - chan->dma.cur; |
| 236 | if (chan->dma.free >= size) |
| 237 | break; |
| 238 | |
| 239 | /* not enough space left at the end of the push buffer, |
| 240 | * instruct the GPU to jump back to the start right |
| 241 | * after processing the currently pending commands. |
| 242 | */ |
| 243 | OUT_RING(chan, chan->push.vma.offset | 0x20000000); |
| 244 | |
| 245 | /* wait for GET to depart from the skips area. |
| 246 | * prevents writing GET==PUT and causing a race |
| 247 | * condition that causes us to think the GPU is |
| 248 | * idle when it's not. |
| 249 | */ |
| 250 | do { |
| 251 | get = READ_GET(chan, &prev_get, &cnt); |
| 252 | if (unlikely(get == -EBUSY)) |
| 253 | return -EBUSY; |
| 254 | if (unlikely(get == -EINVAL)) |
| 255 | continue; |
| 256 | } while (get <= NOUVEAU_DMA_SKIPS); |
| 257 | WRITE_PUT(NOUVEAU_DMA_SKIPS); |
| 258 | |
| 259 | /* we're now submitting commands at the start of |
| 260 | * the push buffer. |
| 261 | */ |
| 262 | chan->dma.cur = |
| 263 | chan->dma.put = NOUVEAU_DMA_SKIPS; |
| 264 | } |
| 265 | |
| 266 | /* engine fetching ahead of us, we have space up until the |
| 267 | * current GET pointer. the "- 1" is to ensure there's |
| 268 | * space left to emit a jump back to the beginning of the |
| 269 | * push buffer if we require it. we can never get GET == PUT |
| 270 | * here, so this is safe. |
| 271 | */ |
| 272 | chan->dma.free = get - chan->dma.cur - 1; |
| 273 | } |
| 274 | |
| 275 | return 0; |
| 276 | } |
| 277 | |
| 278 | |