1 /* $NetBSD: ttm_bo_util.c,v 1.28 2021/12/19 11:34:29 riastradh Exp $ */
2
3 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
4 /**************************************************************************
5 *
6 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
7 * All Rights Reserved.
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the
11 * "Software"), to deal in the Software without restriction, including
12 * without limitation the rights to use, copy, modify, merge, publish,
13 * distribute, sub license, and/or sell copies of the Software, and to
14 * permit persons to whom the Software is furnished to do so, subject to
15 * the following conditions:
16 *
17 * The above copyright notice and this permission notice (including the
18 * next paragraph) shall be included in all copies or substantial portions
19 * of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
24 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
25 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
26 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
27 * USE OR OTHER DEALINGS IN THE SOFTWARE.
28 *
29 **************************************************************************/
30 /*
31 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: ttm_bo_util.c,v 1.28 2021/12/19 11:34:29 riastradh Exp $");
36
37 #include <drm/ttm/ttm_bo_driver.h>
38 #include <drm/ttm/ttm_placement.h>
39 #include <drm/drm_vma_manager.h>
40 #include <linux/io.h>
41 #include <linux/highmem.h>
42 #include <linux/wait.h>
43 #include <linux/slab.h>
44 #include <linux/vmalloc.h>
45 #include <linux/module.h>
46 #include <linux/dma-resv.h>
47
48 struct ttm_transfer_obj {
49 struct ttm_buffer_object base;
50 struct ttm_buffer_object *bo;
51 };
52
53 #ifdef __NetBSD__ /* PMAP_* caching flags for ttm_io_prot */
54 #include <uvm/uvm_pmap.h>
55 #include <linux/nbsd-namespace.h>
56 #endif
57
ttm_bo_free_old_node(struct ttm_buffer_object * bo)58 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
59 {
60 ttm_bo_mem_put(bo, &bo->mem);
61 }
62
ttm_bo_move_ttm(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,struct ttm_mem_reg * new_mem)63 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
64 struct ttm_operation_ctx *ctx,
65 struct ttm_mem_reg *new_mem)
66 {
67 struct ttm_tt *ttm = bo->ttm;
68 struct ttm_mem_reg *old_mem = &bo->mem;
69 int ret;
70
71 if (old_mem->mem_type != TTM_PL_SYSTEM) {
72 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
73
74 if (unlikely(ret != 0)) {
75 if (ret != -ERESTARTSYS)
76 pr_err("Failed to expire sync object before unbinding TTM\n");
77 return ret;
78 }
79
80 ttm_tt_unbind(ttm);
81 ttm_bo_free_old_node(bo);
82 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
83 TTM_PL_MASK_MEM);
84 old_mem->mem_type = TTM_PL_SYSTEM;
85 }
86
87 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
88 if (unlikely(ret != 0))
89 return ret;
90
91 if (new_mem->mem_type != TTM_PL_SYSTEM) {
92 ret = ttm_tt_bind(ttm, new_mem, ctx);
93 if (unlikely(ret != 0))
94 return ret;
95 }
96
97 *old_mem = *new_mem;
98 new_mem->mm_node = NULL;
99
100 return 0;
101 }
102 EXPORT_SYMBOL(ttm_bo_move_ttm);
103
ttm_mem_io_lock(struct ttm_mem_type_manager * man,bool interruptible)104 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
105 {
106 if (likely(man->io_reserve_fastpath))
107 return 0;
108
109 if (interruptible)
110 return mutex_lock_interruptible(&man->io_reserve_mutex);
111
112 mutex_lock(&man->io_reserve_mutex);
113 return 0;
114 }
115
ttm_mem_io_unlock(struct ttm_mem_type_manager * man)116 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
117 {
118 if (likely(man->io_reserve_fastpath))
119 return;
120
121 mutex_unlock(&man->io_reserve_mutex);
122 }
123
ttm_mem_io_evict(struct ttm_mem_type_manager * man)124 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
125 {
126 struct ttm_buffer_object *bo;
127
128 if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
129 return -EAGAIN;
130
131 bo = list_first_entry(&man->io_reserve_lru,
132 struct ttm_buffer_object,
133 io_reserve_lru);
134 list_del_init(&bo->io_reserve_lru);
135 ttm_bo_unmap_virtual_locked(bo);
136
137 return 0;
138 }
139
140
ttm_mem_io_reserve(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem)141 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
142 struct ttm_mem_reg *mem)
143 {
144 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
145 int ret = 0;
146
147 if (!bdev->driver->io_mem_reserve)
148 return 0;
149 if (likely(man->io_reserve_fastpath))
150 return bdev->driver->io_mem_reserve(bdev, mem);
151
152 if (bdev->driver->io_mem_reserve &&
153 mem->bus.io_reserved_count++ == 0) {
154 retry:
155 ret = bdev->driver->io_mem_reserve(bdev, mem);
156 if (ret == -EAGAIN) {
157 ret = ttm_mem_io_evict(man);
158 if (ret == 0)
159 goto retry;
160 }
161 }
162 return ret;
163 }
164
ttm_mem_io_free(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem)165 void ttm_mem_io_free(struct ttm_bo_device *bdev,
166 struct ttm_mem_reg *mem)
167 {
168 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
169
170 if (likely(man->io_reserve_fastpath))
171 return;
172
173 if (bdev->driver->io_mem_reserve &&
174 --mem->bus.io_reserved_count == 0 &&
175 bdev->driver->io_mem_free)
176 bdev->driver->io_mem_free(bdev, mem);
177
178 }
179
ttm_mem_io_reserve_vm(struct ttm_buffer_object * bo)180 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
181 {
182 struct ttm_mem_reg *mem = &bo->mem;
183 int ret;
184
185 if (!mem->bus.io_reserved_vm) {
186 struct ttm_mem_type_manager *man =
187 &bo->bdev->man[mem->mem_type];
188
189 ret = ttm_mem_io_reserve(bo->bdev, mem);
190 if (unlikely(ret != 0))
191 return ret;
192 mem->bus.io_reserved_vm = true;
193 if (man->use_io_reserve_lru)
194 list_add_tail(&bo->io_reserve_lru,
195 &man->io_reserve_lru);
196 }
197 return 0;
198 }
199
ttm_mem_io_free_vm(struct ttm_buffer_object * bo)200 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
201 {
202 struct ttm_mem_reg *mem = &bo->mem;
203
204 if (mem->bus.io_reserved_vm) {
205 mem->bus.io_reserved_vm = false;
206 list_del_init(&bo->io_reserve_lru);
207 ttm_mem_io_free(bo->bdev, mem);
208 }
209 }
210
ttm_mem_reg_ioremap(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem,void ** virtual)211 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
212 void **virtual)
213 {
214 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
215 int ret;
216 void *addr;
217
218 *virtual = NULL;
219 (void) ttm_mem_io_lock(man, false);
220 ret = ttm_mem_io_reserve(bdev, mem);
221 ttm_mem_io_unlock(man);
222 if (ret || !mem->bus.is_iomem)
223 return ret;
224
225 if (mem->bus.addr) {
226 addr = mem->bus.addr;
227 } else {
228 #ifdef __NetBSD__
229 const bus_addr_t bus_addr = (mem->bus.base + mem->bus.offset);
230 int flags = BUS_SPACE_MAP_LINEAR;
231
232 if (ISSET(mem->placement, TTM_PL_FLAG_WC))
233 flags |= BUS_SPACE_MAP_PREFETCHABLE;
234 /* XXX errno NetBSD->Linux */
235 ret = -bus_space_map(bdev->memt, bus_addr, mem->bus.size,
236 flags, &mem->bus.memh);
237 if (ret) {
238 (void) ttm_mem_io_lock(man, false);
239 ttm_mem_io_free(bdev, mem);
240 ttm_mem_io_unlock(man);
241 return ret;
242 }
243 addr = bus_space_vaddr(bdev->memt, mem->bus.memh);
244 #else
245 if (mem->placement & TTM_PL_FLAG_WC)
246 addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
247 else
248 addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size);
249 if (!addr) {
250 (void) ttm_mem_io_lock(man, false);
251 ttm_mem_io_free(bdev, mem);
252 ttm_mem_io_unlock(man);
253 return -ENOMEM;
254 }
255 #endif
256 }
257 *virtual = addr;
258 return 0;
259 }
260
ttm_mem_reg_iounmap(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem,void * virtual)261 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
262 void *virtual)
263 {
264 struct ttm_mem_type_manager *man;
265
266 man = &bdev->man[mem->mem_type];
267
268 if (virtual && mem->bus.addr == NULL)
269 #ifdef __NetBSD__
270 bus_space_unmap(bdev->memt, mem->bus.memh, mem->bus.size);
271 #else
272 iounmap(virtual);
273 #endif
274 (void) ttm_mem_io_lock(man, false);
275 ttm_mem_io_free(bdev, mem);
276 ttm_mem_io_unlock(man);
277 }
278
279 #ifdef __NetBSD__
280 # define ioread32 fake_ioread32
281 # define iowrite32 fake_iowrite32
282
283 static inline uint32_t
ioread32(const volatile uint32_t * p)284 ioread32(const volatile uint32_t *p)
285 {
286 uint32_t v;
287
288 v = *p;
289 __insn_barrier(); /* XXX ttm io barrier */
290
291 return v; /* XXX ttm byte order */
292 }
293
294 static inline void
iowrite32(uint32_t v,volatile uint32_t * p)295 iowrite32(uint32_t v, volatile uint32_t *p)
296 {
297
298 __insn_barrier(); /* XXX ttm io barrier */
299 *p = v; /* XXX ttm byte order */
300 }
301 #endif
302
ttm_copy_io_page(void * dst,void * src,unsigned long page)303 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
304 {
305 uint32_t *dstP =
306 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
307 uint32_t *srcP =
308 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
309
310 int i;
311 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
312 iowrite32(ioread32(srcP++), dstP++);
313 return 0;
314 }
315
316 #ifdef __NetBSD__
317 # undef ioread32
318 # undef iowrite32
319 #endif
320
321 #ifdef CONFIG_X86
322 #define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
323 #define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
324 #else
325 #define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0, __prot)
326 #define __ttm_kunmap_atomic(__addr) vunmap(__addr, 1)
327 #endif
328
329
330 /**
331 * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
332 * specified page protection.
333 *
334 * @page: The page to map.
335 * @prot: The page protection.
336 *
337 * This function maps a TTM page using the kmap_atomic api if available,
338 * otherwise falls back to vmap. The user must make sure that the
339 * specified page does not have an aliased mapping with a different caching
340 * policy unless the architecture explicitly allows it. Also mapping and
341 * unmapping using this api must be correctly nested. Unmapping should
342 * occur in the reverse order of mapping.
343 */
ttm_kmap_atomic_prot(struct page * page,pgprot_t prot)344 void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
345 {
346 if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
347 return kmap_atomic(page);
348 else
349 return __ttm_kmap_atomic_prot(page, prot);
350 }
351 EXPORT_SYMBOL(ttm_kmap_atomic_prot);
352
353 /**
354 * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
355 * ttm_kmap_atomic_prot.
356 *
357 * @addr: The virtual address from the map.
358 * @prot: The page protection.
359 */
ttm_kunmap_atomic_prot(void * addr,pgprot_t prot)360 void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
361 {
362 if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
363 kunmap_atomic(addr);
364 else
365 __ttm_kunmap_atomic(addr);
366 }
367 EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
368
ttm_copy_io_ttm_page(struct ttm_tt * ttm,void * src,unsigned long page,pgprot_t prot)369 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
370 unsigned long page,
371 pgprot_t prot)
372 {
373 struct page *d = ttm->pages[page];
374 void *dst;
375
376 if (!d)
377 return -ENOMEM;
378
379 src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
380 dst = ttm_kmap_atomic_prot(d, prot);
381 if (!dst)
382 return -ENOMEM;
383
384 memcpy_fromio(dst, src, PAGE_SIZE);
385
386 ttm_kunmap_atomic_prot(dst, prot);
387
388 return 0;
389 }
390
ttm_copy_ttm_io_page(struct ttm_tt * ttm,void * dst,unsigned long page,pgprot_t prot)391 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
392 unsigned long page,
393 pgprot_t prot)
394 {
395 struct page *s = ttm->pages[page];
396 void *src;
397
398 if (!s)
399 return -ENOMEM;
400
401 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
402 src = ttm_kmap_atomic_prot(s, prot);
403 if (!src)
404 return -ENOMEM;
405
406 memcpy_toio(dst, src, PAGE_SIZE);
407
408 ttm_kunmap_atomic_prot(src, prot);
409
410 return 0;
411 }
412
ttm_bo_move_memcpy(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,struct ttm_mem_reg * new_mem)413 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
414 struct ttm_operation_ctx *ctx,
415 struct ttm_mem_reg *new_mem)
416 {
417 struct ttm_bo_device *bdev = bo->bdev;
418 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
419 struct ttm_tt *ttm = bo->ttm;
420 struct ttm_mem_reg *old_mem = &bo->mem;
421 struct ttm_mem_reg old_copy = *old_mem;
422 void *old_iomap;
423 void *new_iomap;
424 int ret;
425 unsigned long i;
426 unsigned long page;
427 unsigned long add = 0;
428 int dir;
429
430 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
431 if (ret)
432 return ret;
433
434 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
435 if (ret)
436 return ret;
437 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
438 if (ret)
439 goto out;
440
441 /*
442 * Single TTM move. NOP.
443 */
444 if (old_iomap == NULL && new_iomap == NULL)
445 goto out2;
446
447 /*
448 * Don't move nonexistent data. Clear destination instead.
449 */
450 if (old_iomap == NULL &&
451 (ttm == NULL || (ttm->state == tt_unpopulated &&
452 !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
453 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
454 goto out2;
455 }
456
457 /*
458 * TTM might be null for moves within the same region.
459 */
460 if (ttm) {
461 ret = ttm_tt_populate(ttm, ctx);
462 if (ret)
463 goto out1;
464 }
465
466 add = 0;
467 dir = 1;
468
469 if ((old_mem->mem_type == new_mem->mem_type) &&
470 (new_mem->start < old_mem->start + old_mem->size)) {
471 dir = -1;
472 add = new_mem->num_pages - 1;
473 }
474
475 for (i = 0; i < new_mem->num_pages; ++i) {
476 page = i * dir + add;
477 if (old_iomap == NULL) {
478 pgprot_t prot = ttm_io_prot(old_mem->placement,
479 PAGE_KERNEL);
480 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
481 prot);
482 } else if (new_iomap == NULL) {
483 pgprot_t prot = ttm_io_prot(new_mem->placement,
484 PAGE_KERNEL);
485 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
486 prot);
487 } else {
488 ret = ttm_copy_io_page(new_iomap, old_iomap, page);
489 }
490 if (ret)
491 goto out1;
492 }
493 mb();
494 out2:
495 old_copy = *old_mem;
496 *old_mem = *new_mem;
497 new_mem->mm_node = NULL;
498
499 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
500 ttm_tt_destroy(ttm);
501 bo->ttm = NULL;
502 }
503
504 out1:
505 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
506 out:
507 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
508
509 /*
510 * On error, keep the mm node!
511 */
512 if (!ret)
513 ttm_bo_mem_put(bo, &old_copy);
514 return ret;
515 }
516 EXPORT_SYMBOL(ttm_bo_move_memcpy);
517
ttm_transfered_destroy(struct ttm_buffer_object * bo)518 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
519 {
520 struct ttm_transfer_obj *fbo;
521
522 fbo = container_of(bo, struct ttm_transfer_obj, base);
523 ttm_bo_put(fbo->bo);
524 dma_resv_fini(&fbo->base.base._resv);
525 if (ttm_bo_uses_embedded_gem_object(bo)) {
526 /*
527 * Initialization is unconditional, but we don't go
528 * through drm_gem_object_release, and destruction in
529 * ttm_bo_release is conditional, so do this
530 * conditionally with the reverse sense.
531 *
532 * Yes, this is a kludge.
533 */
534 drm_vma_node_destroy(&fbo->base.base.vma_node);
535 }
536 kfree(fbo);
537 }
538
539 /**
540 * ttm_buffer_object_transfer
541 *
542 * @bo: A pointer to a struct ttm_buffer_object.
543 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
544 * holding the data of @bo with the old placement.
545 *
546 * This is a utility function that may be called after an accelerated move
547 * has been scheduled. A new buffer object is created as a placeholder for
548 * the old data while it's being copied. When that buffer object is idle,
549 * it can be destroyed, releasing the space of the old placement.
550 * Returns:
551 * !0: Failure.
552 */
553
ttm_buffer_object_transfer(struct ttm_buffer_object * bo,struct ttm_buffer_object ** new_obj)554 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
555 struct ttm_buffer_object **new_obj)
556 {
557 struct ttm_transfer_obj *fbo;
558 int ret;
559
560 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
561 if (!fbo)
562 return -ENOMEM;
563
564 fbo->base = *bo;
565 fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT;
566
567 ttm_bo_get(bo);
568 fbo->bo = bo;
569
570 /**
571 * Fix up members that we shouldn't copy directly:
572 * TODO: Explicit member copy would probably be better here.
573 */
574
575 atomic_inc(&ttm_bo_glob.bo_count);
576 INIT_LIST_HEAD(&fbo->base.ddestroy);
577 INIT_LIST_HEAD(&fbo->base.lru);
578 INIT_LIST_HEAD(&fbo->base.swap);
579 INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
580 fbo->base.moving = NULL;
581 #ifdef __NetBSD__
582 drm_vma_node_init(&fbo->base.base.vma_node);
583 uvm_obj_init(&fbo->base.uvmobj, bo->bdev->driver->ttm_uvm_ops, true, 1);
584 rw_obj_hold(bo->uvmobj.vmobjlock);
585 uvm_obj_setlock(&fbo->base.uvmobj, bo->uvmobj.vmobjlock);
586 #else
587 drm_vma_node_reset(&fbo->base.base.vma_node);
588 #endif
589
590 kref_init(&fbo->base.list_kref);
591 kref_init(&fbo->base.kref);
592 fbo->base.destroy = &ttm_transfered_destroy;
593 fbo->base.acc_size = 0;
594 if (bo->base.resv == &bo->base._resv)
595 fbo->base.base.resv = &fbo->base.base._resv;
596
597 dma_resv_init(&fbo->base.base._resv);
598 ret = dma_resv_trylock(&fbo->base.base._resv);
599 WARN_ON(!ret);
600
601 *new_obj = &fbo->base;
602 return 0;
603 }
604
ttm_io_prot(uint32_t caching_flags,pgprot_t tmp)605 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
606 {
607 /* Cached mappings need no adjustment */
608 if (caching_flags & TTM_PL_FLAG_CACHED)
609 return tmp;
610
611 #ifdef __NetBSD__
612 tmp &= ~PMAP_CACHE_MASK;
613 if (caching_flags & TTM_PL_FLAG_WC)
614 return (tmp | PMAP_WRITE_COMBINE);
615 else
616 return (tmp | PMAP_NOCACHE);
617 #else
618 #if defined(__i386__) || defined(__x86_64__)
619 if (caching_flags & TTM_PL_FLAG_WC)
620 tmp = pgprot_writecombine(tmp);
621 else if (boot_cpu_data.x86 > 3)
622 tmp = pgprot_noncached(tmp);
623 #endif
624 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
625 defined(__powerpc__) || defined(__mips__)
626 if (caching_flags & TTM_PL_FLAG_WC)
627 tmp = pgprot_writecombine(tmp);
628 else
629 tmp = pgprot_noncached(tmp);
630 #endif
631 #if defined(__sparc__)
632 tmp = pgprot_noncached(tmp);
633 #endif
634 return tmp;
635 #endif
636 }
637 EXPORT_SYMBOL(ttm_io_prot);
638
ttm_bo_ioremap(struct ttm_buffer_object * bo,unsigned long offset,unsigned long size,struct ttm_bo_kmap_obj * map)639 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
640 unsigned long offset,
641 unsigned long size,
642 struct ttm_bo_kmap_obj *map)
643 {
644 struct ttm_mem_reg *mem = &bo->mem;
645
646 if (bo->mem.bus.addr) {
647 map->bo_kmap_type = ttm_bo_map_premapped;
648 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
649 } else {
650 map->bo_kmap_type = ttm_bo_map_iomap;
651 #ifdef __NetBSD__
652 {
653 bus_addr_t addr;
654 int flags = BUS_SPACE_MAP_LINEAR;
655 int ret;
656
657 addr = (bo->mem.bus.base + bo->mem.bus.offset + offset);
658 if (ISSET(mem->placement, TTM_PL_FLAG_WC))
659 flags |= BUS_SPACE_MAP_PREFETCHABLE;
660 /* XXX errno NetBSD->Linux */
661 ret = -bus_space_map(bo->bdev->memt, addr, size, flags,
662 &map->u.io.memh);
663 if (ret)
664 return ret;
665 map->u.io.size = size;
666 map->virtual = bus_space_vaddr(bo->bdev->memt, map->u.io.memh);
667 }
668 #else
669 if (mem->placement & TTM_PL_FLAG_WC)
670 map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
671 size);
672 else
673 map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset,
674 size);
675 #endif
676 }
677 return (!map->virtual) ? -ENOMEM : 0;
678 }
679
ttm_bo_kmap_ttm(struct ttm_buffer_object * bo,unsigned long start_page,unsigned long num_pages,struct ttm_bo_kmap_obj * map)680 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
681 unsigned long start_page,
682 unsigned long num_pages,
683 struct ttm_bo_kmap_obj *map)
684 {
685 struct ttm_mem_reg *mem = &bo->mem;
686 struct ttm_operation_ctx ctx = {
687 .interruptible = false,
688 .no_wait_gpu = false
689 };
690 struct ttm_tt *ttm = bo->ttm;
691 pgprot_t prot;
692 int ret;
693
694 BUG_ON(!ttm);
695
696 ret = ttm_tt_populate(ttm, &ctx);
697 if (ret)
698 return ret;
699
700 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
701 /*
702 * We're mapping a single page, and the desired
703 * page protection is consistent with the bo.
704 */
705
706 map->bo_kmap_type = ttm_bo_map_kmap;
707 #ifdef __NetBSD__
708 map->u.kmapped.page = ttm->pages[start_page];
709 map->virtual = kmap(map->u.kmapped.page);
710 #else
711 map->page = ttm->pages[start_page];
712 map->virtual = kmap(map->page);
713 #endif
714 } else {
715 /*
716 * We need to use vmap to get the desired page protection
717 * or to make the buffer object look contiguous.
718 */
719 prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
720 map->bo_kmap_type = ttm_bo_map_vmap;
721 map->virtual = vmap(ttm->pages + start_page, num_pages,
722 0, prot);
723 #ifdef __NetBSD__
724 map->u.vmapped.vsize = (vsize_t)num_pages << PAGE_SHIFT;
725 #endif
726 }
727 return (!map->virtual) ? -ENOMEM : 0;
728 }
729
ttm_bo_kmap(struct ttm_buffer_object * bo,unsigned long start_page,unsigned long num_pages,struct ttm_bo_kmap_obj * map)730 int ttm_bo_kmap(struct ttm_buffer_object *bo,
731 unsigned long start_page, unsigned long num_pages,
732 struct ttm_bo_kmap_obj *map)
733 {
734 struct ttm_mem_type_manager *man =
735 &bo->bdev->man[bo->mem.mem_type];
736 unsigned long offset, size;
737 int ret;
738
739 map->virtual = NULL;
740 map->bo = bo;
741 if (num_pages > bo->num_pages)
742 return -EINVAL;
743 if (start_page > bo->num_pages)
744 return -EINVAL;
745
746 (void) ttm_mem_io_lock(man, false);
747 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
748 ttm_mem_io_unlock(man);
749 if (ret)
750 return ret;
751 if (!bo->mem.bus.is_iomem) {
752 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
753 } else {
754 offset = start_page << PAGE_SHIFT;
755 size = num_pages << PAGE_SHIFT;
756 return ttm_bo_ioremap(bo, offset, size, map);
757 }
758 }
759 EXPORT_SYMBOL(ttm_bo_kmap);
760
ttm_bo_kunmap(struct ttm_bo_kmap_obj * map)761 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
762 {
763 struct ttm_buffer_object *bo = map->bo;
764 struct ttm_mem_type_manager *man =
765 &bo->bdev->man[bo->mem.mem_type];
766
767 if (!map->virtual)
768 return;
769 switch (map->bo_kmap_type) {
770 case ttm_bo_map_iomap:
771 #ifdef __NetBSD__
772 bus_space_unmap(bo->bdev->memt, map->u.io.memh,
773 map->u.io.size);
774 #else
775 iounmap(map->virtual);
776 #endif
777 break;
778 case ttm_bo_map_vmap:
779 #ifdef __NetBSD__
780 vunmap(map->virtual, map->u.vmapped.vsize >> PAGE_SHIFT);
781 #else
782 vunmap(map->virtual);
783 #endif
784 break;
785 case ttm_bo_map_kmap:
786 #ifdef __NetBSD__
787 kunmap(map->u.kmapped.page);
788 #else
789 kunmap(map->page);
790 #endif
791 break;
792 case ttm_bo_map_premapped:
793 break;
794 default:
795 BUG();
796 }
797 (void) ttm_mem_io_lock(man, false);
798 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
799 ttm_mem_io_unlock(man);
800 map->virtual = NULL;
801 #ifndef __NetBSD__
802 map->page = NULL;
803 #endif
804 }
805 EXPORT_SYMBOL(ttm_bo_kunmap);
806
ttm_bo_move_accel_cleanup(struct ttm_buffer_object * bo,struct dma_fence * fence,bool evict,struct ttm_mem_reg * new_mem)807 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
808 struct dma_fence *fence,
809 bool evict,
810 struct ttm_mem_reg *new_mem)
811 {
812 struct ttm_bo_device *bdev = bo->bdev;
813 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
814 struct ttm_mem_reg *old_mem = &bo->mem;
815 int ret;
816 struct ttm_buffer_object *ghost_obj;
817
818 dma_resv_add_excl_fence(bo->base.resv, fence);
819 if (evict) {
820 ret = ttm_bo_wait(bo, false, false);
821 if (ret)
822 return ret;
823
824 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
825 ttm_tt_destroy(bo->ttm);
826 bo->ttm = NULL;
827 }
828 ttm_bo_free_old_node(bo);
829 } else {
830 /**
831 * This should help pipeline ordinary buffer moves.
832 *
833 * Hang old buffer memory on a new buffer object,
834 * and leave it to be released when the GPU
835 * operation has completed.
836 */
837
838 dma_fence_put(bo->moving);
839 bo->moving = dma_fence_get(fence);
840
841 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
842 if (ret)
843 return ret;
844
845 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
846
847 /**
848 * If we're not moving to fixed memory, the TTM object
849 * needs to stay alive. Otherwhise hang it on the ghost
850 * bo to be unbound and destroyed.
851 */
852
853 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
854 ghost_obj->ttm = NULL;
855 else
856 bo->ttm = NULL;
857
858 dma_resv_unlock(&ghost_obj->base._resv);
859 ttm_bo_put(ghost_obj);
860 }
861
862 *old_mem = *new_mem;
863 new_mem->mm_node = NULL;
864
865 return 0;
866 }
867 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
868
ttm_bo_pipeline_move(struct ttm_buffer_object * bo,struct dma_fence * fence,bool evict,struct ttm_mem_reg * new_mem)869 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
870 struct dma_fence *fence, bool evict,
871 struct ttm_mem_reg *new_mem)
872 {
873 struct ttm_bo_device *bdev = bo->bdev;
874 struct ttm_mem_reg *old_mem = &bo->mem;
875
876 struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
877 struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
878
879 int ret;
880
881 dma_resv_add_excl_fence(bo->base.resv, fence);
882
883 if (!evict) {
884 struct ttm_buffer_object *ghost_obj;
885
886 /**
887 * This should help pipeline ordinary buffer moves.
888 *
889 * Hang old buffer memory on a new buffer object,
890 * and leave it to be released when the GPU
891 * operation has completed.
892 */
893
894 dma_fence_put(bo->moving);
895 bo->moving = dma_fence_get(fence);
896
897 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
898 if (ret)
899 return ret;
900
901 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
902
903 /**
904 * If we're not moving to fixed memory, the TTM object
905 * needs to stay alive. Otherwhise hang it on the ghost
906 * bo to be unbound and destroyed.
907 */
908
909 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
910 ghost_obj->ttm = NULL;
911 else
912 bo->ttm = NULL;
913
914 dma_resv_unlock(&ghost_obj->base._resv);
915 ttm_bo_put(ghost_obj);
916
917 } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
918
919 /**
920 * BO doesn't have a TTM we need to bind/unbind. Just remember
921 * this eviction and free up the allocation
922 */
923
924 spin_lock(&from->move_lock);
925 if (!from->move || dma_fence_is_later(fence, from->move)) {
926 dma_fence_put(from->move);
927 from->move = dma_fence_get(fence);
928 }
929 spin_unlock(&from->move_lock);
930
931 ttm_bo_free_old_node(bo);
932
933 dma_fence_put(bo->moving);
934 bo->moving = dma_fence_get(fence);
935
936 } else {
937 /**
938 * Last resort, wait for the move to be completed.
939 *
940 * Should never happen in pratice.
941 */
942
943 ret = ttm_bo_wait(bo, false, false);
944 if (ret)
945 return ret;
946
947 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
948 ttm_tt_destroy(bo->ttm);
949 bo->ttm = NULL;
950 }
951 ttm_bo_free_old_node(bo);
952 }
953
954 *old_mem = *new_mem;
955 new_mem->mm_node = NULL;
956
957 return 0;
958 }
959 EXPORT_SYMBOL(ttm_bo_pipeline_move);
960
ttm_bo_pipeline_gutting(struct ttm_buffer_object * bo)961 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
962 {
963 struct ttm_buffer_object *ghost;
964 int ret;
965
966 ret = ttm_buffer_object_transfer(bo, &ghost);
967 if (ret)
968 return ret;
969
970 ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);
971 /* Last resort, wait for the BO to be idle when we are OOM */
972 if (ret)
973 ttm_bo_wait(bo, false, false);
974
975 memset(&bo->mem, 0, sizeof(bo->mem));
976 bo->mem.mem_type = TTM_PL_SYSTEM;
977 bo->ttm = NULL;
978
979 dma_resv_unlock(&ghost->base._resv);
980 ttm_bo_put(ghost);
981
982 return 0;
983 }
984