1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 * Authors: Christian König <christian.koenig@amd.com>
26 */
27
28 #include <linux/firmware.h>
29 #include <linux/module.h>
30 #include <drm/drmP.h>
31 #include <drm/drm.h>
32
33 #include "amdgpu.h"
34 #include "amdgpu_pm.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37
38 /* 1 second timeout */
39 #define VCE_IDLE_TIMEOUT msecs_to_jiffies(1000)
40
41 /* Firmware Names */
42 #ifdef CONFIG_DRM_AMDGPU_CIK
43 #define FIRMWARE_BONAIRE "amdgpufw_bonaire_vce"
44 #define FIRMWARE_KABINI "amdgpufw_kabini_vce"
45 #define FIRMWARE_KAVERI "amdgpufw_kaveri_vce"
46 #define FIRMWARE_HAWAII "amdgpufw_hawaii_vce"
47 #define FIRMWARE_MULLINS "amdgpufw_mullins_vce"
48 #endif
49 #define FIRMWARE_TONGA "amdgpufw_tonga_vce"
50 #define FIRMWARE_CARRIZO "amdgpufw_carrizo_vce"
51 #define FIRMWARE_FIJI "amdgpufw_fiji_vce"
52 #define FIRMWARE_STONEY "amdgpufw_stoney_vce"
53 #define FIRMWARE_POLARIS10 "amdgpufw_polaris10_vce"
54 #define FIRMWARE_POLARIS11 "amdgpufw_polaris11_vce"
55 #define FIRMWARE_POLARIS12 "amdgpufw_polaris12_vce"
56 #define FIRMWARE_VEGAM "amdgpufw_vegam_vce"
57
58 #define FIRMWARE_VEGA10 "amdgpufw_vega10_vce"
59 #define FIRMWARE_VEGA12 "amdgpufw_vega12_vce"
60 #define FIRMWARE_VEGA20 "amdgpufw_vega20_vce"
61
62 #ifdef CONFIG_DRM_AMDGPU_CIK
63 MODULE_FIRMWARE(FIRMWARE_BONAIRE);
64 MODULE_FIRMWARE(FIRMWARE_KABINI);
65 MODULE_FIRMWARE(FIRMWARE_KAVERI);
66 MODULE_FIRMWARE(FIRMWARE_HAWAII);
67 MODULE_FIRMWARE(FIRMWARE_MULLINS);
68 #endif
69 MODULE_FIRMWARE(FIRMWARE_TONGA);
70 MODULE_FIRMWARE(FIRMWARE_CARRIZO);
71 MODULE_FIRMWARE(FIRMWARE_FIJI);
72 MODULE_FIRMWARE(FIRMWARE_STONEY);
73 MODULE_FIRMWARE(FIRMWARE_POLARIS10);
74 MODULE_FIRMWARE(FIRMWARE_POLARIS11);
75 MODULE_FIRMWARE(FIRMWARE_POLARIS12);
76 MODULE_FIRMWARE(FIRMWARE_VEGAM);
77
78 MODULE_FIRMWARE(FIRMWARE_VEGA10);
79 MODULE_FIRMWARE(FIRMWARE_VEGA12);
80 MODULE_FIRMWARE(FIRMWARE_VEGA20);
81
82 static void amdgpu_vce_idle_work_handler(struct work_struct *work);
83
84 /**
85 * amdgpu_vce_init - allocate memory, load vce firmware
86 *
87 * @adev: amdgpu_device pointer
88 *
89 * First step to get VCE online, allocate memory and load the firmware
90 */
amdgpu_vce_sw_init(struct amdgpu_device * adev,unsigned long size)91 int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
92 {
93 const char *fw_name;
94 const struct common_firmware_header *hdr;
95 unsigned ucode_version, version_major, version_minor, binary_id;
96 int i, r;
97
98 switch (adev->asic_type) {
99 #ifdef CONFIG_DRM_AMDGPU_CIK
100 case CHIP_BONAIRE:
101 fw_name = FIRMWARE_BONAIRE;
102 break;
103 case CHIP_KAVERI:
104 fw_name = FIRMWARE_KAVERI;
105 break;
106 case CHIP_KABINI:
107 fw_name = FIRMWARE_KABINI;
108 break;
109 case CHIP_HAWAII:
110 fw_name = FIRMWARE_HAWAII;
111 break;
112 case CHIP_MULLINS:
113 fw_name = FIRMWARE_MULLINS;
114 break;
115 #endif
116 case CHIP_TONGA:
117 fw_name = FIRMWARE_TONGA;
118 break;
119 case CHIP_CARRIZO:
120 fw_name = FIRMWARE_CARRIZO;
121 break;
122 case CHIP_FIJI:
123 fw_name = FIRMWARE_FIJI;
124 break;
125 case CHIP_STONEY:
126 fw_name = FIRMWARE_STONEY;
127 break;
128 case CHIP_POLARIS10:
129 fw_name = FIRMWARE_POLARIS10;
130 break;
131 case CHIP_POLARIS11:
132 fw_name = FIRMWARE_POLARIS11;
133 break;
134 case CHIP_POLARIS12:
135 fw_name = FIRMWARE_POLARIS12;
136 break;
137 case CHIP_VEGAM:
138 fw_name = FIRMWARE_VEGAM;
139 break;
140 case CHIP_VEGA10:
141 fw_name = FIRMWARE_VEGA10;
142 break;
143 case CHIP_VEGA12:
144 fw_name = FIRMWARE_VEGA12;
145 break;
146 case CHIP_VEGA20:
147 fw_name = FIRMWARE_VEGA20;
148 break;
149
150 default:
151 return -EINVAL;
152 }
153
154 r = request_firmware(&adev->vce.fw, fw_name, adev->dev);
155 if (r) {
156 dev_err(adev->dev, "amdgpu_vce: Can't load firmware \"%s\"\n",
157 fw_name);
158 return r;
159 }
160
161 r = amdgpu_ucode_validate(adev->vce.fw);
162 if (r) {
163 dev_err(adev->dev, "amdgpu_vce: Can't validate firmware \"%s\"\n",
164 fw_name);
165 release_firmware(adev->vce.fw);
166 adev->vce.fw = NULL;
167 return r;
168 }
169
170 hdr = (const struct common_firmware_header *)adev->vce.fw->data;
171
172 ucode_version = le32_to_cpu(hdr->ucode_version);
173 version_major = (ucode_version >> 20) & 0xfff;
174 version_minor = (ucode_version >> 8) & 0xfff;
175 binary_id = ucode_version & 0xff;
176 DRM_INFO("Found VCE firmware Version: %hhd.%hhd Binary ID: %hhd\n",
177 version_major, version_minor, binary_id);
178 adev->vce.fw_version = ((version_major << 24) | (version_minor << 16) |
179 (binary_id << 8));
180
181 r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
182 AMDGPU_GEM_DOMAIN_VRAM, &adev->vce.vcpu_bo,
183 (u64 *)&adev->vce.gpu_addr, &adev->vce.cpu_addr);
184 if (r) {
185 dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
186 return r;
187 }
188
189 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
190 atomic_set(&adev->vce.handles[i], 0);
191 adev->vce.filp[i] = NULL;
192 }
193
194 INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler);
195 lockinit(&adev->vce.idle_mutex, "agavim", 0, LK_CANRECURSE);
196
197 return 0;
198 }
199
200 /**
201 * amdgpu_vce_fini - free memory
202 *
203 * @adev: amdgpu_device pointer
204 *
205 * Last step on VCE teardown, free firmware memory
206 */
amdgpu_vce_sw_fini(struct amdgpu_device * adev)207 int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
208 {
209 unsigned i;
210
211 if (adev->vce.vcpu_bo == NULL)
212 return 0;
213
214 drm_sched_entity_destroy(&adev->vce.entity);
215
216 amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, (u64 *)&adev->vce.gpu_addr,
217 (void **)&adev->vce.cpu_addr);
218
219 for (i = 0; i < adev->vce.num_rings; i++)
220 amdgpu_ring_fini(&adev->vce.ring[i]);
221
222 release_firmware(adev->vce.fw);
223 mutex_destroy(&adev->vce.idle_mutex);
224
225 return 0;
226 }
227
228 /**
229 * amdgpu_vce_entity_init - init entity
230 *
231 * @adev: amdgpu_device pointer
232 *
233 */
amdgpu_vce_entity_init(struct amdgpu_device * adev)234 int amdgpu_vce_entity_init(struct amdgpu_device *adev)
235 {
236 struct amdgpu_ring *ring;
237 struct drm_sched_rq *rq;
238 int r;
239
240 ring = &adev->vce.ring[0];
241 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
242 r = drm_sched_entity_init(&adev->vce.entity, &rq, 1, NULL);
243 if (r != 0) {
244 DRM_ERROR("Failed setting up VCE run queue.\n");
245 return r;
246 }
247
248 return 0;
249 }
250
251 /**
252 * amdgpu_vce_suspend - unpin VCE fw memory
253 *
254 * @adev: amdgpu_device pointer
255 *
256 */
amdgpu_vce_suspend(struct amdgpu_device * adev)257 int amdgpu_vce_suspend(struct amdgpu_device *adev)
258 {
259 int i;
260
261 cancel_delayed_work_sync(&adev->vce.idle_work);
262
263 if (adev->vce.vcpu_bo == NULL)
264 return 0;
265
266 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
267 if (atomic_read(&adev->vce.handles[i]))
268 break;
269
270 if (i == AMDGPU_MAX_VCE_HANDLES)
271 return 0;
272
273 /* TODO: suspending running encoding sessions isn't supported */
274 return -EINVAL;
275 }
276
277 /**
278 * amdgpu_vce_resume - pin VCE fw memory
279 *
280 * @adev: amdgpu_device pointer
281 *
282 */
amdgpu_vce_resume(struct amdgpu_device * adev)283 int amdgpu_vce_resume(struct amdgpu_device *adev)
284 {
285 void *cpu_addr;
286 const struct common_firmware_header *hdr;
287 unsigned offset;
288 int r;
289
290 if (adev->vce.vcpu_bo == NULL)
291 return -EINVAL;
292
293 r = amdgpu_bo_reserve(adev->vce.vcpu_bo, false);
294 if (r) {
295 dev_err(adev->dev, "(%d) failed to reserve VCE bo\n", r);
296 return r;
297 }
298
299 r = amdgpu_bo_kmap(adev->vce.vcpu_bo, &cpu_addr);
300 if (r) {
301 amdgpu_bo_unreserve(adev->vce.vcpu_bo);
302 dev_err(adev->dev, "(%d) VCE map failed\n", r);
303 return r;
304 }
305
306 hdr = (const struct common_firmware_header *)adev->vce.fw->data;
307 offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
308 memcpy_toio(cpu_addr, (adev->vce.fw->data) + offset,
309 (adev->vce.fw->datasize) - offset);
310
311 amdgpu_bo_kunmap(adev->vce.vcpu_bo);
312
313 amdgpu_bo_unreserve(adev->vce.vcpu_bo);
314
315 return 0;
316 }
317
318 /**
319 * amdgpu_vce_idle_work_handler - power off VCE
320 *
321 * @work: pointer to work structure
322 *
323 * power of VCE when it's not used any more
324 */
amdgpu_vce_idle_work_handler(struct work_struct * work)325 static void amdgpu_vce_idle_work_handler(struct work_struct *work)
326 {
327 struct amdgpu_device *adev =
328 container_of(work, struct amdgpu_device, vce.idle_work.work);
329 unsigned i, count = 0;
330
331 for (i = 0; i < adev->vce.num_rings; i++)
332 count += amdgpu_fence_count_emitted(&adev->vce.ring[i]);
333
334 if (count == 0) {
335 if (adev->pm.dpm_enabled) {
336 amdgpu_dpm_enable_vce(adev, false);
337 } else {
338 amdgpu_asic_set_vce_clocks(adev, 0, 0);
339 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
340 AMD_PG_STATE_GATE);
341 amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
342 AMD_CG_STATE_GATE);
343 }
344 } else {
345 schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT);
346 }
347 }
348
349 /**
350 * amdgpu_vce_ring_begin_use - power up VCE
351 *
352 * @ring: amdgpu ring
353 *
354 * Make sure VCE is powerd up when we want to use it
355 */
amdgpu_vce_ring_begin_use(struct amdgpu_ring * ring)356 void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring)
357 {
358 struct amdgpu_device *adev = ring->adev;
359 bool set_clocks;
360
361 if (amdgpu_sriov_vf(adev))
362 return;
363
364 mutex_lock(&adev->vce.idle_mutex);
365 set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work);
366 if (set_clocks) {
367 if (adev->pm.dpm_enabled) {
368 amdgpu_dpm_enable_vce(adev, true);
369 } else {
370 amdgpu_asic_set_vce_clocks(adev, 53300, 40000);
371 amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
372 AMD_CG_STATE_UNGATE);
373 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
374 AMD_PG_STATE_UNGATE);
375
376 }
377 }
378 mutex_unlock(&adev->vce.idle_mutex);
379 }
380
381 /**
382 * amdgpu_vce_ring_end_use - power VCE down
383 *
384 * @ring: amdgpu ring
385 *
386 * Schedule work to power VCE down again
387 */
amdgpu_vce_ring_end_use(struct amdgpu_ring * ring)388 void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring)
389 {
390 if (!amdgpu_sriov_vf(ring->adev))
391 schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT);
392 }
393
394 /**
395 * amdgpu_vce_free_handles - free still open VCE handles
396 *
397 * @adev: amdgpu_device pointer
398 * @filp: drm file pointer
399 *
400 * Close all VCE handles still open by this file pointer
401 */
amdgpu_vce_free_handles(struct amdgpu_device * adev,struct drm_file * filp)402 void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
403 {
404 struct amdgpu_ring *ring = &adev->vce.ring[0];
405 int i, r;
406 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
407 uint32_t handle = atomic_read(&adev->vce.handles[i]);
408
409 if (!handle || adev->vce.filp[i] != filp)
410 continue;
411
412 r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL);
413 if (r)
414 DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
415
416 adev->vce.filp[i] = NULL;
417 atomic_set(&adev->vce.handles[i], 0);
418 }
419 }
420
421 /**
422 * amdgpu_vce_get_create_msg - generate a VCE create msg
423 *
424 * @adev: amdgpu_device pointer
425 * @ring: ring we should submit the msg to
426 * @handle: VCE session handle to use
427 * @fence: optional fence to return
428 *
429 * Open up a stream for HW test
430 */
amdgpu_vce_get_create_msg(struct amdgpu_ring * ring,uint32_t handle,struct dma_fence ** fence)431 int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
432 struct dma_fence **fence)
433 {
434 const unsigned ib_size_dw = 1024;
435 struct amdgpu_job *job;
436 struct amdgpu_ib *ib;
437 struct dma_fence *f = NULL;
438 uint64_t dummy;
439 int i, r;
440
441 r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
442 if (r)
443 return r;
444
445 ib = &job->ibs[0];
446
447 dummy = ib->gpu_addr + 1024;
448
449 /* stitch together an VCE create msg */
450 ib->length_dw = 0;
451 ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
452 ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
453 ib->ptr[ib->length_dw++] = handle;
454
455 if ((ring->adev->vce.fw_version >> 24) >= 52)
456 ib->ptr[ib->length_dw++] = 0x00000040; /* len */
457 else
458 ib->ptr[ib->length_dw++] = 0x00000030; /* len */
459 ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */
460 ib->ptr[ib->length_dw++] = 0x00000000;
461 ib->ptr[ib->length_dw++] = 0x00000042;
462 ib->ptr[ib->length_dw++] = 0x0000000a;
463 ib->ptr[ib->length_dw++] = 0x00000001;
464 ib->ptr[ib->length_dw++] = 0x00000080;
465 ib->ptr[ib->length_dw++] = 0x00000060;
466 ib->ptr[ib->length_dw++] = 0x00000100;
467 ib->ptr[ib->length_dw++] = 0x00000100;
468 ib->ptr[ib->length_dw++] = 0x0000000c;
469 ib->ptr[ib->length_dw++] = 0x00000000;
470 if ((ring->adev->vce.fw_version >> 24) >= 52) {
471 ib->ptr[ib->length_dw++] = 0x00000000;
472 ib->ptr[ib->length_dw++] = 0x00000000;
473 ib->ptr[ib->length_dw++] = 0x00000000;
474 ib->ptr[ib->length_dw++] = 0x00000000;
475 }
476
477 ib->ptr[ib->length_dw++] = 0x00000014; /* len */
478 ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
479 ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
480 ib->ptr[ib->length_dw++] = dummy;
481 ib->ptr[ib->length_dw++] = 0x00000001;
482
483 for (i = ib->length_dw; i < ib_size_dw; ++i)
484 ib->ptr[i] = 0x0;
485
486 r = amdgpu_job_submit_direct(job, ring, &f);
487 if (r)
488 goto err;
489
490 if (fence)
491 *fence = dma_fence_get(f);
492 dma_fence_put(f);
493 return 0;
494
495 err:
496 amdgpu_job_free(job);
497 return r;
498 }
499
500 /**
501 * amdgpu_vce_get_destroy_msg - generate a VCE destroy msg
502 *
503 * @adev: amdgpu_device pointer
504 * @ring: ring we should submit the msg to
505 * @handle: VCE session handle to use
506 * @fence: optional fence to return
507 *
508 * Close up a stream for HW test or if userspace failed to do so
509 */
amdgpu_vce_get_destroy_msg(struct amdgpu_ring * ring,uint32_t handle,bool direct,struct dma_fence ** fence)510 int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
511 bool direct, struct dma_fence **fence)
512 {
513 const unsigned ib_size_dw = 1024;
514 struct amdgpu_job *job;
515 struct amdgpu_ib *ib;
516 struct dma_fence *f = NULL;
517 int i, r;
518
519 r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
520 if (r)
521 return r;
522
523 ib = &job->ibs[0];
524
525 /* stitch together an VCE destroy msg */
526 ib->length_dw = 0;
527 ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
528 ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
529 ib->ptr[ib->length_dw++] = handle;
530
531 ib->ptr[ib->length_dw++] = 0x00000020; /* len */
532 ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
533 ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */
534 ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */
535 ib->ptr[ib->length_dw++] = 0x00000000;
536 ib->ptr[ib->length_dw++] = 0x00000000;
537 ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */
538 ib->ptr[ib->length_dw++] = 0x00000000;
539
540 ib->ptr[ib->length_dw++] = 0x00000008; /* len */
541 ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */
542
543 for (i = ib->length_dw; i < ib_size_dw; ++i)
544 ib->ptr[i] = 0x0;
545
546 if (direct)
547 r = amdgpu_job_submit_direct(job, ring, &f);
548 else
549 r = amdgpu_job_submit(job, &ring->adev->vce.entity,
550 AMDGPU_FENCE_OWNER_UNDEFINED, &f);
551 if (r)
552 goto err;
553
554 if (fence)
555 *fence = dma_fence_get(f);
556 dma_fence_put(f);
557 return 0;
558
559 err:
560 amdgpu_job_free(job);
561 return r;
562 }
563
564 /**
565 * amdgpu_vce_cs_validate_bo - make sure not to cross 4GB boundary
566 *
567 * @p: parser context
568 * @lo: address of lower dword
569 * @hi: address of higher dword
570 * @size: minimum size
571 * @index: bs/fb index
572 *
573 * Make sure that no BO cross a 4GB boundary.
574 */
amdgpu_vce_validate_bo(struct amdgpu_cs_parser * p,uint32_t ib_idx,int lo,int hi,unsigned size,int32_t index)575 static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser *p, uint32_t ib_idx,
576 int lo, int hi, unsigned size, int32_t index)
577 {
578 int64_t offset = ((uint64_t)size) * ((int64_t)index);
579 struct ttm_operation_ctx ctx = { false, false };
580 struct amdgpu_bo_va_mapping *mapping;
581 unsigned i, fpfn, lpfn;
582 struct amdgpu_bo *bo;
583 uint64_t addr;
584 int r;
585
586 addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
587 ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
588 if (index >= 0) {
589 addr += offset;
590 fpfn = PAGE_ALIGN(offset) >> PAGE_SHIFT;
591 lpfn = 0x100000000ULL >> PAGE_SHIFT;
592 } else {
593 fpfn = 0;
594 lpfn = (0x100000000ULL - PAGE_ALIGN(offset)) >> PAGE_SHIFT;
595 }
596
597 r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
598 if (r) {
599 DRM_ERROR("Can't find BO for addr 0x%010lx %d %d %d %d\n",
600 addr, lo, hi, size, index);
601 return r;
602 }
603
604 for (i = 0; i < bo->placement.num_placement; ++i) {
605 bo->placements[i].fpfn = max(bo->placements[i].fpfn, fpfn);
606 bo->placements[i].lpfn = bo->placements[i].lpfn ?
607 min(bo->placements[i].lpfn, lpfn) : lpfn;
608 }
609 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
610 }
611
612
613 /**
614 * amdgpu_vce_cs_reloc - command submission relocation
615 *
616 * @p: parser context
617 * @lo: address of lower dword
618 * @hi: address of higher dword
619 * @size: minimum size
620 *
621 * Patch relocation inside command stream with real buffer address
622 */
amdgpu_vce_cs_reloc(struct amdgpu_cs_parser * p,uint32_t ib_idx,int lo,int hi,unsigned size,uint32_t index)623 static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
624 int lo, int hi, unsigned size, uint32_t index)
625 {
626 struct amdgpu_bo_va_mapping *mapping;
627 struct amdgpu_bo *bo;
628 uint64_t addr;
629 int r;
630
631 if (index == 0xffffffff)
632 index = 0;
633
634 addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
635 ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
636 addr += ((uint64_t)size) * ((uint64_t)index);
637
638 r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
639 if (r) {
640 DRM_ERROR("Can't find BO for addr 0x%010lx %d %d %d %d\n",
641 addr, lo, hi, size, index);
642 return r;
643 }
644
645 if ((addr + (uint64_t)size) >
646 (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
647 DRM_ERROR("BO to small for addr 0x%010lx %d %d\n",
648 addr, lo, hi);
649 return -EINVAL;
650 }
651
652 addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE;
653 addr += amdgpu_bo_gpu_offset(bo);
654 addr -= ((uint64_t)size) * ((uint64_t)index);
655
656 amdgpu_set_ib_value(p, ib_idx, lo, lower_32_bits(addr));
657 amdgpu_set_ib_value(p, ib_idx, hi, upper_32_bits(addr));
658
659 return 0;
660 }
661
662 /**
663 * amdgpu_vce_validate_handle - validate stream handle
664 *
665 * @p: parser context
666 * @handle: handle to validate
667 * @allocated: allocated a new handle?
668 *
669 * Validates the handle and return the found session index or -EINVAL
670 * we we don't have another free session index.
671 */
amdgpu_vce_validate_handle(struct amdgpu_cs_parser * p,uint32_t handle,uint32_t * allocated)672 static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p,
673 uint32_t handle, uint32_t *allocated)
674 {
675 unsigned i;
676
677 /* validate the handle */
678 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
679 if (atomic_read(&p->adev->vce.handles[i]) == handle) {
680 if (p->adev->vce.filp[i] != p->filp) {
681 DRM_ERROR("VCE handle collision detected!\n");
682 return -EINVAL;
683 }
684 return i;
685 }
686 }
687
688 /* handle not found try to alloc a new one */
689 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
690 if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) {
691 p->adev->vce.filp[i] = p->filp;
692 p->adev->vce.img_size[i] = 0;
693 *allocated |= 1 << i;
694 return i;
695 }
696 }
697
698 DRM_ERROR("No more free VCE handles!\n");
699 return -EINVAL;
700 }
701
702 /**
703 * amdgpu_vce_cs_parse - parse and validate the command stream
704 *
705 * @p: parser context
706 *
707 */
amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser * p,uint32_t ib_idx)708 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
709 {
710 struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
711 unsigned fb_idx = 0, bs_idx = 0;
712 int session_idx = -1;
713 uint32_t destroyed = 0;
714 uint32_t created = 0;
715 uint32_t allocated = 0;
716 uint32_t tmp, handle = 0;
717 uint32_t *size = &tmp;
718 unsigned idx;
719 int i, r = 0;
720
721 p->job->vm = NULL;
722 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
723
724 for (idx = 0; idx < ib->length_dw;) {
725 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
726 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
727
728 if ((len < 8) || (len & 3)) {
729 DRM_ERROR("invalid VCE command length (%d)!\n", len);
730 r = -EINVAL;
731 goto out;
732 }
733
734 switch (cmd) {
735 case 0x00000002: /* task info */
736 fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
737 bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
738 break;
739
740 case 0x03000001: /* encode */
741 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 10,
742 idx + 9, 0, 0);
743 if (r)
744 goto out;
745
746 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 12,
747 idx + 11, 0, 0);
748 if (r)
749 goto out;
750 break;
751
752 case 0x05000001: /* context buffer */
753 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
754 idx + 2, 0, 0);
755 if (r)
756 goto out;
757 break;
758
759 case 0x05000004: /* video bitstream buffer */
760 tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
761 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
762 tmp, bs_idx);
763 if (r)
764 goto out;
765 break;
766
767 case 0x05000005: /* feedback buffer */
768 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
769 4096, fb_idx);
770 if (r)
771 goto out;
772 break;
773
774 case 0x0500000d: /* MV buffer */
775 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
776 idx + 2, 0, 0);
777 if (r)
778 goto out;
779
780 r = amdgpu_vce_validate_bo(p, ib_idx, idx + 8,
781 idx + 7, 0, 0);
782 if (r)
783 goto out;
784 break;
785 }
786
787 idx += len / 4;
788 }
789
790 for (idx = 0; idx < ib->length_dw;) {
791 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
792 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
793
794 switch (cmd) {
795 case 0x00000001: /* session */
796 handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
797 session_idx = amdgpu_vce_validate_handle(p, handle,
798 &allocated);
799 if (session_idx < 0) {
800 r = session_idx;
801 goto out;
802 }
803 size = &p->adev->vce.img_size[session_idx];
804 break;
805
806 case 0x00000002: /* task info */
807 fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
808 bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
809 break;
810
811 case 0x01000001: /* create */
812 created |= 1 << session_idx;
813 if (destroyed & (1 << session_idx)) {
814 destroyed &= ~(1 << session_idx);
815 allocated |= 1 << session_idx;
816
817 } else if (!(allocated & (1 << session_idx))) {
818 DRM_ERROR("Handle already in use!\n");
819 r = -EINVAL;
820 goto out;
821 }
822
823 *size = amdgpu_get_ib_value(p, ib_idx, idx + 8) *
824 amdgpu_get_ib_value(p, ib_idx, idx + 10) *
825 8 * 3 / 2;
826 break;
827
828 case 0x04000001: /* config extension */
829 case 0x04000002: /* pic control */
830 case 0x04000005: /* rate control */
831 case 0x04000007: /* motion estimation */
832 case 0x04000008: /* rdo */
833 case 0x04000009: /* vui */
834 case 0x05000002: /* auxiliary buffer */
835 case 0x05000009: /* clock table */
836 break;
837
838 case 0x0500000c: /* hw config */
839 switch (p->adev->asic_type) {
840 #ifdef CONFIG_DRM_AMDGPU_CIK
841 case CHIP_KAVERI:
842 case CHIP_MULLINS:
843 #endif
844 case CHIP_CARRIZO:
845 break;
846 default:
847 r = -EINVAL;
848 goto out;
849 }
850 break;
851
852 case 0x03000001: /* encode */
853 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9,
854 *size, 0);
855 if (r)
856 goto out;
857
858 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 12, idx + 11,
859 *size / 3, 0);
860 if (r)
861 goto out;
862 break;
863
864 case 0x02000001: /* destroy */
865 destroyed |= 1 << session_idx;
866 break;
867
868 case 0x05000001: /* context buffer */
869 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
870 *size * 2, 0);
871 if (r)
872 goto out;
873 break;
874
875 case 0x05000004: /* video bitstream buffer */
876 tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
877 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
878 tmp, bs_idx);
879 if (r)
880 goto out;
881 break;
882
883 case 0x05000005: /* feedback buffer */
884 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
885 4096, fb_idx);
886 if (r)
887 goto out;
888 break;
889
890 case 0x0500000d: /* MV buffer */
891 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3,
892 idx + 2, *size, 0);
893 if (r)
894 goto out;
895
896 r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 8,
897 idx + 7, *size / 12, 0);
898 if (r)
899 goto out;
900 break;
901
902 default:
903 DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
904 r = -EINVAL;
905 goto out;
906 }
907
908 if (session_idx == -1) {
909 DRM_ERROR("no session command at start of IB\n");
910 r = -EINVAL;
911 goto out;
912 }
913
914 idx += len / 4;
915 }
916
917 if (allocated & ~created) {
918 DRM_ERROR("New session without create command!\n");
919 r = -ENOENT;
920 }
921
922 out:
923 if (!r) {
924 /* No error, free all destroyed handle slots */
925 tmp = destroyed;
926 } else {
927 /* Error during parsing, free all allocated handle slots */
928 tmp = allocated;
929 }
930
931 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
932 if (tmp & (1 << i))
933 atomic_set(&p->adev->vce.handles[i], 0);
934
935 return r;
936 }
937
938 /**
939 * amdgpu_vce_cs_parse_vm - parse the command stream in VM mode
940 *
941 * @p: parser context
942 *
943 */
amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser * p,uint32_t ib_idx)944 int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx)
945 {
946 struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
947 int session_idx = -1;
948 uint32_t destroyed = 0;
949 uint32_t created = 0;
950 uint32_t allocated = 0;
951 uint32_t tmp, handle = 0;
952 int i, r = 0, idx = 0;
953
954 while (idx < ib->length_dw) {
955 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
956 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
957
958 if ((len < 8) || (len & 3)) {
959 DRM_ERROR("invalid VCE command length (%d)!\n", len);
960 r = -EINVAL;
961 goto out;
962 }
963
964 switch (cmd) {
965 case 0x00000001: /* session */
966 handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
967 session_idx = amdgpu_vce_validate_handle(p, handle,
968 &allocated);
969 if (session_idx < 0) {
970 r = session_idx;
971 goto out;
972 }
973 break;
974
975 case 0x01000001: /* create */
976 created |= 1 << session_idx;
977 if (destroyed & (1 << session_idx)) {
978 destroyed &= ~(1 << session_idx);
979 allocated |= 1 << session_idx;
980
981 } else if (!(allocated & (1 << session_idx))) {
982 DRM_ERROR("Handle already in use!\n");
983 r = -EINVAL;
984 goto out;
985 }
986
987 break;
988
989 case 0x02000001: /* destroy */
990 destroyed |= 1 << session_idx;
991 break;
992
993 default:
994 break;
995 }
996
997 if (session_idx == -1) {
998 DRM_ERROR("no session command at start of IB\n");
999 r = -EINVAL;
1000 goto out;
1001 }
1002
1003 idx += len / 4;
1004 }
1005
1006 if (allocated & ~created) {
1007 DRM_ERROR("New session without create command!\n");
1008 r = -ENOENT;
1009 }
1010
1011 out:
1012 if (!r) {
1013 /* No error, free all destroyed handle slots */
1014 tmp = destroyed;
1015 amdgpu_ib_free(p->adev, ib, NULL);
1016 } else {
1017 /* Error during parsing, free all allocated handle slots */
1018 tmp = allocated;
1019 }
1020
1021 for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
1022 if (tmp & (1 << i))
1023 atomic_set(&p->adev->vce.handles[i], 0);
1024
1025 return r;
1026 }
1027
1028 /**
1029 * amdgpu_vce_ring_emit_ib - execute indirect buffer
1030 *
1031 * @ring: engine to use
1032 * @ib: the IB to execute
1033 *
1034 */
amdgpu_vce_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_ib * ib,unsigned vmid,bool ctx_switch)1035 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib,
1036 unsigned vmid, bool ctx_switch)
1037 {
1038 amdgpu_ring_write(ring, VCE_CMD_IB);
1039 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1040 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1041 amdgpu_ring_write(ring, ib->length_dw);
1042 }
1043
1044 /**
1045 * amdgpu_vce_ring_emit_fence - add a fence command to the ring
1046 *
1047 * @ring: engine to use
1048 * @fence: the fence
1049 *
1050 */
amdgpu_vce_ring_emit_fence(struct amdgpu_ring * ring,uint64_t addr,uint64_t seq,unsigned flags)1051 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, uint64_t addr, uint64_t seq,
1052 unsigned flags)
1053 {
1054 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1055
1056 amdgpu_ring_write(ring, VCE_CMD_FENCE);
1057 amdgpu_ring_write(ring, addr);
1058 amdgpu_ring_write(ring, upper_32_bits(addr));
1059 amdgpu_ring_write(ring, seq);
1060 amdgpu_ring_write(ring, VCE_CMD_TRAP);
1061 amdgpu_ring_write(ring, VCE_CMD_END);
1062 }
1063
1064 /**
1065 * amdgpu_vce_ring_test_ring - test if VCE ring is working
1066 *
1067 * @ring: the engine to test on
1068 *
1069 */
amdgpu_vce_ring_test_ring(struct amdgpu_ring * ring)1070 int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
1071 {
1072 struct amdgpu_device *adev = ring->adev;
1073 uint32_t rptr;
1074 unsigned i;
1075 int r, timeout = adev->usec_timeout;
1076
1077 /* skip ring test for sriov*/
1078 if (amdgpu_sriov_vf(adev))
1079 return 0;
1080
1081 r = amdgpu_ring_alloc(ring, 16);
1082 if (r) {
1083 DRM_ERROR("amdgpu: vce failed to lock ring %d (%d).\n",
1084 ring->idx, r);
1085 return r;
1086 }
1087
1088 rptr = amdgpu_ring_get_rptr(ring);
1089
1090 amdgpu_ring_write(ring, VCE_CMD_END);
1091 amdgpu_ring_commit(ring);
1092
1093 for (i = 0; i < timeout; i++) {
1094 if (amdgpu_ring_get_rptr(ring) != rptr)
1095 break;
1096 DRM_UDELAY(1);
1097 }
1098
1099 if (i < timeout) {
1100 DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
1101 ring->idx, i);
1102 } else {
1103 DRM_ERROR("amdgpu: ring %d test failed\n",
1104 ring->idx);
1105 r = -ETIMEDOUT;
1106 }
1107
1108 return r;
1109 }
1110
1111 /**
1112 * amdgpu_vce_ring_test_ib - test if VCE IBs are working
1113 *
1114 * @ring: the engine to test on
1115 *
1116 */
amdgpu_vce_ring_test_ib(struct amdgpu_ring * ring,long timeout)1117 int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1118 {
1119 struct dma_fence *fence = NULL;
1120 long r;
1121
1122 /* skip vce ring1/2 ib test for now, since it's not reliable */
1123 if (ring != &ring->adev->vce.ring[0])
1124 return 0;
1125
1126 r = amdgpu_vce_get_create_msg(ring, 1, NULL);
1127 if (r) {
1128 DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
1129 goto error;
1130 }
1131
1132 r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence);
1133 if (r) {
1134 DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
1135 goto error;
1136 }
1137
1138 r = dma_fence_wait_timeout(fence, false, timeout);
1139 if (r == 0) {
1140 DRM_ERROR("amdgpu: IB test timed out.\n");
1141 r = -ETIMEDOUT;
1142 } else if (r < 0) {
1143 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1144 } else {
1145 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
1146 r = 0;
1147 }
1148 error:
1149 dma_fence_put(fence);
1150 return r;
1151 }
1152