1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #include <drm/drmP.h>
26 #include "amdgpu.h"
27
28 struct amdgpu_vram_mgr {
29 struct drm_mm mm;
30 struct spinlock lock;
31 atomic64_t usage;
32 atomic64_t vis_usage;
33 };
34
35 /**
36 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
37 *
38 * @man: TTM memory type manager
39 * @p_size: maximum size of VRAM
40 *
41 * Allocate and initialize the VRAM manager.
42 */
amdgpu_vram_mgr_init(struct ttm_mem_type_manager * man,unsigned long p_size)43 static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
44 unsigned long p_size)
45 {
46 struct amdgpu_vram_mgr *mgr;
47
48 mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
49 if (!mgr)
50 return -ENOMEM;
51
52 drm_mm_init(&mgr->mm, 0, p_size);
53 spin_init(&mgr->lock, "agvrml");
54 man->priv = mgr;
55 return 0;
56 }
57
58 /**
59 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
60 *
61 * @man: TTM memory type manager
62 *
63 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
64 * allocated inside it.
65 */
amdgpu_vram_mgr_fini(struct ttm_mem_type_manager * man)66 static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
67 {
68 struct amdgpu_vram_mgr *mgr = man->priv;
69
70 spin_lock(&mgr->lock);
71 drm_mm_takedown(&mgr->mm);
72 spin_unlock(&mgr->lock);
73 kfree(mgr);
74 man->priv = NULL;
75 return 0;
76 }
77
78 /**
79 * amdgpu_vram_mgr_vis_size - Calculate visible node size
80 *
81 * @adev: amdgpu device structure
82 * @node: MM node structure
83 *
84 * Calculate how many bytes of the MM node are inside visible VRAM
85 */
amdgpu_vram_mgr_vis_size(struct amdgpu_device * adev,struct drm_mm_node * node)86 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
87 struct drm_mm_node *node)
88 {
89 uint64_t start = node->start << PAGE_SHIFT;
90 uint64_t end = (node->size + node->start) << PAGE_SHIFT;
91
92 if (start >= adev->gmc.visible_vram_size)
93 return 0;
94
95 return (end > adev->gmc.visible_vram_size ?
96 adev->gmc.visible_vram_size : end) - start;
97 }
98
99 /**
100 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
101 *
102 * @bo: &amdgpu_bo buffer object (must be in VRAM)
103 *
104 * Returns:
105 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
106 */
amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo * bo)107 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
108 {
109 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
110 struct ttm_mem_reg *mem = &bo->tbo.mem;
111 struct drm_mm_node *nodes = mem->mm_node;
112 unsigned pages = mem->num_pages;
113 u64 usage;
114
115 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
116 return amdgpu_bo_size(bo);
117
118 if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
119 return 0;
120
121 for (usage = 0; nodes && pages; pages -= nodes->size, nodes++)
122 usage += amdgpu_vram_mgr_vis_size(adev, nodes);
123
124 return usage;
125 }
126
127 /**
128 * amdgpu_vram_mgr_new - allocate new ranges
129 *
130 * @man: TTM memory type manager
131 * @tbo: TTM BO we need this range for
132 * @place: placement flags and restrictions
133 * @mem: the resulting mem object
134 *
135 * Allocate VRAM for the given BO.
136 */
amdgpu_vram_mgr_new(struct ttm_mem_type_manager * man,struct ttm_buffer_object * tbo,const struct ttm_place * place,struct ttm_mem_reg * mem)137 static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
138 struct ttm_buffer_object *tbo,
139 const struct ttm_place *place,
140 struct ttm_mem_reg *mem)
141 {
142 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
143 struct amdgpu_vram_mgr *mgr = man->priv;
144 struct drm_mm *mm = &mgr->mm;
145 struct drm_mm_node *nodes;
146 enum drm_mm_insert_mode mode;
147 unsigned long lpfn, num_nodes, pages_per_node, pages_left;
148 uint64_t usage = 0, vis_usage = 0;
149 unsigned i;
150 int r;
151
152 lpfn = place->lpfn;
153 if (!lpfn)
154 lpfn = man->size;
155
156 if (place->flags & TTM_PL_FLAG_CONTIGUOUS ||
157 amdgpu_vram_page_split == -1) {
158 pages_per_node = ~0ul;
159 num_nodes = 1;
160 } else {
161 pages_per_node = max((uint32_t)amdgpu_vram_page_split,
162 mem->page_alignment);
163 num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
164 }
165
166 nodes = kvmalloc_array(num_nodes, sizeof(*nodes),
167 GFP_KERNEL | __GFP_ZERO);
168 if (!nodes)
169 return -ENOMEM;
170
171 mode = DRM_MM_INSERT_BEST;
172 if (place->flags & TTM_PL_FLAG_TOPDOWN)
173 mode = DRM_MM_INSERT_HIGH;
174
175 mem->start = 0;
176 pages_left = mem->num_pages;
177
178 spin_lock(&mgr->lock);
179 for (i = 0; i < num_nodes; ++i) {
180 unsigned long pages = min(pages_left, pages_per_node);
181 uint32_t alignment = mem->page_alignment;
182 unsigned long start;
183
184 if (pages == pages_per_node)
185 alignment = pages_per_node;
186
187 r = drm_mm_insert_node_in_range(mm, &nodes[i],
188 pages, alignment, 0,
189 place->fpfn, lpfn,
190 mode);
191 if (unlikely(r))
192 goto error;
193
194 usage += nodes[i].size << PAGE_SHIFT;
195 vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
196
197 /* Calculate a virtual BO start address to easily check if
198 * everything is CPU accessible.
199 */
200 start = nodes[i].start + nodes[i].size;
201 if (start > mem->num_pages)
202 start -= mem->num_pages;
203 else
204 start = 0;
205 mem->start = max(mem->start, start);
206 pages_left -= pages;
207 }
208 spin_unlock(&mgr->lock);
209
210 atomic64_add(usage, &mgr->usage);
211 atomic64_add(vis_usage, &mgr->vis_usage);
212
213 mem->mm_node = nodes;
214
215 return 0;
216
217 error:
218 while (i--)
219 drm_mm_remove_node(&nodes[i]);
220 spin_unlock(&mgr->lock);
221
222 kvfree(nodes);
223 return r == -ENOSPC ? 0 : r;
224 }
225
226 /**
227 * amdgpu_vram_mgr_del - free ranges
228 *
229 * @man: TTM memory type manager
230 * @tbo: TTM BO we need this range for
231 * @place: placement flags and restrictions
232 * @mem: TTM memory object
233 *
234 * Free the allocated VRAM again.
235 */
amdgpu_vram_mgr_del(struct ttm_mem_type_manager * man,struct ttm_mem_reg * mem)236 static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
237 struct ttm_mem_reg *mem)
238 {
239 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
240 struct amdgpu_vram_mgr *mgr = man->priv;
241 struct drm_mm_node *nodes = mem->mm_node;
242 uint64_t usage = 0, vis_usage = 0;
243 unsigned pages = mem->num_pages;
244
245 if (!mem->mm_node)
246 return;
247
248 spin_lock(&mgr->lock);
249 while (pages) {
250 pages -= nodes->size;
251 drm_mm_remove_node(nodes);
252 usage += nodes->size << PAGE_SHIFT;
253 vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
254 ++nodes;
255 }
256 spin_unlock(&mgr->lock);
257
258 atomic64_sub(usage, &mgr->usage);
259 atomic64_sub(vis_usage, &mgr->vis_usage);
260
261 kvfree(mem->mm_node);
262 mem->mm_node = NULL;
263 }
264
265 /**
266 * amdgpu_vram_mgr_usage - how many bytes are used in this domain
267 *
268 * @man: TTM memory type manager
269 *
270 * Returns how many bytes are used in this domain.
271 */
amdgpu_vram_mgr_usage(struct ttm_mem_type_manager * man)272 uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man)
273 {
274 struct amdgpu_vram_mgr *mgr = man->priv;
275
276 return atomic64_read(&mgr->usage);
277 }
278
279 /**
280 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
281 *
282 * @man: TTM memory type manager
283 *
284 * Returns how many bytes are used in the visible part of VRAM
285 */
amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager * man)286 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man)
287 {
288 struct amdgpu_vram_mgr *mgr = man->priv;
289
290 return atomic64_read(&mgr->vis_usage);
291 }
292
293 /**
294 * amdgpu_vram_mgr_debug - dump VRAM table
295 *
296 * @man: TTM memory type manager
297 * @printer: DRM printer to use
298 *
299 * Dump the table content using printk.
300 */
amdgpu_vram_mgr_debug(struct ttm_mem_type_manager * man,struct drm_printer * printer)301 static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
302 struct drm_printer *printer)
303 {
304 struct amdgpu_vram_mgr *mgr = man->priv;
305
306 spin_lock(&mgr->lock);
307 drm_mm_print(&mgr->mm, printer);
308 spin_unlock(&mgr->lock);
309
310 drm_printf(printer, "man size:%lu pages, ram usage:%luMB, vis usage:%luMB\n",
311 man->size, amdgpu_vram_mgr_usage(man) >> 20,
312 amdgpu_vram_mgr_vis_usage(man) >> 20);
313 }
314
315 const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
316 .init = amdgpu_vram_mgr_init,
317 .takedown = amdgpu_vram_mgr_fini,
318 .get_node = amdgpu_vram_mgr_new,
319 .put_node = amdgpu_vram_mgr_del,
320 .debug = amdgpu_vram_mgr_debug
321 };
322