1 /* $NetBSD: uvm_object.c,v 1.25 2020/08/15 07:24:09 chs Exp $ */
2
3 /*
4 * Copyright (c) 2006, 2010, 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Mindaugas Rasiukevicius.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * uvm_object.c: operate with memory objects
34 *
35 * TODO:
36 * 1. Support PG_RELEASED-using objects
37 */
38
39 #include <sys/cdefs.h>
40 __KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.25 2020/08/15 07:24:09 chs Exp $");
41
42 #ifdef _KERNEL_OPT
43 #include "opt_ddb.h"
44 #endif
45
46 #include <sys/param.h>
47 #include <sys/rwlock.h>
48 #include <sys/queue.h>
49
50 #include <uvm/uvm.h>
51 #include <uvm/uvm_ddb.h>
52 #include <uvm/uvm_page_array.h>
53
54 /* Page count to fetch per single step. */
55 #define FETCH_PAGECOUNT 16
56
57 /*
58 * uvm_obj_init: initialize UVM memory object.
59 */
60 void
uvm_obj_init(struct uvm_object * uo,const struct uvm_pagerops * ops,bool alock,u_int refs)61 uvm_obj_init(struct uvm_object *uo, const struct uvm_pagerops *ops,
62 bool alock, u_int refs)
63 {
64
65 #if 0 /* notyet */
66 KASSERT(ops);
67 #endif
68 if (alock) {
69 /* Allocate and assign a lock. */
70 uo->vmobjlock = rw_obj_alloc();
71 } else {
72 /* The lock will need to be set via uvm_obj_setlock(). */
73 uo->vmobjlock = NULL;
74 }
75 uo->pgops = ops;
76 LIST_INIT(&uo->uo_ubc);
77 uo->uo_npages = 0;
78 uo->uo_refs = refs;
79 radix_tree_init_tree(&uo->uo_pages);
80 }
81
82 /*
83 * uvm_obj_destroy: destroy UVM memory object.
84 */
85 void
uvm_obj_destroy(struct uvm_object * uo,bool dlock)86 uvm_obj_destroy(struct uvm_object *uo, bool dlock)
87 {
88
89 KASSERT(radix_tree_empty_tree_p(&uo->uo_pages));
90
91 /* Purge any UBC entries associated with this object. */
92 ubc_purge(uo);
93
94 /* Destroy the lock, if requested. */
95 if (dlock) {
96 rw_obj_free(uo->vmobjlock);
97 }
98 radix_tree_fini_tree(&uo->uo_pages);
99 }
100
101 /*
102 * uvm_obj_setlock: assign a vmobjlock to the UVM object.
103 *
104 * => Caller is responsible to ensure that UVM objects is not use.
105 * => Only dynamic lock may be previously set. We drop the reference then.
106 */
107 void
uvm_obj_setlock(struct uvm_object * uo,krwlock_t * lockptr)108 uvm_obj_setlock(struct uvm_object *uo, krwlock_t *lockptr)
109 {
110 krwlock_t *olockptr = uo->vmobjlock;
111
112 if (olockptr) {
113 /* Drop the reference on the old lock. */
114 rw_obj_free(olockptr);
115 }
116 if (lockptr == NULL) {
117 /* If new lock is not passed - allocate default one. */
118 lockptr = rw_obj_alloc();
119 }
120 uo->vmobjlock = lockptr;
121 }
122
123 /*
124 * uvm_obj_wirepages: wire the pages of entire UVM object.
125 *
126 * => NOTE: this function should only be used for types of objects
127 * where PG_RELEASED flag is never set (aobj objects)
128 * => caller must pass page-aligned start and end values
129 */
130 int
uvm_obj_wirepages(struct uvm_object * uobj,off_t start,off_t end,struct pglist * list)131 uvm_obj_wirepages(struct uvm_object *uobj, off_t start, off_t end,
132 struct pglist *list)
133 {
134 int i, npages, error;
135 struct vm_page *pgs[FETCH_PAGECOUNT], *pg = NULL;
136 off_t offset = start, left;
137
138 left = (end - start) >> PAGE_SHIFT;
139
140 rw_enter(uobj->vmobjlock, RW_WRITER);
141 while (left) {
142
143 npages = MIN(FETCH_PAGECOUNT, left);
144
145 /* Get the pages */
146 memset(pgs, 0, sizeof(pgs));
147 error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0,
148 VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL,
149 PGO_SYNCIO);
150
151 if (error)
152 goto error;
153
154 rw_enter(uobj->vmobjlock, RW_WRITER);
155 for (i = 0; i < npages; i++) {
156
157 KASSERT(pgs[i] != NULL);
158 KASSERT(!(pgs[i]->flags & PG_RELEASED));
159
160 /*
161 * Loan break
162 */
163 if (pgs[i]->loan_count) {
164 while (pgs[i]->loan_count) {
165 pg = uvm_loanbreak(pgs[i]);
166 if (!pg) {
167 rw_exit(uobj->vmobjlock);
168 uvm_wait("uobjwirepg");
169 rw_enter(uobj->vmobjlock, RW_WRITER);
170 continue;
171 }
172 }
173 pgs[i] = pg;
174 }
175
176 if (pgs[i]->flags & PG_AOBJ) {
177 uvm_pagemarkdirty(pgs[i],
178 UVM_PAGE_STATUS_DIRTY);
179 uao_dropswap(uobj, i);
180 }
181 }
182
183 /* Wire the pages */
184 for (i = 0; i < npages; i++) {
185 uvm_pagelock(pgs[i]);
186 uvm_pagewire(pgs[i]);
187 uvm_pageunlock(pgs[i]);
188 if (list != NULL)
189 TAILQ_INSERT_TAIL(list, pgs[i], pageq.queue);
190 }
191
192 /* Unbusy the pages */
193 uvm_page_unbusy(pgs, npages);
194
195 left -= npages;
196 offset += npages << PAGE_SHIFT;
197 }
198 rw_exit(uobj->vmobjlock);
199
200 return 0;
201
202 error:
203 /* Unwire the pages which has been wired */
204 uvm_obj_unwirepages(uobj, start, offset);
205
206 return error;
207 }
208
209 /*
210 * uvm_obj_unwirepages: unwire the pages of entire UVM object.
211 *
212 * => NOTE: this function should only be used for types of objects
213 * where PG_RELEASED flag is never set
214 * => caller must pass page-aligned start and end values
215 */
216 void
uvm_obj_unwirepages(struct uvm_object * uobj,off_t start,off_t end)217 uvm_obj_unwirepages(struct uvm_object *uobj, off_t start, off_t end)
218 {
219 struct vm_page *pg;
220 off_t offset;
221
222 rw_enter(uobj->vmobjlock, RW_WRITER);
223 for (offset = start; offset < end; offset += PAGE_SIZE) {
224 pg = uvm_pagelookup(uobj, offset);
225
226 KASSERT(pg != NULL);
227 KASSERT(!(pg->flags & PG_RELEASED));
228
229 uvm_pagelock(pg);
230 uvm_pageunwire(pg);
231 uvm_pageunlock(pg);
232 }
233 rw_exit(uobj->vmobjlock);
234 }
235
236 static inline bool
uvm_obj_notag_p(struct uvm_object * uobj,int tag)237 uvm_obj_notag_p(struct uvm_object *uobj, int tag)
238 {
239
240 KASSERT(rw_lock_held(uobj->vmobjlock));
241 return radix_tree_empty_tagged_tree_p(&uobj->uo_pages, tag);
242 }
243
244 bool
uvm_obj_clean_p(struct uvm_object * uobj)245 uvm_obj_clean_p(struct uvm_object *uobj)
246 {
247
248 return uvm_obj_notag_p(uobj, UVM_PAGE_DIRTY_TAG);
249 }
250
251 bool
uvm_obj_nowriteback_p(struct uvm_object * uobj)252 uvm_obj_nowriteback_p(struct uvm_object *uobj)
253 {
254
255 return uvm_obj_notag_p(uobj, UVM_PAGE_WRITEBACK_TAG);
256 }
257
258 static inline bool
uvm_obj_page_tag_p(struct vm_page * pg,int tag)259 uvm_obj_page_tag_p(struct vm_page *pg, int tag)
260 {
261 struct uvm_object *uobj = pg->uobject;
262 uint64_t pgidx = pg->offset >> PAGE_SHIFT;
263
264 KASSERT(uobj != NULL);
265 KASSERT(rw_lock_held(uobj->vmobjlock));
266 return radix_tree_get_tag(&uobj->uo_pages, pgidx, tag) != 0;
267 }
268
269 static inline void
uvm_obj_page_set_tag(struct vm_page * pg,int tag)270 uvm_obj_page_set_tag(struct vm_page *pg, int tag)
271 {
272 struct uvm_object *uobj = pg->uobject;
273 uint64_t pgidx = pg->offset >> PAGE_SHIFT;
274
275 KASSERT(uobj != NULL);
276 KASSERT(rw_write_held(uobj->vmobjlock));
277 radix_tree_set_tag(&uobj->uo_pages, pgidx, tag);
278 }
279
280 static inline void
uvm_obj_page_clear_tag(struct vm_page * pg,int tag)281 uvm_obj_page_clear_tag(struct vm_page *pg, int tag)
282 {
283 struct uvm_object *uobj = pg->uobject;
284 uint64_t pgidx = pg->offset >> PAGE_SHIFT;
285
286 KASSERT(uobj != NULL);
287 KASSERT(rw_write_held(uobj->vmobjlock));
288 radix_tree_clear_tag(&uobj->uo_pages, pgidx, tag);
289 }
290
291 bool
uvm_obj_page_dirty_p(struct vm_page * pg)292 uvm_obj_page_dirty_p(struct vm_page *pg)
293 {
294
295 return uvm_obj_page_tag_p(pg, UVM_PAGE_DIRTY_TAG);
296 }
297
298 void
uvm_obj_page_set_dirty(struct vm_page * pg)299 uvm_obj_page_set_dirty(struct vm_page *pg)
300 {
301
302 uvm_obj_page_set_tag(pg, UVM_PAGE_DIRTY_TAG);
303 }
304
305 void
uvm_obj_page_clear_dirty(struct vm_page * pg)306 uvm_obj_page_clear_dirty(struct vm_page *pg)
307 {
308
309 uvm_obj_page_clear_tag(pg, UVM_PAGE_DIRTY_TAG);
310 }
311
312 bool
uvm_obj_page_writeback_p(struct vm_page * pg)313 uvm_obj_page_writeback_p(struct vm_page *pg)
314 {
315
316 return uvm_obj_page_tag_p(pg, UVM_PAGE_WRITEBACK_TAG);
317 }
318
319 void
uvm_obj_page_set_writeback(struct vm_page * pg)320 uvm_obj_page_set_writeback(struct vm_page *pg)
321 {
322
323 uvm_obj_page_set_tag(pg, UVM_PAGE_WRITEBACK_TAG);
324 }
325
326 void
uvm_obj_page_clear_writeback(struct vm_page * pg)327 uvm_obj_page_clear_writeback(struct vm_page *pg)
328 {
329
330 uvm_obj_page_clear_tag(pg, UVM_PAGE_WRITEBACK_TAG);
331 }
332
333 #if defined(DDB) || defined(DEBUGPRINT)
334
335 /*
336 * uvm_object_printit: actually prints the object
337 */
338 void
uvm_object_printit(struct uvm_object * uobj,bool full,void (* pr)(const char *,...))339 uvm_object_printit(struct uvm_object *uobj, bool full,
340 void (*pr)(const char *, ...))
341 {
342 struct uvm_page_array a;
343 struct vm_page *pg;
344 int cnt = 0;
345 voff_t off;
346
347 (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
348 uobj, rw_write_held(uobj->vmobjlock), uobj->pgops, uobj->uo_npages);
349 if (UVM_OBJ_IS_KERN_OBJECT(uobj))
350 (*pr)("refs=<SYSTEM>\n");
351 else
352 (*pr)("refs=%d\n", uobj->uo_refs);
353
354 if (!full) {
355 return;
356 }
357 (*pr)(" PAGES <pg,offset>:\n ");
358 uvm_page_array_init(&a, uobj, 0);
359 off = 0;
360 while ((pg = uvm_page_array_fill_and_peek(&a, off, 0)) != NULL) {
361 cnt++;
362 (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
363 if ((cnt % 3) == 0) {
364 (*pr)("\n ");
365 }
366 off = pg->offset + PAGE_SIZE;
367 uvm_page_array_advance(&a);
368 }
369 if ((cnt % 3) != 0) {
370 (*pr)("\n");
371 }
372 uvm_page_array_fini(&a);
373 }
374
375 #endif /* DDB || DEBUGPRINT */
376