1 /* $NetBSD: uvm_vnode.c,v 1.121 2024/04/05 13:05:41 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993
6 * The Regents of the University of California.
7 * Copyright (c) 1990 University of Utah.
8 *
9 * All rights reserved.
10 *
11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 *
39 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94
40 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp
41 */
42
43 /*
44 * uvm_vnode.c: the vnode pager.
45 */
46
47 #include <sys/cdefs.h>
48 __KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.121 2024/04/05 13:05:41 riastradh Exp $");
49
50 #ifdef _KERNEL_OPT
51 #include "opt_uvmhist.h"
52 #endif
53
54 #include <sys/atomic.h>
55 #include <sys/param.h>
56 #include <sys/systm.h>
57 #include <sys/kernel.h>
58 #include <sys/vnode.h>
59 #include <sys/disklabel.h>
60 #include <sys/ioctl.h>
61 #include <sys/fcntl.h>
62 #include <sys/conf.h>
63 #include <sys/pool.h>
64 #include <sys/mount.h>
65
66 #include <miscfs/specfs/specdev.h>
67
68 #include <uvm/uvm.h>
69 #include <uvm/uvm_readahead.h>
70 #include <uvm/uvm_page_array.h>
71
72 #ifdef UVMHIST
73 UVMHIST_DEFINE(ubchist);
74 #endif
75
76 /*
77 * functions
78 */
79
80 static void uvn_alloc_ractx(struct uvm_object *);
81 static void uvn_detach(struct uvm_object *);
82 static int uvn_get(struct uvm_object *, voff_t, struct vm_page **, int *,
83 int, vm_prot_t, int, int);
84 static void uvn_markdirty(struct uvm_object *);
85 static int uvn_put(struct uvm_object *, voff_t, voff_t, int);
86 static void uvn_reference(struct uvm_object *);
87
88 static int uvn_findpage(struct uvm_object *, voff_t, struct vm_page **,
89 unsigned int, struct uvm_page_array *a,
90 unsigned int);
91
92 /*
93 * master pager structure
94 */
95
96 const struct uvm_pagerops uvm_vnodeops = {
97 .pgo_reference = uvn_reference,
98 .pgo_detach = uvn_detach,
99 .pgo_get = uvn_get,
100 .pgo_put = uvn_put,
101 .pgo_markdirty = uvn_markdirty,
102 };
103
104 /*
105 * the ops!
106 */
107
108 /*
109 * uvn_reference
110 *
111 * duplicate a reference to a VM object. Note that the reference
112 * count must already be at least one (the passed in reference) so
113 * there is no chance of the uvn being killed or locked out here.
114 *
115 * => caller must call with object unlocked.
116 * => caller must be using the same accessprot as was used at attach time
117 */
118
119 static void
uvn_reference(struct uvm_object * uobj)120 uvn_reference(struct uvm_object *uobj)
121 {
122 vref((struct vnode *)uobj);
123 }
124
125
126 /*
127 * uvn_detach
128 *
129 * remove a reference to a VM object.
130 *
131 * => caller must call with object unlocked and map locked.
132 */
133
134 static void
uvn_detach(struct uvm_object * uobj)135 uvn_detach(struct uvm_object *uobj)
136 {
137 vrele((struct vnode *)uobj);
138 }
139
140 /*
141 * uvn_put: flush page data to backing store.
142 *
143 * => object must be locked on entry! VOP_PUTPAGES must unlock it.
144 * => flags: PGO_SYNCIO -- use sync. I/O
145 */
146
147 static int
uvn_put(struct uvm_object * uobj,voff_t offlo,voff_t offhi,int flags)148 uvn_put(struct uvm_object *uobj, voff_t offlo, voff_t offhi, int flags)
149 {
150 struct vnode *vp = (struct vnode *)uobj;
151 int error;
152
153 KASSERT(rw_write_held(uobj->vmobjlock));
154 error = VOP_PUTPAGES(vp, offlo, offhi, flags);
155
156 return error;
157 }
158
159 /*
160 * uvn_get: get pages (synchronously) from backing store
161 *
162 * => prefer map unlocked (not required)
163 * => object must be locked! we will _unlock_ it before starting any I/O.
164 * => flags: PGO_LOCKED: fault data structures are locked
165 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
166 * => NOTE: caller must check for released pages!!
167 */
168
169 static int
uvn_get(struct uvm_object * uobj,voff_t offset,struct vm_page ** pps,int * npagesp,int centeridx,vm_prot_t access_type,int advice,int flags)170 uvn_get(struct uvm_object *uobj, voff_t offset,
171 struct vm_page **pps /* IN/OUT */,
172 int *npagesp /* IN (OUT if PGO_LOCKED)*/,
173 int centeridx, vm_prot_t access_type, int advice, int flags)
174 {
175 struct vnode *vp = (struct vnode *)uobj;
176 int error;
177
178 UVMHIST_FUNC(__func__);
179 UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)vp, offset,
180 0, 0);
181
182 if (vp->v_type == VREG && (access_type & VM_PROT_WRITE) == 0
183 && (flags & PGO_LOCKED) == 0 && vp->v_tag != VT_TMPFS) {
184 uvn_alloc_ractx(uobj);
185 uvm_ra_request(vp->v_ractx, advice, uobj, offset,
186 *npagesp << PAGE_SHIFT);
187 }
188
189 error = VOP_GETPAGES(vp, offset, pps, npagesp, centeridx,
190 access_type, advice, flags);
191
192 if (flags & PGO_LOCKED)
193 KASSERT(rw_lock_held(uobj->vmobjlock));
194 return error;
195 }
196
197 /*
198 * uvn_markdirty: called when the object gains first dirty page
199 *
200 * => uobj must be write locked.
201 */
202
203 static void
uvn_markdirty(struct uvm_object * uobj)204 uvn_markdirty(struct uvm_object *uobj)
205 {
206 struct vnode *vp = (struct vnode *)uobj;
207
208 KASSERT(rw_write_held(uobj->vmobjlock));
209
210 mutex_enter(vp->v_interlock);
211 if ((vp->v_iflag & VI_ONWORKLST) == 0) {
212 vn_syncer_add_to_worklist(vp, filedelay);
213 }
214 mutex_exit(vp->v_interlock);
215 }
216
217 /*
218 * uvn_findpages:
219 * return the page for the uobj and offset requested, allocating if needed.
220 * => uobj must be locked.
221 * => returned pages will be BUSY.
222 */
223
224 int
uvn_findpages(struct uvm_object * uobj,voff_t offset,unsigned int * npagesp,struct vm_page ** pgs,struct uvm_page_array * a,unsigned int flags)225 uvn_findpages(struct uvm_object *uobj, voff_t offset, unsigned int *npagesp,
226 struct vm_page **pgs, struct uvm_page_array *a, unsigned int flags)
227 {
228 unsigned int count, found, npages;
229 int i, rv;
230 struct uvm_page_array a_store;
231
232 if (a == NULL) {
233 /*
234 * XXX fragile API
235 * note that the array can be the one supplied by the caller of
236 * uvn_findpages. in that case, fillflags used by the caller
237 * might not match strictly with ours.
238 * in particular, the caller might have filled the array
239 * without DENSE but passed us UFP_DIRTYONLY (thus DENSE).
240 */
241 const unsigned int fillflags =
242 ((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) |
243 ((flags & UFP_DIRTYONLY) ?
244 (UVM_PAGE_ARRAY_FILL_DIRTY|UVM_PAGE_ARRAY_FILL_DENSE) : 0);
245 a = &a_store;
246 uvm_page_array_init(a, uobj, fillflags);
247 }
248 count = found = 0;
249 npages = *npagesp;
250 if (flags & UFP_BACKWARD) {
251 for (i = npages - 1; i >= 0; i--, offset -= PAGE_SIZE) {
252 rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
253 i + 1);
254 if (rv == 0) {
255 if (flags & UFP_DIRTYONLY)
256 break;
257 } else
258 found++;
259 count++;
260 }
261 } else {
262 for (i = 0; i < npages; i++, offset += PAGE_SIZE) {
263 rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
264 npages - i);
265 if (rv == 0) {
266 if (flags & UFP_DIRTYONLY)
267 break;
268 } else
269 found++;
270 count++;
271 }
272 }
273 if (a == &a_store) {
274 uvm_page_array_fini(a);
275 }
276 *npagesp = count;
277 return (found);
278 }
279
280 /*
281 * uvn_findpage: find a single page
282 *
283 * if a suitable page was found, put it in *pgp and return 1.
284 * otherwise return 0.
285 */
286
287 static int
uvn_findpage(struct uvm_object * uobj,voff_t offset,struct vm_page ** pgp,unsigned int flags,struct uvm_page_array * a,unsigned int nleft)288 uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
289 unsigned int flags, struct uvm_page_array *a, unsigned int nleft)
290 {
291 struct vm_page *pg;
292 UVMHIST_FUNC(__func__);
293 UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)uobj, offset,
294 0, 0);
295
296 /*
297 * NOBUSY must come with NOWAIT and NOALLOC. if NOBUSY is
298 * specified, this may be called with a reader lock.
299 */
300
301 KASSERT(rw_lock_held(uobj->vmobjlock));
302 KASSERT((flags & UFP_NOBUSY) == 0 || (flags & UFP_NOWAIT) != 0);
303 KASSERT((flags & UFP_NOBUSY) == 0 || (flags & UFP_NOALLOC) != 0);
304 KASSERT((flags & UFP_NOBUSY) != 0 || rw_write_held(uobj->vmobjlock));
305
306 if (*pgp != NULL) {
307 UVMHIST_LOG(ubchist, "dontcare", 0,0,0,0);
308 goto skip_offset;
309 }
310 for (;;) {
311 /*
312 * look for an existing page.
313 */
314 pg = uvm_page_array_fill_and_peek(a, offset, nleft);
315 if (pg != NULL && pg->offset != offset) {
316 struct vm_page __diagused *tpg;
317 KASSERT(
318 ((a->ar_flags & UVM_PAGE_ARRAY_FILL_BACKWARD) != 0)
319 == (pg->offset < offset));
320 KASSERT((tpg = uvm_pagelookup(uobj, offset)) == NULL ||
321 ((a->ar_flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0 &&
322 !uvm_obj_page_dirty_p(tpg)));
323 pg = NULL;
324 if ((a->ar_flags & UVM_PAGE_ARRAY_FILL_DENSE) != 0) {
325 UVMHIST_LOG(ubchist, "dense", 0,0,0,0);
326 return 0;
327 }
328 }
329
330 /* nope? allocate one now */
331 if (pg == NULL) {
332 if (flags & UFP_NOALLOC) {
333 UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0);
334 return 0;
335 }
336 pg = uvm_pagealloc(uobj, offset, NULL,
337 UVM_FLAG_COLORMATCH);
338 if (pg == NULL) {
339 if (flags & UFP_NOWAIT) {
340 UVMHIST_LOG(ubchist, "nowait",0,0,0,0);
341 return 0;
342 }
343 rw_exit(uobj->vmobjlock);
344 uvm_wait("uvnfp1");
345 uvm_page_array_clear(a);
346 rw_enter(uobj->vmobjlock, RW_WRITER);
347 continue;
348 }
349 UVMHIST_LOG(ubchist, "alloced %#jx (color %ju)",
350 (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
351 KASSERTMSG(uvm_pagegetdirty(pg) ==
352 UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg);
353 break;
354 } else if (flags & UFP_NOCACHE) {
355 UVMHIST_LOG(ubchist, "nocache",0,0,0,0);
356 goto skip;
357 }
358
359 /* page is there, see if we need to wait on it */
360 if ((pg->flags & PG_BUSY) != 0) {
361 if (flags & UFP_NOWAIT) {
362 UVMHIST_LOG(ubchist, "nowait",0,0,0,0);
363 goto skip;
364 }
365 UVMHIST_LOG(ubchist, "wait %#jx (color %ju)",
366 (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
367 uvm_pagewait(pg, uobj->vmobjlock, "uvnfp2");
368 uvm_page_array_clear(a);
369 rw_enter(uobj->vmobjlock, RW_WRITER);
370 continue;
371 }
372
373 /* skip PG_RDONLY pages if requested */
374 if ((flags & UFP_NORDONLY) && (pg->flags & PG_RDONLY)) {
375 UVMHIST_LOG(ubchist, "nordonly",0,0,0,0);
376 goto skip;
377 }
378
379 /* stop on clean pages if requested */
380 if (flags & UFP_DIRTYONLY) {
381 const bool dirty = uvm_pagecheckdirty(pg, false);
382 if (!dirty) {
383 UVMHIST_LOG(ubchist, "dirtonly", 0,0,0,0);
384 return 0;
385 }
386 }
387
388 /* mark the page BUSY and we're done. */
389 if ((flags & UFP_NOBUSY) == 0) {
390 pg->flags |= PG_BUSY;
391 UVM_PAGE_OWN(pg, "uvn_findpage");
392 }
393 UVMHIST_LOG(ubchist, "found %#jx (color %ju)",
394 (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
395 uvm_page_array_advance(a);
396 break;
397 }
398 *pgp = pg;
399 return 1;
400
401 skip_offset:
402 /*
403 * skip this offset
404 */
405 pg = uvm_page_array_peek(a);
406 if (pg != NULL) {
407 if (pg->offset == offset) {
408 uvm_page_array_advance(a);
409 } else {
410 KASSERT((a->ar_flags & UVM_PAGE_ARRAY_FILL_DENSE) == 0);
411 }
412 }
413 return 0;
414
415 skip:
416 /*
417 * skip this page
418 */
419 KASSERT(pg != NULL);
420 uvm_page_array_advance(a);
421 return 0;
422 }
423
424 /*
425 * uvm_vnp_setsize: grow or shrink a vnode uobj
426 *
427 * grow => just update size value
428 * shrink => toss un-needed pages
429 *
430 * => we assume that the caller has a reference of some sort to the
431 * vnode in question so that it will not be yanked out from under
432 * us.
433 */
434
435 void
uvm_vnp_setsize(struct vnode * vp,voff_t newsize)436 uvm_vnp_setsize(struct vnode *vp, voff_t newsize)
437 {
438 struct uvm_object *uobj = &vp->v_uobj;
439 voff_t pgend = round_page(newsize);
440 voff_t oldsize;
441 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
442
443 rw_enter(uobj->vmobjlock, RW_WRITER);
444 UVMHIST_LOG(ubchist, "vp %#jx old %#jx new %#jx",
445 (uintptr_t)vp, vp->v_size, newsize, 0);
446
447 /*
448 * now check if the size has changed: if we shrink we had better
449 * toss some pages...
450 */
451
452 KASSERT(newsize != VSIZENOTSET);
453 KASSERT(newsize >= 0);
454 KASSERTMSG(vp->v_size <= vp->v_writesize, "vp=%p"
455 " v_size=0x%llx v_writesize=0x%llx", vp,
456 (unsigned long long)vp->v_size,
457 (unsigned long long)vp->v_writesize);
458 KASSERTMSG((vp->v_size == vp->v_writesize ||
459 newsize == vp->v_writesize || newsize <= vp->v_size),
460 "vp=%p v_size=0x%llx v_writesize=0x%llx newsize=0x%llx",
461 vp,
462 (unsigned long long)vp->v_size,
463 (unsigned long long)vp->v_writesize,
464 (unsigned long long)newsize);
465
466 oldsize = vp->v_writesize;
467
468 /*
469 * check whether size shrinks
470 * if old size hasn't been set, there are no pages to drop
471 * if there was an integer overflow in pgend, then this is no shrink
472 */
473 if (oldsize > pgend && oldsize != VSIZENOTSET && pgend >= 0) {
474 (void) uvn_put(uobj, pgend, 0, PGO_FREE | PGO_SYNCIO);
475 rw_enter(uobj->vmobjlock, RW_WRITER);
476 }
477 mutex_enter(vp->v_interlock);
478 vp->v_size = vp->v_writesize = newsize;
479 mutex_exit(vp->v_interlock);
480 rw_exit(uobj->vmobjlock);
481 }
482
483 void
uvm_vnp_setwritesize(struct vnode * vp,voff_t newsize)484 uvm_vnp_setwritesize(struct vnode *vp, voff_t newsize)
485 {
486
487 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
488 KASSERT(newsize != VSIZENOTSET);
489 KASSERT(newsize >= 0);
490 KASSERT(vp->v_size != VSIZENOTSET);
491 KASSERT(vp->v_writesize != VSIZENOTSET);
492 KASSERTMSG(vp->v_size <= vp->v_writesize, "vp=%p"
493 " v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", vp,
494 (unsigned long long)vp->v_size,
495 (unsigned long long)vp->v_writesize,
496 (unsigned long long)newsize);
497 KASSERTMSG(vp->v_size <= newsize, "vp=%p"
498 " v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", vp,
499 (unsigned long long)vp->v_size,
500 (unsigned long long)vp->v_writesize,
501 (unsigned long long)newsize);
502 mutex_enter(vp->v_interlock);
503 vp->v_writesize = newsize;
504 mutex_exit(vp->v_interlock);
505 rw_exit(vp->v_uobj.vmobjlock);
506 }
507
508 bool
uvn_text_p(struct uvm_object * uobj)509 uvn_text_p(struct uvm_object *uobj)
510 {
511 struct vnode *vp = (struct vnode *)uobj;
512 int iflag;
513
514 /*
515 * v_interlock is not held here, but VI_EXECMAP is only ever changed
516 * with the vmobjlock held too.
517 */
518 iflag = atomic_load_relaxed(&vp->v_iflag);
519 return (iflag & VI_EXECMAP) != 0;
520 }
521
522 static void
uvn_alloc_ractx(struct uvm_object * uobj)523 uvn_alloc_ractx(struct uvm_object *uobj)
524 {
525 struct vnode *vp = (struct vnode *)uobj;
526 struct uvm_ractx *ra = NULL;
527
528 KASSERT(rw_write_held(uobj->vmobjlock));
529
530 if (vp->v_type != VREG) {
531 return;
532 }
533 if (vp->v_ractx != NULL) {
534 return;
535 }
536 if (vp->v_ractx == NULL) {
537 rw_exit(uobj->vmobjlock);
538 ra = uvm_ra_allocctx();
539 rw_enter(uobj->vmobjlock, RW_WRITER);
540 if (ra != NULL && vp->v_ractx == NULL) {
541 vp->v_ractx = ra;
542 ra = NULL;
543 }
544 }
545 if (ra != NULL) {
546 uvm_ra_freectx(ra);
547 }
548 }
549