1 /* $NetBSD: kern_physio.c,v 1.102 2022/07/10 23:11:55 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 1982, 1986, 1990, 1993
5 * The Regents of the University of California. All rights reserved.
6 * (c) UNIX System Laboratories, Inc.
7 * All or some portions of this file are derived from material licensed
8 * to the University of California by American Telephone and Telegraph
9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10 * the permission of UNIX System Laboratories, Inc.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)kern_physio.c 8.1 (Berkeley) 6/10/93
37 */
38
39 /*-
40 * Copyright (c) 1994 Christopher G. Demetriou
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. All advertising materials mentioning features or use of this software
51 * must display the following acknowledgement:
52 * This product includes software developed by the University of
53 * California, Berkeley and its contributors.
54 * 4. Neither the name of the University nor the names of its contributors
55 * may be used to endorse or promote products derived from this software
56 * without specific prior written permission.
57 *
58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68 * SUCH DAMAGE.
69 *
70 * @(#)kern_physio.c 8.1 (Berkeley) 6/10/93
71 */
72
73 #include <sys/cdefs.h>
74 __KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.102 2022/07/10 23:11:55 riastradh Exp $");
75
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/conf.h>
79 #include <sys/buf.h>
80 #include <sys/proc.h>
81 #include <sys/once.h>
82 #include <sys/workqueue.h>
83 #include <sys/kmem.h>
84
85 #include <uvm/uvm_extern.h>
86
87 ONCE_DECL(physio_initialized);
88 struct workqueue *physio_workqueue;
89
90 int physio_concurrency = 16;
91
92 /* #define PHYSIO_DEBUG */
93 #if defined(PHYSIO_DEBUG)
94 #define DPRINTF(a) printf a
95 #else /* defined(PHYSIO_DEBUG) */
96 #define DPRINTF(a) /* nothing */
97 #endif /* defined(PHYSIO_DEBUG) */
98
99 struct physio_stat {
100 int ps_running;
101 int ps_error;
102 int ps_failed;
103 off_t ps_endoffset;
104 size_t ps_resid;
105 buf_t *ps_orig_bp;
106 kmutex_t ps_lock;
107 kcondvar_t ps_cv;
108 };
109
110 static void
physio_done(struct work * wk,void * dummy)111 physio_done(struct work *wk, void *dummy)
112 {
113 struct buf *bp = (void *)wk;
114 size_t todo = bp->b_bufsize;
115 size_t done = bp->b_bcount - bp->b_resid;
116 struct physio_stat *ps = bp->b_private;
117 bool is_iobuf;
118
119 KASSERT(&bp->b_work == wk);
120 KASSERT(bp->b_bcount <= todo);
121 KASSERT(bp->b_resid <= bp->b_bcount);
122 KASSERT((bp->b_flags & B_PHYS) != 0);
123 KASSERT(dummy == NULL);
124
125 vunmapbuf(bp, todo);
126 uvm_vsunlock(bp->b_proc->p_vmspace, bp->b_data, todo);
127
128 mutex_enter(&ps->ps_lock);
129 is_iobuf = (bp != ps->ps_orig_bp);
130 if (__predict_false(done != todo)) {
131 off_t endoffset = dbtob(bp->b_blkno) + done;
132
133 /*
134 * we got an error or hit EOM.
135 *
136 * we only care about the first one.
137 * ie. the one at the lowest offset.
138 */
139
140 KASSERT(ps->ps_endoffset != endoffset);
141 DPRINTF(("%s: error=%d at %" PRIu64 " - %" PRIu64
142 ", blkno=%" PRIu64 ", bcount=%d, flags=0x%x\n",
143 __func__, bp->b_error, dbtob(bp->b_blkno), endoffset,
144 bp->b_blkno, bp->b_bcount, bp->b_flags));
145
146 if (ps->ps_endoffset == -1 || endoffset < ps->ps_endoffset) {
147 DPRINTF(("%s: ps=%p, error %d -> %d, endoff %" PRIu64
148 " -> %" PRIu64 "\n",
149 __func__, ps,
150 ps->ps_error, bp->b_error,
151 ps->ps_endoffset, endoffset));
152
153 ps->ps_endoffset = endoffset;
154 ps->ps_error = bp->b_error;
155 }
156 ps->ps_failed++;
157
158 ps->ps_resid += todo - done;
159 } else {
160 KASSERT(bp->b_error == 0);
161 }
162
163 ps->ps_running--;
164 cv_signal(&ps->ps_cv);
165 mutex_exit(&ps->ps_lock);
166
167 if (is_iobuf)
168 putiobuf(bp);
169 }
170
171 static void
physio_biodone(struct buf * bp)172 physio_biodone(struct buf *bp)
173 {
174 #if defined(DIAGNOSTIC)
175 struct physio_stat *ps = bp->b_private;
176 size_t todo = bp->b_bufsize;
177 size_t done = bp->b_bcount - bp->b_resid;
178
179 KASSERT(ps->ps_running > 0);
180 KASSERT(bp->b_bcount <= todo);
181 KASSERT(bp->b_resid <= bp->b_bcount);
182 if (done == todo)
183 KASSERTMSG(bp->b_error == 0, "error=%d", bp->b_error);
184 #endif /* defined(DIAGNOSTIC) */
185
186 workqueue_enqueue(physio_workqueue, &bp->b_work, NULL);
187 }
188
189 static void
physio_wait(struct physio_stat * ps,int n)190 physio_wait(struct physio_stat *ps, int n)
191 {
192
193 KASSERT(mutex_owned(&ps->ps_lock));
194
195 while (ps->ps_running > n)
196 cv_wait(&ps->ps_cv, &ps->ps_lock);
197 }
198
199 static int
physio_init(void)200 physio_init(void)
201 {
202 int error;
203
204 KASSERT(physio_workqueue == NULL);
205
206 error = workqueue_create(&physio_workqueue, "physiod",
207 physio_done, NULL, PRI_BIO, IPL_BIO, WQ_MPSAFE);
208
209 return error;
210 }
211
212 /*
213 * Do "physical I/O" on behalf of a user. "Physical I/O" is I/O directly
214 * from the raw device to user buffers, and bypasses the buffer cache.
215 */
216 int
physio(void (* strategy)(struct buf *),struct buf * obp,dev_t dev,int flags,void (* min_phys)(struct buf *),struct uio * uio)217 physio(void (*strategy)(struct buf *), struct buf *obp, dev_t dev, int flags,
218 void (*min_phys)(struct buf *), struct uio *uio)
219 {
220 struct iovec *iovp;
221 struct lwp *l = curlwp;
222 struct proc *p = l->l_proc;
223 int i, error;
224 struct buf *bp = NULL;
225 struct physio_stat *ps;
226 int concurrency = physio_concurrency - 1;
227 int isdisk;
228
229 error = RUN_ONCE(&physio_initialized, physio_init);
230 if (__predict_false(error != 0)) {
231 return error;
232 }
233
234 DPRINTF(("%s: called: off=%" PRIu64 ", resid=%zu\n",
235 __func__, uio->uio_offset, uio->uio_resid));
236
237 flags &= B_READ | B_WRITE;
238
239 ps = kmem_zalloc(sizeof(*ps), KM_SLEEP);
240 /* ps->ps_running = 0; */
241 /* ps->ps_error = 0; */
242 /* ps->ps_failed = 0; */
243 ps->ps_orig_bp = obp;
244 ps->ps_endoffset = -1;
245 ps->ps_resid = 0;
246 mutex_init(&ps->ps_lock, MUTEX_DEFAULT, IPL_NONE);
247 cv_init(&ps->ps_cv, "physio");
248
249 /* Allow concurrent I/O only for disks */
250 isdisk = cdev_type(dev) == D_DISK;
251 if (!isdisk)
252 concurrency = 0;
253
254 /* Make sure we have a buffer, creating one if necessary. */
255 if (obp != NULL) {
256 mutex_enter(&bufcache_lock);
257 /* Mark it busy, so nobody else will use it. */
258 while (bbusy(obp, false, 0, NULL) == EPASSTHROUGH)
259 ;
260 mutex_exit(&bufcache_lock);
261 concurrency = 0; /* see "XXXkludge" comment below */
262 }
263
264 for (i = 0; i < uio->uio_iovcnt; i++) {
265 bool sync = true;
266
267 iovp = &uio->uio_iov[i];
268 while (iovp->iov_len > 0) {
269 size_t todo;
270 vaddr_t endp;
271
272 mutex_enter(&ps->ps_lock);
273 if (ps->ps_failed != 0) {
274 goto done_locked;
275 }
276 physio_wait(ps, sync ? 0 : concurrency);
277 mutex_exit(&ps->ps_lock);
278 if (obp != NULL) {
279 /*
280 * XXXkludge
281 * some drivers use "obp" as an identifier.
282 */
283 bp = obp;
284 } else {
285 bp = getiobuf(NULL, true);
286 bp->b_cflags |= BC_BUSY;
287 }
288 bp->b_dev = dev;
289 bp->b_proc = p;
290 bp->b_private = ps;
291
292 /*
293 * Mrk the buffer busy for physical I/O. Also set
294 * B_PHYS because it's an I/O to user memory, and
295 * B_RAW because B_RAW is to be "set by physio for
296 * raw transfers".
297 */
298 bp->b_oflags = 0;
299 bp->b_cflags |= BC_BUSY;
300 bp->b_flags = flags | B_PHYS | B_RAW;
301 bp->b_iodone = physio_biodone;
302
303 /* Set up the buffer for a maximum-sized transfer. */
304 bp->b_blkno = btodb(uio->uio_offset);
305 if (isdisk) {
306 /*
307 * For disks, check that offsets are at least block
308 * aligned, the block addresses are used to track
309 * errors of finished requests.
310 */
311 if (uio->uio_offset & (DEV_BSIZE - 1)) {
312 error = EINVAL;
313 goto done;
314 }
315 /*
316 * Split request into MAXPHYS chunks
317 */
318 bp->b_bcount = MIN(MAXPHYS, iovp->iov_len);
319 } else {
320 bp->b_bcount = MIN(INT_MAX, iovp->iov_len);
321 }
322 bp->b_data = iovp->iov_base;
323
324 /*
325 * Call minphys to bound the transfer size,
326 * and remember the amount of data to transfer,
327 * for later comparison.
328 */
329 (*min_phys)(bp);
330 todo = bp->b_bufsize = bp->b_bcount;
331 #if defined(DIAGNOSTIC)
332 if (todo > MAXPHYS)
333 panic("todo(%zu) > MAXPHYS; minphys broken",
334 todo);
335 #endif /* defined(DIAGNOSTIC) */
336
337 sync = false;
338 endp = (vaddr_t)bp->b_data + todo;
339 if (trunc_page(endp) != endp) {
340 /*
341 * Following requests can overlap.
342 * note that uvm_vslock does round_page.
343 */
344 sync = true;
345 }
346
347 /*
348 * Lock the part of the user address space involved
349 * in the transfer.
350 */
351 error = uvm_vslock(p->p_vmspace, bp->b_data, todo,
352 (flags & B_READ) ? VM_PROT_WRITE : VM_PROT_READ);
353 if (error) {
354 goto done;
355 }
356
357 /*
358 * Beware vmapbuf(); if successful it clobbers
359 * b_data and saves it in b_saveaddr.
360 * However, vunmapbuf() restores b_data.
361 */
362 if ((error = vmapbuf(bp, todo)) != 0) {
363 uvm_vsunlock(p->p_vmspace, bp->b_data, todo);
364 goto done;
365 }
366
367 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
368
369 mutex_enter(&ps->ps_lock);
370 ps->ps_running++;
371 mutex_exit(&ps->ps_lock);
372
373 /* Call strategy to start the transfer. */
374 (*strategy)(bp);
375 bp = NULL;
376
377 iovp->iov_len -= todo;
378 iovp->iov_base = (char *)iovp->iov_base + todo;
379 uio->uio_offset += todo;
380 uio->uio_resid -= todo;
381 }
382 }
383
384 done:
385 mutex_enter(&ps->ps_lock);
386 done_locked:
387 physio_wait(ps, 0);
388 mutex_exit(&ps->ps_lock);
389
390 KASSERT(ps->ps_failed || ps->ps_endoffset == -1);
391
392 /*
393 * Compute residual, for disks adjust for the
394 * lowest numbered block that returned an error.
395 */
396 if (isdisk) {
397 if (ps->ps_failed != 0) {
398 off_t delta;
399
400 delta = uio->uio_offset - ps->ps_endoffset;
401 KASSERT(delta > 0);
402 uio->uio_resid += delta;
403 /* uio->uio_offset = ps->ps_endoffset; */
404 }
405 } else {
406 uio->uio_resid += ps->ps_resid;
407 }
408
409 if (bp != NULL && bp != obp) {
410 putiobuf(bp);
411 }
412 if (error == 0) {
413 error = ps->ps_error;
414 }
415 mutex_destroy(&ps->ps_lock);
416 cv_destroy(&ps->ps_cv);
417 kmem_free(ps, sizeof(*ps));
418
419 /*
420 * Clean up the state of the buffer. Remember if somebody wants
421 * it, so we can wake them up below. Also, if we had to steal it,
422 * give it back.
423 */
424 if (obp != NULL) {
425 KASSERT((obp->b_cflags & BC_BUSY) != 0);
426
427 /*
428 * If another process is waiting for the raw I/O buffer,
429 * wake up processes waiting to do physical I/O;
430 */
431 mutex_enter(&bufcache_lock);
432 obp->b_cflags &= ~(BC_BUSY | BC_WANTED);
433 obp->b_flags &= ~(B_PHYS | B_RAW);
434 obp->b_iodone = NULL;
435 cv_broadcast(&obp->b_busy);
436 mutex_exit(&bufcache_lock);
437 }
438
439 DPRINTF(("%s: done: off=%" PRIu64 ", resid=%zu\n",
440 __func__, uio->uio_offset, uio->uio_resid));
441
442 return error;
443 }
444
445 /*
446 * A minphys() routine is called by physio() to adjust the size of each
447 * I/O transfer before the latter is passed to the strategy routine.
448 *
449 * This minphys() is a default that must be called to enforce limits
450 * that are applicable to all devices, because of limitations in the
451 * kernel or the hardware platform.
452 */
453 void
minphys(struct buf * bp)454 minphys(struct buf *bp)
455 {
456
457 if (bp->b_bcount > MAXPHYS)
458 bp->b_bcount = MAXPHYS;
459 }
460