blob: 9a09f2034fcc53a2733d931f33e82d7c3d79fa97 [file] [log] [blame]
Al Viro4f18cd32014-02-05 19:11:33 -05001#include <linux/export.h>
2#include <linux/uio.h>
3#include <linux/pagemap.h>
Al Viro91f79c42014-03-21 04:58:33 -04004#include <linux/slab.h>
5#include <linux/vmalloc.h>
Al Viro4f18cd32014-02-05 19:11:33 -05006
Al Viro62a80672014-04-04 23:12:29 -04007static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
Al Viro4f18cd32014-02-05 19:11:33 -05008 struct iov_iter *i)
9{
10 size_t skip, copy, left, wanted;
11 const struct iovec *iov;
12 char __user *buf;
13 void *kaddr, *from;
14
15 if (unlikely(bytes > i->count))
16 bytes = i->count;
17
18 if (unlikely(!bytes))
19 return 0;
20
21 wanted = bytes;
22 iov = i->iov;
23 skip = i->iov_offset;
24 buf = iov->iov_base + skip;
25 copy = min(bytes, iov->iov_len - skip);
26
27 if (!fault_in_pages_writeable(buf, copy)) {
28 kaddr = kmap_atomic(page);
29 from = kaddr + offset;
30
31 /* first chunk, usually the only one */
32 left = __copy_to_user_inatomic(buf, from, copy);
33 copy -= left;
34 skip += copy;
35 from += copy;
36 bytes -= copy;
37
38 while (unlikely(!left && bytes)) {
39 iov++;
40 buf = iov->iov_base;
41 copy = min(bytes, iov->iov_len);
42 left = __copy_to_user_inatomic(buf, from, copy);
43 copy -= left;
44 skip = copy;
45 from += copy;
46 bytes -= copy;
47 }
48 if (likely(!bytes)) {
49 kunmap_atomic(kaddr);
50 goto done;
51 }
52 offset = from - kaddr;
53 buf += copy;
54 kunmap_atomic(kaddr);
55 copy = min(bytes, iov->iov_len - skip);
56 }
57 /* Too bad - revert to non-atomic kmap */
58 kaddr = kmap(page);
59 from = kaddr + offset;
60 left = __copy_to_user(buf, from, copy);
61 copy -= left;
62 skip += copy;
63 from += copy;
64 bytes -= copy;
65 while (unlikely(!left && bytes)) {
66 iov++;
67 buf = iov->iov_base;
68 copy = min(bytes, iov->iov_len);
69 left = __copy_to_user(buf, from, copy);
70 copy -= left;
71 skip = copy;
72 from += copy;
73 bytes -= copy;
74 }
75 kunmap(page);
76done:
Al Viro81055e52014-04-04 19:23:46 -040077 if (skip == iov->iov_len) {
78 iov++;
79 skip = 0;
80 }
Al Viro4f18cd32014-02-05 19:11:33 -050081 i->count -= wanted - bytes;
82 i->nr_segs -= iov - i->iov;
83 i->iov = iov;
84 i->iov_offset = skip;
85 return wanted - bytes;
86}
Al Viro4f18cd32014-02-05 19:11:33 -050087
Al Viro62a80672014-04-04 23:12:29 -040088static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
Al Virof0d1bec2014-04-03 15:05:18 -040089 struct iov_iter *i)
90{
91 size_t skip, copy, left, wanted;
92 const struct iovec *iov;
93 char __user *buf;
94 void *kaddr, *to;
95
96 if (unlikely(bytes > i->count))
97 bytes = i->count;
98
99 if (unlikely(!bytes))
100 return 0;
101
102 wanted = bytes;
103 iov = i->iov;
104 skip = i->iov_offset;
105 buf = iov->iov_base + skip;
106 copy = min(bytes, iov->iov_len - skip);
107
108 if (!fault_in_pages_readable(buf, copy)) {
109 kaddr = kmap_atomic(page);
110 to = kaddr + offset;
111
112 /* first chunk, usually the only one */
113 left = __copy_from_user_inatomic(to, buf, copy);
114 copy -= left;
115 skip += copy;
116 to += copy;
117 bytes -= copy;
118
119 while (unlikely(!left && bytes)) {
120 iov++;
121 buf = iov->iov_base;
122 copy = min(bytes, iov->iov_len);
123 left = __copy_from_user_inatomic(to, buf, copy);
124 copy -= left;
125 skip = copy;
126 to += copy;
127 bytes -= copy;
128 }
129 if (likely(!bytes)) {
130 kunmap_atomic(kaddr);
131 goto done;
132 }
133 offset = to - kaddr;
134 buf += copy;
135 kunmap_atomic(kaddr);
136 copy = min(bytes, iov->iov_len - skip);
137 }
138 /* Too bad - revert to non-atomic kmap */
139 kaddr = kmap(page);
140 to = kaddr + offset;
141 left = __copy_from_user(to, buf, copy);
142 copy -= left;
143 skip += copy;
144 to += copy;
145 bytes -= copy;
146 while (unlikely(!left && bytes)) {
147 iov++;
148 buf = iov->iov_base;
149 copy = min(bytes, iov->iov_len);
150 left = __copy_from_user(to, buf, copy);
151 copy -= left;
152 skip = copy;
153 to += copy;
154 bytes -= copy;
155 }
156 kunmap(page);
157done:
Al Viro81055e52014-04-04 19:23:46 -0400158 if (skip == iov->iov_len) {
159 iov++;
160 skip = 0;
161 }
Al Virof0d1bec2014-04-03 15:05:18 -0400162 i->count -= wanted - bytes;
163 i->nr_segs -= iov - i->iov;
164 i->iov = iov;
165 i->iov_offset = skip;
166 return wanted - bytes;
167}
Al Virof0d1bec2014-04-03 15:05:18 -0400168
Al Viro4f18cd32014-02-05 19:11:33 -0500169static size_t __iovec_copy_from_user_inatomic(char *vaddr,
170 const struct iovec *iov, size_t base, size_t bytes)
171{
172 size_t copied = 0, left = 0;
173
174 while (bytes) {
175 char __user *buf = iov->iov_base + base;
176 int copy = min(bytes, iov->iov_len - base);
177
178 base = 0;
179 left = __copy_from_user_inatomic(vaddr, buf, copy);
180 copied += copy;
181 bytes -= copy;
182 vaddr += copy;
183 iov++;
184
185 if (unlikely(left))
186 break;
187 }
188 return copied - left;
189}
190
191/*
192 * Copy as much as we can into the page and return the number of bytes which
193 * were successfully copied. If a fault is encountered then return the number of
194 * bytes which were copied.
195 */
Al Viro62a80672014-04-04 23:12:29 -0400196static size_t copy_from_user_atomic_iovec(struct page *page,
Al Viro4f18cd32014-02-05 19:11:33 -0500197 struct iov_iter *i, unsigned long offset, size_t bytes)
198{
199 char *kaddr;
200 size_t copied;
201
202 kaddr = kmap_atomic(page);
203 if (likely(i->nr_segs == 1)) {
204 int left;
205 char __user *buf = i->iov->iov_base + i->iov_offset;
206 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
207 copied = bytes - left;
208 } else {
209 copied = __iovec_copy_from_user_inatomic(kaddr + offset,
210 i->iov, i->iov_offset, bytes);
211 }
212 kunmap_atomic(kaddr);
213
214 return copied;
215}
Al Viro4f18cd32014-02-05 19:11:33 -0500216
Al Viro62a80672014-04-04 23:12:29 -0400217static void advance_iovec(struct iov_iter *i, size_t bytes)
Al Viro4f18cd32014-02-05 19:11:33 -0500218{
219 BUG_ON(i->count < bytes);
220
221 if (likely(i->nr_segs == 1)) {
222 i->iov_offset += bytes;
223 i->count -= bytes;
224 } else {
225 const struct iovec *iov = i->iov;
226 size_t base = i->iov_offset;
227 unsigned long nr_segs = i->nr_segs;
228
229 /*
230 * The !iov->iov_len check ensures we skip over unlikely
231 * zero-length segments (without overruning the iovec).
232 */
233 while (bytes || unlikely(i->count && !iov->iov_len)) {
234 int copy;
235
236 copy = min(bytes, iov->iov_len - base);
237 BUG_ON(!i->count || i->count < copy);
238 i->count -= copy;
239 bytes -= copy;
240 base += copy;
241 if (iov->iov_len == base) {
242 iov++;
243 nr_segs--;
244 base = 0;
245 }
246 }
247 i->iov = iov;
248 i->iov_offset = base;
249 i->nr_segs = nr_segs;
250 }
251}
Al Viro4f18cd32014-02-05 19:11:33 -0500252
253/*
254 * Fault in the first iovec of the given iov_iter, to a maximum length
255 * of bytes. Returns 0 on success, or non-zero if the memory could not be
256 * accessed (ie. because it is an invalid address).
257 *
258 * writev-intensive code may want this to prefault several iovecs -- that
259 * would be possible (callers must not rely on the fact that _only_ the
260 * first iovec will be faulted with the current implementation).
261 */
262int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
263{
Al Viro62a80672014-04-04 23:12:29 -0400264 if (!(i->type & ITER_BVEC)) {
265 char __user *buf = i->iov->iov_base + i->iov_offset;
266 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
267 return fault_in_pages_readable(buf, bytes);
268 }
269 return 0;
Al Viro4f18cd32014-02-05 19:11:33 -0500270}
271EXPORT_SYMBOL(iov_iter_fault_in_readable);
272
Al Viro62a80672014-04-04 23:12:29 -0400273static unsigned long alignment_iovec(const struct iov_iter *i)
Al Viro886a3912014-03-05 13:50:45 -0500274{
275 const struct iovec *iov = i->iov;
276 unsigned long res;
277 size_t size = i->count;
278 size_t n;
279
280 if (!size)
281 return 0;
282
283 res = (unsigned long)iov->iov_base + i->iov_offset;
284 n = iov->iov_len - i->iov_offset;
285 if (n >= size)
286 return res | size;
287 size -= n;
288 res |= n;
289 while (size > (++iov)->iov_len) {
290 res |= (unsigned long)iov->iov_base | iov->iov_len;
291 size -= iov->iov_len;
292 }
293 res |= (unsigned long)iov->iov_base | size;
294 return res;
295}
Al Viro71d8e532014-03-05 19:28:09 -0500296
297void iov_iter_init(struct iov_iter *i, int direction,
298 const struct iovec *iov, unsigned long nr_segs,
299 size_t count)
300{
301 /* It will get better. Eventually... */
302 if (segment_eq(get_fs(), KERNEL_DS))
Al Viro62a80672014-04-04 23:12:29 -0400303 direction |= ITER_KVEC;
Al Viro71d8e532014-03-05 19:28:09 -0500304 i->type = direction;
305 i->iov = iov;
306 i->nr_segs = nr_segs;
307 i->iov_offset = 0;
308 i->count = count;
309}
310EXPORT_SYMBOL(iov_iter_init);
Al Viro7b2c99d2014-03-15 04:05:57 -0400311
Al Viro62a80672014-04-04 23:12:29 -0400312static ssize_t get_pages_iovec(struct iov_iter *i,
Miklos Szeredi2c809292014-09-24 17:09:11 +0200313 struct page **pages, size_t maxsize, unsigned maxpages,
Al Viro7b2c99d2014-03-15 04:05:57 -0400314 size_t *start)
315{
316 size_t offset = i->iov_offset;
317 const struct iovec *iov = i->iov;
318 size_t len;
319 unsigned long addr;
320 int n;
321 int res;
322
323 len = iov->iov_len - offset;
324 if (len > i->count)
325 len = i->count;
Miklos Szeredi2c809292014-09-24 17:09:11 +0200326 if (len > maxsize)
327 len = maxsize;
Al Viro7b2c99d2014-03-15 04:05:57 -0400328 addr = (unsigned long)iov->iov_base + offset;
329 len += *start = addr & (PAGE_SIZE - 1);
Al Viroc7f38882014-06-18 20:34:33 -0400330 if (len > maxpages * PAGE_SIZE)
331 len = maxpages * PAGE_SIZE;
Al Viro7b2c99d2014-03-15 04:05:57 -0400332 addr &= ~(PAGE_SIZE - 1);
333 n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
334 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
335 if (unlikely(res < 0))
336 return res;
337 return (res == n ? len : res * PAGE_SIZE) - *start;
338}
Al Virof67da302014-03-19 01:16:16 -0400339
Al Viro62a80672014-04-04 23:12:29 -0400340static ssize_t get_pages_alloc_iovec(struct iov_iter *i,
Al Viro91f79c42014-03-21 04:58:33 -0400341 struct page ***pages, size_t maxsize,
342 size_t *start)
343{
344 size_t offset = i->iov_offset;
345 const struct iovec *iov = i->iov;
346 size_t len;
347 unsigned long addr;
348 void *p;
349 int n;
350 int res;
351
352 len = iov->iov_len - offset;
353 if (len > i->count)
354 len = i->count;
355 if (len > maxsize)
356 len = maxsize;
357 addr = (unsigned long)iov->iov_base + offset;
358 len += *start = addr & (PAGE_SIZE - 1);
359 addr &= ~(PAGE_SIZE - 1);
360 n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
361
362 p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
363 if (!p)
364 p = vmalloc(n * sizeof(struct page *));
365 if (!p)
366 return -ENOMEM;
367
368 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
369 if (unlikely(res < 0)) {
370 kvfree(p);
371 return res;
372 }
373 *pages = p;
374 return (res == n ? len : res * PAGE_SIZE) - *start;
375}
Al Viro91f79c42014-03-21 04:58:33 -0400376
Al Viro62a80672014-04-04 23:12:29 -0400377static int iov_iter_npages_iovec(const struct iov_iter *i, int maxpages)
Al Virof67da302014-03-19 01:16:16 -0400378{
379 size_t offset = i->iov_offset;
380 size_t size = i->count;
381 const struct iovec *iov = i->iov;
382 int npages = 0;
383 int n;
384
385 for (n = 0; size && n < i->nr_segs; n++, iov++) {
386 unsigned long addr = (unsigned long)iov->iov_base + offset;
387 size_t len = iov->iov_len - offset;
388 offset = 0;
389 if (unlikely(!len)) /* empty segment */
390 continue;
391 if (len > size)
392 len = size;
393 npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
394 - addr / PAGE_SIZE;
395 if (npages >= maxpages) /* don't bother going further */
396 return maxpages;
397 size -= len;
398 offset = 0;
399 }
400 return min(npages, maxpages);
401}
Al Viro62a80672014-04-04 23:12:29 -0400402
403static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
404{
405 char *from = kmap_atomic(page);
406 memcpy(to, from + offset, len);
407 kunmap_atomic(from);
408}
409
410static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
411{
412 char *to = kmap_atomic(page);
413 memcpy(to + offset, from, len);
414 kunmap_atomic(to);
415}
416
417static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t bytes,
418 struct iov_iter *i)
419{
420 size_t skip, copy, wanted;
421 const struct bio_vec *bvec;
422 void *kaddr, *from;
423
424 if (unlikely(bytes > i->count))
425 bytes = i->count;
426
427 if (unlikely(!bytes))
428 return 0;
429
430 wanted = bytes;
431 bvec = i->bvec;
432 skip = i->iov_offset;
433 copy = min_t(size_t, bytes, bvec->bv_len - skip);
434
435 kaddr = kmap_atomic(page);
436 from = kaddr + offset;
437 memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy);
438 skip += copy;
439 from += copy;
440 bytes -= copy;
441 while (bytes) {
442 bvec++;
443 copy = min(bytes, (size_t)bvec->bv_len);
444 memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy);
445 skip = copy;
446 from += copy;
447 bytes -= copy;
448 }
449 kunmap_atomic(kaddr);
450 if (skip == bvec->bv_len) {
451 bvec++;
452 skip = 0;
453 }
454 i->count -= wanted - bytes;
455 i->nr_segs -= bvec - i->bvec;
456 i->bvec = bvec;
457 i->iov_offset = skip;
458 return wanted - bytes;
459}
460
461static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t bytes,
462 struct iov_iter *i)
463{
464 size_t skip, copy, wanted;
465 const struct bio_vec *bvec;
466 void *kaddr, *to;
467
468 if (unlikely(bytes > i->count))
469 bytes = i->count;
470
471 if (unlikely(!bytes))
472 return 0;
473
474 wanted = bytes;
475 bvec = i->bvec;
476 skip = i->iov_offset;
477
478 kaddr = kmap_atomic(page);
479
480 to = kaddr + offset;
481
482 copy = min(bytes, bvec->bv_len - skip);
483
484 memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy);
485
486 to += copy;
487 skip += copy;
488 bytes -= copy;
489
490 while (bytes) {
491 bvec++;
492 copy = min(bytes, (size_t)bvec->bv_len);
493 memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy);
494 skip = copy;
495 to += copy;
496 bytes -= copy;
497 }
498 kunmap_atomic(kaddr);
499 if (skip == bvec->bv_len) {
500 bvec++;
501 skip = 0;
502 }
503 i->count -= wanted;
504 i->nr_segs -= bvec - i->bvec;
505 i->bvec = bvec;
506 i->iov_offset = skip;
507 return wanted;
508}
509
510static size_t copy_from_user_bvec(struct page *page,
511 struct iov_iter *i, unsigned long offset, size_t bytes)
512{
513 char *kaddr;
514 size_t left;
515 const struct bio_vec *bvec;
516 size_t base = i->iov_offset;
517
518 kaddr = kmap_atomic(page);
519 for (left = bytes, bvec = i->bvec; left; bvec++, base = 0) {
520 size_t copy = min(left, bvec->bv_len - base);
521 if (!bvec->bv_len)
522 continue;
523 memcpy_from_page(kaddr + offset, bvec->bv_page,
524 bvec->bv_offset + base, copy);
525 offset += copy;
526 left -= copy;
527 }
528 kunmap_atomic(kaddr);
529 return bytes;
530}
531
532static void advance_bvec(struct iov_iter *i, size_t bytes)
533{
534 BUG_ON(i->count < bytes);
535
536 if (likely(i->nr_segs == 1)) {
537 i->iov_offset += bytes;
538 i->count -= bytes;
539 } else {
540 const struct bio_vec *bvec = i->bvec;
541 size_t base = i->iov_offset;
542 unsigned long nr_segs = i->nr_segs;
543
544 /*
545 * The !iov->iov_len check ensures we skip over unlikely
546 * zero-length segments (without overruning the iovec).
547 */
548 while (bytes || unlikely(i->count && !bvec->bv_len)) {
549 int copy;
550
551 copy = min(bytes, bvec->bv_len - base);
552 BUG_ON(!i->count || i->count < copy);
553 i->count -= copy;
554 bytes -= copy;
555 base += copy;
556 if (bvec->bv_len == base) {
557 bvec++;
558 nr_segs--;
559 base = 0;
560 }
561 }
562 i->bvec = bvec;
563 i->iov_offset = base;
564 i->nr_segs = nr_segs;
565 }
566}
567
568static unsigned long alignment_bvec(const struct iov_iter *i)
569{
570 const struct bio_vec *bvec = i->bvec;
571 unsigned long res;
572 size_t size = i->count;
573 size_t n;
574
575 if (!size)
576 return 0;
577
578 res = bvec->bv_offset + i->iov_offset;
579 n = bvec->bv_len - i->iov_offset;
580 if (n >= size)
581 return res | size;
582 size -= n;
583 res |= n;
584 while (size > (++bvec)->bv_len) {
585 res |= bvec->bv_offset | bvec->bv_len;
586 size -= bvec->bv_len;
587 }
588 res |= bvec->bv_offset | size;
589 return res;
590}
591
592static ssize_t get_pages_bvec(struct iov_iter *i,
Miklos Szeredi2c809292014-09-24 17:09:11 +0200593 struct page **pages, size_t maxsize, unsigned maxpages,
Al Viro62a80672014-04-04 23:12:29 -0400594 size_t *start)
595{
596 const struct bio_vec *bvec = i->bvec;
597 size_t len = bvec->bv_len - i->iov_offset;
598 if (len > i->count)
599 len = i->count;
Miklos Szeredi2c809292014-09-24 17:09:11 +0200600 if (len > maxsize)
601 len = maxsize;
Al Viroc7f38882014-06-18 20:34:33 -0400602 /* can't be more than PAGE_SIZE */
Al Viro62a80672014-04-04 23:12:29 -0400603 *start = bvec->bv_offset + i->iov_offset;
604
605 get_page(*pages = bvec->bv_page);
606
607 return len;
608}
609
610static ssize_t get_pages_alloc_bvec(struct iov_iter *i,
611 struct page ***pages, size_t maxsize,
612 size_t *start)
613{
614 const struct bio_vec *bvec = i->bvec;
615 size_t len = bvec->bv_len - i->iov_offset;
616 if (len > i->count)
617 len = i->count;
618 if (len > maxsize)
619 len = maxsize;
620 *start = bvec->bv_offset + i->iov_offset;
621
622 *pages = kmalloc(sizeof(struct page *), GFP_KERNEL);
623 if (!*pages)
624 return -ENOMEM;
625
626 get_page(**pages = bvec->bv_page);
627
628 return len;
629}
630
631static int iov_iter_npages_bvec(const struct iov_iter *i, int maxpages)
632{
633 size_t offset = i->iov_offset;
634 size_t size = i->count;
635 const struct bio_vec *bvec = i->bvec;
636 int npages = 0;
637 int n;
638
639 for (n = 0; size && n < i->nr_segs; n++, bvec++) {
640 size_t len = bvec->bv_len - offset;
641 offset = 0;
642 if (unlikely(!len)) /* empty segment */
643 continue;
644 if (len > size)
645 len = size;
646 npages++;
647 if (npages >= maxpages) /* don't bother going further */
648 return maxpages;
649 size -= len;
650 offset = 0;
651 }
652 return min(npages, maxpages);
653}
654
655size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
656 struct iov_iter *i)
657{
658 if (i->type & ITER_BVEC)
659 return copy_page_to_iter_bvec(page, offset, bytes, i);
660 else
661 return copy_page_to_iter_iovec(page, offset, bytes, i);
662}
663EXPORT_SYMBOL(copy_page_to_iter);
664
665size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
666 struct iov_iter *i)
667{
668 if (i->type & ITER_BVEC)
669 return copy_page_from_iter_bvec(page, offset, bytes, i);
670 else
671 return copy_page_from_iter_iovec(page, offset, bytes, i);
672}
673EXPORT_SYMBOL(copy_page_from_iter);
674
675size_t iov_iter_copy_from_user_atomic(struct page *page,
676 struct iov_iter *i, unsigned long offset, size_t bytes)
677{
678 if (i->type & ITER_BVEC)
679 return copy_from_user_bvec(page, i, offset, bytes);
680 else
681 return copy_from_user_atomic_iovec(page, i, offset, bytes);
682}
683EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
684
685void iov_iter_advance(struct iov_iter *i, size_t size)
686{
687 if (i->type & ITER_BVEC)
688 advance_bvec(i, size);
689 else
690 advance_iovec(i, size);
691}
692EXPORT_SYMBOL(iov_iter_advance);
693
694/*
695 * Return the count of just the current iov_iter segment.
696 */
697size_t iov_iter_single_seg_count(const struct iov_iter *i)
698{
699 if (i->nr_segs == 1)
700 return i->count;
701 else if (i->type & ITER_BVEC)
702 return min(i->count, i->iov->iov_len - i->iov_offset);
703 else
704 return min(i->count, i->bvec->bv_len - i->iov_offset);
705}
706EXPORT_SYMBOL(iov_iter_single_seg_count);
707
708unsigned long iov_iter_alignment(const struct iov_iter *i)
709{
710 if (i->type & ITER_BVEC)
711 return alignment_bvec(i);
712 else
713 return alignment_iovec(i);
714}
715EXPORT_SYMBOL(iov_iter_alignment);
716
717ssize_t iov_iter_get_pages(struct iov_iter *i,
Miklos Szeredi2c809292014-09-24 17:09:11 +0200718 struct page **pages, size_t maxsize, unsigned maxpages,
Al Viro62a80672014-04-04 23:12:29 -0400719 size_t *start)
720{
721 if (i->type & ITER_BVEC)
Miklos Szeredi2c809292014-09-24 17:09:11 +0200722 return get_pages_bvec(i, pages, maxsize, maxpages, start);
Al Viro62a80672014-04-04 23:12:29 -0400723 else
Miklos Szeredi2c809292014-09-24 17:09:11 +0200724 return get_pages_iovec(i, pages, maxsize, maxpages, start);
Al Viro62a80672014-04-04 23:12:29 -0400725}
726EXPORT_SYMBOL(iov_iter_get_pages);
727
728ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
729 struct page ***pages, size_t maxsize,
730 size_t *start)
731{
732 if (i->type & ITER_BVEC)
733 return get_pages_alloc_bvec(i, pages, maxsize, start);
734 else
735 return get_pages_alloc_iovec(i, pages, maxsize, start);
736}
737EXPORT_SYMBOL(iov_iter_get_pages_alloc);
738
739int iov_iter_npages(const struct iov_iter *i, int maxpages)
740{
741 if (i->type & ITER_BVEC)
742 return iov_iter_npages_bvec(i, maxpages);
743 else
744 return iov_iter_npages_iovec(i, maxpages);
745}
Al Virof67da302014-03-19 01:16:16 -0400746EXPORT_SYMBOL(iov_iter_npages);