blob: 5287be18633bc7c2b6621fc20c2f5ce97fa8e561 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
6
7/*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
20
21#include <linux/config.h>
22#include <linux/kernel.h>
23#include <linux/syscalls.h>
24#include <linux/fs.h>
25#include <linux/mm.h>
26#include <linux/percpu.h>
27#include <linux/slab.h>
28#include <linux/smp_lock.h>
29#include <linux/blkdev.h>
30#include <linux/file.h>
31#include <linux/quotaops.h>
32#include <linux/highmem.h>
33#include <linux/module.h>
34#include <linux/writeback.h>
35#include <linux/hash.h>
36#include <linux/suspend.h>
37#include <linux/buffer_head.h>
38#include <linux/bio.h>
39#include <linux/notifier.h>
40#include <linux/cpu.h>
41#include <linux/bitops.h>
42#include <linux/mpage.h>
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070043#include <linux/bit_spinlock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46static void invalidate_bh_lrus(void);
47
48#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
49
50inline void
51init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52{
53 bh->b_end_io = handler;
54 bh->b_private = private;
55}
56
57static int sync_buffer(void *word)
58{
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
62
63 smp_mb();
64 bd = bh->b_bdev;
65 if (bd)
66 blk_run_address_space(bd->bd_inode->i_mapping);
67 io_schedule();
68 return 0;
69}
70
71void fastcall __lock_buffer(struct buffer_head *bh)
72{
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
75}
76EXPORT_SYMBOL(__lock_buffer);
77
78void fastcall unlock_buffer(struct buffer_head *bh)
79{
80 clear_buffer_locked(bh);
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
83}
84
85/*
86 * Block until a buffer comes unlocked. This doesn't stop it
87 * from becoming locked again - you have to lock it yourself
88 * if you want to preserve its state.
89 */
90void __wait_on_buffer(struct buffer_head * bh)
91{
92 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
93}
94
95static void
96__clear_page_buffers(struct page *page)
97{
98 ClearPagePrivate(page);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -070099 set_page_private(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 page_cache_release(page);
101}
102
103static void buffer_io_error(struct buffer_head *bh)
104{
105 char b[BDEVNAME_SIZE];
106
107 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
108 bdevname(bh->b_bdev, b),
109 (unsigned long long)bh->b_blocknr);
110}
111
112/*
113 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
114 * unlock the buffer. This is what ll_rw_block uses too.
115 */
116void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
117{
118 if (uptodate) {
119 set_buffer_uptodate(bh);
120 } else {
121 /* This happens, due to failed READA attempts. */
122 clear_buffer_uptodate(bh);
123 }
124 unlock_buffer(bh);
125 put_bh(bh);
126}
127
128void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
129{
130 char b[BDEVNAME_SIZE];
131
132 if (uptodate) {
133 set_buffer_uptodate(bh);
134 } else {
135 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) {
136 buffer_io_error(bh);
137 printk(KERN_WARNING "lost page write due to "
138 "I/O error on %s\n",
139 bdevname(bh->b_bdev, b));
140 }
141 set_buffer_write_io_error(bh);
142 clear_buffer_uptodate(bh);
143 }
144 unlock_buffer(bh);
145 put_bh(bh);
146}
147
148/*
149 * Write out and wait upon all the dirty data associated with a block
150 * device via its mapping. Does not take the superblock lock.
151 */
152int sync_blockdev(struct block_device *bdev)
153{
154 int ret = 0;
155
156 if (bdev) {
157 int err;
158
159 ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
160 err = filemap_fdatawait(bdev->bd_inode->i_mapping);
161 if (!ret)
162 ret = err;
163 }
164 return ret;
165}
166EXPORT_SYMBOL(sync_blockdev);
167
168/*
169 * Write out and wait upon all dirty data associated with this
170 * superblock. Filesystem data as well as the underlying block
171 * device. Takes the superblock lock.
172 */
173int fsync_super(struct super_block *sb)
174{
175 sync_inodes_sb(sb, 0);
176 DQUOT_SYNC(sb);
177 lock_super(sb);
178 if (sb->s_dirt && sb->s_op->write_super)
179 sb->s_op->write_super(sb);
180 unlock_super(sb);
181 if (sb->s_op->sync_fs)
182 sb->s_op->sync_fs(sb, 1);
183 sync_blockdev(sb->s_bdev);
184 sync_inodes_sb(sb, 1);
185
186 return sync_blockdev(sb->s_bdev);
187}
188
189/*
190 * Write out and wait upon all dirty data associated with this
191 * device. Filesystem data as well as the underlying block
192 * device. Takes the superblock lock.
193 */
194int fsync_bdev(struct block_device *bdev)
195{
196 struct super_block *sb = get_super(bdev);
197 if (sb) {
198 int res = fsync_super(sb);
199 drop_super(sb);
200 return res;
201 }
202 return sync_blockdev(bdev);
203}
204
205/**
206 * freeze_bdev -- lock a filesystem and force it into a consistent state
207 * @bdev: blockdevice to lock
208 *
209 * This takes the block device bd_mount_sem to make sure no new mounts
210 * happen on bdev until thaw_bdev() is called.
211 * If a superblock is found on this device, we take the s_umount semaphore
212 * on it to make sure nobody unmounts until the snapshot creation is done.
213 */
214struct super_block *freeze_bdev(struct block_device *bdev)
215{
216 struct super_block *sb;
217
218 down(&bdev->bd_mount_sem);
219 sb = get_super(bdev);
220 if (sb && !(sb->s_flags & MS_RDONLY)) {
221 sb->s_frozen = SB_FREEZE_WRITE;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700222 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223
224 sync_inodes_sb(sb, 0);
225 DQUOT_SYNC(sb);
226
227 lock_super(sb);
228 if (sb->s_dirt && sb->s_op->write_super)
229 sb->s_op->write_super(sb);
230 unlock_super(sb);
231
232 if (sb->s_op->sync_fs)
233 sb->s_op->sync_fs(sb, 1);
234
235 sync_blockdev(sb->s_bdev);
236 sync_inodes_sb(sb, 1);
237
238 sb->s_frozen = SB_FREEZE_TRANS;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700239 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
241 sync_blockdev(sb->s_bdev);
242
243 if (sb->s_op->write_super_lockfs)
244 sb->s_op->write_super_lockfs(sb);
245 }
246
247 sync_blockdev(bdev);
248 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */
249}
250EXPORT_SYMBOL(freeze_bdev);
251
252/**
253 * thaw_bdev -- unlock filesystem
254 * @bdev: blockdevice to unlock
255 * @sb: associated superblock
256 *
257 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
258 */
259void thaw_bdev(struct block_device *bdev, struct super_block *sb)
260{
261 if (sb) {
262 BUG_ON(sb->s_bdev != bdev);
263
264 if (sb->s_op->unlockfs)
265 sb->s_op->unlockfs(sb);
266 sb->s_frozen = SB_UNFROZEN;
akpm@osdl.orgd59dd462005-05-01 08:58:47 -0700267 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 wake_up(&sb->s_wait_unfrozen);
269 drop_super(sb);
270 }
271
272 up(&bdev->bd_mount_sem);
273}
274EXPORT_SYMBOL(thaw_bdev);
275
276/*
277 * sync everything. Start out by waking pdflush, because that writes back
278 * all queues in parallel.
279 */
280static void do_sync(unsigned long wait)
281{
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700282 wakeup_pdflush(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 sync_inodes(0); /* All mappings, inodes and their blockdevs */
284 DQUOT_SYNC(NULL);
285 sync_supers(); /* Write the superblocks */
286 sync_filesystems(0); /* Start syncing the filesystems */
287 sync_filesystems(wait); /* Waitingly sync the filesystems */
288 sync_inodes(wait); /* Mappings, inodes and blockdevs, again. */
289 if (!wait)
290 printk("Emergency Sync complete\n");
291 if (unlikely(laptop_mode))
292 laptop_sync_completion();
293}
294
295asmlinkage long sys_sync(void)
296{
297 do_sync(1);
298 return 0;
299}
300
301void emergency_sync(void)
302{
303 pdflush_operation(do_sync, 0);
304}
305
306/*
307 * Generic function to fsync a file.
308 *
309 * filp may be NULL if called via the msync of a vma.
310 */
311
312int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
313{
314 struct inode * inode = dentry->d_inode;
315 struct super_block * sb;
316 int ret, err;
317
318 /* sync the inode to buffers */
319 ret = write_inode_now(inode, 0);
320
321 /* sync the superblock to buffers */
322 sb = inode->i_sb;
323 lock_super(sb);
324 if (sb->s_op->write_super)
325 sb->s_op->write_super(sb);
326 unlock_super(sb);
327
328 /* .. finally sync the buffers to disk */
329 err = sync_blockdev(sb->s_bdev);
330 if (!ret)
331 ret = err;
332 return ret;
333}
334
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700335static long do_fsync(unsigned int fd, int datasync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
337 struct file * file;
338 struct address_space *mapping;
339 int ret, err;
340
341 ret = -EBADF;
342 file = fget(fd);
343 if (!file)
344 goto out;
345
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 ret = -EINVAL;
347 if (!file->f_op || !file->f_op->fsync) {
348 /* Why? We can still call filemap_fdatawrite */
349 goto out_putf;
350 }
351
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700352 mapping = file->f_mapping;
353
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 current->flags |= PF_SYNCWRITE;
355 ret = filemap_fdatawrite(mapping);
356
357 /*
358 * We need to protect against concurrent writers,
359 * which could cause livelocks in fsync_buffers_list
360 */
361 down(&mapping->host->i_sem);
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700362 err = file->f_op->fsync(file, file->f_dentry, datasync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 if (!ret)
364 ret = err;
365 up(&mapping->host->i_sem);
366 err = filemap_fdatawait(mapping);
367 if (!ret)
368 ret = err;
369 current->flags &= ~PF_SYNCWRITE;
370
371out_putf:
372 fput(file);
373out:
374 return ret;
375}
376
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700377asmlinkage long sys_fsync(unsigned int fd)
378{
379 return do_fsync(fd, 0);
380}
381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382asmlinkage long sys_fdatasync(unsigned int fd)
383{
Oleg Nesterovdfb388b2005-06-23 00:10:02 -0700384 return do_fsync(fd, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
387/*
388 * Various filesystems appear to want __find_get_block to be non-blocking.
389 * But it's the page lock which protects the buffers. To get around this,
390 * we get exclusion from try_to_free_buffers with the blockdev mapping's
391 * private_lock.
392 *
393 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
394 * may be quite high. This code could TryLock the page, and if that
395 * succeeds, there is no need to take private_lock. (But if
396 * private_lock is contended then so is mapping->tree_lock).
397 */
398static struct buffer_head *
Coywolf Qi Hunt385fd4c2005-11-07 00:59:39 -0800399__find_get_block_slow(struct block_device *bdev, sector_t block)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400{
401 struct inode *bd_inode = bdev->bd_inode;
402 struct address_space *bd_mapping = bd_inode->i_mapping;
403 struct buffer_head *ret = NULL;
404 pgoff_t index;
405 struct buffer_head *bh;
406 struct buffer_head *head;
407 struct page *page;
408 int all_mapped = 1;
409
410 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
411 page = find_get_page(bd_mapping, index);
412 if (!page)
413 goto out;
414
415 spin_lock(&bd_mapping->private_lock);
416 if (!page_has_buffers(page))
417 goto out_unlock;
418 head = page_buffers(page);
419 bh = head;
420 do {
421 if (bh->b_blocknr == block) {
422 ret = bh;
423 get_bh(bh);
424 goto out_unlock;
425 }
426 if (!buffer_mapped(bh))
427 all_mapped = 0;
428 bh = bh->b_this_page;
429 } while (bh != head);
430
431 /* we might be here because some of the buffers on this page are
432 * not mapped. This is due to various races between
433 * file io on the block device and getblk. It gets dealt with
434 * elsewhere, don't buffer_error if we had some unmapped buffers
435 */
436 if (all_mapped) {
437 printk("__find_get_block_slow() failed. "
438 "block=%llu, b_blocknr=%llu\n",
439 (unsigned long long)block, (unsigned long long)bh->b_blocknr);
440 printk("b_state=0x%08lx, b_size=%u\n", bh->b_state, bh->b_size);
441 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
442 }
443out_unlock:
444 spin_unlock(&bd_mapping->private_lock);
445 page_cache_release(page);
446out:
447 return ret;
448}
449
450/* If invalidate_buffers() will trash dirty buffers, it means some kind
451 of fs corruption is going on. Trashing dirty data always imply losing
452 information that was supposed to be just stored on the physical layer
453 by the user.
454
455 Thus invalidate_buffers in general usage is not allwowed to trash
456 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
457 be preserved. These buffers are simply skipped.
458
459 We also skip buffers which are still in use. For example this can
460 happen if a userspace program is reading the block device.
461
462 NOTE: In the case where the user removed a removable-media-disk even if
463 there's still dirty data not synced on disk (due a bug in the device driver
464 or due an error of the user), by not destroying the dirty buffers we could
465 generate corruption also on the next media inserted, thus a parameter is
466 necessary to handle this case in the most safe way possible (trying
467 to not corrupt also the new disk inserted with the data belonging to
468 the old now corrupted disk). Also for the ramdisk the natural thing
469 to do in order to release the ramdisk memory is to destroy dirty buffers.
470
471 These are two special cases. Normal usage imply the device driver
472 to issue a sync on the device (without waiting I/O completion) and
473 then an invalidate_buffers call that doesn't trash dirty buffers.
474
475 For handling cache coherency with the blkdev pagecache the 'update' case
476 is been introduced. It is needed to re-read from disk any pinned
477 buffer. NOTE: re-reading from disk is destructive so we can do it only
478 when we assume nobody is changing the buffercache under our I/O and when
479 we think the disk contains more recent information than the buffercache.
480 The update == 1 pass marks the buffers we need to update, the update == 2
481 pass does the actual I/O. */
482void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
483{
484 invalidate_bh_lrus();
485 /*
486 * FIXME: what about destroy_dirty_buffers?
487 * We really want to use invalidate_inode_pages2() for
488 * that, but not until that's cleaned up.
489 */
490 invalidate_inode_pages(bdev->bd_inode->i_mapping);
491}
492
493/*
494 * Kick pdflush then try to free up some ZONE_NORMAL memory.
495 */
496static void free_more_memory(void)
497{
498 struct zone **zones;
499 pg_data_t *pgdat;
500
Pekka J Enberg687a21c2005-06-28 20:44:55 -0700501 wakeup_pdflush(1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 yield();
503
504 for_each_pgdat(pgdat) {
Al Viroaf4ca452005-10-21 02:55:38 -0400505 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 if (*zones)
Darren Hart1ad539b2005-06-21 17:14:53 -0700507 try_to_free_pages(zones, GFP_NOFS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 }
509}
510
511/*
512 * I/O completion handler for block_read_full_page() - pages
513 * which come unlocked at the end of I/O.
514 */
515static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
516{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700518 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 struct buffer_head *tmp;
520 struct page *page;
521 int page_uptodate = 1;
522
523 BUG_ON(!buffer_async_read(bh));
524
525 page = bh->b_page;
526 if (uptodate) {
527 set_buffer_uptodate(bh);
528 } else {
529 clear_buffer_uptodate(bh);
530 if (printk_ratelimit())
531 buffer_io_error(bh);
532 SetPageError(page);
533 }
534
535 /*
536 * Be _very_ careful from here on. Bad things can happen if
537 * two buffer heads end IO at almost the same time and both
538 * decide that the page is now completely done.
539 */
Nick Piggina3972202005-07-07 17:56:56 -0700540 first = page_buffers(page);
541 local_irq_save(flags);
542 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 clear_buffer_async_read(bh);
544 unlock_buffer(bh);
545 tmp = bh;
546 do {
547 if (!buffer_uptodate(tmp))
548 page_uptodate = 0;
549 if (buffer_async_read(tmp)) {
550 BUG_ON(!buffer_locked(tmp));
551 goto still_busy;
552 }
553 tmp = tmp->b_this_page;
554 } while (tmp != bh);
Nick Piggina3972202005-07-07 17:56:56 -0700555 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
556 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
558 /*
559 * If none of the buffers had errors and they are all
560 * uptodate then we can set the page uptodate.
561 */
562 if (page_uptodate && !PageError(page))
563 SetPageUptodate(page);
564 unlock_page(page);
565 return;
566
567still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700568 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
569 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 return;
571}
572
573/*
574 * Completion handler for block_write_full_page() - pages which are unlocked
575 * during I/O, and which have PageWriteback cleared upon I/O completion.
576 */
577void end_buffer_async_write(struct buffer_head *bh, int uptodate)
578{
579 char b[BDEVNAME_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 unsigned long flags;
Nick Piggina3972202005-07-07 17:56:56 -0700581 struct buffer_head *first;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 struct buffer_head *tmp;
583 struct page *page;
584
585 BUG_ON(!buffer_async_write(bh));
586
587 page = bh->b_page;
588 if (uptodate) {
589 set_buffer_uptodate(bh);
590 } else {
591 if (printk_ratelimit()) {
592 buffer_io_error(bh);
593 printk(KERN_WARNING "lost page write due to "
594 "I/O error on %s\n",
595 bdevname(bh->b_bdev, b));
596 }
597 set_bit(AS_EIO, &page->mapping->flags);
598 clear_buffer_uptodate(bh);
599 SetPageError(page);
600 }
601
Nick Piggina3972202005-07-07 17:56:56 -0700602 first = page_buffers(page);
603 local_irq_save(flags);
604 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
605
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 clear_buffer_async_write(bh);
607 unlock_buffer(bh);
608 tmp = bh->b_this_page;
609 while (tmp != bh) {
610 if (buffer_async_write(tmp)) {
611 BUG_ON(!buffer_locked(tmp));
612 goto still_busy;
613 }
614 tmp = tmp->b_this_page;
615 }
Nick Piggina3972202005-07-07 17:56:56 -0700616 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
617 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 end_page_writeback(page);
619 return;
620
621still_busy:
Nick Piggina3972202005-07-07 17:56:56 -0700622 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
623 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 return;
625}
626
627/*
628 * If a page's buffers are under async readin (end_buffer_async_read
629 * completion) then there is a possibility that another thread of
630 * control could lock one of the buffers after it has completed
631 * but while some of the other buffers have not completed. This
632 * locked buffer would confuse end_buffer_async_read() into not unlocking
633 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
634 * that this buffer is not under async I/O.
635 *
636 * The page comes unlocked when it has no locked buffer_async buffers
637 * left.
638 *
639 * PageLocked prevents anyone starting new async I/O reads any of
640 * the buffers.
641 *
642 * PageWriteback is used to prevent simultaneous writeout of the same
643 * page.
644 *
645 * PageLocked prevents anyone from starting writeback of a page which is
646 * under read I/O (PageWriteback is only ever set against a locked page).
647 */
648static void mark_buffer_async_read(struct buffer_head *bh)
649{
650 bh->b_end_io = end_buffer_async_read;
651 set_buffer_async_read(bh);
652}
653
654void mark_buffer_async_write(struct buffer_head *bh)
655{
656 bh->b_end_io = end_buffer_async_write;
657 set_buffer_async_write(bh);
658}
659EXPORT_SYMBOL(mark_buffer_async_write);
660
661
662/*
663 * fs/buffer.c contains helper functions for buffer-backed address space's
664 * fsync functions. A common requirement for buffer-based filesystems is
665 * that certain data from the backing blockdev needs to be written out for
666 * a successful fsync(). For example, ext2 indirect blocks need to be
667 * written back and waited upon before fsync() returns.
668 *
669 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
670 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
671 * management of a list of dependent buffers at ->i_mapping->private_list.
672 *
673 * Locking is a little subtle: try_to_free_buffers() will remove buffers
674 * from their controlling inode's queue when they are being freed. But
675 * try_to_free_buffers() will be operating against the *blockdev* mapping
676 * at the time, not against the S_ISREG file which depends on those buffers.
677 * So the locking for private_list is via the private_lock in the address_space
678 * which backs the buffers. Which is different from the address_space
679 * against which the buffers are listed. So for a particular address_space,
680 * mapping->private_lock does *not* protect mapping->private_list! In fact,
681 * mapping->private_list will always be protected by the backing blockdev's
682 * ->private_lock.
683 *
684 * Which introduces a requirement: all buffers on an address_space's
685 * ->private_list must be from the same address_space: the blockdev's.
686 *
687 * address_spaces which do not place buffers at ->private_list via these
688 * utility functions are free to use private_lock and private_list for
689 * whatever they want. The only requirement is that list_empty(private_list)
690 * be true at clear_inode() time.
691 *
692 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
693 * filesystems should do that. invalidate_inode_buffers() should just go
694 * BUG_ON(!list_empty).
695 *
696 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
697 * take an address_space, not an inode. And it should be called
698 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
699 * queued up.
700 *
701 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
702 * list if it is already on a list. Because if the buffer is on a list,
703 * it *must* already be on the right one. If not, the filesystem is being
704 * silly. This will save a ton of locking. But first we have to ensure
705 * that buffers are taken *off* the old inode's list when they are freed
706 * (presumably in truncate). That requires careful auditing of all
707 * filesystems (do it inside bforget()). It could also be done by bringing
708 * b_inode back.
709 */
710
711/*
712 * The buffer's backing address_space's private_lock must be held
713 */
714static inline void __remove_assoc_queue(struct buffer_head *bh)
715{
716 list_del_init(&bh->b_assoc_buffers);
717}
718
719int inode_has_buffers(struct inode *inode)
720{
721 return !list_empty(&inode->i_data.private_list);
722}
723
724/*
725 * osync is designed to support O_SYNC io. It waits synchronously for
726 * all already-submitted IO to complete, but does not queue any new
727 * writes to the disk.
728 *
729 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
730 * you dirty the buffers, and then use osync_inode_buffers to wait for
731 * completion. Any other dirty buffers which are not yet queued for
732 * write will not be flushed to disk by the osync.
733 */
734static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
735{
736 struct buffer_head *bh;
737 struct list_head *p;
738 int err = 0;
739
740 spin_lock(lock);
741repeat:
742 list_for_each_prev(p, list) {
743 bh = BH_ENTRY(p);
744 if (buffer_locked(bh)) {
745 get_bh(bh);
746 spin_unlock(lock);
747 wait_on_buffer(bh);
748 if (!buffer_uptodate(bh))
749 err = -EIO;
750 brelse(bh);
751 spin_lock(lock);
752 goto repeat;
753 }
754 }
755 spin_unlock(lock);
756 return err;
757}
758
759/**
760 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
761 * buffers
Martin Waitz67be2dd2005-05-01 08:59:26 -0700762 * @mapping: the mapping which wants those buffers written
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 *
764 * Starts I/O against the buffers at mapping->private_list, and waits upon
765 * that I/O.
766 *
Martin Waitz67be2dd2005-05-01 08:59:26 -0700767 * Basically, this is a convenience function for fsync().
768 * @mapping is a file or directory which needs those buffers to be written for
769 * a successful fsync().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 */
771int sync_mapping_buffers(struct address_space *mapping)
772{
773 struct address_space *buffer_mapping = mapping->assoc_mapping;
774
775 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
776 return 0;
777
778 return fsync_buffers_list(&buffer_mapping->private_lock,
779 &mapping->private_list);
780}
781EXPORT_SYMBOL(sync_mapping_buffers);
782
783/*
784 * Called when we've recently written block `bblock', and it is known that
785 * `bblock' was for a buffer_boundary() buffer. This means that the block at
786 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
787 * dirty, schedule it for IO. So that indirects merge nicely with their data.
788 */
789void write_boundary_block(struct block_device *bdev,
790 sector_t bblock, unsigned blocksize)
791{
792 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
793 if (bh) {
794 if (buffer_dirty(bh))
795 ll_rw_block(WRITE, 1, &bh);
796 put_bh(bh);
797 }
798}
799
800void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
801{
802 struct address_space *mapping = inode->i_mapping;
803 struct address_space *buffer_mapping = bh->b_page->mapping;
804
805 mark_buffer_dirty(bh);
806 if (!mapping->assoc_mapping) {
807 mapping->assoc_mapping = buffer_mapping;
808 } else {
809 if (mapping->assoc_mapping != buffer_mapping)
810 BUG();
811 }
812 if (list_empty(&bh->b_assoc_buffers)) {
813 spin_lock(&buffer_mapping->private_lock);
814 list_move_tail(&bh->b_assoc_buffers,
815 &mapping->private_list);
816 spin_unlock(&buffer_mapping->private_lock);
817 }
818}
819EXPORT_SYMBOL(mark_buffer_dirty_inode);
820
821/*
822 * Add a page to the dirty page list.
823 *
824 * It is a sad fact of life that this function is called from several places
825 * deeply under spinlocking. It may not sleep.
826 *
827 * If the page has buffers, the uptodate buffers are set dirty, to preserve
828 * dirty-state coherency between the page and the buffers. It the page does
829 * not have buffers then when they are later attached they will all be set
830 * dirty.
831 *
832 * The buffers are dirtied before the page is dirtied. There's a small race
833 * window in which a writepage caller may see the page cleanness but not the
834 * buffer dirtiness. That's fine. If this code were to set the page dirty
835 * before the buffers, a concurrent writepage caller could clear the page dirty
836 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
837 * page on the dirty page list.
838 *
839 * We use private_lock to lock against try_to_free_buffers while using the
840 * page's buffer list. Also use this to protect against clean buffers being
841 * added to the page after it was set dirty.
842 *
843 * FIXME: may need to call ->reservepage here as well. That's rather up to the
844 * address_space though.
845 */
846int __set_page_dirty_buffers(struct page *page)
847{
848 struct address_space * const mapping = page->mapping;
849
850 spin_lock(&mapping->private_lock);
851 if (page_has_buffers(page)) {
852 struct buffer_head *head = page_buffers(page);
853 struct buffer_head *bh = head;
854
855 do {
856 set_buffer_dirty(bh);
857 bh = bh->b_this_page;
858 } while (bh != head);
859 }
860 spin_unlock(&mapping->private_lock);
861
862 if (!TestSetPageDirty(page)) {
863 write_lock_irq(&mapping->tree_lock);
864 if (page->mapping) { /* Race with truncate? */
865 if (mapping_cap_account_dirty(mapping))
866 inc_page_state(nr_dirty);
867 radix_tree_tag_set(&mapping->page_tree,
868 page_index(page),
869 PAGECACHE_TAG_DIRTY);
870 }
871 write_unlock_irq(&mapping->tree_lock);
872 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
873 }
874
875 return 0;
876}
877EXPORT_SYMBOL(__set_page_dirty_buffers);
878
879/*
880 * Write out and wait upon a list of buffers.
881 *
882 * We have conflicting pressures: we want to make sure that all
883 * initially dirty buffers get waited on, but that any subsequently
884 * dirtied buffers don't. After all, we don't want fsync to last
885 * forever if somebody is actively writing to the file.
886 *
887 * Do this in two main stages: first we copy dirty buffers to a
888 * temporary inode list, queueing the writes as we go. Then we clean
889 * up, waiting for those writes to complete.
890 *
891 * During this second stage, any subsequent updates to the file may end
892 * up refiling the buffer on the original inode's dirty list again, so
893 * there is a chance we will end up with a buffer queued for write but
894 * not yet completed on that list. So, as a final cleanup we go through
895 * the osync code to catch these locked, dirty buffers without requeuing
896 * any newly dirty buffers for write.
897 */
898static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
899{
900 struct buffer_head *bh;
901 struct list_head tmp;
902 int err = 0, err2;
903
904 INIT_LIST_HEAD(&tmp);
905
906 spin_lock(lock);
907 while (!list_empty(list)) {
908 bh = BH_ENTRY(list->next);
909 list_del_init(&bh->b_assoc_buffers);
910 if (buffer_dirty(bh) || buffer_locked(bh)) {
911 list_add(&bh->b_assoc_buffers, &tmp);
912 if (buffer_dirty(bh)) {
913 get_bh(bh);
914 spin_unlock(lock);
915 /*
916 * Ensure any pending I/O completes so that
917 * ll_rw_block() actually writes the current
918 * contents - it is a noop if I/O is still in
919 * flight on potentially older contents.
920 */
Jan Karaa7662232005-09-06 15:19:10 -0700921 ll_rw_block(SWRITE, 1, &bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 brelse(bh);
923 spin_lock(lock);
924 }
925 }
926 }
927
928 while (!list_empty(&tmp)) {
929 bh = BH_ENTRY(tmp.prev);
930 __remove_assoc_queue(bh);
931 get_bh(bh);
932 spin_unlock(lock);
933 wait_on_buffer(bh);
934 if (!buffer_uptodate(bh))
935 err = -EIO;
936 brelse(bh);
937 spin_lock(lock);
938 }
939
940 spin_unlock(lock);
941 err2 = osync_buffers_list(lock, list);
942 if (err)
943 return err;
944 else
945 return err2;
946}
947
948/*
949 * Invalidate any and all dirty buffers on a given inode. We are
950 * probably unmounting the fs, but that doesn't mean we have already
951 * done a sync(). Just drop the buffers from the inode list.
952 *
953 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
954 * assumes that all the buffers are against the blockdev. Not true
955 * for reiserfs.
956 */
957void invalidate_inode_buffers(struct inode *inode)
958{
959 if (inode_has_buffers(inode)) {
960 struct address_space *mapping = &inode->i_data;
961 struct list_head *list = &mapping->private_list;
962 struct address_space *buffer_mapping = mapping->assoc_mapping;
963
964 spin_lock(&buffer_mapping->private_lock);
965 while (!list_empty(list))
966 __remove_assoc_queue(BH_ENTRY(list->next));
967 spin_unlock(&buffer_mapping->private_lock);
968 }
969}
970
971/*
972 * Remove any clean buffers from the inode's buffer list. This is called
973 * when we're trying to free the inode itself. Those buffers can pin it.
974 *
975 * Returns true if all buffers were removed.
976 */
977int remove_inode_buffers(struct inode *inode)
978{
979 int ret = 1;
980
981 if (inode_has_buffers(inode)) {
982 struct address_space *mapping = &inode->i_data;
983 struct list_head *list = &mapping->private_list;
984 struct address_space *buffer_mapping = mapping->assoc_mapping;
985
986 spin_lock(&buffer_mapping->private_lock);
987 while (!list_empty(list)) {
988 struct buffer_head *bh = BH_ENTRY(list->next);
989 if (buffer_dirty(bh)) {
990 ret = 0;
991 break;
992 }
993 __remove_assoc_queue(bh);
994 }
995 spin_unlock(&buffer_mapping->private_lock);
996 }
997 return ret;
998}
999
1000/*
1001 * Create the appropriate buffers when given a page for data area and
1002 * the size of each buffer.. Use the bh->b_this_page linked list to
1003 * follow the buffers created. Return NULL if unable to create more
1004 * buffers.
1005 *
1006 * The retry flag is used to differentiate async IO (paging, swapping)
1007 * which may not fail from ordinary buffer allocations.
1008 */
1009struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
1010 int retry)
1011{
1012 struct buffer_head *bh, *head;
1013 long offset;
1014
1015try_again:
1016 head = NULL;
1017 offset = PAGE_SIZE;
1018 while ((offset -= size) >= 0) {
1019 bh = alloc_buffer_head(GFP_NOFS);
1020 if (!bh)
1021 goto no_grow;
1022
1023 bh->b_bdev = NULL;
1024 bh->b_this_page = head;
1025 bh->b_blocknr = -1;
1026 head = bh;
1027
1028 bh->b_state = 0;
1029 atomic_set(&bh->b_count, 0);
1030 bh->b_size = size;
1031
1032 /* Link the buffer to its page */
1033 set_bh_page(bh, page, offset);
1034
1035 bh->b_end_io = NULL;
1036 }
1037 return head;
1038/*
1039 * In case anything failed, we just free everything we got.
1040 */
1041no_grow:
1042 if (head) {
1043 do {
1044 bh = head;
1045 head = head->b_this_page;
1046 free_buffer_head(bh);
1047 } while (head);
1048 }
1049
1050 /*
1051 * Return failure for non-async IO requests. Async IO requests
1052 * are not allowed to fail, so we have to wait until buffer heads
1053 * become available. But we don't want tasks sleeping with
1054 * partially complete buffers, so all were released above.
1055 */
1056 if (!retry)
1057 return NULL;
1058
1059 /* We're _really_ low on memory. Now we just
1060 * wait for old buffer heads to become free due to
1061 * finishing IO. Since this is an async request and
1062 * the reserve list is empty, we're sure there are
1063 * async buffer heads in use.
1064 */
1065 free_more_memory();
1066 goto try_again;
1067}
1068EXPORT_SYMBOL_GPL(alloc_page_buffers);
1069
1070static inline void
1071link_dev_buffers(struct page *page, struct buffer_head *head)
1072{
1073 struct buffer_head *bh, *tail;
1074
1075 bh = head;
1076 do {
1077 tail = bh;
1078 bh = bh->b_this_page;
1079 } while (bh);
1080 tail->b_this_page = head;
1081 attach_page_buffers(page, head);
1082}
1083
1084/*
1085 * Initialise the state of a blockdev page's buffers.
1086 */
1087static void
1088init_page_buffers(struct page *page, struct block_device *bdev,
1089 sector_t block, int size)
1090{
1091 struct buffer_head *head = page_buffers(page);
1092 struct buffer_head *bh = head;
1093 int uptodate = PageUptodate(page);
1094
1095 do {
1096 if (!buffer_mapped(bh)) {
1097 init_buffer(bh, NULL, NULL);
1098 bh->b_bdev = bdev;
1099 bh->b_blocknr = block;
1100 if (uptodate)
1101 set_buffer_uptodate(bh);
1102 set_buffer_mapped(bh);
1103 }
1104 block++;
1105 bh = bh->b_this_page;
1106 } while (bh != head);
1107}
1108
1109/*
1110 * Create the page-cache page that contains the requested block.
1111 *
1112 * This is user purely for blockdev mappings.
1113 */
1114static struct page *
1115grow_dev_page(struct block_device *bdev, sector_t block,
1116 pgoff_t index, int size)
1117{
1118 struct inode *inode = bdev->bd_inode;
1119 struct page *page;
1120 struct buffer_head *bh;
1121
1122 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
1123 if (!page)
1124 return NULL;
1125
1126 if (!PageLocked(page))
1127 BUG();
1128
1129 if (page_has_buffers(page)) {
1130 bh = page_buffers(page);
1131 if (bh->b_size == size) {
1132 init_page_buffers(page, bdev, block, size);
1133 return page;
1134 }
1135 if (!try_to_free_buffers(page))
1136 goto failed;
1137 }
1138
1139 /*
1140 * Allocate some buffers for this page
1141 */
1142 bh = alloc_page_buffers(page, size, 0);
1143 if (!bh)
1144 goto failed;
1145
1146 /*
1147 * Link the page to the buffers and initialise them. Take the
1148 * lock to be atomic wrt __find_get_block(), which does not
1149 * run under the page lock.
1150 */
1151 spin_lock(&inode->i_mapping->private_lock);
1152 link_dev_buffers(page, bh);
1153 init_page_buffers(page, bdev, block, size);
1154 spin_unlock(&inode->i_mapping->private_lock);
1155 return page;
1156
1157failed:
1158 BUG();
1159 unlock_page(page);
1160 page_cache_release(page);
1161 return NULL;
1162}
1163
1164/*
1165 * Create buffers for the specified block device block's page. If
1166 * that page was dirty, the buffers are set dirty also.
1167 *
1168 * Except that's a bug. Attaching dirty buffers to a dirty
1169 * blockdev's page can result in filesystem corruption, because
1170 * some of those buffers may be aliases of filesystem data.
1171 * grow_dev_page() will go BUG() if this happens.
1172 */
1173static inline int
1174grow_buffers(struct block_device *bdev, sector_t block, int size)
1175{
1176 struct page *page;
1177 pgoff_t index;
1178 int sizebits;
1179
1180 sizebits = -1;
1181 do {
1182 sizebits++;
1183 } while ((size << sizebits) < PAGE_SIZE);
1184
1185 index = block >> sizebits;
1186 block = index << sizebits;
1187
1188 /* Create a page with the proper size buffers.. */
1189 page = grow_dev_page(bdev, block, index, size);
1190 if (!page)
1191 return 0;
1192 unlock_page(page);
1193 page_cache_release(page);
1194 return 1;
1195}
1196
Adrian Bunk75c96f82005-05-05 16:16:09 -07001197static struct buffer_head *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198__getblk_slow(struct block_device *bdev, sector_t block, int size)
1199{
1200 /* Size must be multiple of hard sectorsize */
1201 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1202 (size < 512 || size > PAGE_SIZE))) {
1203 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1204 size);
1205 printk(KERN_ERR "hardsect size: %d\n",
1206 bdev_hardsect_size(bdev));
1207
1208 dump_stack();
1209 return NULL;
1210 }
1211
1212 for (;;) {
1213 struct buffer_head * bh;
1214
1215 bh = __find_get_block(bdev, block, size);
1216 if (bh)
1217 return bh;
1218
1219 if (!grow_buffers(bdev, block, size))
1220 free_more_memory();
1221 }
1222}
1223
1224/*
1225 * The relationship between dirty buffers and dirty pages:
1226 *
1227 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1228 * the page is tagged dirty in its radix tree.
1229 *
1230 * At all times, the dirtiness of the buffers represents the dirtiness of
1231 * subsections of the page. If the page has buffers, the page dirty bit is
1232 * merely a hint about the true dirty state.
1233 *
1234 * When a page is set dirty in its entirety, all its buffers are marked dirty
1235 * (if the page has buffers).
1236 *
1237 * When a buffer is marked dirty, its page is dirtied, but the page's other
1238 * buffers are not.
1239 *
1240 * Also. When blockdev buffers are explicitly read with bread(), they
1241 * individually become uptodate. But their backing page remains not
1242 * uptodate - even if all of its buffers are uptodate. A subsequent
1243 * block_read_full_page() against that page will discover all the uptodate
1244 * buffers, will set the page uptodate and will perform no I/O.
1245 */
1246
1247/**
1248 * mark_buffer_dirty - mark a buffer_head as needing writeout
Martin Waitz67be2dd2005-05-01 08:59:26 -07001249 * @bh: the buffer_head to mark dirty
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 *
1251 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1252 * backing page dirty, then tag the page as dirty in its address_space's radix
1253 * tree and then attach the address_space's inode to its superblock's dirty
1254 * inode list.
1255 *
1256 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1257 * mapping->tree_lock and the global inode_lock.
1258 */
1259void fastcall mark_buffer_dirty(struct buffer_head *bh)
1260{
1261 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1262 __set_page_dirty_nobuffers(bh->b_page);
1263}
1264
1265/*
1266 * Decrement a buffer_head's reference count. If all buffers against a page
1267 * have zero reference count, are clean and unlocked, and if the page is clean
1268 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1269 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1270 * a page but it ends up not being freed, and buffers may later be reattached).
1271 */
1272void __brelse(struct buffer_head * buf)
1273{
1274 if (atomic_read(&buf->b_count)) {
1275 put_bh(buf);
1276 return;
1277 }
1278 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1279 WARN_ON(1);
1280}
1281
1282/*
1283 * bforget() is like brelse(), except it discards any
1284 * potentially dirty data.
1285 */
1286void __bforget(struct buffer_head *bh)
1287{
1288 clear_buffer_dirty(bh);
1289 if (!list_empty(&bh->b_assoc_buffers)) {
1290 struct address_space *buffer_mapping = bh->b_page->mapping;
1291
1292 spin_lock(&buffer_mapping->private_lock);
1293 list_del_init(&bh->b_assoc_buffers);
1294 spin_unlock(&buffer_mapping->private_lock);
1295 }
1296 __brelse(bh);
1297}
1298
1299static struct buffer_head *__bread_slow(struct buffer_head *bh)
1300{
1301 lock_buffer(bh);
1302 if (buffer_uptodate(bh)) {
1303 unlock_buffer(bh);
1304 return bh;
1305 } else {
1306 get_bh(bh);
1307 bh->b_end_io = end_buffer_read_sync;
1308 submit_bh(READ, bh);
1309 wait_on_buffer(bh);
1310 if (buffer_uptodate(bh))
1311 return bh;
1312 }
1313 brelse(bh);
1314 return NULL;
1315}
1316
1317/*
1318 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1319 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1320 * refcount elevated by one when they're in an LRU. A buffer can only appear
1321 * once in a particular CPU's LRU. A single buffer can be present in multiple
1322 * CPU's LRUs at the same time.
1323 *
1324 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1325 * sb_find_get_block().
1326 *
1327 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1328 * a local interrupt disable for that.
1329 */
1330
1331#define BH_LRU_SIZE 8
1332
1333struct bh_lru {
1334 struct buffer_head *bhs[BH_LRU_SIZE];
1335};
1336
1337static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1338
1339#ifdef CONFIG_SMP
1340#define bh_lru_lock() local_irq_disable()
1341#define bh_lru_unlock() local_irq_enable()
1342#else
1343#define bh_lru_lock() preempt_disable()
1344#define bh_lru_unlock() preempt_enable()
1345#endif
1346
1347static inline void check_irqs_on(void)
1348{
1349#ifdef irqs_disabled
1350 BUG_ON(irqs_disabled());
1351#endif
1352}
1353
1354/*
1355 * The LRU management algorithm is dopey-but-simple. Sorry.
1356 */
1357static void bh_lru_install(struct buffer_head *bh)
1358{
1359 struct buffer_head *evictee = NULL;
1360 struct bh_lru *lru;
1361
1362 check_irqs_on();
1363 bh_lru_lock();
1364 lru = &__get_cpu_var(bh_lrus);
1365 if (lru->bhs[0] != bh) {
1366 struct buffer_head *bhs[BH_LRU_SIZE];
1367 int in;
1368 int out = 0;
1369
1370 get_bh(bh);
1371 bhs[out++] = bh;
1372 for (in = 0; in < BH_LRU_SIZE; in++) {
1373 struct buffer_head *bh2 = lru->bhs[in];
1374
1375 if (bh2 == bh) {
1376 __brelse(bh2);
1377 } else {
1378 if (out >= BH_LRU_SIZE) {
1379 BUG_ON(evictee != NULL);
1380 evictee = bh2;
1381 } else {
1382 bhs[out++] = bh2;
1383 }
1384 }
1385 }
1386 while (out < BH_LRU_SIZE)
1387 bhs[out++] = NULL;
1388 memcpy(lru->bhs, bhs, sizeof(bhs));
1389 }
1390 bh_lru_unlock();
1391
1392 if (evictee)
1393 __brelse(evictee);
1394}
1395
1396/*
1397 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1398 */
1399static inline struct buffer_head *
1400lookup_bh_lru(struct block_device *bdev, sector_t block, int size)
1401{
1402 struct buffer_head *ret = NULL;
1403 struct bh_lru *lru;
1404 int i;
1405
1406 check_irqs_on();
1407 bh_lru_lock();
1408 lru = &__get_cpu_var(bh_lrus);
1409 for (i = 0; i < BH_LRU_SIZE; i++) {
1410 struct buffer_head *bh = lru->bhs[i];
1411
1412 if (bh && bh->b_bdev == bdev &&
1413 bh->b_blocknr == block && bh->b_size == size) {
1414 if (i) {
1415 while (i) {
1416 lru->bhs[i] = lru->bhs[i - 1];
1417 i--;
1418 }
1419 lru->bhs[0] = bh;
1420 }
1421 get_bh(bh);
1422 ret = bh;
1423 break;
1424 }
1425 }
1426 bh_lru_unlock();
1427 return ret;
1428}
1429
1430/*
1431 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1432 * it in the LRU and mark it as accessed. If it is not present then return
1433 * NULL
1434 */
1435struct buffer_head *
1436__find_get_block(struct block_device *bdev, sector_t block, int size)
1437{
1438 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1439
1440 if (bh == NULL) {
Coywolf Qi Hunt385fd4c2005-11-07 00:59:39 -08001441 bh = __find_get_block_slow(bdev, block);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442 if (bh)
1443 bh_lru_install(bh);
1444 }
1445 if (bh)
1446 touch_buffer(bh);
1447 return bh;
1448}
1449EXPORT_SYMBOL(__find_get_block);
1450
1451/*
1452 * __getblk will locate (and, if necessary, create) the buffer_head
1453 * which corresponds to the passed block_device, block and size. The
1454 * returned buffer has its reference count incremented.
1455 *
1456 * __getblk() cannot fail - it just keeps trying. If you pass it an
1457 * illegal block number, __getblk() will happily return a buffer_head
1458 * which represents the non-existent block. Very weird.
1459 *
1460 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1461 * attempt is failing. FIXME, perhaps?
1462 */
1463struct buffer_head *
1464__getblk(struct block_device *bdev, sector_t block, int size)
1465{
1466 struct buffer_head *bh = __find_get_block(bdev, block, size);
1467
1468 might_sleep();
1469 if (bh == NULL)
1470 bh = __getblk_slow(bdev, block, size);
1471 return bh;
1472}
1473EXPORT_SYMBOL(__getblk);
1474
1475/*
1476 * Do async read-ahead on a buffer..
1477 */
1478void __breadahead(struct block_device *bdev, sector_t block, int size)
1479{
1480 struct buffer_head *bh = __getblk(bdev, block, size);
Andrew Mortona3e713b2005-10-30 15:03:15 -08001481 if (likely(bh)) {
1482 ll_rw_block(READA, 1, &bh);
1483 brelse(bh);
1484 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485}
1486EXPORT_SYMBOL(__breadahead);
1487
1488/**
1489 * __bread() - reads a specified block and returns the bh
Martin Waitz67be2dd2005-05-01 08:59:26 -07001490 * @bdev: the block_device to read from
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 * @block: number of block
1492 * @size: size (in bytes) to read
1493 *
1494 * Reads a specified block, and returns buffer head that contains it.
1495 * It returns NULL if the block was unreadable.
1496 */
1497struct buffer_head *
1498__bread(struct block_device *bdev, sector_t block, int size)
1499{
1500 struct buffer_head *bh = __getblk(bdev, block, size);
1501
Andrew Mortona3e713b2005-10-30 15:03:15 -08001502 if (likely(bh) && !buffer_uptodate(bh))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503 bh = __bread_slow(bh);
1504 return bh;
1505}
1506EXPORT_SYMBOL(__bread);
1507
1508/*
1509 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1510 * This doesn't race because it runs in each cpu either in irq
1511 * or with preempt disabled.
1512 */
1513static void invalidate_bh_lru(void *arg)
1514{
1515 struct bh_lru *b = &get_cpu_var(bh_lrus);
1516 int i;
1517
1518 for (i = 0; i < BH_LRU_SIZE; i++) {
1519 brelse(b->bhs[i]);
1520 b->bhs[i] = NULL;
1521 }
1522 put_cpu_var(bh_lrus);
1523}
1524
1525static void invalidate_bh_lrus(void)
1526{
1527 on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
1528}
1529
1530void set_bh_page(struct buffer_head *bh,
1531 struct page *page, unsigned long offset)
1532{
1533 bh->b_page = page;
1534 if (offset >= PAGE_SIZE)
1535 BUG();
1536 if (PageHighMem(page))
1537 /*
1538 * This catches illegal uses and preserves the offset:
1539 */
1540 bh->b_data = (char *)(0 + offset);
1541 else
1542 bh->b_data = page_address(page) + offset;
1543}
1544EXPORT_SYMBOL(set_bh_page);
1545
1546/*
1547 * Called when truncating a buffer on a page completely.
1548 */
1549static inline void discard_buffer(struct buffer_head * bh)
1550{
1551 lock_buffer(bh);
1552 clear_buffer_dirty(bh);
1553 bh->b_bdev = NULL;
1554 clear_buffer_mapped(bh);
1555 clear_buffer_req(bh);
1556 clear_buffer_new(bh);
1557 clear_buffer_delay(bh);
1558 unlock_buffer(bh);
1559}
1560
1561/**
1562 * try_to_release_page() - release old fs-specific metadata on a page
1563 *
1564 * @page: the page which the kernel is trying to free
1565 * @gfp_mask: memory allocation flags (and I/O mode)
1566 *
1567 * The address_space is to try to release any data against the page
1568 * (presumably at page->private). If the release was successful, return `1'.
1569 * Otherwise return zero.
1570 *
1571 * The @gfp_mask argument specifies whether I/O may be performed to release
1572 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1573 *
1574 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1575 */
Al Viro27496a82005-10-21 03:20:48 -04001576int try_to_release_page(struct page *page, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577{
1578 struct address_space * const mapping = page->mapping;
1579
1580 BUG_ON(!PageLocked(page));
1581 if (PageWriteback(page))
1582 return 0;
1583
1584 if (mapping && mapping->a_ops->releasepage)
1585 return mapping->a_ops->releasepage(page, gfp_mask);
1586 return try_to_free_buffers(page);
1587}
1588EXPORT_SYMBOL(try_to_release_page);
1589
1590/**
1591 * block_invalidatepage - invalidate part of all of a buffer-backed page
1592 *
1593 * @page: the page which is affected
1594 * @offset: the index of the truncation point
1595 *
1596 * block_invalidatepage() is called when all or part of the page has become
1597 * invalidatedby a truncate operation.
1598 *
1599 * block_invalidatepage() does not have to release all buffers, but it must
1600 * ensure that no dirty buffer is left outside @offset and that no I/O
1601 * is underway against any of the blocks which are outside the truncation
1602 * point. Because the caller is about to free (and possibly reuse) those
1603 * blocks on-disk.
1604 */
1605int block_invalidatepage(struct page *page, unsigned long offset)
1606{
1607 struct buffer_head *head, *bh, *next;
1608 unsigned int curr_off = 0;
1609 int ret = 1;
1610
1611 BUG_ON(!PageLocked(page));
1612 if (!page_has_buffers(page))
1613 goto out;
1614
1615 head = page_buffers(page);
1616 bh = head;
1617 do {
1618 unsigned int next_off = curr_off + bh->b_size;
1619 next = bh->b_this_page;
1620
1621 /*
1622 * is this block fully invalidated?
1623 */
1624 if (offset <= curr_off)
1625 discard_buffer(bh);
1626 curr_off = next_off;
1627 bh = next;
1628 } while (bh != head);
1629
1630 /*
1631 * We release buffers only if the entire page is being invalidated.
1632 * The get_block cached value has been unconditionally invalidated,
1633 * so real IO is not possible anymore.
1634 */
1635 if (offset == 0)
1636 ret = try_to_release_page(page, 0);
1637out:
1638 return ret;
1639}
1640EXPORT_SYMBOL(block_invalidatepage);
1641
Jan Karaaaa40592005-10-30 15:00:16 -08001642int do_invalidatepage(struct page *page, unsigned long offset)
1643{
1644 int (*invalidatepage)(struct page *, unsigned long);
1645 invalidatepage = page->mapping->a_ops->invalidatepage;
1646 if (invalidatepage == NULL)
1647 invalidatepage = block_invalidatepage;
1648 return (*invalidatepage)(page, offset);
1649}
1650
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651/*
1652 * We attach and possibly dirty the buffers atomically wrt
1653 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1654 * is already excluded via the page lock.
1655 */
1656void create_empty_buffers(struct page *page,
1657 unsigned long blocksize, unsigned long b_state)
1658{
1659 struct buffer_head *bh, *head, *tail;
1660
1661 head = alloc_page_buffers(page, blocksize, 1);
1662 bh = head;
1663 do {
1664 bh->b_state |= b_state;
1665 tail = bh;
1666 bh = bh->b_this_page;
1667 } while (bh);
1668 tail->b_this_page = head;
1669
1670 spin_lock(&page->mapping->private_lock);
1671 if (PageUptodate(page) || PageDirty(page)) {
1672 bh = head;
1673 do {
1674 if (PageDirty(page))
1675 set_buffer_dirty(bh);
1676 if (PageUptodate(page))
1677 set_buffer_uptodate(bh);
1678 bh = bh->b_this_page;
1679 } while (bh != head);
1680 }
1681 attach_page_buffers(page, head);
1682 spin_unlock(&page->mapping->private_lock);
1683}
1684EXPORT_SYMBOL(create_empty_buffers);
1685
1686/*
1687 * We are taking a block for data and we don't want any output from any
1688 * buffer-cache aliases starting from return from that function and
1689 * until the moment when something will explicitly mark the buffer
1690 * dirty (hopefully that will not happen until we will free that block ;-)
1691 * We don't even need to mark it not-uptodate - nobody can expect
1692 * anything from a newly allocated buffer anyway. We used to used
1693 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1694 * don't want to mark the alias unmapped, for example - it would confuse
1695 * anyone who might pick it with bread() afterwards...
1696 *
1697 * Also.. Note that bforget() doesn't lock the buffer. So there can
1698 * be writeout I/O going on against recently-freed buffers. We don't
1699 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1700 * only if we really need to. That happens here.
1701 */
1702void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1703{
1704 struct buffer_head *old_bh;
1705
1706 might_sleep();
1707
Coywolf Qi Hunt385fd4c2005-11-07 00:59:39 -08001708 old_bh = __find_get_block_slow(bdev, block);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 if (old_bh) {
1710 clear_buffer_dirty(old_bh);
1711 wait_on_buffer(old_bh);
1712 clear_buffer_req(old_bh);
1713 __brelse(old_bh);
1714 }
1715}
1716EXPORT_SYMBOL(unmap_underlying_metadata);
1717
1718/*
1719 * NOTE! All mapped/uptodate combinations are valid:
1720 *
1721 * Mapped Uptodate Meaning
1722 *
1723 * No No "unknown" - must do get_block()
1724 * No Yes "hole" - zero-filled
1725 * Yes No "allocated" - allocated on disk, not read in
1726 * Yes Yes "valid" - allocated and up-to-date in memory.
1727 *
1728 * "Dirty" is valid only with the last case (mapped+uptodate).
1729 */
1730
1731/*
1732 * While block_write_full_page is writing back the dirty buffers under
1733 * the page lock, whoever dirtied the buffers may decide to clean them
1734 * again at any time. We handle that by only looking at the buffer
1735 * state inside lock_buffer().
1736 *
1737 * If block_write_full_page() is called for regular writeback
1738 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1739 * locked buffer. This only can happen if someone has written the buffer
1740 * directly, with submit_bh(). At the address_space level PageWriteback
1741 * prevents this contention from occurring.
1742 */
1743static int __block_write_full_page(struct inode *inode, struct page *page,
1744 get_block_t *get_block, struct writeback_control *wbc)
1745{
1746 int err;
1747 sector_t block;
1748 sector_t last_block;
Andrew Mortonf0fbd5f2005-05-05 16:15:48 -07001749 struct buffer_head *bh, *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750 int nr_underway = 0;
1751
1752 BUG_ON(!PageLocked(page));
1753
1754 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1755
1756 if (!page_has_buffers(page)) {
1757 create_empty_buffers(page, 1 << inode->i_blkbits,
1758 (1 << BH_Dirty)|(1 << BH_Uptodate));
1759 }
1760
1761 /*
1762 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1763 * here, and the (potentially unmapped) buffers may become dirty at
1764 * any time. If a buffer becomes dirty here after we've inspected it
1765 * then we just miss that fact, and the page stays dirty.
1766 *
1767 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1768 * handle that here by just cleaning them.
1769 */
1770
1771 block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1772 head = page_buffers(page);
1773 bh = head;
1774
1775 /*
1776 * Get all the dirty buffers mapped to disk addresses and
1777 * handle any aliases from the underlying blockdev's mapping.
1778 */
1779 do {
1780 if (block > last_block) {
1781 /*
1782 * mapped buffers outside i_size will occur, because
1783 * this page can be outside i_size when there is a
1784 * truncate in progress.
1785 */
1786 /*
1787 * The buffer was zeroed by block_write_full_page()
1788 */
1789 clear_buffer_dirty(bh);
1790 set_buffer_uptodate(bh);
1791 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
1792 err = get_block(inode, block, bh, 1);
1793 if (err)
1794 goto recover;
1795 if (buffer_new(bh)) {
1796 /* blockdev mappings never come here */
1797 clear_buffer_new(bh);
1798 unmap_underlying_metadata(bh->b_bdev,
1799 bh->b_blocknr);
1800 }
1801 }
1802 bh = bh->b_this_page;
1803 block++;
1804 } while (bh != head);
1805
1806 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807 if (!buffer_mapped(bh))
1808 continue;
1809 /*
1810 * If it's a fully non-blocking write attempt and we cannot
1811 * lock the buffer then redirty the page. Note that this can
1812 * potentially cause a busy-wait loop from pdflush and kswapd
1813 * activity, but those code paths have their own higher-level
1814 * throttling.
1815 */
1816 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1817 lock_buffer(bh);
1818 } else if (test_set_buffer_locked(bh)) {
1819 redirty_page_for_writepage(wbc, page);
1820 continue;
1821 }
1822 if (test_clear_buffer_dirty(bh)) {
1823 mark_buffer_async_write(bh);
1824 } else {
1825 unlock_buffer(bh);
1826 }
1827 } while ((bh = bh->b_this_page) != head);
1828
1829 /*
1830 * The page and its buffers are protected by PageWriteback(), so we can
1831 * drop the bh refcounts early.
1832 */
1833 BUG_ON(PageWriteback(page));
1834 set_page_writeback(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001835
1836 do {
1837 struct buffer_head *next = bh->b_this_page;
1838 if (buffer_async_write(bh)) {
1839 submit_bh(WRITE, bh);
1840 nr_underway++;
1841 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842 bh = next;
1843 } while (bh != head);
Andrew Morton05937ba2005-05-05 16:15:47 -07001844 unlock_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001845
1846 err = 0;
1847done:
1848 if (nr_underway == 0) {
1849 /*
1850 * The page was marked dirty, but the buffers were
1851 * clean. Someone wrote them back by hand with
1852 * ll_rw_block/submit_bh. A rare case.
1853 */
1854 int uptodate = 1;
1855 do {
1856 if (!buffer_uptodate(bh)) {
1857 uptodate = 0;
1858 break;
1859 }
1860 bh = bh->b_this_page;
1861 } while (bh != head);
1862 if (uptodate)
1863 SetPageUptodate(page);
1864 end_page_writeback(page);
1865 /*
1866 * The page and buffer_heads can be released at any time from
1867 * here on.
1868 */
1869 wbc->pages_skipped++; /* We didn't write this page */
1870 }
1871 return err;
1872
1873recover:
1874 /*
1875 * ENOSPC, or some other error. We may already have added some
1876 * blocks to the file, so we need to write these out to avoid
1877 * exposing stale data.
1878 * The page is currently locked and not marked for writeback
1879 */
1880 bh = head;
1881 /* Recovery: lock and submit the mapped buffers */
1882 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001883 if (buffer_mapped(bh) && buffer_dirty(bh)) {
1884 lock_buffer(bh);
1885 mark_buffer_async_write(bh);
1886 } else {
1887 /*
1888 * The buffer may have been set dirty during
1889 * attachment to a dirty page.
1890 */
1891 clear_buffer_dirty(bh);
1892 }
1893 } while ((bh = bh->b_this_page) != head);
1894 SetPageError(page);
1895 BUG_ON(PageWriteback(page));
1896 set_page_writeback(page);
1897 unlock_page(page);
1898 do {
1899 struct buffer_head *next = bh->b_this_page;
1900 if (buffer_async_write(bh)) {
1901 clear_buffer_dirty(bh);
1902 submit_bh(WRITE, bh);
1903 nr_underway++;
1904 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 bh = next;
1906 } while (bh != head);
1907 goto done;
1908}
1909
1910static int __block_prepare_write(struct inode *inode, struct page *page,
1911 unsigned from, unsigned to, get_block_t *get_block)
1912{
1913 unsigned block_start, block_end;
1914 sector_t block;
1915 int err = 0;
1916 unsigned blocksize, bbits;
1917 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1918
1919 BUG_ON(!PageLocked(page));
1920 BUG_ON(from > PAGE_CACHE_SIZE);
1921 BUG_ON(to > PAGE_CACHE_SIZE);
1922 BUG_ON(from > to);
1923
1924 blocksize = 1 << inode->i_blkbits;
1925 if (!page_has_buffers(page))
1926 create_empty_buffers(page, blocksize, 0);
1927 head = page_buffers(page);
1928
1929 bbits = inode->i_blkbits;
1930 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1931
1932 for(bh = head, block_start = 0; bh != head || !block_start;
1933 block++, block_start=block_end, bh = bh->b_this_page) {
1934 block_end = block_start + blocksize;
1935 if (block_end <= from || block_start >= to) {
1936 if (PageUptodate(page)) {
1937 if (!buffer_uptodate(bh))
1938 set_buffer_uptodate(bh);
1939 }
1940 continue;
1941 }
1942 if (buffer_new(bh))
1943 clear_buffer_new(bh);
1944 if (!buffer_mapped(bh)) {
1945 err = get_block(inode, block, bh, 1);
1946 if (err)
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001947 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 if (buffer_new(bh)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001949 unmap_underlying_metadata(bh->b_bdev,
1950 bh->b_blocknr);
1951 if (PageUptodate(page)) {
1952 set_buffer_uptodate(bh);
1953 continue;
1954 }
1955 if (block_end > to || block_start < from) {
1956 void *kaddr;
1957
1958 kaddr = kmap_atomic(page, KM_USER0);
1959 if (block_end > to)
1960 memset(kaddr+to, 0,
1961 block_end-to);
1962 if (block_start < from)
1963 memset(kaddr+block_start,
1964 0, from-block_start);
1965 flush_dcache_page(page);
1966 kunmap_atomic(kaddr, KM_USER0);
1967 }
1968 continue;
1969 }
1970 }
1971 if (PageUptodate(page)) {
1972 if (!buffer_uptodate(bh))
1973 set_buffer_uptodate(bh);
1974 continue;
1975 }
1976 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1977 (block_start < from || block_end > to)) {
1978 ll_rw_block(READ, 1, &bh);
1979 *wait_bh++=bh;
1980 }
1981 }
1982 /*
1983 * If we issued read requests - let them complete.
1984 */
1985 while(wait_bh > wait) {
1986 wait_on_buffer(*--wait_bh);
1987 if (!buffer_uptodate(*wait_bh))
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001988 err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 }
Anton Altaparmakov152becd2005-06-23 00:10:21 -07001990 if (!err) {
1991 bh = head;
1992 do {
1993 if (buffer_new(bh))
1994 clear_buffer_new(bh);
1995 } while ((bh = bh->b_this_page) != head);
1996 return 0;
1997 }
Nick Pigginf3ddbdc2005-05-05 16:15:45 -07001998 /* Error case: */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001999 /*
2000 * Zero out any newly allocated blocks to avoid exposing stale
2001 * data. If BH_New is set, we know that the block was newly
2002 * allocated in the above loop.
2003 */
2004 bh = head;
2005 block_start = 0;
2006 do {
2007 block_end = block_start+blocksize;
2008 if (block_end <= from)
2009 goto next_bh;
2010 if (block_start >= to)
2011 break;
2012 if (buffer_new(bh)) {
2013 void *kaddr;
2014
2015 clear_buffer_new(bh);
2016 kaddr = kmap_atomic(page, KM_USER0);
2017 memset(kaddr+block_start, 0, bh->b_size);
2018 kunmap_atomic(kaddr, KM_USER0);
2019 set_buffer_uptodate(bh);
2020 mark_buffer_dirty(bh);
2021 }
2022next_bh:
2023 block_start = block_end;
2024 bh = bh->b_this_page;
2025 } while (bh != head);
2026 return err;
2027}
2028
2029static int __block_commit_write(struct inode *inode, struct page *page,
2030 unsigned from, unsigned to)
2031{
2032 unsigned block_start, block_end;
2033 int partial = 0;
2034 unsigned blocksize;
2035 struct buffer_head *bh, *head;
2036
2037 blocksize = 1 << inode->i_blkbits;
2038
2039 for(bh = head = page_buffers(page), block_start = 0;
2040 bh != head || !block_start;
2041 block_start=block_end, bh = bh->b_this_page) {
2042 block_end = block_start + blocksize;
2043 if (block_end <= from || block_start >= to) {
2044 if (!buffer_uptodate(bh))
2045 partial = 1;
2046 } else {
2047 set_buffer_uptodate(bh);
2048 mark_buffer_dirty(bh);
2049 }
2050 }
2051
2052 /*
2053 * If this is a partial write which happened to make all buffers
2054 * uptodate then we can optimize away a bogus readpage() for
2055 * the next read(). Here we 'discover' whether the page went
2056 * uptodate as a result of this (potentially partial) write.
2057 */
2058 if (!partial)
2059 SetPageUptodate(page);
2060 return 0;
2061}
2062
2063/*
2064 * Generic "read page" function for block devices that have the normal
2065 * get_block functionality. This is most of the block device filesystems.
2066 * Reads the page asynchronously --- the unlock_buffer() and
2067 * set/clear_buffer_uptodate() functions propagate buffer state into the
2068 * page struct once IO has completed.
2069 */
2070int block_read_full_page(struct page *page, get_block_t *get_block)
2071{
2072 struct inode *inode = page->mapping->host;
2073 sector_t iblock, lblock;
2074 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2075 unsigned int blocksize;
2076 int nr, i;
2077 int fully_mapped = 1;
2078
Matt Mackallcd7619d2005-05-01 08:59:01 -07002079 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 blocksize = 1 << inode->i_blkbits;
2081 if (!page_has_buffers(page))
2082 create_empty_buffers(page, blocksize, 0);
2083 head = page_buffers(page);
2084
2085 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2086 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2087 bh = head;
2088 nr = 0;
2089 i = 0;
2090
2091 do {
2092 if (buffer_uptodate(bh))
2093 continue;
2094
2095 if (!buffer_mapped(bh)) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002096 int err = 0;
2097
Linus Torvalds1da177e2005-04-16 15:20:36 -07002098 fully_mapped = 0;
2099 if (iblock < lblock) {
Andrew Mortonc64610b2005-05-16 21:53:49 -07002100 err = get_block(inode, iblock, bh, 0);
2101 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002102 SetPageError(page);
2103 }
2104 if (!buffer_mapped(bh)) {
2105 void *kaddr = kmap_atomic(page, KM_USER0);
2106 memset(kaddr + i * blocksize, 0, blocksize);
2107 flush_dcache_page(page);
2108 kunmap_atomic(kaddr, KM_USER0);
Andrew Mortonc64610b2005-05-16 21:53:49 -07002109 if (!err)
2110 set_buffer_uptodate(bh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002111 continue;
2112 }
2113 /*
2114 * get_block() might have updated the buffer
2115 * synchronously
2116 */
2117 if (buffer_uptodate(bh))
2118 continue;
2119 }
2120 arr[nr++] = bh;
2121 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2122
2123 if (fully_mapped)
2124 SetPageMappedToDisk(page);
2125
2126 if (!nr) {
2127 /*
2128 * All buffers are uptodate - we can set the page uptodate
2129 * as well. But not if get_block() returned an error.
2130 */
2131 if (!PageError(page))
2132 SetPageUptodate(page);
2133 unlock_page(page);
2134 return 0;
2135 }
2136
2137 /* Stage two: lock the buffers */
2138 for (i = 0; i < nr; i++) {
2139 bh = arr[i];
2140 lock_buffer(bh);
2141 mark_buffer_async_read(bh);
2142 }
2143
2144 /*
2145 * Stage 3: start the IO. Check for uptodateness
2146 * inside the buffer lock in case another process reading
2147 * the underlying blockdev brought it uptodate (the sct fix).
2148 */
2149 for (i = 0; i < nr; i++) {
2150 bh = arr[i];
2151 if (buffer_uptodate(bh))
2152 end_buffer_async_read(bh, 1);
2153 else
2154 submit_bh(READ, bh);
2155 }
2156 return 0;
2157}
2158
2159/* utility function for filesystems that need to do work on expanding
2160 * truncates. Uses prepare/commit_write to allow the filesystem to
2161 * deal with the hole.
2162 */
2163int generic_cont_expand(struct inode *inode, loff_t size)
2164{
2165 struct address_space *mapping = inode->i_mapping;
2166 struct page *page;
2167 unsigned long index, offset, limit;
2168 int err;
2169
2170 err = -EFBIG;
2171 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2172 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2173 send_sig(SIGXFSZ, current, 0);
2174 goto out;
2175 }
2176 if (size > inode->i_sb->s_maxbytes)
2177 goto out;
2178
2179 offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */
2180
2181 /* ugh. in prepare/commit_write, if from==to==start of block, we
2182 ** skip the prepare. make sure we never send an offset for the start
2183 ** of a block
2184 */
2185 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
2186 offset++;
2187 }
2188 index = size >> PAGE_CACHE_SHIFT;
2189 err = -ENOMEM;
2190 page = grab_cache_page(mapping, index);
2191 if (!page)
2192 goto out;
2193 err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
2194 if (!err) {
2195 err = mapping->a_ops->commit_write(NULL, page, offset, offset);
2196 }
2197 unlock_page(page);
2198 page_cache_release(page);
2199 if (err > 0)
2200 err = 0;
2201out:
2202 return err;
2203}
2204
2205/*
2206 * For moronic filesystems that do not allow holes in file.
2207 * We may have to extend the file.
2208 */
2209
2210int cont_prepare_write(struct page *page, unsigned offset,
2211 unsigned to, get_block_t *get_block, loff_t *bytes)
2212{
2213 struct address_space *mapping = page->mapping;
2214 struct inode *inode = mapping->host;
2215 struct page *new_page;
2216 pgoff_t pgpos;
2217 long status;
2218 unsigned zerofrom;
2219 unsigned blocksize = 1 << inode->i_blkbits;
2220 void *kaddr;
2221
2222 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
2223 status = -ENOMEM;
2224 new_page = grab_cache_page(mapping, pgpos);
2225 if (!new_page)
2226 goto out;
2227 /* we might sleep */
2228 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
2229 unlock_page(new_page);
2230 page_cache_release(new_page);
2231 continue;
2232 }
2233 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2234 if (zerofrom & (blocksize-1)) {
2235 *bytes |= (blocksize-1);
2236 (*bytes)++;
2237 }
2238 status = __block_prepare_write(inode, new_page, zerofrom,
2239 PAGE_CACHE_SIZE, get_block);
2240 if (status)
2241 goto out_unmap;
2242 kaddr = kmap_atomic(new_page, KM_USER0);
2243 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
2244 flush_dcache_page(new_page);
2245 kunmap_atomic(kaddr, KM_USER0);
2246 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE);
2247 unlock_page(new_page);
2248 page_cache_release(new_page);
2249 }
2250
2251 if (page->index < pgpos) {
2252 /* completely inside the area */
2253 zerofrom = offset;
2254 } else {
2255 /* page covers the boundary, find the boundary offset */
2256 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2257
2258 /* if we will expand the thing last block will be filled */
2259 if (to > zerofrom && (zerofrom & (blocksize-1))) {
2260 *bytes |= (blocksize-1);
2261 (*bytes)++;
2262 }
2263
2264 /* starting below the boundary? Nothing to zero out */
2265 if (offset <= zerofrom)
2266 zerofrom = offset;
2267 }
2268 status = __block_prepare_write(inode, page, zerofrom, to, get_block);
2269 if (status)
2270 goto out1;
2271 if (zerofrom < offset) {
2272 kaddr = kmap_atomic(page, KM_USER0);
2273 memset(kaddr+zerofrom, 0, offset-zerofrom);
2274 flush_dcache_page(page);
2275 kunmap_atomic(kaddr, KM_USER0);
2276 __block_commit_write(inode, page, zerofrom, offset);
2277 }
2278 return 0;
2279out1:
2280 ClearPageUptodate(page);
2281 return status;
2282
2283out_unmap:
2284 ClearPageUptodate(new_page);
2285 unlock_page(new_page);
2286 page_cache_release(new_page);
2287out:
2288 return status;
2289}
2290
2291int block_prepare_write(struct page *page, unsigned from, unsigned to,
2292 get_block_t *get_block)
2293{
2294 struct inode *inode = page->mapping->host;
2295 int err = __block_prepare_write(inode, page, from, to, get_block);
2296 if (err)
2297 ClearPageUptodate(page);
2298 return err;
2299}
2300
2301int block_commit_write(struct page *page, unsigned from, unsigned to)
2302{
2303 struct inode *inode = page->mapping->host;
2304 __block_commit_write(inode,page,from,to);
2305 return 0;
2306}
2307
2308int generic_commit_write(struct file *file, struct page *page,
2309 unsigned from, unsigned to)
2310{
2311 struct inode *inode = page->mapping->host;
2312 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2313 __block_commit_write(inode,page,from,to);
2314 /*
2315 * No need to use i_size_read() here, the i_size
2316 * cannot change under us because we hold i_sem.
2317 */
2318 if (pos > inode->i_size) {
2319 i_size_write(inode, pos);
2320 mark_inode_dirty(inode);
2321 }
2322 return 0;
2323}
2324
2325
2326/*
2327 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2328 * immediately, while under the page lock. So it needs a special end_io
2329 * handler which does not touch the bh after unlocking it.
2330 *
2331 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2332 * a race there is benign: unlock_buffer() only use the bh's address for
2333 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2334 * itself.
2335 */
2336static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2337{
2338 if (uptodate) {
2339 set_buffer_uptodate(bh);
2340 } else {
2341 /* This happens, due to failed READA attempts. */
2342 clear_buffer_uptodate(bh);
2343 }
2344 unlock_buffer(bh);
2345}
2346
2347/*
2348 * On entry, the page is fully not uptodate.
2349 * On exit the page is fully uptodate in the areas outside (from,to)
2350 */
2351int nobh_prepare_write(struct page *page, unsigned from, unsigned to,
2352 get_block_t *get_block)
2353{
2354 struct inode *inode = page->mapping->host;
2355 const unsigned blkbits = inode->i_blkbits;
2356 const unsigned blocksize = 1 << blkbits;
2357 struct buffer_head map_bh;
2358 struct buffer_head *read_bh[MAX_BUF_PER_PAGE];
2359 unsigned block_in_page;
2360 unsigned block_start;
2361 sector_t block_in_file;
2362 char *kaddr;
2363 int nr_reads = 0;
2364 int i;
2365 int ret = 0;
2366 int is_mapped_to_disk = 1;
2367 int dirtied_it = 0;
2368
2369 if (PageMappedToDisk(page))
2370 return 0;
2371
2372 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2373 map_bh.b_page = page;
2374
2375 /*
2376 * We loop across all blocks in the page, whether or not they are
2377 * part of the affected region. This is so we can discover if the
2378 * page is fully mapped-to-disk.
2379 */
2380 for (block_start = 0, block_in_page = 0;
2381 block_start < PAGE_CACHE_SIZE;
2382 block_in_page++, block_start += blocksize) {
2383 unsigned block_end = block_start + blocksize;
2384 int create;
2385
2386 map_bh.b_state = 0;
2387 create = 1;
2388 if (block_start >= to)
2389 create = 0;
2390 ret = get_block(inode, block_in_file + block_in_page,
2391 &map_bh, create);
2392 if (ret)
2393 goto failed;
2394 if (!buffer_mapped(&map_bh))
2395 is_mapped_to_disk = 0;
2396 if (buffer_new(&map_bh))
2397 unmap_underlying_metadata(map_bh.b_bdev,
2398 map_bh.b_blocknr);
2399 if (PageUptodate(page))
2400 continue;
2401 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) {
2402 kaddr = kmap_atomic(page, KM_USER0);
2403 if (block_start < from) {
2404 memset(kaddr+block_start, 0, from-block_start);
2405 dirtied_it = 1;
2406 }
2407 if (block_end > to) {
2408 memset(kaddr + to, 0, block_end - to);
2409 dirtied_it = 1;
2410 }
2411 flush_dcache_page(page);
2412 kunmap_atomic(kaddr, KM_USER0);
2413 continue;
2414 }
2415 if (buffer_uptodate(&map_bh))
2416 continue; /* reiserfs does this */
2417 if (block_start < from || block_end > to) {
2418 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS);
2419
2420 if (!bh) {
2421 ret = -ENOMEM;
2422 goto failed;
2423 }
2424 bh->b_state = map_bh.b_state;
2425 atomic_set(&bh->b_count, 0);
2426 bh->b_this_page = NULL;
2427 bh->b_page = page;
2428 bh->b_blocknr = map_bh.b_blocknr;
2429 bh->b_size = blocksize;
2430 bh->b_data = (char *)(long)block_start;
2431 bh->b_bdev = map_bh.b_bdev;
2432 bh->b_private = NULL;
2433 read_bh[nr_reads++] = bh;
2434 }
2435 }
2436
2437 if (nr_reads) {
2438 struct buffer_head *bh;
2439
2440 /*
2441 * The page is locked, so these buffers are protected from
2442 * any VM or truncate activity. Hence we don't need to care
2443 * for the buffer_head refcounts.
2444 */
2445 for (i = 0; i < nr_reads; i++) {
2446 bh = read_bh[i];
2447 lock_buffer(bh);
2448 bh->b_end_io = end_buffer_read_nobh;
2449 submit_bh(READ, bh);
2450 }
2451 for (i = 0; i < nr_reads; i++) {
2452 bh = read_bh[i];
2453 wait_on_buffer(bh);
2454 if (!buffer_uptodate(bh))
2455 ret = -EIO;
2456 free_buffer_head(bh);
2457 read_bh[i] = NULL;
2458 }
2459 if (ret)
2460 goto failed;
2461 }
2462
2463 if (is_mapped_to_disk)
2464 SetPageMappedToDisk(page);
2465 SetPageUptodate(page);
2466
2467 /*
2468 * Setting the page dirty here isn't necessary for the prepare_write
2469 * function - commit_write will do that. But if/when this function is
2470 * used within the pagefault handler to ensure that all mmapped pages
2471 * have backing space in the filesystem, we will need to dirty the page
2472 * if its contents were altered.
2473 */
2474 if (dirtied_it)
2475 set_page_dirty(page);
2476
2477 return 0;
2478
2479failed:
2480 for (i = 0; i < nr_reads; i++) {
2481 if (read_bh[i])
2482 free_buffer_head(read_bh[i]);
2483 }
2484
2485 /*
2486 * Error recovery is pretty slack. Clear the page and mark it dirty
2487 * so we'll later zero out any blocks which _were_ allocated.
2488 */
2489 kaddr = kmap_atomic(page, KM_USER0);
2490 memset(kaddr, 0, PAGE_CACHE_SIZE);
2491 kunmap_atomic(kaddr, KM_USER0);
2492 SetPageUptodate(page);
2493 set_page_dirty(page);
2494 return ret;
2495}
2496EXPORT_SYMBOL(nobh_prepare_write);
2497
2498int nobh_commit_write(struct file *file, struct page *page,
2499 unsigned from, unsigned to)
2500{
2501 struct inode *inode = page->mapping->host;
2502 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2503
2504 set_page_dirty(page);
2505 if (pos > inode->i_size) {
2506 i_size_write(inode, pos);
2507 mark_inode_dirty(inode);
2508 }
2509 return 0;
2510}
2511EXPORT_SYMBOL(nobh_commit_write);
2512
2513/*
2514 * nobh_writepage() - based on block_full_write_page() except
2515 * that it tries to operate without attaching bufferheads to
2516 * the page.
2517 */
2518int nobh_writepage(struct page *page, get_block_t *get_block,
2519 struct writeback_control *wbc)
2520{
2521 struct inode * const inode = page->mapping->host;
2522 loff_t i_size = i_size_read(inode);
2523 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2524 unsigned offset;
2525 void *kaddr;
2526 int ret;
2527
2528 /* Is the page fully inside i_size? */
2529 if (page->index < end_index)
2530 goto out;
2531
2532 /* Is the page fully outside i_size? (truncate in progress) */
2533 offset = i_size & (PAGE_CACHE_SIZE-1);
2534 if (page->index >= end_index+1 || !offset) {
2535 /*
2536 * The page may have dirty, unmapped buffers. For example,
2537 * they may have been added in ext3_writepage(). Make them
2538 * freeable here, so the page does not leak.
2539 */
2540#if 0
2541 /* Not really sure about this - do we need this ? */
2542 if (page->mapping->a_ops->invalidatepage)
2543 page->mapping->a_ops->invalidatepage(page, offset);
2544#endif
2545 unlock_page(page);
2546 return 0; /* don't care */
2547 }
2548
2549 /*
2550 * The page straddles i_size. It must be zeroed out on each and every
2551 * writepage invocation because it may be mmapped. "A file is mapped
2552 * in multiples of the page size. For a file that is not a multiple of
2553 * the page size, the remaining memory is zeroed when mapped, and
2554 * writes to that region are not written out to the file."
2555 */
2556 kaddr = kmap_atomic(page, KM_USER0);
2557 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2558 flush_dcache_page(page);
2559 kunmap_atomic(kaddr, KM_USER0);
2560out:
2561 ret = mpage_writepage(page, get_block, wbc);
2562 if (ret == -EAGAIN)
2563 ret = __block_write_full_page(inode, page, get_block, wbc);
2564 return ret;
2565}
2566EXPORT_SYMBOL(nobh_writepage);
2567
2568/*
2569 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2570 */
2571int nobh_truncate_page(struct address_space *mapping, loff_t from)
2572{
2573 struct inode *inode = mapping->host;
2574 unsigned blocksize = 1 << inode->i_blkbits;
2575 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2576 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2577 unsigned to;
2578 struct page *page;
2579 struct address_space_operations *a_ops = mapping->a_ops;
2580 char *kaddr;
2581 int ret = 0;
2582
2583 if ((offset & (blocksize - 1)) == 0)
2584 goto out;
2585
2586 ret = -ENOMEM;
2587 page = grab_cache_page(mapping, index);
2588 if (!page)
2589 goto out;
2590
2591 to = (offset + blocksize) & ~(blocksize - 1);
2592 ret = a_ops->prepare_write(NULL, page, offset, to);
2593 if (ret == 0) {
2594 kaddr = kmap_atomic(page, KM_USER0);
2595 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2596 flush_dcache_page(page);
2597 kunmap_atomic(kaddr, KM_USER0);
2598 set_page_dirty(page);
2599 }
2600 unlock_page(page);
2601 page_cache_release(page);
2602out:
2603 return ret;
2604}
2605EXPORT_SYMBOL(nobh_truncate_page);
2606
2607int block_truncate_page(struct address_space *mapping,
2608 loff_t from, get_block_t *get_block)
2609{
2610 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2611 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2612 unsigned blocksize;
2613 pgoff_t iblock;
2614 unsigned length, pos;
2615 struct inode *inode = mapping->host;
2616 struct page *page;
2617 struct buffer_head *bh;
2618 void *kaddr;
2619 int err;
2620
2621 blocksize = 1 << inode->i_blkbits;
2622 length = offset & (blocksize - 1);
2623
2624 /* Block boundary? Nothing to do */
2625 if (!length)
2626 return 0;
2627
2628 length = blocksize - length;
2629 iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2630
2631 page = grab_cache_page(mapping, index);
2632 err = -ENOMEM;
2633 if (!page)
2634 goto out;
2635
2636 if (!page_has_buffers(page))
2637 create_empty_buffers(page, blocksize, 0);
2638
2639 /* Find the buffer that contains "offset" */
2640 bh = page_buffers(page);
2641 pos = blocksize;
2642 while (offset >= pos) {
2643 bh = bh->b_this_page;
2644 iblock++;
2645 pos += blocksize;
2646 }
2647
2648 err = 0;
2649 if (!buffer_mapped(bh)) {
2650 err = get_block(inode, iblock, bh, 0);
2651 if (err)
2652 goto unlock;
2653 /* unmapped? It's a hole - nothing to do */
2654 if (!buffer_mapped(bh))
2655 goto unlock;
2656 }
2657
2658 /* Ok, it's mapped. Make sure it's up-to-date */
2659 if (PageUptodate(page))
2660 set_buffer_uptodate(bh);
2661
2662 if (!buffer_uptodate(bh) && !buffer_delay(bh)) {
2663 err = -EIO;
2664 ll_rw_block(READ, 1, &bh);
2665 wait_on_buffer(bh);
2666 /* Uhhuh. Read error. Complain and punt. */
2667 if (!buffer_uptodate(bh))
2668 goto unlock;
2669 }
2670
2671 kaddr = kmap_atomic(page, KM_USER0);
2672 memset(kaddr + offset, 0, length);
2673 flush_dcache_page(page);
2674 kunmap_atomic(kaddr, KM_USER0);
2675
2676 mark_buffer_dirty(bh);
2677 err = 0;
2678
2679unlock:
2680 unlock_page(page);
2681 page_cache_release(page);
2682out:
2683 return err;
2684}
2685
2686/*
2687 * The generic ->writepage function for buffer-backed address_spaces
2688 */
2689int block_write_full_page(struct page *page, get_block_t *get_block,
2690 struct writeback_control *wbc)
2691{
2692 struct inode * const inode = page->mapping->host;
2693 loff_t i_size = i_size_read(inode);
2694 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2695 unsigned offset;
2696 void *kaddr;
2697
2698 /* Is the page fully inside i_size? */
2699 if (page->index < end_index)
2700 return __block_write_full_page(inode, page, get_block, wbc);
2701
2702 /* Is the page fully outside i_size? (truncate in progress) */
2703 offset = i_size & (PAGE_CACHE_SIZE-1);
2704 if (page->index >= end_index+1 || !offset) {
2705 /*
2706 * The page may have dirty, unmapped buffers. For example,
2707 * they may have been added in ext3_writepage(). Make them
2708 * freeable here, so the page does not leak.
2709 */
Jan Karaaaa40592005-10-30 15:00:16 -08002710 do_invalidatepage(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002711 unlock_page(page);
2712 return 0; /* don't care */
2713 }
2714
2715 /*
2716 * The page straddles i_size. It must be zeroed out on each and every
2717 * writepage invokation because it may be mmapped. "A file is mapped
2718 * in multiples of the page size. For a file that is not a multiple of
2719 * the page size, the remaining memory is zeroed when mapped, and
2720 * writes to that region are not written out to the file."
2721 */
2722 kaddr = kmap_atomic(page, KM_USER0);
2723 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
2724 flush_dcache_page(page);
2725 kunmap_atomic(kaddr, KM_USER0);
2726 return __block_write_full_page(inode, page, get_block, wbc);
2727}
2728
2729sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2730 get_block_t *get_block)
2731{
2732 struct buffer_head tmp;
2733 struct inode *inode = mapping->host;
2734 tmp.b_state = 0;
2735 tmp.b_blocknr = 0;
2736 get_block(inode, block, &tmp, 0);
2737 return tmp.b_blocknr;
2738}
2739
2740static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
2741{
2742 struct buffer_head *bh = bio->bi_private;
2743
2744 if (bio->bi_size)
2745 return 1;
2746
2747 if (err == -EOPNOTSUPP) {
2748 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2749 set_bit(BH_Eopnotsupp, &bh->b_state);
2750 }
2751
2752 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2753 bio_put(bio);
2754 return 0;
2755}
2756
2757int submit_bh(int rw, struct buffer_head * bh)
2758{
2759 struct bio *bio;
2760 int ret = 0;
2761
2762 BUG_ON(!buffer_locked(bh));
2763 BUG_ON(!buffer_mapped(bh));
2764 BUG_ON(!bh->b_end_io);
2765
2766 if (buffer_ordered(bh) && (rw == WRITE))
2767 rw = WRITE_BARRIER;
2768
2769 /*
2770 * Only clear out a write error when rewriting, should this
2771 * include WRITE_SYNC as well?
2772 */
2773 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER))
2774 clear_buffer_write_io_error(bh);
2775
2776 /*
2777 * from here on down, it's all bio -- do the initial mapping,
2778 * submit_bio -> generic_make_request may further map this bio around
2779 */
2780 bio = bio_alloc(GFP_NOIO, 1);
2781
2782 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2783 bio->bi_bdev = bh->b_bdev;
2784 bio->bi_io_vec[0].bv_page = bh->b_page;
2785 bio->bi_io_vec[0].bv_len = bh->b_size;
2786 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2787
2788 bio->bi_vcnt = 1;
2789 bio->bi_idx = 0;
2790 bio->bi_size = bh->b_size;
2791
2792 bio->bi_end_io = end_bio_bh_io_sync;
2793 bio->bi_private = bh;
2794
2795 bio_get(bio);
2796 submit_bio(rw, bio);
2797
2798 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2799 ret = -EOPNOTSUPP;
2800
2801 bio_put(bio);
2802 return ret;
2803}
2804
2805/**
2806 * ll_rw_block: low-level access to block devices (DEPRECATED)
Jan Karaa7662232005-09-06 15:19:10 -07002807 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002808 * @nr: number of &struct buffer_heads in the array
2809 * @bhs: array of pointers to &struct buffer_head
2810 *
Jan Karaa7662232005-09-06 15:19:10 -07002811 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2812 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2813 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2814 * are sent to disk. The fourth %READA option is described in the documentation
2815 * for generic_make_request() which ll_rw_block() calls.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002816 *
2817 * This function drops any buffer that it cannot get a lock on (with the
Jan Karaa7662232005-09-06 15:19:10 -07002818 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2819 * clean when doing a write request, and any buffer that appears to be
2820 * up-to-date when doing read request. Further it marks as clean buffers that
2821 * are processed for writing (the buffer cache won't assume that they are
2822 * actually clean until the buffer gets unlocked).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823 *
2824 * ll_rw_block sets b_end_io to simple completion handler that marks
2825 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2826 * any waiters.
2827 *
2828 * All of the buffers must be for the same device, and must also be a
2829 * multiple of the current approved size for the device.
2830 */
2831void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2832{
2833 int i;
2834
2835 for (i = 0; i < nr; i++) {
2836 struct buffer_head *bh = bhs[i];
2837
Jan Karaa7662232005-09-06 15:19:10 -07002838 if (rw == SWRITE)
2839 lock_buffer(bh);
2840 else if (test_set_buffer_locked(bh))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002841 continue;
2842
2843 get_bh(bh);
Jan Karaa7662232005-09-06 15:19:10 -07002844 if (rw == WRITE || rw == SWRITE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002845 if (test_clear_buffer_dirty(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002846 bh->b_end_io = end_buffer_write_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002847 submit_bh(WRITE, bh);
2848 continue;
2849 }
2850 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851 if (!buffer_uptodate(bh)) {
akpm@osdl.org76c30732005-04-16 15:24:07 -07002852 bh->b_end_io = end_buffer_read_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853 submit_bh(rw, bh);
2854 continue;
2855 }
2856 }
2857 unlock_buffer(bh);
2858 put_bh(bh);
2859 }
2860}
2861
2862/*
2863 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2864 * and then start new I/O and then wait upon it. The caller must have a ref on
2865 * the buffer_head.
2866 */
2867int sync_dirty_buffer(struct buffer_head *bh)
2868{
2869 int ret = 0;
2870
2871 WARN_ON(atomic_read(&bh->b_count) < 1);
2872 lock_buffer(bh);
2873 if (test_clear_buffer_dirty(bh)) {
2874 get_bh(bh);
2875 bh->b_end_io = end_buffer_write_sync;
2876 ret = submit_bh(WRITE, bh);
2877 wait_on_buffer(bh);
2878 if (buffer_eopnotsupp(bh)) {
2879 clear_buffer_eopnotsupp(bh);
2880 ret = -EOPNOTSUPP;
2881 }
2882 if (!ret && !buffer_uptodate(bh))
2883 ret = -EIO;
2884 } else {
2885 unlock_buffer(bh);
2886 }
2887 return ret;
2888}
2889
2890/*
2891 * try_to_free_buffers() checks if all the buffers on this particular page
2892 * are unused, and releases them if so.
2893 *
2894 * Exclusion against try_to_free_buffers may be obtained by either
2895 * locking the page or by holding its mapping's private_lock.
2896 *
2897 * If the page is dirty but all the buffers are clean then we need to
2898 * be sure to mark the page clean as well. This is because the page
2899 * may be against a block device, and a later reattachment of buffers
2900 * to a dirty page will set *all* buffers dirty. Which would corrupt
2901 * filesystem data on the same device.
2902 *
2903 * The same applies to regular filesystem pages: if all the buffers are
2904 * clean then we set the page clean and proceed. To do that, we require
2905 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2906 * private_lock.
2907 *
2908 * try_to_free_buffers() is non-blocking.
2909 */
2910static inline int buffer_busy(struct buffer_head *bh)
2911{
2912 return atomic_read(&bh->b_count) |
2913 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2914}
2915
2916static int
2917drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
2918{
2919 struct buffer_head *head = page_buffers(page);
2920 struct buffer_head *bh;
2921
2922 bh = head;
2923 do {
akpm@osdl.orgde7d5a32005-05-01 08:58:39 -07002924 if (buffer_write_io_error(bh) && page->mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002925 set_bit(AS_EIO, &page->mapping->flags);
2926 if (buffer_busy(bh))
2927 goto failed;
2928 bh = bh->b_this_page;
2929 } while (bh != head);
2930
2931 do {
2932 struct buffer_head *next = bh->b_this_page;
2933
2934 if (!list_empty(&bh->b_assoc_buffers))
2935 __remove_assoc_queue(bh);
2936 bh = next;
2937 } while (bh != head);
2938 *buffers_to_free = head;
2939 __clear_page_buffers(page);
2940 return 1;
2941failed:
2942 return 0;
2943}
2944
2945int try_to_free_buffers(struct page *page)
2946{
2947 struct address_space * const mapping = page->mapping;
2948 struct buffer_head *buffers_to_free = NULL;
2949 int ret = 0;
2950
2951 BUG_ON(!PageLocked(page));
2952 if (PageWriteback(page))
2953 return 0;
2954
2955 if (mapping == NULL) { /* can this still happen? */
2956 ret = drop_buffers(page, &buffers_to_free);
2957 goto out;
2958 }
2959
2960 spin_lock(&mapping->private_lock);
2961 ret = drop_buffers(page, &buffers_to_free);
2962 if (ret) {
2963 /*
2964 * If the filesystem writes its buffers by hand (eg ext3)
2965 * then we can have clean buffers against a dirty page. We
2966 * clean the page here; otherwise later reattachment of buffers
2967 * could encounter a non-uptodate page, which is unresolvable.
2968 * This only applies in the rare case where try_to_free_buffers
2969 * succeeds but the page is not freed.
2970 */
2971 clear_page_dirty(page);
2972 }
2973 spin_unlock(&mapping->private_lock);
2974out:
2975 if (buffers_to_free) {
2976 struct buffer_head *bh = buffers_to_free;
2977
2978 do {
2979 struct buffer_head *next = bh->b_this_page;
2980 free_buffer_head(bh);
2981 bh = next;
2982 } while (bh != buffers_to_free);
2983 }
2984 return ret;
2985}
2986EXPORT_SYMBOL(try_to_free_buffers);
2987
2988int block_sync_page(struct page *page)
2989{
2990 struct address_space *mapping;
2991
2992 smp_mb();
2993 mapping = page_mapping(page);
2994 if (mapping)
2995 blk_run_backing_dev(mapping->backing_dev_info, page);
2996 return 0;
2997}
2998
2999/*
3000 * There are no bdflush tunables left. But distributions are
3001 * still running obsolete flush daemons, so we terminate them here.
3002 *
3003 * Use of bdflush() is deprecated and will be removed in a future kernel.
3004 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3005 */
3006asmlinkage long sys_bdflush(int func, long data)
3007{
3008 static int msg_count;
3009
3010 if (!capable(CAP_SYS_ADMIN))
3011 return -EPERM;
3012
3013 if (msg_count < 5) {
3014 msg_count++;
3015 printk(KERN_INFO
3016 "warning: process `%s' used the obsolete bdflush"
3017 " system call\n", current->comm);
3018 printk(KERN_INFO "Fix your initscripts?\n");
3019 }
3020
3021 if (func == 1)
3022 do_exit(0);
3023 return 0;
3024}
3025
3026/*
3027 * Buffer-head allocation
3028 */
3029static kmem_cache_t *bh_cachep;
3030
3031/*
3032 * Once the number of bh's in the machine exceeds this level, we start
3033 * stripping them in writeback.
3034 */
3035static int max_buffer_heads;
3036
3037int buffer_heads_over_limit;
3038
3039struct bh_accounting {
3040 int nr; /* Number of live bh's */
3041 int ratelimit; /* Limit cacheline bouncing */
3042};
3043
3044static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3045
3046static void recalc_bh_state(void)
3047{
3048 int i;
3049 int tot = 0;
3050
3051 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3052 return;
3053 __get_cpu_var(bh_accounting).ratelimit = 0;
3054 for_each_cpu(i)
3055 tot += per_cpu(bh_accounting, i).nr;
3056 buffer_heads_over_limit = (tot > max_buffer_heads);
3057}
3058
Al Virodd0fc662005-10-07 07:46:04 +01003059struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003060{
3061 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3062 if (ret) {
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003063 get_cpu_var(bh_accounting).nr++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003064 recalc_bh_state();
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003065 put_cpu_var(bh_accounting);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003066 }
3067 return ret;
3068}
3069EXPORT_SYMBOL(alloc_buffer_head);
3070
3071void free_buffer_head(struct buffer_head *bh)
3072{
3073 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3074 kmem_cache_free(bh_cachep, bh);
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003075 get_cpu_var(bh_accounting).nr--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003076 recalc_bh_state();
Coywolf Qi Hunt736c7b82005-09-06 15:18:17 -07003077 put_cpu_var(bh_accounting);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003078}
3079EXPORT_SYMBOL(free_buffer_head);
3080
3081static void
3082init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
3083{
3084 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
3085 SLAB_CTOR_CONSTRUCTOR) {
3086 struct buffer_head * bh = (struct buffer_head *)data;
3087
3088 memset(bh, 0, sizeof(*bh));
3089 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3090 }
3091}
3092
3093#ifdef CONFIG_HOTPLUG_CPU
3094static void buffer_exit_cpu(int cpu)
3095{
3096 int i;
3097 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3098
3099 for (i = 0; i < BH_LRU_SIZE; i++) {
3100 brelse(b->bhs[i]);
3101 b->bhs[i] = NULL;
3102 }
3103}
3104
3105static int buffer_cpu_notify(struct notifier_block *self,
3106 unsigned long action, void *hcpu)
3107{
3108 if (action == CPU_DEAD)
3109 buffer_exit_cpu((unsigned long)hcpu);
3110 return NOTIFY_OK;
3111}
3112#endif /* CONFIG_HOTPLUG_CPU */
3113
3114void __init buffer_init(void)
3115{
3116 int nrpages;
3117
3118 bh_cachep = kmem_cache_create("buffer_head",
3119 sizeof(struct buffer_head), 0,
Andrea Arcangelie422fd22005-05-05 16:15:04 -07003120 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC, init_buffer_head, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003121
3122 /*
3123 * Limit the bh occupancy to 10% of ZONE_NORMAL
3124 */
3125 nrpages = (nr_free_buffer_pages() * 10) / 100;
3126 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3127 hotcpu_notifier(buffer_cpu_notify, 0);
3128}
3129
3130EXPORT_SYMBOL(__bforget);
3131EXPORT_SYMBOL(__brelse);
3132EXPORT_SYMBOL(__wait_on_buffer);
3133EXPORT_SYMBOL(block_commit_write);
3134EXPORT_SYMBOL(block_prepare_write);
3135EXPORT_SYMBOL(block_read_full_page);
3136EXPORT_SYMBOL(block_sync_page);
3137EXPORT_SYMBOL(block_truncate_page);
3138EXPORT_SYMBOL(block_write_full_page);
3139EXPORT_SYMBOL(cont_prepare_write);
3140EXPORT_SYMBOL(end_buffer_async_write);
3141EXPORT_SYMBOL(end_buffer_read_sync);
3142EXPORT_SYMBOL(end_buffer_write_sync);
3143EXPORT_SYMBOL(file_fsync);
3144EXPORT_SYMBOL(fsync_bdev);
3145EXPORT_SYMBOL(generic_block_bmap);
3146EXPORT_SYMBOL(generic_commit_write);
3147EXPORT_SYMBOL(generic_cont_expand);
3148EXPORT_SYMBOL(init_buffer);
3149EXPORT_SYMBOL(invalidate_bdev);
3150EXPORT_SYMBOL(ll_rw_block);
3151EXPORT_SYMBOL(mark_buffer_dirty);
3152EXPORT_SYMBOL(submit_bh);
3153EXPORT_SYMBOL(sync_dirty_buffer);
3154EXPORT_SYMBOL(unlock_buffer);