linux写入磁盘过程经历VFS -> 页缓存（page cache） -> 具体的文件系统（ext2/3/4、XFS、ReiserFS等） -> Block IO ->设备驱动 -> SCSI指令（或者其他指令），总体来说linux文件写入磁盘过程比较复杂
1、VFS（虚拟文件系统）
Linux中采用了VFS的方式屏蔽了多个文件系统的差别， 当需要不同的设备或者其他文件系统时，采用挂载mount的方式访问其他设备或者其他文件系统（这里可以把文件系统理解为具体的设备）。正是因为使用了VFS，所以所有的文件系统设备使用统一的文件目录树视图访问，整个存储空间采用一个文件系统目录树来管理，屏蔽了底层多个文件系统之间的差别。当然，如果你需要把你自己编写的文件系统集成到Linux内核，采用VFS的方式进行访问，你需要采用模块加载的方式进行处理，相应的文件系统模块文件需要编入到系统目录/lib/modules/your-system-name/kernel/fs当中。当然VFS的作用远不止这些，通过VFS也进行访问设备，在Linux下所有的对象都是文件，简化了系统的访问。
1.1 正常情况下，所有的文件操作通过系统调用进入到VFS中，特殊的处理，直接操作原始设备。文件系统写入的系统调用为：
#include <unistd.h>
ssize_t write(int fd, const void * buffer, size_t count);
1.2 当采用系统调用进入VFS时，接下来的处理交给VFS层。处理过程比较中要的是vfs_write、generic_file_aio_write
1 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) 2 { 3 ssize_t ret; 4 5 if (!(file->f_mode & FMODE_WRITE)) 6 return -EBADF; 7 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write)) 8 return -EINVAL; 9 if (unlikely(!access_ok(VERIFY_READ, buf, count))) 10 return -EFAULT; 11 12 ret = rw_verify_area(WRITE, file, pos, count); 13 if (ret >= 0) { 14 count = ret; 15 if (file->f_op->write) 16 ret = file->f_op->write(file, buf, count, pos); 17 else 18 ret = do_sync_write(file, buf, count, pos); 19 if (ret > 0) { 20 fsnotify_modify(file->f_path.dentry); 21 add_wchar(current, ret); 22 } 23 inc_syscw(current); 24 } 25 26 return ret; 27 }
1 /** 2 * generic_file_aio_write - write data to a file 3 * @iocb: IO state structure 4 * @iov: vector with data to write 5 * @nr_segs: number of segments in the vector 6 * @pos: position in file where to write 7 * 8 * This is a wrapper around __generic_file_aio_write() to be used by most 9 * filesystems. It takes care of syncing the file in case of O_SYNC file 10 * and acquires i_mutex as needed. 11 */ 12 ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 13 unsigned long nr_segs, loff_t pos) 14 { 15 struct file *file = iocb->ki_filp; 16 struct inode *inode = file->f_mapping->host; 17 ssize_t ret; 18 19 BUG_ON(iocb->ki_pos != pos); 20 21 mutex_lock(&inode->i_mutex); 22 ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); 23 mutex_unlock(&inode->i_mutex); 24 25 if (ret > 0 || ret == -EIOCBQUEUED) { 26 ssize_t err; 27 28 err = generic_write_sync(file, pos, ret); 29 if (err < 0 && ret > 0) 30 ret = err; 31 } 32 return ret; 33 }
2、 对于VFS层也有采用page cache和非page cache两种，下面重要介绍采用page cache的处理。
在VFS中， 每个打开操作的文件对应内核都有一个address_space 数据结构， 该数据结构是用来表示系统中打开的文件，并且一个打开的文件只有一个address_space数据结构。
如下：
1 struct address_space { 2 struct inode *host; /* owner: inode, block_device */ 3 struct radix_tree_root page_tree; /* radix tree of all pages */ 4 spinlock_t tree_lock; /* and lock protecting it */ 5 unsigned int i_mmap_writable;/* count VM_SHARED mappings */ 6 struct prio_tree_root i_mmap; /* tree of private and shared mappings */ 7 struct list_head i_mmap_nonlinear;/*list VM_NONLINEAR mappings */ 8 spinlock_t i_mmap_lock; /* protect tree, count, list */ 9 unsigned int truncate_count; /* Cover race condition with truncate */ 10 unsigned long nrpages; /* number of total pages */ 11 pgoff_t writeback_index;/* writeback starts here */ 12 const struct address_space_operations *a_ops; /* methods */ 13 unsigned long flags; /* error bits/gfp mask */ 14 struct backing_dev_info *backing_dev_info; /* device readahead, etc */ 15 spinlock_t private_lock; /* for use by the address_space */ 16 struct list_head private_list; /* ditto */ 17 struct address_space *assoc_mapping; /* ditto */ 18 struct mutex unmap_mutex; /* to protect unmapping */ 19 } __attribute__((aligned(sizeof(long))));
对于文件中的文件内容缓存采用的是基数树的方式来保存的，在成员变量page_tree中，关于基数树的介绍参考[1]和[2]。 下面是关于page cache写处理的几个重要的函数
1 ssize_t 2 generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, 3 unsigned long nr_segs, loff_t pos, loff_t *ppos, 4 size_t count, ssize_t written) 5 { 6 struct file *file = iocb->ki_filp; 7 struct address_space *mapping = file->f_mapping; 8 ssize_t status; 9 struct iov_iter i; 10 11 iov_iter_init(&i, iov, nr_segs, count, written); 12 status = generic_perform_write(file, &i, pos); 13 14 if (likely(status >= 0)) { 15 written += status; 16 *ppos = pos + status; 17 } 18 19 /* 20 * If we get here for O_DIRECT writes then we must have fallen through 21 * to buffered writes (block instantiation inside i_size). So we sync 22 * the file data here, to try to honour O_DIRECT expectations. 23 */ 24 if (unlikely(file->f_flags & O_DIRECT) && written) 25 status = filemap_write_and_wait_range(mapping, 26 pos, pos + written - 1); 27 28 return written ? written : status; 29 }
调用page cache中的write_begin 和write_end
Note： 在进行VFS系统调用写入文件过程中，可以允许在文件中的任何位置写入，这其中就包括当写入的过程中写入的起始位置不是一个block的开始位置，这时需要特殊的处理，上述的过程都在write_begin这个函数调用过程中处理完毕。
3、ext2/3/4中文件的处理。
当在page cache中进行到write_begin时，需要ext4中的ext4_write_begin处理， 如下：
1 static int ext4_write_begin(struct file *file, struct address_space *mapping, 2 loff_t pos, unsigned len, unsigned flags, 3 struct page **pagep, void **fsdata) 4 { 5 struct inode *inode = mapping->host; 6 int ret, needed_blocks; 7 handle_t *handle; 8 int retries = 0; 9 struct page *page; 10 pgoff_t index; 11 unsigned from, to; 12 ......... 13 14 index = pos >> PAGE_CACHE_SHIFT; 15 from = pos & (PAGE_CACHE_SIZE - 1); 16 to = from + len; 17 18 retry: 19 handle = ext4_journal_start(inode, needed_blocks); 20 if (IS_ERR(handle)) { 21 ret = PTR_ERR(handle); 22 goto out; 23 } 24 25 /* We cannot recurse into the filesystem as the transaction is already 26 * started */ 27 flags |= AOP_FLAG_NOFS; 28 29 page = grab_cache_page_write_begin(mapping, index, flags); 30 if (!page) { 31 ext4_journal_stop(handle); 32 ret = -ENOMEM; 33 goto out; 34 } 35 *pagep = page; 36 37 ret = block_write_begin(file, mapping, pos, len, flags, pagep, fsdata, 38 ext4_get_block); 39 40 if (!ret && ext4_should_journal_data(inode)) { 41 ret = walk_page_buffers(handle, page_buffers(page), 42 from, to, NULL, do_journal_get_write_access); 43 } 44 45 if (ret) { 46 unlock_page(page); 47 page_cache_release(page); 48 /* 49 * block_write_begin may have instantiated a few blocks 50 * outside i_size. Trim these off again. Don't need 51 * i_size_read because we hold i_mutex. 52 * 53 * Add inode to orphan list in case we crash before 54 * truncate finishes 55 */ 56 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 57 ext4_orphan_add(handle, inode); 58 59 ext4_journal_stop(handle); 60 if (pos + len > inode->i_size) { 61 ext4_truncate_failed_write(inode); 62 /* 63 * If truncate failed early the inode might 64 * still be on the orphan list; we need to 65 * make sure the inode is removed from the 66 * orphan list in that case. 67 */ 68 if (inode->i_nlink) 69 ext4_orphan_del(NULL, inode); 70 } 71 } 72 73 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 74 goto retry; 75 out: 76 return ret; 77 }
其中在ext4_write_begin中包含了很多的处理功能，包括文件物理块的分配（假设ext4中的delay allocation特性没有开启）、文件块的部分写过程的处理等。下面是在ext_write_begin函数调用过程中比较重要的几个函数。
1 /* 2 * block_write_begin takes care of the basic task of block allocation and 3 * bringing partial write blocks uptodate first. 4 * 5 * If *pagep is not NULL, then block_write_begin uses the locked page 6 * at *pagep rather than allocating its own. In this case, the page will 7 * not be unlocked or deallocated on failure. 8 */ 9 int block_write_begin(struct file *file, struct address_space *mapping, 10 loff_t pos, unsigned len, unsigned flags, 11 struct page **pagep, void **fsdata, 12 get_block_t *get_block) 13 { 14 struct inode *inode = mapping->host; 15 int status = 0; 16 struct page *page; 17 pgoff_t index; 18 unsigned start, end; 19 int ownpage = 0; 20 21 index = pos >> PAGE_CACHE_SHIFT; 22 start = pos & (PAGE_CACHE_SIZE - 1); 23 end = start + len; 24 25 page = *pagep; 26 if (page == NULL) { 27 ownpage = 1; 28 page = grab_cache_page_write_begin(mapping, index, flags); 29 if (!page) { 30 status = -ENOMEM; 31 goto out; 32 } 33 *pagep = page; 34 } else 35 BUG_ON(!PageLocked(page)); 36 37 status = __block_prepare_write(inode, page, start, end, get_block); 38 if (unlikely(status)) { 39 ClearPageUptodate(page); 40 41 if (ownpage) { 42 unlock_page(page); 43 page_cache_release(page); 44 *pagep = NULL; 45 46 /* 47 * prepare_write() may have instantiated a few blocks 48 * outside i_size. Trim these off again. Don't need 49 * i_size_read because we hold i_mutex. 50 */ 51 if (pos + len > inode->i_size) 52 vmtruncate(inode, inode->i_size); 53 } 54 } 55 56 out: 57 return status; 58 }
1 static int __block_prepare_write(struct inode *inode, struct page *page, 2 unsigned from, unsigned to, get_block_t *get_block) 3 { 4 unsigned block_start, block_end; 5 sector_t block; 6 int err = 0; 7 unsigned blocksize, bbits; 8 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; 9 10 BUG_ON(!PageLocked(page)); 11 BUG_ON(from > PAGE_CACHE_SIZE); 12 BUG_ON(to > PAGE_CACHE_SIZE); 13 BUG_ON(from > to); 14 15 blocksize = 1 << inode->i_blkbits; 16 if (!page_has_buffers(page)) 17 create_empty_buffers(page, blocksize, 0); 18 head = page_buffers(page); 19 20 bbits = inode->i_blkbits; 21 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits); 22 23 for(bh = head, block_start = 0; bh != head || !block_start; 24 block++, block_start=block_end, bh = bh->b_this_page) { 25 block_end = block_start + blocksize; 26 if (block_end <= from || block_start >= to) { 27 if (PageUptodate(page)) { 28 if (!buffer_uptodate(bh)) 29 set_buffer_uptodate(bh); 30 } 31 continue; 32 } 33 if (buffer_new(bh)) 34 clear_buffer_new(bh); 35 if (!buffer_mapped(bh)) { 36 WARN_ON(bh->b_size != blocksize); 37 err = get_block(inode, block, bh, 1); 38 if (err) 39 break; 40 if (buffer_new(bh)) { 41 unmap_underlying_metadata(bh->b_bdev, 42 bh->b_blocknr); 43 if (PageUptodate(page)) { 44 clear_buffer_new(bh); 45 set_buffer_uptodate(bh); 46 mark_buffer_dirty(bh); 47 continue; 48 } 49 if (block_end > to || block_start < from) 50 zero_user_segments(page, 51 to, block_end, 52 block_start, from); 53 continue; 54 } 55 } 56 if (PageUptodate(page)) { 57 if (!buffer_uptodate(bh)) 58 set_buffer_uptodate(bh); 59 continue; 60 } 61 if (!buffer_uptodate(bh) && !buffer_delay(bh) && 62 !buffer_unwritten(bh) && 63 (block_start < from || block_end > to)) { 64 ll_rw_block(READ, 1, &bh); 65 *wait_bh++=bh; 66 } 67 } 68 /* 69 * If we issued read requests - let them complete. 70 */ 71 while(wait_bh > wait) { 72 wait_on_buffer(*--wait_bh); 73 if (!buffer_uptodate(*wait_bh)) 74 err = -EIO; 75 } 76 if (unlikely(err)) 77 page_zero_new_buffers(page, from, to); 78 return err; 79 }
1 /** 2 * ll_rw_block: low-level access to block devices (DEPRECATED) 3 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead) 4 * @nr: number of &struct buffer_heads in the array 5 * @bhs: array of pointers to &struct buffer_head 6 * 7 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and 8 * requests an I/O operation on them, either a %READ or a %WRITE. The third 9 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers 10 * are sent to disk. The fourth %READA option is described in the documentation 11 * for generic_make_request() which ll_rw_block() calls. 12 * 13 * This function drops any buffer that it cannot get a lock on (with the 14 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be 15 * clean when doing a write request, and any buffer that appears to be 16 * up-to-date when doing read request. Further it marks as clean buffers that 17 * are processed for writing (the buffer cache won't assume that they are 18 * actually clean until the buffer gets unlocked). 19 * 20 * ll_rw_block sets b_end_io to simple completion handler that marks 21 * the buffer up-to-date (if approriate), unlocks the buffer and wakes 22 * any waiters. 23 * 24 * All of the buffers must be for the same device, and must also be a 25 * multiple of the current approved size for the device. 26 */ 27 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[]) 28 { 29 int i; 30 31 for (i = 0; i < nr; i++) { 32 struct buffer_head *bh = bhs[i]; 33 34 if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG) 35 lock_buffer(bh); 36 else if (!trylock_buffer(bh)) 37 continue; 38 39 if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC || 40 rw == SWRITE_SYNC_PLUG) { 41 if (test_clear_buffer_dirty(bh)) { 42 bh->b_end_io = end_buffer_write_sync; 43 get_bh(bh); 44 if (rw == SWRITE_SYNC) 45 submit_bh(WRITE_SYNC, bh); 46 else 47 submit_bh(WRITE, bh); 48 continue; 49 } 50 } else { 51 if (!buffer_uptodate(bh)) { 52 bh->b_end_io = end_buffer_read_sync; 53 get_bh(bh); 54 submit_bh(rw, bh); 55 continue; 56 } 57 } 58 unlock_buffer(bh); 59 } 60 }
其中在ext4中块的分配过程中，管理块分配处理的函数实现在fs/ext4/balloc.c fs/ext4/mballoc.c
4、当page cache中的数据需要刷新到disk上的时候，这时处理的过程由Block IO接管。
在进行文件page cache刷新到disk上的过程中比较重要的数据结构有如下两个buffer_head 和 bio
1 struct buffer_head { 2 unsigned long b_state; /* buffer state bitmap (see above) */ 3 struct buffer_head *b_this_page;/* circular list of page's buffers */ 4 struct page *b_page; /* the page this bh is mapped to */ 5 6 sector_t b_blocknr; /* start block number */ 7 size_t b_size; /* size of mapping */ 8 char *b_data; /* pointer to data within the page */ 9 10 struct block_device *b_bdev; 11 bh_end_io_t *b_end_io; /* I/O completion */ 12 void *b_private; /* reserved for b_end_io */ 13 struct list_head b_assoc_buffers; /* associated with another mapping */ 14 struct address_space *b_assoc_map; /* mapping this buffer is 15 associated with */ 16 atomic_t b_count; /* users using this buffer_head */ 17 };
1 /* 2 * main unit of I/O for the block layer and lower layers (ie drivers and 3 * stacking drivers) 4 */ 5 struct bio { 6 sector_t bi_sector; /* device address in 512 byte 7 sectors */ 8 struct bio *bi_next; /* request queue link */ 9 struct block_device *bi_bdev; 10 unsigned long bi_flags; /* status, command, etc */ 11 unsigned long bi_rw; /* bottom bits READ/WRITE, 12 * top bits priority 13 */ 14 15 unsigned short bi_vcnt; /* how many bio_vec's */ 16 unsigned short bi_idx; /* current index into bvl_vec */ 17 ............... 18 19 /* 20 * We can inline a number of vecs at the end of the bio, to avoid 21 * double allocations for a small number of bio_vecs. This member 22 * MUST obviously be kept at the very end of the bio. 23 */ 24 struct bio_vec bi_inline_vecs[0]; 25 };
在Block IO层进行基本的IO request的合并和处理调度， 基本的层由elevator管理， 具体的调度算法有noop、deadline和anticipate等多种调度算法，现在默认的调度算法是deadline，当然调度算法可调，根据系统可以调成系统最有的处理。
[1] 基数树(radix tree). http://blog.csdn.net/joker0910/article/details/8250085
[2] Radix Tree. http://en.wikipedia.org/wiki/Radix_tree
