hadoop源码剖析--RawLocalFileSystem
RawLocalFileSystem是hadoop中实现的本地文件系统，在该类中与文件元数据和目录相关的操作，都是通过适配方式适配到java.io.File的对应API来完成的，适配过程简单，代码清晰。
1.文件元数据和目录相关的操作分析
下面主要以mkDirs()方法为例来窥探该类的实现和一些独到之处。
/**************************************************************** * Implement the FileSystem API for the raw local filesystem. * * 本地文件系统实现，文件元数据和目录相关的操作都是通过适配到java.io.File的对应API完成的。 *****************************************************************/ public class RawLocalFileSystem extends FileSystem { static final URI NAME = URI.create("file:///"); //本地文件系统的uri scheme private Path workingDir; /** * Creates the specified directory hierarchy. Does not * treat existence as an error. */ //递归创建目录，是个幂等操作 public boolean mkdirs(Path f) throws IOException { Path parent = f.getParent(); File p2f = pathToFile(f); //如果父目录为空，试图先创建父目录 //通过File创建目录，并判断成功创建目录 return (parent == null || mkdirs(parent)) && (p2f.mkdir() || p2f.isDirectory()); } /** {@inheritDoc} */ //递归创建目录，并为目录设置访问权限（通过调用shell的"chmod "命令来完成的） //问答：奇怪java的文件操作中没有提供chmod的api吗？？？查看java.io.File后发现提供了相应的api， //但控制粒度太粗了，相关api为：setReadOnly,setWritable,setReadable,setExecutable。对用户权限的控制只到了owner和other的区分对待，没有“chmod ”控制的精细 @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { boolean b = mkdirs(f); setPermission(f, permission); return b; }
再看一下RawLocalFileSystem中的一个内部类RowLocalFileStatus：
static class RawLocalFileStatus extends FileStatus { /* We can add extra fields here. It breaks at least CopyFiles.FilePair(). * We recognize if the information is already loaded by check if * onwer.equals(""). */ private boolean isPermissionLoaded() { return !super.getOwner().equals(""); } RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { super(f.length(), f.isDirectory(), 1, defaultBlockSize, f.lastModified(), new Path(f.getPath()).makeQualified(fs)); } @Override public FsPermission getPermission() { if (!isPermissionLoaded()) { loadPermissionInfo(); } return super.getPermission(); } //使用'ls -ld'命令来获取权限信息 private void loadPermissionInfo() { IOException e = null; try { StringTokenizer t = new StringTokenizer( FileUtil.execCommand(new File(getPath().toUri()), Shell.getGET_PERMISSION_COMMAND())); //expected format //-rw------- 1 username groupname ... String permission = t.nextToken(); if (permission.length() > 10) { //files with ACLs might have a '+' permission = permission.substring(0, 10); } setPermission(FsPermission.valueOf(permission)); t.nextToken(); setOwner(t.nextToken()); setGroup(t.nextToken()); } catch (Shell.ExitCodeException ioe) { if (ioe.getExitCode() != 1) { e = ioe; } else { setPermission(null); setOwner(null); setGroup(null); } } catch (IOException ioe) { e = ioe; } finally { if (e != null) { throw new RuntimeException("Error while running command to get " + "file permissions : " + StringUtils.stringifyException(e)); } } }
通过以上两段代码可以看出hadoop的本地文件系统的实现，在利用java语言提供的File类的基础上，做了一些适合自身的变化来达到目标。调用linux的shell命令，需要在linux系统中创建一个新的java虚拟机而消耗大量的资源。
2. 文件的读分析
RawLocalFileSystem使用LocalFSFileInputStream和LocalFSFileOutputStream进行读写。
/******************************************************* * For open()'s FSInputStream *******************************************************/ //本地文件系统读取流 class LocalFSFileInputStream extends FSInputStream { FileInputStream fis; //文件读取流 private long position; //记录当前读取的数据在文件中的位置 public LocalFSFileInputStream(Path f) throws IOException { this.fis = new TrackingFileInputStream(pathToFile(f)); //实际使用的是文件读取流是TrackingFileInputStream } //系统文件当前位置 public void seek(long pos) throws IOException { fis.getChannel().position(pos); this.position = pos; } //获取位置 public long getPos() throws IOException { return this.position; } //定位到新的block（本地文件系统没有这样的功能，所以简单返回失败） public boolean seekToNewSource(long targetPos) throws IOException { return false; } /* * Just forward to the fis */ //获取剩余可读或可跳过的字节数 public int available() throws IOException { return fis.available(); } //关闭输入流，并释放系统分配的资源 public void close() throws IOException { fis.close(); } public boolean markSupport() { return false; } //read()方法需要随时更新position，以保证getPos()能返回正确的值 public int read() throws IOException { try { int value = fis.read(); if (value >= 0) { this.position++; //更新文件当前位置 } return value; } catch (IOException e) { // unexpected exception throw new FSError(e); // assume native fs error } } public int read(byte[] b, int off, int len) throws IOException { try { int value = fis.read(b, off, len); if (value > 0) { this.position += value; } return value; } catch (IOException e) { // unexpected exception throw new FSError(e); // assume native fs error } } public int read(long position, byte[] b, int off, int len) throws IOException { ByteBuffer bb = ByteBuffer.wrap(b, off, len); try { return fis.getChannel().read(bb, position); } catch (IOException e) { throw new FSError(e); } } public long skip(long n) throws IOException { long value = fis.skip(n); if (value > 0) { this.position += value; } return value; } }
可以看到LocalFSFileInputStream类实际使用的读流是TrackingFileInputStream
//重写了FileInputStream中的所有read方法，提供文件读取字节数的统计功能。 //TrackingFileInputStream使用修饰器模式 class TrackingFileInputStream extends FileInputStream { public TrackingFileInputStream(File f) throws IOException { super(f); } public int read() throws IOException { int result = super.read(); if (result != -1) { statistics.incrementBytesRead(1); } return result; } public int read(byte[] data) throws IOException { int result = super.read(data); if (result != -1) { statistics.incrementBytesRead(result); } return result; } public int read(byte[] data, int offset, int length) throws IOException { int result = super.read(data, offset, length); if (result != -1) { statistics.incrementBytesRead(result); } return result; } }
那么RawLocalFileSystem和LocalFSFileInputStream是如何对接起来进行读操作的呢，当然还是和java的api一致（使用open()和create()方法来创建LocalFSFileInputStream）。下面以LocalFSFileInputStream的open()方法为例进行分析：
public FSDataInputStream open(Path f, int bufferSize) throws IOException { if (!exists(f)) { throw new FileNotFoundException(f.toString()); } return new FSDataInputStream(new BufferedFSInputStream( //包装LocalFSFileInputStream new LocalFSFileInputStream(f), bufferSize)); }
public class FSDataInputStream extends DataInputStream implements Seekable, PositionedReadable, Closeable { public FSDataInputStream(InputStream in) throws IOException { super(in); if( !(in instanceof Seekable) || !(in instanceof PositionedReadable) ) { throw new IllegalArgumentException( "In is not an instance of Seekable or PositionedReadable"); } } public synchronized void seek(long desired) throws IOException { ((Seekable)in).seek(desired); } public long getPos() throws IOException { return ((Seekable)in).getPos(); } public int read(long position, byte[] buffer, int offset, int length) throws IOException { return ((PositionedReadable)in).read(position, buffer, offset, length); } public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { ((PositionedReadable)in).readFully(position, buffer, offset, length); } public void readFully(long position, byte[] buffer) throws IOException { ((PositionedReadable)in).readFully(position, buffer, 0, buffer.length); } public boolean seekToNewSource(long targetPos) throws IOException { return ((Seekable)in).seekToNewSource(targetPos); } }
获取到读流后就可以调用流的读取方法进行读取了。
3. 文件的写分析
至于写操作，还是和java中的写保持一致的，支持append和随机写两种方式。
