hadoop 操作 hdfs
package com.picc.test; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URI; import java.net.URL; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.apache.hadoop.fs.Path; public class UrlCat { static{ URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); } public static void main(String[] args) { InputStream in =null; try {//new FileInputStream("/home/liucheng/file/student.txt"); in= new URL("hdfs://localhost:9000/user/liucheng/input/student.txt").openStream(); } catch (IOException e) { e.printStackTrace(); } BufferedReader read = new BufferedReader(new InputStreamReader(in)); String line=null; try { while((line=read.readLine())!=null){ System.out.println("result:"+line.trim()); } } catch (IOException e) { e.printStackTrace(); } try { read.close(); } catch (IOException e) { e.printStackTrace(); } run("hdfs://localhost:9000/user/liucheng/input/student.txt"); } public static void run(String hdfs){ Configuration conf = new Configuration(); InputStream in =null; try { FileSystem fs = FileSystem.get(URI.create(hdfs), conf); in = fs.open(new Path(hdfs)); BufferedReader read = new BufferedReader(new InputStreamReader(in)); String line=null; try { while((line=read.readLine())!=null){ System.out.println("result:"+line.trim()); } } catch (IOException e) { e.printStackTrace(); } } catch (IOException e) { e.printStackTrace(); }finally{ try { in.close(); } catch (IOException e) { e.printStackTrace(); } } } }
示例是读取hdfs 的Input 文件夹下的student.txt 文件。示例。
java 操作 hdfs
package com.picc.test; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URI; import java.net.URL; import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.apache.hadoop.fs.Path; public class UrlCat { static{ URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); } public static void main(String[] args) { run("hdfs://localhost:9000/user/liucheng/input/file01.txt"); //doubleCat("hdfs://localhost:9000/user/liucheng/input/student.txt"); // statusTest("hdfs://localhost:9000/user/liucheng/input"); } public static void run(String hdfs){ Configuration conf = new Configuration(); InputStream in =null; try { FileSystem fs = FileSystem.get(URI.create(hdfs), conf); in = fs.open(new Path(hdfs)); BufferedReader read = new BufferedReader(new InputStreamReader(in)); String line=null; try { while((line=read.readLine())!=null){ System.out.println("result:"+line.trim()); } } catch (IOException e) { e.printStackTrace(); } } catch (IOException e) { e.printStackTrace(); }finally{ try { in.close(); } catch (IOException e) { e.printStackTrace(); } } } //FSDataInputStream extents DataInputStream extents InputStream /** * 其实doubleCat 和 getInputStreamtoText 方法需要放到一起 * 在有异常时,把位置保存到一个properties文件中 * @param hdfs */ public static void doubleCat(String hdfs){ Configuration conf = new Configuration(); FSDataInputStream fsdis =null; try { FileSystem fs = FileSystem.get(URI.create(hdfs),conf); fsdis= fs.open(new Path(hdfs)); long rs = fsdis.skip(1000);//跳过1000个位置 fsdis.seek(rs);//查找rs的位子 String result = getInputStreamtoText(fsdis); System.out.println("result:"+result); } catch (IOException e) { e.printStackTrace(); }finally{ if(fsdis!=null){ try { fsdis.close(); } catch (IOException e) { e.printStackTrace(); } } } } /** * inputStream to Text * @param in * @return */ public static String getInputStreamtoText(InputStream in ){ BufferedReader read = new BufferedReader(new InputStreamReader(in)); String line=null; StringBuffer result=new StringBuffer(); try { while((line=read.readLine())!=null){ // System.out.println("result:"+line.trim()); String lineSeparator =System.getProperty("line.separator"); result.append(line).append(lineSeparator); } } catch (IOException e) { e.printStackTrace(); }finally{ try { in.close(); } catch (IOException e) { e.printStackTrace(); } } return result.toString(); } public static void statusTest(String url){ Configuration conf = new Configuration(); try { FileSystem fs = FileSystem.get(URI.create(url), conf); Path [] paths = new Path[url.length()]; for(int i=0;i<paths.length;i++){ // paths[i]=new Path[String.url]; FileStatus [] filestatus = fs.listStatus(paths[i]); Path [] listedPaths = FileUtil.stat2Paths(filestatus); for(Path p:listedPaths){ System.out.println(p); } } // /tmp/hadoop-liucheng/dfs/name/current/edits // bin/mysqld } catch (IOException e) { e.printStackTrace(); } } }