HDFS 工具类
读取HDFS上文件数据
import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; import java.net.URI; import java.util.ArrayList; import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.util.Progressable; /** * @author 作者 E-mail: * @version 创建时间：2016年3月8日 上午9:37:49 类说明 * 读取hdfs文件数据 */ public class ReadHDFSDatas { static Configuration conf = new Configuration(); /** * * * @param location * @param conf * @return * @throws Exception */ public static List<String> readLines( Path location, Configuration conf ) throws Exception { // StringBuffer sb = new StringBuffer(); FileSystem fileSystem = FileSystem.get( location.toUri(), conf ); CompressionCodecFactory factory = new CompressionCodecFactory( conf ); FileStatus[] items = fileSystem.listStatus( location ); if ( items == null ) return new ArrayList<String>(); List<String> results = new ArrayList<String>(); for ( FileStatus item : items ) { // ignoring files like _SUCCESS if ( item.getPath().getName().startsWith( "_" ) ) { continue; } CompressionCodec codec = factory.getCodec( item.getPath() ); InputStream stream = null; if ( codec != null ) { stream = codec.createInputStream( fileSystem.open( item.getPath() ) ); } else { stream = fileSystem.open( item.getPath() ); } StringWriter writer = new StringWriter(); IOUtils.copy( stream, writer, "UTF-8" ); String raw = writer.toString(); // String[] resulting = raw.split( "\n" ); for ( String str : raw.split( "\t" ) ) { results.add( str ); System.out.println( "start..." + results + "....." ); } } return results; } public String ReadFile( String hdfs ) throws IOException { StringBuffer sb = new StringBuffer(); FileSystem fs = FileSystem.get( URI.create( hdfs ), conf ); FSDataInputStream hdfsInStream = fs.open( new Path( hdfs ) ); try { fs = FileSystem.get( conf ); hdfsInStream = fs.open( new Path( hdfs ) ); byte[] b = new byte[10240]; int numBytes = 0; // Windows os error while ( ( numBytes = hdfsInStream.read( b ) ) > 0 ) { numBytes = hdfsInStream.read( b ); } } catch ( IOException e ) { e.printStackTrace(); } hdfsInStream.close(); fs.close(); return sb.toString(); } /** * * @param filePath * @return * @throws IOException */ public static String getFile( String filePath ) throws IOException { String line = ""; try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get( URI.create( filePath ), conf ); Path pathq = new Path( filePath ); FSDataInputStream fsr = fs.open( pathq ); while ( line != null ) { line = fsr.readLine(); if ( line != null ) { System.out.println( line ); } } } catch ( Exception e ) { e.printStackTrace(); } return line; } /* * */ public static List<String> getDatas( String filePath ) { List<String> list = new ArrayList<String>(); try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get( URI.create( filePath ), conf ); Path pathq = new Path( filePath ); FSDataInputStream fsr = fs.open( pathq ); String line =""; while ( line != null ) { line = fsr.readLine(); if ( line != null ) { list.add( line ); } } } catch ( Exception e ) { e.printStackTrace(); } return list; } public static void main( String[] args ){ //String hdfs = "hdfs://node4:9000/hive/warehouse/u_data/u.data"; //String hdfs = "/datas/t1"; String hdfs = "/datas/u.data"; Path path = new Path( hdfs ); // String hdfs = "/datas"; // String hdfs = "/hive/warehouse/u_data/u.data"; // getFile(hdfs); /** * userid INT, movieid INT, rating INT, weekday INT) */ List<String> listDatas = getDatas(hdfs); for (int i = 0; i < listDatas.size(); i++){ String[] split = listDatas.get(i).split("\t"); String userid = split[0]; String movieid = split[1]; String rating = split[2]; String weekday = split[3]; String makeRowKey = RegionSeverSplit.makeRowKey(userid);
// 用put API实现批量入库 //System.out.println("userid--"+ userid + ".."+ "movieid--"+ movieid + ".." +"rating--"+ rating + ".."+"weekday--"+ weekday + "...."); HBaseUtils.addRows("t1", makeRowKey, "f1", "weekday-rating", (movieid+"-"+rating+"-"+weekday).getBytes()); } System.out.println("success......"); } }
HBase 随机生成rowkey 前置处理
import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import org.apache.commons.codec.binary.Hex; public class RegionSeverSplit { public static String makeRowKey(String id){ String md5_content = null; try { MessageDigest messageDigest = MessageDigest.getInstance("MD5"); messageDigest.reset(); messageDigest.update(id.getBytes()); byte[] bytes = messageDigest.digest(); md5_content = new String(Hex.encodeHex(bytes)); } catch (NoSuchAlgorithmException e1) { e1.printStackTrace(); } //turn right md5 String right_md5_id = Integer.toHexString(Integer.parseInt(md5_content.substring(0,7),16)>>1); while(right_md5_id.length()<7){ right_md5_id = "0" + right_md5_id; } return right_md5_id + "-" + id; } public static void main(String[] args){ String rowky = makeRowKey("asdfasdf"); System.out.println(rowky); } }
HBase Util工具类，用put方式批量或者单条数据入库
import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.HTablePool; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.mapreduce.InputSplit; import cn.tansun.bd.hdfs.ReadHDFSDatas; /** * * @author root * */ public class HBaseUtils { private static HBaseAdmin hadmin = null; private static Configuration conf; private static HTable htable = null; static { conf = new Configuration(); String filePath = "hbase-site.xml"; Path path = new Path(filePath); conf.addResource(path); conf = HBaseConfiguration.create(conf); } /** * insert one row * * @param tableName * @param rowkey * @param columnFinaly * @param columnName * @param values * @return */ public static boolean addRow(String tableName, String rowkey, String columnFinaly, String columnName, byte[] values) { boolean flag = true; if (tableName != null) { HTablePool hTpool = new HTablePool(conf, 1000); HTableInterface table = hTpool.getTable(tableName); Put put = new Put(rowkey.getBytes()); put.addColumn(columnFinaly.getBytes(), columnName.getBytes(), values); try { table.put(put); System.out.print("addRow success..." + "tableName....." + tableName); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } else { System.out.println(" please select tableName"); } return flag; } public static void main(String[] args) { /*String makeRowKey = RegionSeverSplit.makeRowKey("adcdfef"); String tableName = "student"; String columnfianly = "info"; String columnName = "name"; String values = "zhangsan"; addRow(tableName, makeRowKey, columnfianly, columnName, values.getBytes());*/ ReadHDFSDatas readh = new ReadHDFSDatas(); String hdfs = "/datas/u.data"; List<String> getDatas = readh.getDatas(hdfs); for (int i = 0; i < getDatas.size(); i++){ if (i < 100){ System.out.println(getDatas.get(i)); } } } /** * put many rows * * @param tableName * @param rowkey * @param columnFinaly * @param columnName * @param values * @return */ public static List<Put> addRows(String tableName, String rowkey, String columnFinaly, String columnName, byte[] values) { List<Put> lists = null; long start = System.currentTimeMillis(); if (tableName != null || rowkey != null) { HTablePool hTablePool = new HTablePool(conf, 1000); HTableInterface table = hTablePool.getTable(tableName); try { table.setAutoFlush(false); table.setWriteBufferSize(1024 * 1024 * 1); lists = new ArrayList<Put>(); Random random = new Random(); byte[] buffers = new byte[256]; int count = 100; for (int i = 0; i < count; i++){ Put put = new Put(rowkey.getBytes()); random.nextBytes(buffers); put.add(columnFinaly.getBytes(), columnName.toString().getBytes(), values); put.getDurability(); //table.setAutoFlush(false); if ( i % 100 == 0){ lists.add(put); try { table.batch(lists); } catch (InterruptedException e) { System.out.println("error......"); e.printStackTrace(); } table.put(lists); lists.clear(); table.flushCommits(); } } } catch (IOException e) { e.printStackTrace(); } } else { System.out.println("..tableName not null"); } long end = System.currentTimeMillis(); long times = end - start; System.out.println(times * 1.0 / 1000 +"..... finsh........" ); return lists; } /** * read datas by fileName * @param fileName * @return */ public List<String> getFileDatas(String fileName){ return null; } /** * read hdfs datas by fileName * @param fileName * @return */ public static List<String> getHdfsDatas(String fileName){ /* List<String> getDatas = ReadHDFSDatas.getDatas(fileName); for (int i = 0; i < getDatas.size(); i++){ if (i < 100){ System.out.println(getDatas.get(i)); } } return getDatas;*/ return null; } /** * * @param startKey * @param endKey * @return */ public List<InputSplit> getSplits(byte[] startKey, byte[] endKey) { return null; } }
