用java读取一个文件往hbase里插入数据(List<PUT>)
package com.asp; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTablePool; import org.apache.hadoop.hbase.client.Put; public class Test { public static Configuration conf = HBaseConfiguration.create(); static { Configuration HBASE_CONFIG = new Configuration(); HBASE_CONFIG.set("hbase.zookeeper.quorum", "ugcserver3,ugcserver4,ugcserver5"); HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181"); conf = HBaseConfiguration.create(HBASE_CONFIG); } public static void creatTable(String tableName, String[] familys) throws Exception { HBaseAdmin admin = new HBaseAdmin(conf); if (admin.tableExists(tableName)) { System.out.println("table already exists!"); } else { HTableDescriptor tableDesc = new HTableDescriptor(tableName); for (int i = 0; i < familys.length; i++) { tableDesc.addFamily(new HColumnDescriptor(familys[i])); } admin.createTable(tableDesc); System.out.println("create table " + tableName + " ok"); } } public static void main(String[] args) { String tableName = "test"; String[] fa = { "c1", "c2" }; int i = 0; List<Put> list = new ArrayList<Put>(); try { Test.creatTable(tableName, fa); } catch (Exception e) { e.printStackTrace(); } System.out.println("测试开始插入数据"); long start = System.currentTimeMillis(); HTablePool pool = new HTablePool(conf, 1000); //HTableInterface table = pool.getTable(tableName);
HTable table = (HTable) pool.getTable(tableName); table.setAutoFlush(false); try { table.setWriteBufferSize(24*1024*1024); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } File file = new File("/opt/pmdce/datamining/123.txt"); BufferedReader reader = null; String lineString = null; try { reader = new BufferedReader(new FileReader(file)); while ((lineString = reader.readLine()) != null) { i++; String[] lines = lineString.split("\\,"); Put put = new Put(lines[0].getBytes()); put.add("c1".getBytes(), "name".getBytes(), lines[1].getBytes()); put.add("c1".getBytes(), "age".getBytes(), lines[2].getBytes()); put.add("c2".getBytes(), "class".getBytes(),lines[3].getBytes()); list.add(put); if (i % 10000 == 0) { table.put(list); list.clear(); table.flushCommits(); } } reader.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { if (reader != null) { try { reader.close(); } catch (IOException e) { e.printStackTrace(); } } } long stop = System.currentTimeMillis(); System.out.println("插入数据共耗时:" + (stop - start) * 1.0 / 1000 + "s"); } }
【推荐】国内首个AI IDE,深度理解中文开发场景,立即下载体验Trae
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· SQL Server 2025 AI相关能力初探
· Linux系列:如何用 C#调用 C方法造成内存泄露
· AI与.NET技术实操系列(二):开始使用ML.NET
· 记一次.NET内存居高不下排查解决与启示
· 探究高空视频全景AR技术的实现原理
· 阿里最新开源QwQ-32B,效果媲美deepseek-r1满血版,部署成本又又又降低了!
· AI编程工具终极对决:字节Trae VS Cursor,谁才是开发者新宠?
· 开源Multi-agent AI智能体框架aevatar.ai,欢迎大家贡献代码
· Manus重磅发布:全球首款通用AI代理技术深度解析与实战指南
· 被坑几百块钱后,我竟然真的恢复了删除的微信聊天记录!