通过HA方式操作HDFS
之前操作hdfs的时候，都是固定namenode的地址，然后去操作。这个时候就必须判断namenode的状态为active还是standby，比较繁琐，如果集群使用了HA的形式，就很方便了
直接上代码，看注释：
package com.ideal.template.openbigdata.util;
import java.io.IOException;
import java.net.URI;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.LinkedList;
import java.util.List;
//import org.anarres.lzo.LzoAlgorithm;
//import org.anarres.lzo.LzoDecompressor;
//import org.anarres.lzo.LzoInputStream;
//import org.anarres.lzo.LzoLibrary;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Logger;
public class HadoopUse
{
private static Log log = LogFactory.getLog(HadoopUse.class);
/**
* 设置hdfs配置信息
* @return
*/
private static Configuration getConf()
{
Configuration conf = new Configuration();
//设置配置相关的信息，分别对应hdfs-site.xml core-site.xml
conf.set("fs.defaultFS", "hdfs://dragoncluster");
conf.set("dfs.nameservices", "dragoncluster");
conf.set("dfs.ha.namenodes.dragoncluster", "nn1,nn2");
conf.set("dfs.namenode.rpc-address.dragoncluster.nn1", "n01.dragon.com:8020");
conf.set("dfs.namenode.rpc-address.dragoncluster.nn2", "n02.dragon.com:8020");
conf.set("dfs.client.failover.proxy.provider.dragoncluster", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
//设置实现类，因为会出现类覆盖的问题
conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
return conf;
}
/**
* 设置kerberos认证
* @param conf
* @throws Exception
*/
private static void kerberosLogin(Configuration conf) throws Exception
{
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab("openbigdata@DRAGON.COM", "/etc/security/keytabs/openbigdata.keytab");
}
public static long getSize(String uri, String user)
{
Path path = new Path(URI.create(uri));
Configuration conf = new Configuration();
try
{
FileSystem fs = FileSystem.get(URI.create(uri), conf, user);
return fs.getContentSummary(path).getLength() / 1024 / 1024; // 单位为MB
}
catch (Exception ex)
{
log.error("HadoopUse.getSize" + ex.getMessage(), ex);
return 0;
}
}
/**
* 在hdfs上创建文件，并写入内容
*
* @param uri
* @param content
* @param user
* @return
*/
public static boolean createHdfsFile(String uri, String user, String fullName, String content)
{
if (fullName == null || fullName.length() == 0)
{// 本地路径不正确
return false;
}
if (content == null || content.length() == 0)
{// hdfs路径不正确
return false;
}
try
{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri), conf, user);
FSDataOutputStream os = null;
if (fs.exists(new Path(fullName)) == true)
{// 如果该路径存在
// os = fs.append(new Path(fullName));
fs.delete(new Path(fullName), true);
}
os = fs.create(new Path(fullName));
os.write(content.getBytes());
os.close();
fs.close();
return true;
}
catch (Exception ex)
{
log.error("HadoopUse.createHdfsFile" + ex.getMessage(), ex);
return false;
}
}
/**
* 删除hdfs上的文件
* @param uri
* @param user
* @param fullName
* @return
*/
public static boolean deleteHdfsFile(String uri, String user, String fullName)
{
if (fullName == null || fullName.length() == 0)
{// 本地路径不正确
log.error("HadoopUse.deleteHdfsFile文件名不合法");
return false;
}
try
{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri), conf, user);
//FSDataOutputStream os = null;
if (fs.exists(new Path(fullName)) == true)
{// 如果该路径存在
// os = fs.append(new Path(fullName));
fs.delete(new Path(fullName), true);
}
return true;
}
catch (Exception ex)
{
log.error("HadoopUse.createHdfsFile" + ex.getMessage(), ex);
}
return false;
}
/**
* 根据resultset将值写入到hdfs上
* @param uri
* @param user
* @param fullName
* @param resultSet
* @param terminated
* @return
* @throws InterruptedException
* @throws IOException
* @throws SQLException
*/
public void createHdfsFile(String fullName, ResultSet resultSet, String terminated, FlagUtil flag)
throws IOException, InterruptedException, SQLException, Exception
{
if (resultSet == null)
{ // 如果查询出来的游标为空，直接退出
return;
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
FileSystem fs = null;
FSDataOutputStream out = null;
Configuration conf = getConf();
kerberosLogin(conf);
fs = FileSystem.get(conf);
if (fs.exists(new Path(fullName)) == true)
{// 如果该路径存在
fs.delete(new Path(fullName), true);
}
// 获取文件句柄
out = fs.create(new Path(fullName));
// 写入文件内容
ResultSetMetaData rsmd = resultSet.getMetaData();
int rowCnt = rsmd.getColumnCount();
int count = 0;
while (resultSet.next())
{
count++;
if(count >= 1000)
{//每1000条记录检查一次需要终止任务
if(flag.getTeminalStatus() == true)
{
break;
}
count = 0;
}
for (int i = 1; i <= rowCnt; i++)
{
if (resultSet.getObject(i) == null)
{// 如果是空的数据
out.write("".getBytes("utf-8"));
}
else
{
String item = null;
if("DATE".equals(rsmd.getColumnTypeName(i).toUpperCase()))
{//如果是日期类型
Timestamp date = resultSet.getTimestamp(i);
item = sdf.format(date);
}
else
{
item = String.valueOf(resultSet.getObject(i));
}
if (item != null)
{
out.write(item.getBytes("utf-8"));
}
else
{
out.write("".getBytes("utf-8"));
}
}
if (i < rowCnt)
{// 如果写完一列，则插入分隔符
out.write(terminated.getBytes("utf-8"));
}
}
// 切换到下一行
out.write("\r\n".getBytes("utf-8"));
}
log.info("fullName:" + fullName + "写入成功");
if (out != null)
{
out.flush();
out.close();
}
if (fs != null)
{
fs.close();
}
}
/**
* 查询路径
* @param path
* @return
* @throws Exception
*/
public static List<String> listDir(String path) throws Exception
{
Configuration conf = getConf();
kerberosLogin(conf);
FileSystem fs = FileSystem.get(conf);
Path hdfs = new Path(path);
List<String> pathList = null;
FileStatus files[] = fs.listStatus(hdfs);
if(files!=null && files.length >0)
{
pathList = new LinkedList<String>();
for (FileStatus file : files)
{
pathList.add(file.getPath().toString());
}
}
return pathList;
}
public static void main(String[] args) throws Exception
{
List<String> pathList = listDir(args[0]);
for(String path: pathList)
{
System.out.println(path);
}
}
}
注意，这用到了HA，以及kerberos认证，
