读写hdfs文件(工作笔记)
import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Maps; public class LoadHdfsTemplate { static final Logger logger = LoggerFactory .getLogger(LoadHdfsTemplate.class); protected FileSystem fileSystem ; private String hdfsPath = "/template/template.txt"; public final boolean checkFile(String filePath) { boolean exists = false; try { Path path = new Path(filePath); exists = fileSystem.exists(path); } catch (IOException e) { logger.error("接口模板文件不存在!", e); } catch (Exception e) { logger.error("", e); } return exists; } public Map<String,Object> readHdfsFile() throws IOException{ fileSystem = FileSystem.get(new Configuration()); Path path = new Path(hdfsPath); InputStream in = fileSystem.open(path); List<String> lines = IOUtils.readLines(in); if(null == lines || lines.isEmpty()){ return null; } Map<String,Object> map = Maps.newConcurrentMap(); int rowNum = 0; for(String line : lines){ rowNum++; String[] content = line.split(" "); String code = content[0].trim(); String template = content[1].trim(); if(StringUtils.isEmpty(line) || StringUtils.isEmpty(template)){ logger.error("第{}条模板格式错误!内容为:{}",rowNum,line); continue; } map.put(code, template); } return map; } public void setHdfsPath(String hdfsPath) { this.hdfsPath = hdfsPath; } }
应用场景为,读取hdfs上的文件,并转化成<k,v>形式存入内存。