【mapreudce】6.对Nginx的access日志进行数据清洗，我们提取出文件数据的ip，时间，url
1.首先我们需要一个util辅助类
package cn.cutter.demo.hadoop.mapreduce.nginxlog.util; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; /** * @ClassName AccessLogUtil * @Description * @Author xiaof * @Date 2019/5/15 22:07 * @Version 1.0 **/ public class AccessLogUtil { public static final SimpleDateFormat FORMAT = new SimpleDateFormat( "d/MMM/yyyy:HH:mm:ss", Locale.ENGLISH); public static final SimpleDateFormat dateformat1 = new SimpleDateFormat( "yyyyMMddHHmmss");/** * 解析英文时间字符串 * * @param string * @return * @throws ParseException */ private static Date parseDateFormat(String string) { Date parse = null; try { parse = FORMAT.parse(string); } catch (ParseException e) { e.printStackTrace(); } return parse; } /** * 解析日志的行记录 * * @param line * @return 数组含有5个元素，分别是ip、时间、url、状态、流量,请求来源 */ public static String[] parse(String line) { String ip = parseIP(line); String time = parseTime(line); String url = parseURL(line); String status = parseStatus(line); String traffic = parseTraffic(line); String sourcePath = parseSource(line); return new String[] { ip, time, url, status, traffic, sourcePath }; } private static String parseTraffic(String line) { // final String trim = line.substring(line.lastIndexOf("-") + 1) // .trim(); int start = line.indexOf("\""); int second = line.indexOf("\"", start + 1); int three = line.indexOf("\"", second + 1); final String trim = line.substring(second + 1, three) .trim(); String traffic = trim.split(" ")[1]; return traffic; } private static String parseStatus(String line) { int start = line.indexOf("\""); int second = line.indexOf("\"", start + 1); int three = line.indexOf("\"", second + 1); final String trim = line.substring(second + 1, three) .trim(); String status = trim.split(" ")[0]; return status; } private static String parseURL(String line) { final int first = line.indexOf("\""); final int second = line.indexOf("\"", first + 1); final int last = line.lastIndexOf("\""); String url = line.substring(first + 1, second); return url; } private static String parseTime(String line) { final int first = line.indexOf("["); final int last = line.indexOf("+0800]"); String time = line.substring(first + 1, last).trim(); Date date = parseDateFormat(time); return dateformat1.format(date); } private static String parseIP(String line) { String ip = line.substring(0, line.indexOf("-")).trim(); return ip; } private static String parseSource(String line) { final int end = line.lastIndexOf("\""); final int start = line.lastIndexOf("\"", end - 1); String sourcePath = line.substring(start + 1, end).trim(); return sourcePath; } public static void main(String args[]) { String s1 = "10.25.24.133 - admin [07/Mar/2019:14:19:53 +0800] \"GET /oss-eureka-server/console HTTP/1.1\" 200 21348 \"http://218.200.65.200:9425/oss-web/main.jsp\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36\"\n"; String result[] = AccessLogUtil.parse(s1); for(int i = 0; i < result.length; ++i) { System.out.println(result[i]); } } }
2.map类
package cn.cutter.demo.hadoop.mapreduce.nginxlog.map; import cn.cutter.demo.hadoop.mapreduce.nginxlog.util.AccessLogUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; /** * @ProjectName: cutter-point * @Package: cn.cutter.demo.hadoop.mapreduce.nginxlog.map * @ClassName: NginxAccessLogMap * @Author: xiaof * @Description: ${description} * @Date: 2019/5/17 11:12 * @Version: 1.0 */ public class NginxAccessLogCleanMap extends Mapper<LongWritable, Text, LongWritable, Text> { Text outputValue = new Text(); @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { //解析没行数据，获取不同的数据 String data[] = AccessLogUtil.parse(value.toString()); //组装前三个数据信息，输出到combine outputValue.set(data[0] + "\t" + data[1] + "\t" + data[2]); context.write(key, outputValue); } }
3.reduce类
package cn.cutter.demo.hadoop.mapreduce.nginxlog.reduce; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; /** * @ProjectName: cutter-point * @Package: cn.cutter.demo.hadoop.mapreduce.nginxlog.reduce * @ClassName: NginxAccessLogReduce * @Author: xiaof * @Description: 进行数据清洗 * @Date: 2019/5/17 11:21 * @Version: 1.0 */ public class NginxAccessLogCleanReduce extends Reducer<LongWritable, Text, Text, NullWritable> { @Override protected void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { //循环所有遍历到的数据并输出 for(Text v : values) { context.write(v, NullWritable.get()); } } }
4.启动类
package cn.cutter.demo.hadoop.mapreduce.nginxlog; import cn.cutter.demo.hadoop.mapreduce.nginxlog.map.NginxAccessLogCleanMap; import cn.cutter.demo.hadoop.mapreduce.nginxlog.reduce.NginxAccessLogCleanReduce; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; /** * @ProjectName: cutter-point * @Package: cn.cutter.demo.hadoop.mapreduce.nginxlog * @ClassName: NginxAccessLogClean * @Author: xiaof * @Description: hadoop jar ./cutter-point-service1.jar NginxAccessLogClean /user/xiaof/nginx /user/xiaof/nginx/output * @Date: 2019/5/17 11:25 * @Version: 1.0 */ public class NginxAccessLogClean { public static void main(String args[]) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException { System.out.println("进入NginxAccessLogClean方法"); Configuration conf = new Configuration(); // conf.set("mapreduce.job.jar", "cutter-point-service1.jar"); //其中mr01.jar是你的导出的jar文件名。 conf.set("fs.default.name", "hdfs://jyh-zhzw-inline-27:9000"); conf.set("dfs.client.use.datanode.hostname", "true"); GenericOptionsParser optionsParser = new GenericOptionsParser(conf, args); String[] remainingArgs = optionsParser.getRemainingArgs(); //输出参数 for(int i = 0; i < remainingArgs.length; ++i) { System.out.println(remainingArgs[i]); } Job job = Job.getInstance(conf, NginxAccessLogClean.class.getName()); job.setJarByClass(NginxAccessLogClean.class); job.setMapperClass(NginxAccessLogCleanMap.class); job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(Text.class); job.setReducerClass(NginxAccessLogCleanReduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); // 清理已存在的输出文件 FileInputFormat.setInputPaths(job, remainingArgs[0]); FileOutputFormat.setOutputPath(job, new Path(remainingArgs[1])); FileSystem fs = FileSystem.get(new URI(remainingArgs[0]), conf); Path outPath = new Path(remainingArgs[1]); if (fs.exists(outPath)) { fs.delete(outPath, true); } boolean success = job.waitForCompletion(true); if(success){ System.out.println("Clean process success!"); } else{ System.out.println("Clean process failed!"); } } }
数据源：
我们文件原始数据格式展示
我们清洗之后数据展示
展示数据
