课堂测试3第一阶段数据清洗
Result文件数据说明:
Ip:106.39.41.166,(城市)
Date:10/Nov/2016:00:01:02 +0800,(日期)
Day:10,(天数)
Traffic: 54 ,(流量)
Type: video,(类型:视频video或文章article)
Id: 8701(视频或者文章的id)
测试要求:
1、 数据清洗:按照进行数据清洗,并将清洗后的数据导入hive数据库中。
两阶段数据清洗:
(1)第一阶段:把需要的信息从原始日志中提取出来
ip: 199.30.25.88
time: 10/Nov/2016:00:01:03 +0800
traffic: 62
文章: article/11325
视频: video/3235
(2)第二阶段:根据提取出来的信息做精细化操作
ip--->城市 city(IP)
date--> time:2016-11-10 00:01:03
day: 10
traffic:62
type:article/video
id:11325
(3)hive数据库表结构:
create table data( ip string, time string , day string, traffic bigint,
type string, id string )
2、数据处理:
·统计最受欢迎的视频/文章的Top10访问次数 (video/article)
·按照地市统计最受欢迎的Top10课程 (ip)
·按照流量统计最受欢迎的Top10课程 (traffic)
3、数据可视化:将统计结果导入MySql数据库中,通过图形化展示的方式展现出来。
(本次只完成1.数据清洗)
1 package hiveUDF; 2 import java.lang.String; 3 import java.text.SimpleDateFormat; 4 import java.util.Date; 5 import java.util.Locale; 6 import java.io.IOException; 7 import org.apache.hadoop.conf.Configuration; 8 import org.apache.hadoop.fs.Path; 9 import org.apache.hadoop.io.LongWritable; 10 import org.apache.hadoop.io.Text; 11 import org.apache.hadoop.mapreduce.Job; 12 import org.apache.hadoop.mapreduce.Mapper; 13 import org.apache.hadoop.mapreduce.Reducer; 14 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 15 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 public class test1 { 17 18 public static final SimpleDateFormat FORMAT = new SimpleDateFormat("d/MMM/yyyy:HH:mm:ss", Locale.ENGLISH); //原时间格式 19 public static final SimpleDateFormat dateformat1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//现时间格式 20 private static Date parseDateFormat(String string) { //转换时间格式 21 Date parse = null; 22 try { 23 parse = FORMAT.parse(string); 24 } catch (Exception e) { 25 e.printStackTrace(); 26 } 27 return parse; 28 } 29 public static class MyMapper extends Mapper<LongWritable, Text, Text/*map对应键类型*/, Text/*map对应值类型*/> 30 { 31 protected void map(LongWritable key, Text value,Context context)throws IOException, InterruptedException 32 { 33 String[] strNlist = value.toString().split(",");//如何分隔 34 //LongWritable,IntWritable,Text等 35 Date date = parseDateFormat(strNlist[1]); 36 context.write(new Text(strNlist[0])/*map对应键类型*/,new Text(dateformat1.format(date)+","+strNlist[2]+","+strNlist[3]+","+strNlist[4]+","+strNlist[5])/*map对应值类型*/); 37 } 38 } 39 public static class MyReducer extends Reducer<Text/*map对应键类型*/, Text/*map对应值类型*/, Text/*reduce对应键类型*/, Text/*reduce对应值类型*/> 40 { 41 // static No1Info info=new No1Info(); 42 protected void reduce(Text key, Iterable<Text/*map对应值类型*/> values,Context context)throws IOException, InterruptedException 43 { 44 for (/*map对应值类型*/Text init : values) 45 { 46 // String[] strNlist = init.toString().split(","); 47 // dao.add("data", strNlist); 48 context.write( key/*reduce对应键类型*/, new Text(init)/*reduce对应值类型*/); 49 } 50 } 51 } 52 53 public static void main(String[] args) throws Exception { 54 Configuration conf = new Configuration(); 55 56 //将命令行中的参数自动设置到变量conf中 57 // String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs(); 58 // if (otherArgs.length != 2) { 59 // System.err.println("Usage: wordcount <in> <out>"); 60 // System.exit(2); 61 // } 62 63 Job job = Job.getInstance(); 64 //job.setJar("MapReduceDriver.jar"); 65 job.setJarByClass(test1.class); 66 // TODO: specify a mapper 67 job.setMapperClass(MyMapper.class); 68 job.setMapOutputKeyClass(/*map对应键类型*/Text.class); 69 job.setMapOutputValueClass( /*map对应值类型*/Text.class); 70 71 // TODO: specify a reducer 72 job.setReducerClass(MyReducer.class); 73 job.setOutputKeyClass(/*reduce对应键类型*/Text.class); 74 job.setOutputValueClass(/*reduce对应值类型*/Text.class); 75 76 // TODO: specify input and output DIRECTORIES (not files) 77 FileInputFormat.setInputPaths(job, new Path("hdfs://192.168.1.102:9000/user/hadoop/input/result.txt")); 78 FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.1.102:9000/user/hadoop/out")); 79 80 boolean flag = job.waitForCompletion(true); 81 System.out.println("SUCCEED!"+flag); //任务完成提示 82 System.exit(flag ? 0 : 1); 83 System.out.println(); 84 } 85 }
数据清洗前:
数据清洗后:
完成之后,用hive命令,将文件导入
load data inpath '/user/local/hadoop/out/part-r-00000' overwrite into table result;