mapreduce测试

package mapreduceshiyan1;

import java.io.IOException;  
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.NullWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;  
public class Filter{  
    public static class Map extends Mapper<Object , Text , Text , NullWritable>{  
    private static Text newKey=new Text();  
    public void map(Object key,Text value,Context context) throws IOException, InterruptedException{  
    String line=value.toString();  
    System.out.println(line);  
    String arr[]=line.split("\t");  
    newKey.set(arr[1]);  
    context.write(newKey, NullWritable.get());  
    System.out.println(newKey);  
    }  
    }  
    public static class Reduce extends Reducer<Text, NullWritable, Text, NullWritable>{  
    public void reduce(Text key,Iterable<NullWritable> values,Context context) throws IOException, InterruptedException{  
        context.write(key,NullWritable.get());  
        }  
        }  
        @SuppressWarnings("deprecation")
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{  
        Configuration conf=new Configuration();  
        System.out.println("start");  
        Job job =new Job(conf,"filter");  
        job.setJarByClass(Filter.class);  
        job.setMapperClass(Map.class);  
        job.setReducerClass(Reduce.class);  
        job.setOutputKeyClass(Text.class);  
        job.setOutputValueClass(NullWritable.class);  
        job.setInputFormatClass(TextInputFormat.class);  
        job.setOutputFormatClass(TextOutputFormat.class);  
        Path in=new Path("hdfs://hdfs://192.168.198.130:8020/mapreduce/shiyanyi/input/shiyanyi.txt");  
        Path out=new Path("hdfs://hdfs://192.168.198.130:8020/mapreduce/shiyanyi/output");  
        FileInputFormat.addInputPath(job,in);  
        FileOutputFormat.setOutputPath(job,out);  
        System.exit(job.waitForCompletion(true) ? 0 : 1);  
        }  
        }  

mapreduce的一个代码的测试

posted @ 2020-11-12 12:21  凋零_(  阅读(67)  评论(0编辑  收藏  举报