Hadoop组件---HDFS&Yarn&MapReduce

一. HDFS

1. namenode作用

元数据信息存储

2. datanode作用

 

 3. 文件副本机制

4. 机架感知

 

 

 5. 命令操作

 

DEMO

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;

public class HadoopTestDemo {

    // 连接hadoop获取filesystem的四种方式
    @Test
    public void getFileSystem1() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://192.168.217.4:8020/");
        configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
        FileSystem fileSystem = FileSystem.get(configuration);
        System.out.println(fileSystem.toString());
    }

    @Test
    public void getFileSystem2() throws Exception {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        System.out.println(fileSystem.toString());
    }

    @Test
    public void getFileSystem3() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://192.168.217.4:8020/");
        configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
        FileSystem fileSystem = FileSystem.newInstance(configuration);
        System.out.println(fileSystem.toString());
    }

    @Test
    public void getFileSystem4() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        System.out.println(fileSystem.toString());
    }

    // 遍历HDFS中所有文件
    @Test
    public void listMyFiles() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fileSystem.listFiles(new Path("/"), true);
        while (locatedFileStatusRemoteIterator.hasNext()) {
            LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
            System.out.println(next.getPath().toString());
        }
        fileSystem.close();
    }

    // HDFS上创建文件夹
    @Test
    public void mkdirs() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        boolean mkdirs = fileSystem.mkdirs(new Path("/hello/mydir/test"));
        System.out.println(mkdirs);
        fileSystem.close();
    }

    // 下载文件
    @Test
    public void downloadFile() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        FSDataInputStream inputStream = fileSystem.open(new Path("/test/a.txt"));
        FileOutputStream fileOutputStream = new FileOutputStream(new File("./a.txt"));
        IOUtils.copy(inputStream, fileOutputStream);
        IOUtils.closeQuietly(inputStream);
        IOUtils.closeQuietly(fileOutputStream);
        System.out.println("end");
        fileSystem.close();
    }

    // 文件上传
    @Test
    public void uploadFile() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        fileSystem.copyFromLocalFile(new Path("./delubi.jpg"), new Path("/test"));
        fileSystem.close();
    }

    // 小文件合并
    @Test
    public void mergeFile() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/test/lucian.txt"));
        LocalFileSystem local = FileSystem.getLocal(new Configuration());
        FileStatus[] fileStatuses = local.listStatus(new Path("./input"));
        for (FileStatus fileStatus : fileStatuses) {
            FSDataInputStream inputStream = local.open(fileStatus.getPath());
            IOUtils.copy(inputStream, fsDataOutputStream);
            IOUtils.closeQuietly(inputStream);
        }
        IOUtils.closeQuietly(fsDataOutputStream);
        local.close();
        fileSystem.close();
    }

}

二. MapReduce 

1.基本使用

 WordCountMapper.java

package com.luyizhou.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/*
四个泛型
 */
public class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

    // map方法就是将k1/v1转为k2,v2
    // k1 偏移量，v1 每一行的文本数据
    // context 上下文对象
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        Text text = new Text();
        LongWritable longWritable = new LongWritable();
        // 1.将一行的文本数据拆分
        String[] split = value.toString().split(",");
        // 2.遍历数组，组装k2,v2
        for (String word : split) {
            // 3.将k2,v2写入上下文
            text.set(word);
            longWritable.set(1);
            context.write(text, longWritable);
        }

    }
}

 

WordCountReducer.java

package com.luyizhou.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

    /*
    * 将新的k2,v2转为k3,v3,并写入上下文
    * key:新k2
    * values:集合，新v2
    * context:表示上下文对象
    * */
    @Override
    protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
        long count = 0;
        //1.遍历集合，将集合中的数字相加，得到v3
        for (LongWritable value : values) {
            count += value.get();
        }
        //2.将k3和v3写入上下文
        context.write(key, new LongWritable(count));
    }
}

JobMain.java

package com.luyizhou.mapreduce;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.net.URI;


public class JobMain extends Configured implements Tool {

    // 该方法用于指定一个Job任务
    @Override
    public int run(String[] strings) throws Exception {
        // 1.创建一个job任务对象
        Job job = Job.getInstance(super.getConf(), "wordcountOfluyizhou");
        job.setJarByClass(JobMain.class);   // 必须添加，否则打包jar报错,跟版本有关
        // 2.配置job任务对象, 八个步骤

        // map阶段
        // 第一步：指定文件的读取路径和读取方式
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path("hdfs://192.168.217.4:8020/wordcount"));
        // 本地模式
        // TextInputFormat.addInputPath(job, new Path("file:///F:\\tmp\\input"));
        // 第二步：指定map阶段的处理方式
        job.setMapperClass(WordCountMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);

        // shuffle阶段
        // 第三、四、五、六步：采用默认方式，暂时不做处理

        // reduce阶段
        // 第七步：指定reduce阶段处理方式和数据类型
        job.setReducerClass(WordCountReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        // 第八步：设置输出类型和输出路径
        job.setOutputFormatClass(TextOutputFormat.class);
        Path path = new Path("hdfs://192.168.217.4:8020/wordcount_out");
        TextOutputFormat.setOutputPath(job, path);

        // 本地模式运行
        // TextOutputFormat.setOutputPath(job, new Path("file:///F:\\tmp\\output"));

        // 获取filesystem,判断目录是否存在
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.217.4:8020"), new Configuration());
        boolean exists = fileSystem.exists(path);
        if (exists) {
            // 删除目录
            fileSystem.delete(path, true);
        }

        // 等待任务结束
        boolean b = job.waitForCompletion(true);
        return b ? 0:1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        // 启动job任务
        int run = ToolRunner.run(configuration, new JobMain(), args);
        System.exit(run);
    }
}

 

2. 分区

com.luyizhou.partition.PartitionMapper

package com.luyizhou.partition;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/*
* k1 行偏移量
* v1 行文本数据
* k2 一整行文本数据
* v2 占位符
* */
public class PartitionMapper extends Mapper<LongWritable, Text, Text, NullWritable> {

    // map方法将k1，v1 转为k2,v2
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        // 方式一，定义计数器
        Counter counter = context.getCounter("MR_COUNTER", "partition_counter");
        // 每次执行改方法，计数器数据加1
        counter.increment(1L);

        context.write(value, NullWritable.get());
    }
}

com.luyizhou.partition.PartitionReducer

package com.luyizhou.partition;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/*
* k2 Text 一行文本数据
* V2 NullWritable
* k3 text 一行文本数据
* v2 NullWritable
* */
public class PartitionReducer extends Reducer<Text, NullWritable, Text, NullWritable> {

    public static enum Counter{
        MY_INPUT_RECORDS, MY_INPUT_BYTES
    }

    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

        // 方式二：使用枚举来定义计数器
        context.getCounter(Counter.MY_INPUT_RECORDS).increment(1L);

        context.write(key, NullWritable.get());
    }
}

com.luyizhou.partition.MyPartition

package com.luyizhou.partition;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;


public class MyPartition extends Partitioner<Text, NullWritable> {

    /*
    * 定义分区规则
    * 返回对应的分区编号
    *
    * */
    @Override
    public int getPartition(Text text, NullWritable nullWritable, int i) {
        // 1.拆分行文本数据(k2),获取中间字段值
        String[] split = text.toString().split("\t");
        String numStr = split[5];
        // 2.判断中将字段的值和15的关系，返回对应的分区编号
        if (Integer.parseInt(numStr) > 15) {
            return 1;
        } else {
            return 0;
        }
    }
}

com.luyizhou.partition.JobMain

package com.luyizhou.partition;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class JobMain extends Configured implements Tool {

    @Override
    public int run(String[] strings) throws Exception {
        Job job = Job.getInstance(super.getConf(), "partition_mapreduce");
        job.setJarByClass(JobMain.class);

        // map
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path("hdfs://192.168.217.4:8020/input"));
        job.setMapperClass(PartitionMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);

        // shuffle
        // 分区
        job.setPartitionerClass(MyPartition.class);
        // 排序，规约，分组

        // reduce
        job.setReducerClass(PartitionReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        job.setNumReduceTasks(2);

        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, new Path("hdfs://192.168.217.4:8020/output/partition_out"));

        boolean b = job.waitForCompletion(true);

        return b ? 0:1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        int run = ToolRunner.run(configuration, new JobMain(), args);
        System.exit(run);
    }
}

 

3. 排序&序列化

 

4. 规约

理解:规约就是提前进行局部汇总,本质上属于reduce的一部分,减少网络IO次数,提高性能