Hadoop 上使用C 语言编程【转】

转自：https://www.linuxidc.com/Linux/2012-04/58991.htm

今天尝试用C语言在Hadoop上编写统计单词的程序，具体过程如下：

一、编写map和reduce程序

mapper.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUF_SIZE    2048
#define DELIM       '\n'

int main(int argc, char * argv[])
{
    char buffer[BUF_SIZE];
    while(fgets(buffer,BUF_SIZE-1,stdin))
    {
        int len = strlen(buffer);
        if(buffer[len-1] == DELIM) // 将换行符去掉
            buffer[len-1] = 0;

        char *query = NULL;
        query = strtok(buffer, " ");
        while(query)
        {
            printf("%s\t1\n",query);
            query = strtok(NULL," ");
        }
    }
    return 0;
}

reducer.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUFFER_SIZE     1024
#define DELIM       "\t"

int main(int argc, char * argv[])
{
    char str_last_key[BUFFER_SIZE];
    char str_line[BUFFER_SIZE];
    int count = 0;

    *str_last_key = '\0';

    while( fgets(str_line,BUFFER_SIZE-1,stdin) )
    {
        char * str_cur_key = NULL;
        char * str_cur_num = NULL;

        str_cur_key = strtok(str_line,DELIM);
        str_cur_num = strtok(NULL,DELIM);

        if(str_last_key[0] =='\0')
        {
            strcpy(str_last_key,str_cur_key);
        }
        if(strcmp(str_cur_key, str_last_key))// 前后不相等，输出
        {
            printf("%s\t%d\n",str_last_key,count);
            count = atoi(str_cur_num);
        }else{// 相等，则加当前的key的value
            count += atoi(str_cur_num);
        }
        strcpy(str_last_key,str_cur_key);
    }
    printf("%s\t%d\n",str_last_key,count);
    return 0;
}

二、编译

gcc mapper.c -o mapper

gcc reducer.c -o reducer

三、运行

（一）启动hadoop后将待统计单词的输入文件放到 input文件夹中：bin/hadoop fs -put 待统计文件 input

（二）使用contrib/streaming/下的jar工具调用上面的mapper\reducer:

bin/hadoop jar /home/huangkq/Desktop/hadoop/contrib/streaming/hadoop-streaming-0.20.203.0.jar -mapper /home/huangkq/Desktop/hadoop2/mapper -reducer /home/huangkq/Desktop/hadoop2/reducer -input input -output c_output -jobconf mapred.reduce.tasks=2

说明：hadoop-streaming-0.20.203.0.jar是一个管道工具

（三）查看结果：bin/hadoop fs -cat c_output/*