hadoop c++ pipes

环境：hadoop 1.1.2，centos-64bit

1.wordcount.cpp

#include <unistd.h>
#include <string>
#include <vector>
using namespace std;
 
#include <hadoop/Pipes.hh>
#include <hadoop/StringUtils.hh>
#include <hadoop/TemplateFactory.hh>
 
class WordCountMapper : public HadoopPipes::Mapper {
    public:
        WordCountMapper(HadoopPipes::TaskContext& context) {}
        void map(HadoopPipes::MapContext& context)
        {
            vector<string> words = HadoopUtils::splitString(context.getInputValue(), " ");
            for(unsigned int i = 0; i < words.size(); ++i)
                context.emit(words[i], "1");
        }
};
 
class WordCountReducer : public HadoopPipes::Reducer {
    public:
        WordCountReducer(HadoopPipes::TaskContext& context) {}
        void reduce(HadoopPipes::ReduceContext& context)
        {
            int sum = 0;
            while (context.nextValue())
                sum += HadoopUtils::toInt(context.getInputValue());
            context.emit(context.getInputKey(), HadoopUtils::toString(sum));
        }
};
 
int main(void)
{
    return HadoopPipes::runTask(HadoopPipes::TemplateFactory<WordCountMapper, WordCountReducer>());
}

 2.Makefile文件。路径要写正确，还有加上-lhadooppipes -lhadooputils -lpthread -lcrypto -lssl，具体见下。

CXX := g++ 
CXXFLAGS := -g -Wall
#INCLUDE := -I/usr/hadoop/c++/Linux-amd64-64/include
INCLUDE := -I/usr/hadoop/src/c++/install/include
#LIBS := -L/usr/hadoop/c++/Linux-amd64-64/lib -lhadooppipes -lhadooputils -lpthread -lcrypto
LIBS := -L/usr/hadoop/src/c++/install/lib -lhadooppipes -lhadooputils -lpthread -lcrypto -lssl
 
TARGET := wordcount
 
.PHONY: all clean
 
all: $(TARGET)
 
wordcount: wordcount.o
        $(CXX) $(CXXFLAGS) $^ -o $@ $(LIBS)
 
.cpp.o:
        $(CXX) $(CXXFLAGS) -c $< $(INCLUDE)
 
clean:
        rm -f $(TARGET) *.o

 3.make之后，将可执行文件上传到hadoop dfs

hadoop dfs -put wordcount wordcount

 4.创建测试程序的输入文件wordcount-input，里面填些单词。

 5.运行。

hadoop pipes -D hadoop.pipes.java.recordreader=true \
-D hadoop.pipes.java.recordwriter=true \
-input wordcount-input -output wordcount-output \
-program wordcount

 

FAQ:

Hadoop Pipes ("Server failed to authenticate")

   解决方法：

http://www.linuxquestions.org/questions/linux-software-2/hadoop-1-0-3-pipes-server-failed-to-authenticate-4175429779/

 参考文献：

[1]http://ouonline.net/hadoop-notes-2

[2]http://www.linuxquestions.org/questions/linux-software-2/hadoop-1-0-3-pipes-server-failed-to-authenticate-4175429779/