flume与Mosquitto的集成

文章来自：http://www.cnblogs.com/hark0623/p/4173714.html   转发请注明 

 

因业务需求，需要flume收集MQTT(Mosquitto)的数据。  方法就是flume自定义source，source中来订阅(subscribe)MQTT

 

flume source的java代码如下：

package com.yhx.sensor.flume.source;

import java.util.HashMap;
import java.util.Map;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDrivenSource;
import org.apache.flume.conf.Configurable;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.source.AbstractSource;
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.MqttCallback;
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.MqttTopic;

public class MQTTSource extends AbstractSource implements EventDrivenSource,
        Configurable {
    /**
     * The initialization method for the Source. The context contains all the
     * Flume configuration info, and can be used to retrieve any configuration
     * values necessary to set up the Source.
     */
    @Override
    public void configure(Context arg0) {
        // TODO Auto-generated method stub

    }

    SimpleMqttClient client = null;

    /**
     * Start any dependent systems and begin processing events.
     */
    @Override
    public void start() {
        // TODO Auto-generated method stub
        // super.start();
        client = new SimpleMqttClient();
        client.runClient();
    }

    /**
     * Stop processing events and shut any dependent systems down.
     */
    @Override
    public void stop() {
        // TODO Auto-generated method stub
        // super.stop();
        if (client != null) {
            client.closeConn();
        }
    }

    // public static void main(String[] args) {
    // SimpleMqttClient smc = new SimpleMqttClient();
    // smc.runClient();
    // }

    public class SimpleMqttClient implements MqttCallback {

        MqttClient myClient;
        MqttConnectOptions connOpt;

        String BROKER_URL = "tcp://192.168.116.128:1883";
        String M2MIO_DOMAIN = "192.168.116.128";
        String M2MIO_STUFF = "yhx";
        String M2MIO_THING = "yhx_flume";
        // String M2MIO_USERNAME = "<m2m.io username>";
        // String M2MIO_PASSWORD_MD5 =
        // "<m2m.io password (MD5 sum of password)>";

        Boolean subscriber = true;
        Boolean publisher = false;

        /**
         * 
         * connectionLost This callback is invoked upon losing the MQTT
         * connection.
         * 
         */
        @Override
        public void connectionLost(Throwable t) {
            System.out.println("Connection lost!");
            // code to reconnect to the broker would go here if desired
        }

        public void closeConn() {
            if (myClient != null) {
                if (myClient.isConnected()) {
                    try {
                        myClient.disconnect();
                    } catch (MqttException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                    }
                }
            }
        }

        /**
         * 
         * deliveryComplete This callback is invoked when a message published by
         * this client is successfully received by the broker.
         * 
         */
        @Override
        public void deliveryComplete(IMqttDeliveryToken token) {
            // System.out.println("Pub complete" + new
            // String(token.getMessage().getPayload()));
        }

        /**
         * 
         * messageArrived This callback is invoked when a message is received on
         * a subscribed topic.
         * 
         */
        @Override
        public void messageArrived(String topic, MqttMessage message)
                throws Exception {
            // System.out
            // .println("-------------------------------------------------");
            // // System.out.println("| Topic:" + topic.getName());
            // System.out.println("| Topic:" + topic);
            // System.out
            // .println("| Message: " + new String(message.getPayload()));
            // System.out
            // .println("-------------------------------------------------");


            Map<String, String> headers = new HashMap<String, String>();
            //headers.put("curDate", df.format(new Date()));

            Event flumeEvent = EventBuilder.withBody(message.getPayload(),
                    headers);
            try {
                getChannelProcessor().processEvent(flumeEvent);
            } catch (Exception e) {
                // TODO: handle exception
                e.printStackTrace();
            }

        }

        /**
         * 
         * runClient The main functionality of this simple example. Create a
         * MQTT client, connect to broker, pub/sub, disconnect.
         * 
         */
        public void runClient() {
            // setup MQTT Client
            String clientID = M2MIO_THING;
            connOpt = new MqttConnectOptions();

            connOpt.setCleanSession(true);
            connOpt.setKeepAliveInterval(3000);
            // connOpt.setUserName(M2MIO_USERNAME);
            // connOpt.setPassword(M2MIO_PASSWORD_MD5.toCharArray());

            // Connect to Broker
            try {
                myClient = new MqttClient(BROKER_URL, clientID);
                myClient.setCallback(this);
                myClient.connect(connOpt);
            } catch (MqttException e) {
                e.printStackTrace();
                System.exit(-1);
            }

            System.out.println("Connected to " + BROKER_URL);

            // setup topic
            // topics on m2m.io are in the form <domain>/<stuff>/<thing>
            String myTopic = M2MIO_DOMAIN + "/" + M2MIO_STUFF + "/"
                    + M2MIO_THING;
            System.out.println("myTopic:" + myTopic);
            MqttTopic topic = myClient.getTopic(myTopic);

            // subscribe to topic if subscriber
            if (subscriber) {
                try {
                    int subQoS = 0;
                    myClient.subscribe(myTopic, subQoS);

                } catch (Exception e) {
                    e.printStackTrace();
                }
            }

            // publish messages if publisher
            if (publisher) {
                for (int i = 1; i <= 10; i++) {
                    String pubMsg = "{\"pubmsg\":" + i + "}";
                    int pubQoS = 0;
                    MqttMessage message = new MqttMessage(pubMsg.getBytes());
                    message.setQos(pubQoS);
                    message.setRetained(false);

                    // Publish the message
                    System.out.println("Publishing to topic \"" + topic
                            + "\" qos " + pubQoS);
                    MqttDeliveryToken token = null;
                    try {
                        // publish message to broker
                        token = topic.publish(message);
                        // Wait until the message has been delivered to the
                        // broker
                        token.waitForCompletion();
                        Thread.sleep(100);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }

            // disconnect
            try {
                // wait to ensure subscribed messages are delivered
                if (subscriber) {
                    while (true) {
                        Thread.sleep(5000);
                    }
                }
                // myClient.disconnect();
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
            }
        }

    }

}

打JAR包注意要把Class-Path写上，如下：

Manifest-Version: 1.0
Class-Path: flume-ng-configuration-1.5.2.jar flume-ng-core-1.5.2.jar flume-ng-node-1.5.2.jar flume-ng-sdk-1.5.2.jar org.eclipse.paho.client.mqttv3-1.0.0.jar

 

将打好的JAR包放到flume的lib目录（注意，class-path说明的jar包在lib一定要有。 如果没有，则放上去）

 

接着修改一下flume的配置文件，如下(主要是sourceMqtt ，看这个。  因为我这块同时还监听了UDP)：

a1.sources = sourceMqtt sourceUdp
a1.sinks = sinkMqtt sinkUdp
a1.channels = channelMqtt channelUdp

# Describe/configure the source
a1.sources.sourceMqtt.type = com.yhx.sensor.flume.source.MQTTSource

# Describe the sink
a1.sinks.sinkMqtt.type = logger

# Use a channel which buffers events in memory
a1.channels.channelMqtt.type = memory
a1.channels.channelMqtt.capacity = 1000
a1.channels.channelMqtt.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.sourceMqtt.channels = channelMqtt
a1.sinks.sinkMqtt.channel = channelMqtt



# a2.sources = sourceUdp
# a2.sinks = sinkUdp
# a2.channels = channelUdp

# Describe/configure the source
a1.sources.sourceUdp.type = syslogudp
a1.sources.sourceUdp.host = 0.0.0.0
a1.sources.sourceUdp.port = 12459
a1.sources.sourceUdp.interceptors=interceptorUdp

a1.sources.sourceUdp.interceptors.interceptorUdp.type=com.yhx.sensor.flume.intercepter.UDPIntercepter$Builder

# Describe the sink
a1.sinks.sinkUdp.type = logger

# Use a channel which buffers events in memory
a1.channels.channelUdp.type = memory
a1.channels.channelUdp.capacity = 1000
a1.channels.channelUdp.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.sourceUdp.channels = channelUdp
a1.sinks.sinkUdp.channel = channelUdp

 

配置文件保存至flume目录下的conf,叫flume.conf

然后flume启动命令如下 

bin/flume-ng agent --conf conf --conf-file conf/flume.conf --name a1