随笔- 38 文章- 0 评论- 5 阅读- 30540

使用mongoshake进行oplog同步读取，解决乱码问题

mongoshake 是个开源的用户mongo数据迁移和同步的工具，支持往各种目标源头写数据

具体：https://github.com/alibaba/MongoShake

有业务场景想把oplog 写入到kafka 中，如果直接在collector.conf 设置kafka 信息会导致写入kafka 中数据是乱码

官方解释是直接从collector 采集到的oplog是带有控制信息的。直接写入kafka 的内容使用时要进行剥离。

在下载mongoshake 包时他会提供receiver 进行控制信息剥离

mongo --> collector --> kafka --> receiver --> 业务

mongo --> collector --> receiver --> kafka --> 业务

这里更倾向于第二种

collector --> receiver 我采用的是tcp

配置：

collector.conf

tunnel = tcp
tunnel.address = 127.0.0.1:9300

receiver.conf

tunnel = tcp
tunnel.address = 127.0.0.1:9300

这里会很奇怪，也没有设置kafka 的地方啊，这样所有oplog剥离信息都会放在receiver 的log下

这里官方解释是要求我们对源码进行修改、编译，源码是GO 写的，改起来也比较熟悉

下载官方源码

src/mongoshake/receiver/replayer.go

在handler()

/*
 * Users should modify this function according to different demands.
 */
func (er *ExampleReplayer) handler() {
    config := sarama.NewConfig()//kafka配置
    config.Producer.RequiredAcks = sarama.WaitForAll
    config.Producer.Return.Successes = true
    kafkaClient, err := sarama.NewSyncProducer([]string{conf.Options.KafkaHost}, config)
    if err != nil {
        LOG.Info("producer close,err:", err)
        return
    }
 
    defer kafkaClient.Close()
 
 
    for msg := range er.pendingQueue {
        count := uint64(len(msg.message.RawLogs))
        if count == 0 {
            // probe request
            continue
        }
 
        // parse batched message
        oplogs := make([]*oplog.PartialLog, len(msg.message.RawLogs))
        for i, raw := range msg.message.RawLogs {
            oplogs[i] = new(oplog.PartialLog)
            if err := bson.Unmarshal(raw, oplogs[i]); err != nil {
                // impossible switch, need panic and exit
                LOG.Crashf("unmarshal oplog[%v] failed[%v]", raw, err)
                return
            }
            oplogs[i].RawSize = len(raw)
                        //这里是对oplog 做了一些定制化内容
            kafkaOpLog := KafkaOpLog{}
            kafkaOpLog.Namespace = oplogs[i].Namespace
            kafkaOpLog.Query = oplogs[i].Query
            kafkaOpLog.Object = oplogs[i].Object.Map()
            kafkaOpLog.Operation = oplogs[i].Operation
            kafkaOpLog.Timestamp = oplogs[i].Timestamp
 
            msg := &sarama.ProducerMessage{}
            msg.Topic = conf.Options.KafkaTopic
            encode ,err := json.Marshal(kafkaOpLog)
            if err != nil {
                _ = LOG.Error("oplogs bson.MarshalJSON err",err)
                continue
            }
            msg.Value = sarama.StringEncoder(encode)
            msg.Key = sarama.StringEncoder(kafkaOpLog.Namespace)
            _, _, err = kafkaClient.SendMessage(msg)
            if err != nil {
                _ = LOG.Error("send message failed,", err)
                return
            }
                        //原来源码中只是打印了log
            //LOG.Info(oplogs[i]) // just print for test, users can modify to fulfill different needs
        }
 
        if callback := msg.completion; callback != nil {
            callback() // exec callback
        }
 
        // get the newest timestamp
        n := len(oplogs)
        lastTs := utils.TimestampToInt64(oplogs[n-1].Timestamp)
        er.Ack = lastTs
 
        LOG.Debug("handle ack[%v]", er.Ack)
 
        // add logical code below
    }
}