Flink读写Redis（二）-flink-redis-connector代码学习
源码结构
RedisSink
package org.apache.flink.streaming.connectors.redis;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisClusterConfig;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisConfigBase;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisSentinelConfig;
import org.apache.flink.streaming.connectors.redis.common.container.RedisCommandsContainer;
import org.apache.flink.streaming.connectors.redis.common.container.RedisCommandsContainerBuilder;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisDataType;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* A sink that delivers data to a Redis channel using the Jedis client.
* <p> The sink takes two arguments {@link FlinkJedisConfigBase} and {@link RedisMapper}.
* <p> When {@link FlinkJedisPoolConfig} is passed as the first argument,
* the sink will create connection using {@link redis.clients.jedis.JedisPool}. Please use this when
* you want to connect to a single Redis server.
* <p> When {@link FlinkJedisSentinelConfig} is passed as the first argument, the sink will create connection
* using {@link redis.clients.jedis.JedisSentinelPool}. Please use this when you want to connect to Sentinel.
* <p> Please use {@link FlinkJedisClusterConfig} as the first argument if you want to connect to
* a Redis Cluster.
*
* <p>Example:
*
* <pre>
*{@code
*public static class RedisExampleMapper implements RedisMapper<Tuple2<String, String>> {
*
* private RedisCommand redisCommand;
*
* public RedisExampleMapper(RedisCommand redisCommand){
* this.redisCommand = redisCommand;
* }
* public RedisCommandDescription getCommandDescription() {
* return new RedisCommandDescription(redisCommand, REDIS_ADDITIONAL_KEY);
* }
* public String getKeyFromData(Tuple2<String, String> data) {
* return data.f0;
* }
* public String getValueFromData(Tuple2<String, String> data) {
* return data.f1;
* }
*}
*JedisPoolConfig jedisPoolConfig = new JedisPoolConfig.Builder()
* .setHost(REDIS_HOST).setPort(REDIS_PORT).build();
*new RedisSink<String>(jedisPoolConfig, new RedisExampleMapper(RedisCommand.LPUSH));
*}</pre>
*
* @param <IN> Type of the elements emitted by this sink
*/
public class RedisSink<IN> extends RichSinkFunction<IN> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(RedisSink.class);
/**
* This additional key needed for {@link RedisDataType#HASH} and {@link RedisDataType#SORTED_SET}.
* Other {@link RedisDataType} works only with two variable i.e. name of the list and value to be added.
* But for {@link RedisDataType#HASH} and {@link RedisDataType#SORTED_SET} we need three variables.
* <p>For {@link RedisDataType#HASH} we need hash name, hash key and element.
* {@code additionalKey} used as hash name for {@link RedisDataType#HASH}
* <p>For {@link RedisDataType#SORTED_SET} we need set name, the element and it's score.
* {@code additionalKey} used as set name for {@link RedisDataType#SORTED_SET}
*/
private String additionalKey;
private RedisMapper<IN> redisSinkMapper;
private RedisCommand redisCommand;
private FlinkJedisConfigBase flinkJedisConfigBase;
private RedisCommandsContainer redisCommandsContainer;
/**
* Creates a new {@link RedisSink} that connects to the Redis server.
*
* @param flinkJedisConfigBase The configuration of {@link FlinkJedisConfigBase}
* @param redisSinkMapper This is used to generate Redis command and key value from incoming elements.
*/
public RedisSink(FlinkJedisConfigBase flinkJedisConfigBase, RedisMapper<IN> redisSinkMapper) {
Preconditions.checkNotNull(flinkJedisConfigBase, "Redis connection pool config should not be null");
Preconditions.checkNotNull(redisSinkMapper, "Redis Mapper can not be null");
Preconditions.checkNotNull(redisSinkMapper.getCommandDescription(), "Redis Mapper data type description can not be null");
this.flinkJedisConfigBase = flinkJedisConfigBase;
this.redisSinkMapper = redisSinkMapper;
RedisCommandDescription redisCommandDescription = redisSinkMapper.getCommandDescription();
this.redisCommand = redisCommandDescription.getCommand();
this.additionalKey = redisCommandDescription.getAdditionalKey();
}
/**
* Called when new data arrives to the sink, and forwards it to Redis channel.
* Depending on the specified Redis data type (see {@link RedisDataType}),
* a different Redis command will be applied.
* Available commands are RPUSH, LPUSH, SADD, PUBLISH, SET, PFADD, HSET, ZADD.
*
* @param input The incoming data
*/
@Override
public void invoke(IN input) throws Exception {
String key = redisSinkMapper.getKeyFromData(input);
String value = redisSinkMapper.getValueFromData(input);
switch (redisCommand) {
case RPUSH:
this.redisCommandsContainer.rpush(key, value);
break;
case LPUSH:
this.redisCommandsContainer.lpush(key, value);
break;
case SADD:
this.redisCommandsContainer.sadd(key, value);
break;
case SET:
this.redisCommandsContainer.set(key, value);
break;
case PFADD:
this.redisCommandsContainer.pfadd(key, value);
break;
case PUBLISH:
this.redisCommandsContainer.publish(key, value);
break;
case ZADD:
this.redisCommandsContainer.zadd(this.additionalKey, value, key);
break;
case HSET:
this.redisCommandsContainer.hset(this.additionalKey, key, value);
break;
default:
throw new IllegalArgumentException("Cannot process such data type: " + redisCommand);
}
}
/**
* Initializes the connection to Redis by either cluster or sentinels or single server.
*
* @throws IllegalArgumentException if jedisPoolConfig, jedisClusterConfig and jedisSentinelConfig are all null
*/
@Override
public void open(Configuration parameters) throws Exception {
this.redisCommandsContainer = RedisCommandsContainerBuilder.build(this.flinkJedisConfigBase);
}
/**
* Closes commands container.
* @throws IOException if command container is unable to close.
*/
@Override
public void close() throws IOException {
if (redisCommandsContainer != null) {
redisCommandsContainer.close();
}
}
}
RedisSink类继承了RichSinkFunction类，扩展了其中的open、invoke、close方法。open方法在sink打开时执行一次，在RedisSink中，其创建了一个RedisCommandsContainer对象，该对象其实是封装了对redis的操作，包含连接redis以及不同数据类型的写入操作；close方法中执行了RedisCommandsContainer对象的close方法，其实时关闭redis连接；每当数据流入时，就会调用invoke方法，该方法中根据不同的redis数据类型，调用RedisCommandsContainer对象的不同方法将数据写入redis。
FlinkJedisConfigBase
RedisSink的构造方法中需要传入一个FlinkJedisConfigBase对象，该对象主要是用来设置一些redis连接参数，比如IP、用户、密码、连接超时等信息，在后续创建RedisCommandsContainer对象时使用，FlinkJedisConfigBase是一个抽象类，具体的实现类有FlinkJedisPoolConfig、FlinkJedisSentinelConfig、FlinkJedisClusterConfig三种，分别对应了redis、redis哨兵、redis集群不同的连接模式，比较简单，就不贴代码了。
RedisMapper接口
RedisSink的构造方法中还需要RedisMapper实现对象，需要用户自定义类实现该接口，主要是用来定义数据如何映射成redis的key和value，另外是返回一个RedisDataTypeDescription对象，该对象其实是包含了操作的redis数据类型信息
package org.apache.flink.streaming.connectors.redis.common.mapper;
import org.apache.flink.api.common.functions.Function;
import java.io.Serializable;
/**
* Function that creates the description how the input data should be mapped to redis type.
*<p>Example:
*<pre>{@code
*private static class RedisTestMapper implements RedisMapper<Tuple2<String, String>> {
* public RedisDataTypeDescription getCommandDescription() {
* return new RedisDataTypeDescription(RedisCommand.PUBLISH);
* }
* public String getKeyFromData(Tuple2<String, String> data) {
* return data.f0;
* }
* public String getValueFromData(Tuple2<String, String> data) {
* return data.f1;
* }
*}
*}</pre>
*
* @param <T> The type of the element handled by this {@code RedisMapper}
*/
public interface RedisMapper<T> extends Function, Serializable {
/**
* Returns descriptor which defines data type.
*
* @return data type descriptor
*/
RedisCommandDescription getCommandDescription();
/**
* Extracts key from data.
*
* @param data source data
* @return key
*/
String getKeyFromData(T data);
/**
* Extracts value from data.
*
* @param data source data
* @return value
*/
String getValueFromData(T data);
}
RedisCommandDescription、RedisDataType、RedisCommand
RedisCommandDescription、RedisDataType、RedisCommand三个类用了表示操作的redis数据类型，比较简单，不在描述，支持的数据类型在RedisDataType枚举中
package org.apache.flink.streaming.connectors.redis.common.mapper;
/**
* All available data type for Redis.
*/
public enum RedisDataType {
/**
* Strings are the most basic kind of Redis value. Redis Strings are binary safe,
* this means that a Redis string can contain any kind of data, for instance a JPEG image or a serialized Ruby object.
* A String value can be at max 512 Megabytes in length.
*/
STRING,
/**
* Redis Hashes are maps between string fields and string values.
*/
HASH,
/**
* Redis Lists are simply lists of strings, sorted by insertion order.
*/
LIST,
/**
* Redis Sets are an unordered collection of Strings.
*/
SET,
/**
* Redis Sorted Sets are, similarly to Redis Sets, non repeating collections of Strings.
* The difference is that every member of a Sorted Set is associated with score,
* that is used in order to take the sorted set ordered, from the smallest to the greatest score.
* While members are unique, scores may be repeated.
*/
SORTED_SET,
/**
* HyperLogLog is a probabilistic data structure used in order to count unique things.
*/
HYPER_LOG_LOG,
/**
* Redis implementation of publish and subscribe paradigm. Published messages are characterized into channels,
* without knowledge of what (if any) subscribers there may be.
* Subscribers express interest in one or more channels, and only receive messages
* that are of interest, without knowledge of what (if any) publishers there are.
*/
PUBSUB
}
RedisCommandsContainer
RedisCommandsContainer接口定义了redis操作的方法，具体的实现类有RedisContainer、RedisClusterContainer，其中RedisContainer是用在直连redis和redis哨兵模式中，而RedisClusterContainer是用在集群模式中，具体代码不贴了，里面主要是调用了Jedis的API。
package org.apache.flink.streaming.connectors.redis.common.container;
import java.io.IOException;
import java.io.Serializable;
/**
* The container for all available Redis commands.
*/
public interface RedisCommandsContainer extends Serializable {
/**
* Sets field in the hash stored at key to value.
* If key does not exist, a new key holding a hash is created.
* If field already exists in the hash, it is overwritten.
*
* @param key Hash name
* @param hashField Hash field
* @param value Hash value
*/
void hset(String key, String hashField, String value);
/**
* Insert the specified value at the tail of the list stored at key.
* If key does not exist, it is created as empty list before performing the push operation.
*
* @param listName Name of the List
* @param value Value to be added
*/
void rpush(String listName, String value);
/**
* Insert the specified value at the head of the list stored at key.
* If key does not exist, it is created as empty list before performing the push operation.
*
* @param listName Name of the List
* @param value Value to be added
*/
void lpush(String listName, String value);
/**
* Add the specified member to the set stored at key.
* Specified members that are already a member of this set are ignored.
* If key does not exist, a new set is created before adding the specified members.
*
* @param setName Name of the Set
* @param value Value to be added
*/
void sadd(String setName, String value);
/**
* Posts a message to the given channel.
*
* @param channelName Name of the channel to which data will be published
* @param message the message
*/
void publish(String channelName, String message);
/**
* Set key to hold the string value. If key already holds a value, it is overwritten,
* regardless of its type. Any previous time to live associated with the key is
* discarded on successful SET operation.
*
* @param key the key name in which value to be set
* @param value the value
*/
void set(String key, String value);
/**
* Adds all the element arguments to the HyperLogLog data structure
* stored at the variable name specified as first argument.
*
* @param key The name of the key
* @param element the element
*/
void pfadd(String key, String element);
/**
* Adds the specified member with the specified scores to the sorted set stored at key.
*
* @param key The name of the Sorted Set
* @param score Score of the element
* @param element element to be added
*/
void zadd(String key, String score, String element);
/**
* Close the Jedis container.
*
* @throws IOException if the instance can not be closed properly
*/
void close() throws IOException;
}
RedisCommandsContainerBuilder
RedisCommandsContainerBuilder类是根据不同的FlinkJedisConfigBase实现类来创建不同的RedisCommandsContainer对象
package org.apache.flink.streaming.connectors.redis.common.container;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisClusterConfig;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisConfigBase;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisSentinelConfig;
import org.apache.flink.util.Preconditions;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisSentinelPool;
/**
* The builder for {@link RedisCommandsContainer}.
*/
public class RedisCommandsContainerBuilder {
/**
* Initialize the {@link RedisCommandsContainer} based on the instance type.
* @param flinkJedisConfigBase configuration base
* @return @throws IllegalArgumentException if jedisPoolConfig, jedisClusterConfig and jedisSentinelConfig are all null
*/
public static RedisCommandsContainer build(FlinkJedisConfigBase flinkJedisConfigBase){
if(flinkJedisConfigBase instanceof FlinkJedisPoolConfig){
FlinkJedisPoolConfig flinkJedisPoolConfig = (FlinkJedisPoolConfig) flinkJedisConfigBase;
return RedisCommandsContainerBuilder.build(flinkJedisPoolConfig);
} else if (flinkJedisConfigBase instanceof FlinkJedisClusterConfig) {
FlinkJedisClusterConfig flinkJedisClusterConfig = (FlinkJedisClusterConfig) flinkJedisConfigBase;
return RedisCommandsContainerBuilder.build(flinkJedisClusterConfig);
} else if (flinkJedisConfigBase instanceof FlinkJedisSentinelConfig) {
FlinkJedisSentinelConfig flinkJedisSentinelConfig = (FlinkJedisSentinelConfig) flinkJedisConfigBase;
return RedisCommandsContainerBuilder.build(flinkJedisSentinelConfig);
} else {
throw new IllegalArgumentException("Jedis configuration not found");
}
}
/**
* Builds container for single Redis environment.
*
* @param jedisPoolConfig configuration for JedisPool
* @return container for single Redis environment
* @throws NullPointerException if jedisPoolConfig is null
*/
public static RedisCommandsContainer build(FlinkJedisPoolConfig jedisPoolConfig) {
Preconditions.checkNotNull(jedisPoolConfig, "Redis pool config should not be Null");
GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
genericObjectPoolConfig.setMaxIdle(jedisPoolConfig.getMaxIdle());
genericObjectPoolConfig.setMaxTotal(jedisPoolConfig.getMaxTotal());
genericObjectPoolConfig.setMinIdle(jedisPoolConfig.getMinIdle());
JedisPool jedisPool = new JedisPool(genericObjectPoolConfig, jedisPoolConfig.getHost(),
jedisPoolConfig.getPort(), jedisPoolConfig.getConnectionTimeout(), jedisPoolConfig.getPassword(),
jedisPoolConfig.getDatabase());
return new RedisContainer(jedisPool);
}
/**
* Builds container for Redis Cluster environment.
*
* @param jedisClusterConfig configuration for JedisCluster
* @return container for Redis Cluster environment
* @throws NullPointerException if jedisClusterConfig is null
*/
public static RedisCommandsContainer build(FlinkJedisClusterConfig jedisClusterConfig) {
Preconditions.checkNotNull(jedisClusterConfig, "Redis cluster config should not be Null");
GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
genericObjectPoolConfig.setMaxIdle(jedisClusterConfig.getMaxIdle());
genericObjectPoolConfig.setMaxTotal(jedisClusterConfig.getMaxTotal());
genericObjectPoolConfig.setMinIdle(jedisClusterConfig.getMinIdle());
JedisCluster jedisCluster = new JedisCluster(jedisClusterConfig.getNodes(), jedisClusterConfig.getConnectionTimeout(),
jedisClusterConfig.getMaxRedirections(), genericObjectPoolConfig);
return new RedisClusterContainer(jedisCluster);
}
/**
* Builds container for Redis Sentinel environment.
*
* @param jedisSentinelConfig configuration for JedisSentinel
* @return container for Redis sentinel environment
* @throws NullPointerException if jedisSentinelConfig is null
*/
public static RedisCommandsContainer build(FlinkJedisSentinelConfig jedisSentinelConfig) {
Preconditions.checkNotNull(jedisSentinelConfig, "Redis sentinel config should not be Null");
GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
genericObjectPoolConfig.setMaxIdle(jedisSentinelConfig.getMaxIdle());
genericObjectPoolConfig.setMaxTotal(jedisSentinelConfig.getMaxTotal());
genericObjectPoolConfig.setMinIdle(jedisSentinelConfig.getMinIdle());
JedisSentinelPool jedisSentinelPool = new JedisSentinelPool(jedisSentinelConfig.getMasterName(),
jedisSentinelConfig.getSentinels(), genericObjectPoolConfig,
jedisSentinelConfig.getConnectionTimeout(), jedisSentinelConfig.getSoTimeout(),
jedisSentinelConfig.getPassword(), jedisSentinelConfig.getDatabase());
return new RedisContainer(jedisSentinelPool);
}
}
整体下来，flink-redis-connector源码比较简洁，可以作为自定义flink sink的入门学习。
