Redis单机数据迁移至Sentinel集群
1 <?xml version="1.0" encoding="UTF-8"?> 2 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 3 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4 <modelVersion>4.0.0</modelVersion> 5 6 <groupId>com.asdc</groupId> 7 <artifactId>multiRedisDatabase</artifactId> 8 <version>0.0.1-SNAPSHOT</version> 9 <packaging>jar</packaging> 10 11 <name>multiRedisDatabase</name> 12 <description>Demo project for Spring Boot</description> 13 14 <parent> 15 <groupId>org.springframework.boot</groupId> 16 <artifactId>spring-boot-starter-parent</artifactId> 17 <version>1.5.16.RELEASE</version> 18 <relativePath /> <!-- lookup parent from repository --> 19 </parent> 20 21 <properties> 22 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> 23 <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> 24 <java.version>1.8</java.version> 25 </properties> 26 27 <dependencies> 28 <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson --> 29 <dependency> 30 <groupId>com.alibaba</groupId> 31 <artifactId>fastjson</artifactId> 32 <version>1.2.47</version> 33 </dependency> 34 35 <dependency> 36 <groupId>org.springframework.boot</groupId> 37 <artifactId>spring-boot-starter-data-redis</artifactId> 38 </dependency> 39 <dependency> 40 <groupId>org.springframework.boot</groupId> 41 <artifactId>spring-boot-starter-web</artifactId> 42 </dependency> 43 44 <dependency> 45 <groupId>org.springframework.boot</groupId> 46 <artifactId>spring-boot-starter-test</artifactId> 47 <scope>test</scope> 48 </dependency> 49 </dependencies> 50 51 <build> 52 <plugins> 53 <plugin> 54 <groupId>org.springframework.boot</groupId> 55 <artifactId>spring-boot-maven-plugin</artifactId> 56 </plugin> 57 </plugins> 58 </build> 59 60 61 </project>
application.properties
# Redis数据库索引(默认为0) spring.redis.database=10 # Redis服务器地址 spring.redis.hostName=127.0.0.1 # Redis服务器连接端口 spring.redis.port=6379 # Redis服务器连接密码(默认为空) spring.redis.password= # 连接池最大连接数(使用负值表示没有限制) spring.redis.pool.max-active=1024 # 连接池最大阻塞等待时间(使用负值表示没有限制) spring.redis.pool.max-wait=-1 # 连接池中的最大空闲连接 spring.redis.pool.max-idle=200 # 连接池中的最小空闲连接 spring.redis.pool.min-idle=0 # 连接超时时间(毫秒) spring.redis.timeout=15000 spring.redis.sentinel.master=mymaster spring.redis.sentinel.nodes=127.0.0.1,127.0.0.1,127.0.0.1 spring.redis.sentinel.port=26379
1 package com.asdc.multiRedisDatabase.config; 2 3 import org.springframework.boot.context.properties.ConfigurationProperties; 4 import org.springframework.stereotype.Component; 5 6 /** 7 * @Description: Redis哨兵配置实体类 8 * @author: Jennifer 9 * @date: 2018年9月27日 上午10:10:34 10 */ 11 @Component 12 @ConfigurationProperties(prefix = "spring.redis.sentinel") 13 public class RedisSentinelProperties { 14 private String master; 15 private String nodes; 16 private Integer port; 17 18 public String getMaster() { 19 return master; 20 } 21 22 public void setMaster(String master) { 23 this.master = master; 24 } 25 26 public String getNodes() { 27 return nodes; 28 } 29 30 public void setNodes(String nodes) { 31 this.nodes = nodes; 32 } 33 34 @Override 35 public String toString() { 36 return "RedisProperties [master=" + master + ", nodes=" + nodes + ", port=" + port + "]"; 37 } 38 39 public void setPort(Integer port) { 40 this.port = port; 41 } 42 43 public Integer getPort() { 44 return port; 45 } 46 47 }
1 package com.asdc.multiRedisDatabase.config; 2 3 import org.springframework.beans.factory.annotation.Value; 4 import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 5 import org.springframework.context.annotation.Bean; 6 import org.springframework.context.annotation.Configuration; 7 import org.springframework.context.annotation.Primary; 8 import org.springframework.data.redis.connection.RedisConnectionFactory; 9 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; 10 import org.springframework.data.redis.core.RedisTemplate; 11 import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; 12 import org.springframework.data.redis.serializer.RedisSerializer; 13 import org.springframework.data.redis.serializer.StringRedisSerializer; 14 15 import com.fasterxml.jackson.annotation.JsonAutoDetect; 16 import com.fasterxml.jackson.annotation.PropertyAccessor; 17 import com.fasterxml.jackson.databind.ObjectMapper; 18 19 import redis.clients.jedis.JedisPoolConfig; 20 21 @Configuration 22 @EnableAutoConfiguration 23 public class RedisAloneConfig { 24 25 @Bean(name = "aloneStringRedisTemplate") 26 @Primary 27 public RedisTemplate<String, Object> redisTemplate(@Value("${spring.redis.hostName}") String hostName, 28 @Value("${spring.redis.port}") int port, @Value("${spring.redis.password}") String password, 29 @Value("${spring.redis.pool.max-idle}") int maxIdle, @Value("${spring.redis.pool.max-active}") int maxTotal, 30 @Value("${spring.redis.database}") int index, @Value("${spring.redis.pool.max-wait}") long maxWaitMillis) { 31 // 设置序列化 32 Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<Object>( 33 Object.class); 34 ObjectMapper om = new ObjectMapper(); 35 om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); 36 om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); 37 jackson2JsonRedisSerializer.setObjectMapper(om); 38 39 RedisTemplate<String, Object> temple = new RedisTemplate<String, Object>(); 40 temple.setConnectionFactory( 41 this.connectionFactory(hostName, port, password, maxIdle, maxTotal, index, maxWaitMillis)); 42 RedisSerializer<?> stringSerializer = new StringRedisSerializer(); 43 temple.setKeySerializer(stringSerializer);// key序列化 44 temple.setValueSerializer(jackson2JsonRedisSerializer);// value序列化 45 temple.setHashKeySerializer(stringSerializer);// Hash key序列化 46 temple.setHashValueSerializer(jackson2JsonRedisSerializer);// Hash value序列化 47 return temple; 48 } 49 50 public RedisConnectionFactory connectionFactory(String hostName, int port, String password, int maxIdle, 51 int maxTotal, int index, long maxWaitMillis) { 52 JedisConnectionFactory jedis = new JedisConnectionFactory(); 53 jedis.setHostName(hostName); 54 jedis.setPort(port); 55 jedis.setPassword(password); 56 jedis.setDatabase(index); 57 jedis.setPoolConfig(this.poolCofig(maxIdle, maxTotal, maxWaitMillis)); 58 // 初始化连接pool 59 jedis.afterPropertiesSet(); 60 RedisConnectionFactory factory = jedis; 61 return factory; 62 } 63 64 public JedisPoolConfig poolCofig(int maxIdle, int maxTotal, long maxWaitMillis) { 65 JedisPoolConfig poolCofig = new JedisPoolConfig(); 66 poolCofig.setMaxIdle(maxIdle); 67 poolCofig.setMaxTotal(maxTotal); 68 poolCofig.setMaxWaitMillis(maxWaitMillis); 69 poolCofig.setTestOnBorrow(false); 70 return poolCofig; 71 } 72 73 }
1 package com.asdc.multiRedisDatabase.config; 2 3 import org.springframework.beans.factory.annotation.Autowired; 4 import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 5 import org.springframework.context.annotation.Bean; 6 import org.springframework.context.annotation.Configuration; 7 import org.springframework.context.annotation.Primary; 8 import org.springframework.data.redis.connection.RedisConnectionFactory; 9 import org.springframework.data.redis.connection.RedisSentinelConfiguration; 10 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; 11 import org.springframework.data.redis.core.RedisTemplate; 12 import org.springframework.data.redis.core.StringRedisTemplate; 13 import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; 14 import org.springframework.data.redis.serializer.RedisSerializer; 15 import org.springframework.data.redis.serializer.StringRedisSerializer; 16 17 import com.fasterxml.jackson.annotation.JsonAutoDetect; 18 import com.fasterxml.jackson.annotation.PropertyAccessor; 19 import com.fasterxml.jackson.databind.ObjectMapper; 20 21 import redis.clients.jedis.JedisPoolConfig; 22 23 @Configuration 24 @EnableAutoConfiguration 25 public class RedisSentinelConfig { 26 27 @Autowired 28 private RedisSentinelProperties properties; 29 30 @Bean(name = "firstRedisTemplate") 31 @Primary 32 public RedisTemplate<Object, Object> redisTemplate() { 33 return getRedisTemplate(getConnectionFactory()); 34 } 35 36 @Bean(name = "firstStringRedisTemplate") 37 @Primary 38 public StringRedisTemplate stringRedisTemplate() { 39 return getStringRedisTemplate(getConnectionFactory()); 40 } 41 42 private RedisTemplate<Object, Object> getRedisTemplate(RedisConnectionFactory connectionFactory) { 43 RedisTemplate<Object, Object> template = new RedisTemplate<Object, Object>(); 44 template.setConnectionFactory(connectionFactory); 45 setSerializer(template); 46 template.afterPropertiesSet(); 47 return template; 48 } 49 50 private void setSerializer(RedisTemplate<Object, Object> template) { 51 Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<Object>( 52 Object.class); 53 ObjectMapper om = new ObjectMapper(); 54 om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); 55 om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); 56 jackson2JsonRedisSerializer.setObjectMapper(om); 57 template.setKeySerializer(jackson2JsonRedisSerializer); 58 template.setValueSerializer(jackson2JsonRedisSerializer); 59 template.setHashValueSerializer(jackson2JsonRedisSerializer); 60 } 61 62 private StringRedisTemplate getStringRedisTemplate(RedisConnectionFactory connectionFactory) { 63 StringRedisTemplate template = new StringRedisTemplate(); 64 template.setConnectionFactory(connectionFactory); 65 setStringSerializer(template); 66 template.afterPropertiesSet(); 67 return template; 68 } 69 70 private void setStringSerializer(StringRedisTemplate template) { 71 RedisSerializer<String> stringSerializer = new StringRedisSerializer(); 72 template.setKeySerializer(stringSerializer); 73 template.setValueSerializer(stringSerializer); 74 template.setHashKeySerializer(stringSerializer); 75 template.setHashValueSerializer(stringSerializer); 76 template.afterPropertiesSet(); 77 } 78 79 @Bean 80 public JedisConnectionFactory getConnectionFactory() { 81 JedisPoolConfig config = new JedisPoolConfig(); 82 JedisConnectionFactory factory = new JedisConnectionFactory(getRedisSentinelConfig(), config); 83 factory.setPoolConfig(config); 84 return factory; 85 } 86 87 @Bean 88 public RedisSentinelConfiguration getRedisSentinelConfig() { 89 RedisSentinelConfiguration sentinelConfiguration = new RedisSentinelConfiguration(); 90 sentinelConfiguration.setMaster(properties.getMaster()); 91 String sentinelHost = properties.getNodes(); 92 Integer sentinelPort = properties.getPort(); 93 String[] hosts = sentinelHost.split(","); 94 for (int i = 0; i < hosts.length; i++) { 95 sentinelConfiguration.sentinel(hosts[i], sentinelPort); 96 } 97 return sentinelConfiguration; 98 } 99 100 }
1 package com.asdc.multiRedisDatabase.service; 2 3 import java.util.List; 4 import java.util.Map; 5 import java.util.Set; 6 import java.util.concurrent.TimeUnit; 7 8 import javax.annotation.PostConstruct; 9 import javax.annotation.Resource; 10 11 import org.springframework.dao.DataAccessException; 12 import org.springframework.data.redis.connection.RedisConnection; 13 import org.springframework.data.redis.core.HashOperations; 14 import org.springframework.data.redis.core.ListOperations; 15 import org.springframework.data.redis.core.RedisCallback; 16 import org.springframework.data.redis.core.RedisTemplate; 17 import org.springframework.data.redis.core.StringRedisTemplate; 18 import org.springframework.data.redis.core.ValueOperations; 19 import org.springframework.data.redis.core.ZSetOperations; 20 import org.springframework.data.redis.serializer.RedisSerializer; 21 import org.springframework.data.redis.serializer.StringRedisSerializer; 22 import org.springframework.stereotype.Component; 23 24 @Component 25 @SuppressWarnings({ "unchecked", "rawtypes" }) 26 public class RedisService { 27 28 @Resource(name = "firstStringRedisTemplate") 29 private StringRedisTemplate stringRedisTemplate; 30 31 @Resource(name = "firstStringRedisTemplate") 32 private ValueOperations<String, String> stringRedisOperations; 33 34 @Resource(name = "firstRedisTemplate") 35 private RedisTemplate redisTemplate; 36 37 @Resource(name = "firstRedisTemplate") 38 private ValueOperations<String, String> redisOperations; 39 40 @Resource(name = "firstStringRedisTemplate") 41 private HashOperations<String, String, String> hMapOps; 42 43 @Resource(name = "firstStringRedisTemplate") 44 ZSetOperations<String, String> zSetOps; 45 46 @Resource(name = "firstRedisTemplate") 47 ListOperations<String, Object> listOps; 48 49 @Resource(name = "aloneStringRedisTemplate") 50 private RedisTemplate<String, Object> aloneRedisTemplate; 51 52 public RedisTemplate<String, Object> getTemplate() { 53 return aloneRedisTemplate; 54 } 55 56 /** 57 * StringRedisTemplate<br> 58 * 放入redis,使用默认的过期时间<br> 59 * 60 * @param key 61 * @param value 62 */ 63 public void setStringValue(final String key, String value) { 64 stringRedisOperations.set(key, value); 65 } 66 67 /** 68 * StringRedisTemplate<br> 69 * 从redis取出value 70 * 71 * @param key 72 * @return 73 */ 74 public String getStringValue(final String key) { 75 return stringRedisOperations.get(key); 76 } 77 78 /** 79 * StringRedisTemplate<br> 80 * 放入redis,并设置过期时间(分钟) 81 * 82 * @param key 83 * @param value 84 * @param expireTime 85 */ 86 public void setStringValue(final String key, String value, Long expireTime) { 87 stringRedisOperations.set(key, value); 88 stringRedisTemplate.expire(key, expireTime, TimeUnit.MINUTES); 89 } 90 91 /** 92 * StringRedisTemplate<br> 93 * 判断是否存在key 94 * 95 * @param key 96 * @return 97 */ 98 public boolean hasKey(final String key) { 99 return stringRedisTemplate.hasKey(key); 100 } 101 102 /** 103 * StringRedisTemplate<br> 104 * 删除相应的value 105 * 106 * @param key 107 */ 108 public void remove(final String key) { 109 stringRedisTemplate.delete(key); 110 } 111 112 public StringRedisTemplate geStringRedisTemplate() { 113 return this.stringRedisTemplate; 114 } 115 116 public RedisTemplate geRedisTemplate() { 117 return this.redisTemplate; 118 } 119 120 /** 121 * StringRedisTemplate<br> 122 * 批量删除对应的value 123 * 124 * @param keys 125 */ 126 public void remove(final String... keys) { 127 for (String key : keys) { 128 remove(key); 129 } 130 } 131 132 /** 133 * StringRedisTemplate<br> 134 * 返回 StringRedisOperation 135 * 136 * @return 137 */ 138 public ValueOperations<String, String> getOperations() { 139 return this.stringRedisOperations; 140 } 141 142 /** 143 * RedisTemplate<br> 144 * 获取对象类型的值 145 * 146 * @param key 147 * @return 148 */ 149 public Object getObjectValue(String key) { 150 return redisOperations.get(key); 151 } 152 153 /** 154 * RedisTemplate<br> 155 * 添加对象类型数据 156 * 157 * @param key 158 * @param object 159 */ 160 public void setObjectValue(String key, Object object) { 161 redisOperations.set(key, (String) object); 162 } 163 164 /** 165 * RedisTemplate<br> 166 * 167 * @param key 168 * @param object 169 * @param expireTime 170 */ 171 /* @SuppressWarnings("unchecked") 172 public void setObjectValue(String key,Object object ,Long expireTime){ 173 redisOperations.set(key, object); 174 redisTemplate.expire(key, expireTime, TimeUnit.MINUTES); 175 }*/ 176 177 public void increment(String key, Long delta) { 178 redisOperations.increment(key, delta); 179 } 180 181 public void expire(String key, Long expireTime) { 182 redisTemplate.expire(key, expireTime, TimeUnit.MINUTES); 183 } 184 185 public void expire(String key, Long expireTime, TimeUnit unit) { 186 stringRedisTemplate.expire(key, expireTime, unit); 187 } 188 189 190 public Set getKeysByPattern(String pattern) { 191 return redisTemplate.keys(pattern); 192 } 193 194 public void setHash(String key, String hashKey, String value) { 195 hMapOps.put(key, hashKey, value); 196 } 197 198 public String getHashValue(String key, String mapKey) { 199 return hMapOps.get(key, mapKey); 200 } 201 202 public Map<String, String> getHashAll(String key) { 203 return hMapOps.entries(key); 204 } 205 206 public long setHashIncrement(String key, String hashKey, long counts) { 207 return hMapOps.increment(key, hashKey, counts); 208 } 209 210 public long size(String key) { 211 return hMapOps.size(key); 212 } 213 214 public long lookTimeOut(String key, TimeUnit timeUnit) { 215 return stringRedisTemplate.getExpire(key, timeUnit); 216 } 217 218 public boolean zAddZsetElement(String key, String value, long score) { 219 return zSetOps.add(key, value, score); 220 } 221 222 public Double getZsetScore(String key, String value) { 223 return zSetOps.score(key, value); 224 } 225 226 public long getListSize(String key) { 227 return listOps.size(key); 228 } 229 230 public List<Object> getList(String key) { 231 return listOps.range(key, 0L, -1L); 232 } 233 234 public void addList(String key, List<Object> values) { 235 if (values == null || values.size() == 0) { 236 return; 237 } 238 239 listOps.rightPushAll(key, values); 240 } 241 242 /** 243 * 获取自增值 244 * 245 * @param key 246 * @return 247 */ 248 public int getIncrValue(final String key) { 249 250 return stringRedisTemplate.execute(new RedisCallback<Integer>() { 251 @Override 252 public Integer doInRedis(RedisConnection connection) throws DataAccessException { 253 RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer(); 254 byte[] rowkey = serializer.serialize(key); 255 byte[] rowval = connection.get(rowkey); 256 try { 257 String val = serializer.deserialize(rowval); 258 return Integer.parseInt(val); 259 } catch (Exception e) { 260 return 0; 261 } 262 } 263 }); 264 } 265 266 @PostConstruct 267 public void init() { 268 redisTemplate.setKeySerializer(new StringRedisSerializer()); 269 redisTemplate.setValueSerializer(new StringRedisSerializer()); 270 } 271 272 public long delOfflineContent(String contentId, String listId) { 273 return listOps.remove(listId, 1, contentId); 274 } 275 }
1 package com.asdc.multiRedisDatabase; 2 3 import org.springframework.boot.SpringApplication; 4 import org.springframework.boot.autoconfigure.SpringBootApplication; 5 6 @SpringBootApplication 7 public class MultiRedisDatabaseApplication { 8 9 public static void main(String[] args) { 10 SpringApplication.run(MultiRedisDatabaseApplication.class, args); 11 } 12 }
1 package com.asdc.multiRedisDatabase.controller; 2 3 import java.util.Date; 4 import java.util.Iterator; 5 import java.util.Set; 6 7 import org.springframework.beans.factory.annotation.Autowired; 8 import org.springframework.data.redis.core.RedisTemplate; 9 import org.springframework.web.bind.annotation.RequestMapping; 10 import org.springframework.web.bind.annotation.ResponseBody; 11 import org.springframework.web.bind.annotation.RestController; 12 13 import com.asdc.multiRedisDatabase.service.RedisService; 14 15 /** 16 * @Description: 数据同步控制器 17 * @author: Jennifer 18 * @date: 2018年9月27日 上午11:25:24 19 */ 20 @RestController 21 public class SyncController { 22 @Autowired 23 private RedisService redisService; 24 25 private static Long EXPIRE_TIME = 61 * 24 * 60L; 26 27 @RequestMapping("/sync") 28 @ResponseBody 29 public String sync() { 30 RedisTemplate<String, Object> template = redisService.getTemplate(); 31 Set<String> keys = template.keys("*"); 32 int count = 0; 33 for (Iterator<?> iterator = keys.iterator(); iterator.hasNext();) { 34 String key = (String) iterator.next(); 35 try { 36 Integer value = (Integer) template.opsForValue().get(key); 37 if (value != null) { 38 redisService.setStringValue(key, value.toString(), EXPIRE_TIME); 39 count++; 40 } 41 } catch (Exception e) { 42 } 43 } 44 return String.format("单机Redis总记录数: %s ,Sentinel有效导入数: %s ,记录时间: %s", String.valueOf(keys.size()), count, 45 new Date()); 46 } 47 }
1 package com.asdc.multiRedisDatabase; 2 3 import java.util.Date; 4 import java.util.Iterator; 5 import java.util.Set; 6 7 import org.junit.Test; 8 import org.junit.runner.RunWith; 9 import org.slf4j.Logger; 10 import org.slf4j.LoggerFactory; 11 import org.springframework.beans.factory.annotation.Autowired; 12 import org.springframework.boot.test.context.SpringBootTest; 13 import org.springframework.data.redis.core.RedisTemplate; 14 import org.springframework.test.context.junit4.SpringRunner; 15 16 import com.asdc.multiRedisDatabase.config.RedisSentinelConfig; 17 import com.asdc.multiRedisDatabase.service.RedisService; 18 19 @RunWith(SpringRunner.class) 20 @SpringBootTest 21 public class MultiRedisDatabaseApplicationTests { 22 @Autowired 23 private RedisService redisService; 24 25 private static Long EXPIRE_TIME = 61 * 24 * 60L; 26 27 private static Logger logger = LoggerFactory.getLogger(RedisSentinelConfig.class); 28 29 @Test 30 public void getKeys() { 31 RedisTemplate<String, Object> template = redisService.getTemplate(); 32 Set<String> keys = template.keys("*"); 33 int count = 0; 34 for (Iterator<?> iterator = keys.iterator(); iterator.hasNext();) { 35 String key = (String) iterator.next(); 36 try { 37 Integer value = (Integer) template.opsForValue().get(key); 38 if (value != null) { 39 logger.info(count++ + ":" + key + " = " + value); 40 redisService.setStringValue(key, value.toString(), EXPIRE_TIME); 41 } 42 } catch (Exception e) { 43 } 44 } 45 logger.info(String.format("总记录数: %s ,有效导入数: %s ,记录时间: %s", String.valueOf(keys.size()), count, new Date())); 46 } 47 48 }