MongoDB
获取mongo配置代码
package com.kingdee.mongo; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; @Component @PropertySource(value={"classpath:common.properties"}) public class MongoConfig { @Value("${mongo.host}") private String host; @Value("${mongo.port}") private String port; public String user; public String getUser() { return user; } public void setUser(String user) { this.user = user; } public String getPass() { return pass; } public void setPass(String pass) { this.pass = pass; } public String pass; @Value("${mongo.dbname}") private String database; @Value("${mongo.maxConnect}") public String maxConnect; @Value("${mongo.maxWaitThread}") public String maxWaitThread; @Value("${mongo.maxTimeOut}") public String maxTimeOut; @Value("${mongo.maxWaitTime}") public String maxWaitTime; public String getDatabase() { return database; } public void setDatabase(String database) { this.database = database; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public String getPort() { return port; } public void setPort(String port) { this.port = port; } }
生成连接数据库实例代码
package com.kingdee.mongo; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang.StringUtils; import com.mongodb.DB; import com.mongodb.MongoClient; import com.mongodb.ServerAddress; public class MongoFactory { public static DB getMongoDB(String tag, String mongoHost, int mongoPort,String mongoDbName, String mongoUser, String mongoPassword) throws Exception { List<ServerAddress> replicaSetSeeds = new ArrayList<ServerAddress>(); if (StringUtils.isNotEmpty(mongoHost) && !StringUtils.startsWith(mongoHost, "${") && mongoPort > 0) { replicaSetSeeds.add(new ServerAddress(mongoHost, mongoPort)); } DB db; if (!replicaSetSeeds.isEmpty() && StringUtils.isNotEmpty(mongoDbName) && !StringUtils.startsWith(mongoDbName, "${")) { MongoClient client = new MongoClient(replicaSetSeeds); db = client.getDB(mongoDbName); } else { db = null; } return db; } }
构建客户端实例代码
package com.kingdee.mongo; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import com.kingdee.redis.JedisUtils; import com.mongodb.DB; import com.mongodb.DBCollection; public class MongoDBClient { private static Logger logger = LoggerFactory.getLogger(MongoDBClient.class); @Autowired private MongoConfig mongoConfig; private static DB mongoDB; protected void getMongoClient(){ try { String mongoHost = mongoConfig.getHost(); String mongoPort = mongoConfig.getPort(); String mongoDbName = mongoConfig.getDatabase(); String mongoUser = mongoConfig.getUser(); String mongoPassword = mongoConfig.getPass(); mongoDB = MongoFactory.getMongoDB("data", mongoHost, Integer.valueOf(mongoPort),mongoDbName, mongoUser, mongoPassword); } catch (Exception ignore) { mongoDB = null; logger.warn(ignore.getMessage(), ignore); } } public DBCollection getCollection(String collectionName) { if (mongoDB != null) { return mongoDB.getCollection(collectionName); }else{ getMongoClient(); return mongoDB.getCollection(collectionName); } } protected void close() { if (mongoDB != null) { try { mongoDB.getMongo().close(); } catch (Throwable ignore) { } } mongoDB = null; } }
操作mongo数据代码
package com.kingdee.dao.impl; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.util.TypeUtils; import com.kingdee.mongo.MongoDBClient; import com.kingdee.util.ThreadLocalLog; import com.mongodb.BasicDBObject; import com.mongodb.DBCollection; import com.mongodb.DBCursor; import com.mongodb.DBObject; import com.mongodb.ReadPreference; import com.mongodb.WriteResult; import com.mongodb.util.JSON; import net.sf.json.JSONObject; public class BaseDao { private static Logger logger = LoggerFactory.getLogger(BaseDao.class); private Map<String, DBCollection> collections = new HashMap<String, DBCollection>(); private Lock lock = new ReentrantLock(); @Autowired private MongoDBClient mongoDBClient; /**默认都是从副本集操作数据 * @param collectionName * @return */ public DBCollection getCollection(String collectionName) { DBCollection collection = collections.get(collectionName); if (collection == null) { lock.lock(); try { collection = collections.get(collectionName); if (collection == null) { collection = mongoDBClient.getCollection(collectionName); if (collection != null) { collection.setReadPreference(ReadPreference.secondaryPreferred()); collections.put(collectionName, collection); } } } finally { lock.unlock(); } } return collection; } DBObject findOne(String collectionName, String id) { return findOne(collectionName, new BasicDBObject("_id", id), null, true); } DBObject findOne(String collectionName, DBObject query, DBObject orderBy, boolean secondary) { return findOne(collectionName, query, null, orderBy, secondary); } DBObject findOne(String collectionName, DBObject query, DBObject fields, DBObject orderBy, boolean secondary) { DBCollection collection = getCollection(collectionName); if (collection == null) return null; DBObject rtn = null; long begin = System.currentTimeMillis(); try { if (secondary) { try { rtn = collection.findOne(query, fields, orderBy); } catch (Throwable ignore) { logger.warn("findOne at secondary error! collection:" + collectionName + " query:" + query, ignore); } } if (rtn == null) { try { rtn = collection.findOne(query, fields, orderBy, ReadPreference.primaryPreferred()); } catch (Throwable ignore) { logger.warn("findOne at primary error! collection:" + collectionName + " query:" + query, ignore); } } } finally { long t = System.currentTimeMillis() - begin; if (t > 100) { logger.info("--"+"MongoDao.findOne"+"collection:"+collectionName+"query:"+query+"cost:"+t); } } return rtn; } void removeById(String collectionName, String id) { remove(collectionName, new BasicDBObject("_id", id)); } void remove(String collectionName, DBObject query) { DBCollection collection = getCollection(collectionName); if (collection == null) return; try { collection.remove(query); } catch (Throwable ignore) { logger.warn("remove error! collection:" + collectionName + " query:" + query, ignore); } } List<DBObject> findByIds(String collectionName, String... ids) { if (ids == null || ids.length == 0) return null; DBObject query = new BasicDBObject(); query.put("_id", new BasicDBObject("$in", ids)); return find(collectionName, query, null, 0, 0); } List<DBObject> find(String collectionName, DBObject query, DBObject orderBy, int skip, int limit) { return find(collectionName, query, null, orderBy, skip, limit); } List<DBObject> find(String collectionName, DBObject query, DBObject keys, DBObject orderBy, int skip, int limit) { DBCollection collection = getCollection(collectionName); if (collection == null) return null; DBCursor cursor = null; List<DBObject> rtn = new ArrayList<DBObject>(); long begin = System.currentTimeMillis(); try { cursor = collection.find(query, keys); if (orderBy != null) cursor = cursor.sort(orderBy); if (skip > 0) cursor = cursor.skip(skip); if (limit > 0) cursor = cursor.limit(limit); while (cursor.hasNext()) { DBObject next = cursor.next(); if (next != null) rtn.add(next); } } catch (Throwable ignore) { logger.warn("find error! collection:" + collectionName + " query:" + query, ignore); } finally { try { if (cursor != null) cursor.close(); } catch (Throwable ignore) { } long t = System.currentTimeMillis() - begin; if (t > 100) { boolean b = ThreadLocalLog.addLog("--", "MongoDao.find", "collection:" + collectionName, "query:" + query, "skip:" + skip, "limit:" + limit, "cost:" + t); if (!b) { LoggerFactory.getLogger("Slow").info("MongoDao.find collection:{} query:{} skip:{} limit:{} cost:{}", collectionName, query, skip, limit, t); } } } return rtn; } boolean save(String collectionName, Object bean, String idKey) { DBObject dbObject = covertToDBObject(bean, idKey); return save(collectionName, dbObject); } boolean save(String collectionName, DBObject object) { DBCollection collection = getCollection(collectionName); if (collection == null) return false; long begin = System.currentTimeMillis(); try { collection.save(object); return true; } catch (Throwable ignore) { logger.warn("save error! collection:" + collectionName + " object:" + object, ignore); } finally { long t = System.currentTimeMillis() - begin; if (t > 100) { boolean b = ThreadLocalLog.addLog("--", "MongoDao.save", "collection:" + collectionName, "object:" + object, "cost:" + t); if (!b) { LoggerFactory.getLogger("Slow").info("MongoDao.save collection:{} object:{} cost:{}", collectionName, object, t); } } } return false; } boolean update(String collectionName, DBObject query, DBObject object) { DBCollection collection = getCollection(collectionName); if (collection == null) return false; long begin = System.currentTimeMillis(); try { WriteResult result = collection.update(query, new BasicDBObject("$set", object)); return result != null && result.isUpdateOfExisting(); } catch (Throwable ignore) { logger.warn("update error! collection:" + collectionName + " object:" + object, ignore); } finally { long t = System.currentTimeMillis() - begin; if (t > 100) { boolean b = ThreadLocalLog.addLog("--", "MongoDao.update", "collection:" + collectionName, "query:" + query, "object:" + object, "cost:" + t); if (!b) { LoggerFactory.getLogger("Slow").info("MongoDao.update collection:{} query:{} object:{} cost:{}", collectionName, query, object, t); } } } return false; } /** * * @param collectionName * @param query 查询条件 * @param object 修改 * @param upsert 如果匹配不到记录,是否新增一条 * @param multi false 一次更新一条;true 更新匹配到的所有记录 * @return */ boolean batchUpdate(String collectionName, DBObject query, DBObject object, boolean upsert, boolean multi) { DBCollection collection = getCollection(collectionName); if (collection == null) return false; long begin = System.currentTimeMillis(); try { WriteResult result = collection.update(query, new BasicDBObject("$set", object), upsert, multi); return result != null && result.isUpdateOfExisting(); } catch (Throwable ignore) { logger.warn("update error! collection:" + collectionName + " object:" + object, ignore); } finally { long t = System.currentTimeMillis() - begin; if (t > 100) { boolean b = ThreadLocalLog.addLog("--", "MongoDao.update", "collection:" + collectionName, "query:" + query, "object:" + object, "cost:" + t); if (!b) { LoggerFactory.getLogger("Slow").info("MongoDao.update collection:{} query:{} object:{} cost:{}", collectionName, query, object, t); } } } return false; } long count(String collectionName) { DBCollection collection = getCollection(collectionName); if (collection == null) return 0L; try { return collection.count(); } catch (Throwable ignore) { logger.warn("count error! collection:" + collectionName, ignore); } return 0L; } /**按照条件进行数量统计,从副本里面统计 * @param collectionName * @param query * @return */ long count(String collectionName, DBObject query) { DBCollection collection = getCollection(collectionName); if (collection == null) return 0L; try { return collection.count(query); } catch (Throwable ignore) { logger.warn("count error! collection:" + collectionName, ignore); } return 0L; } protected DBObject covertToDBObject(Object bean, String idKey) { JSONObject jsonObject = JSONObject.fromObject(bean); DBObject dbObject = new BasicDBObject(); for (Object key : jsonObject.keySet()) { if (key == null || jsonObject.get(key)==null) continue; dbObject.put(key.toString(), jsonObject.get(key)); } if (org.apache.commons.lang.StringUtils.isNotEmpty(idKey)) { Object id = dbObject.get(idKey); dbObject.removeField(idKey); dbObject.put("_id", id); } return dbObject; } /** * 新增/修改 * @param collectionName * @param query * @param object * @param upsert 如果记录不存在,是否新增一条记录 * @param multi 是否批量修改满足条件的记录 * @return */ boolean saveOrUpdate(String collectionName, DBObject query, Object bean, boolean upsert, boolean multi) { DBCollection collection = getCollection(collectionName); if (collection == null) return false; DBObject object = covertToDBObject(bean, null); long begin = System.currentTimeMillis(); try { WriteResult result = collection.update(query, new BasicDBObject("$set", object), upsert, multi); return result != null && result.isUpdateOfExisting(); } catch (Throwable ignore) { logger.warn("saveOrUpdate error! collection:" + collectionName + " object:" + object, ignore); } finally { long t = System.currentTimeMillis() - begin; if (t > 100) { boolean b = ThreadLocalLog.addLog("--", "MongoDao.saveOrUpdate", "collection:" + collectionName, "query:" + query, "object:" + object, "cost:" + t); if (!b) { LoggerFactory.getLogger("Slow").info("MongoDao.saveOrUpdate collection:{} query:{} object:{} cost:{}", collectionName, query, object, t); } } } return false; } /** * 删除字段 * @param collectionName * @param query * @param object * @return */ boolean unset(String collectionName, DBObject query, DBObject object) { DBCollection collection = getCollection(collectionName); if (collection == null) return false; long begin = System.currentTimeMillis(); try { WriteResult result = collection.update(query, new BasicDBObject("$unset", object)); return result != null && result.isUpdateOfExisting(); } catch (Throwable ignore) { logger.warn("unset error! collection:" + collectionName + " object:" + object, ignore); } finally { long t = System.currentTimeMillis() - begin; if (t > 100) { boolean b = ThreadLocalLog.addLog("--", "MongoDao.unset", "collection:" + collectionName, "query:" + query, "object:" + object, "cost:" + t); if (!b) { LoggerFactory.getLogger("Slow").info("MongoDao.unset collection:{} query:{} object:{} cost:{}", collectionName, query, object, t); } } } return false; } }
Redis
redis配置代码
package com.kingdee.redis; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.PropertySource; @PropertySource(value={"classpath:common.properties"}) public class RedisConfig { @Value("${redis.host}") private String host; @Value("${redis.pool.active}") private String maxactive; @Value("redis.timeout") private String timeout; public String getTimeout() { return timeout; } public void setTimeout(String timeout) { this.timeout = timeout; } public String getMaxactive() { return maxactive; } public void setMaxactive(String maxactive) { this.maxactive = maxactive; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } @Value("${redis.password}") private String password; public String getHost() { return host; } public void setHost(String host) { this.host = host; } public String getPort() { return port; } public void setPort(String port) { this.port = port; } @Value("${redis.port}") private String port; }
操作redis数据代码
package com.kingdee.redis; import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import javax.management.MBeanServer; import javax.management.ObjectName; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import net.sf.json.JSONArray; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; import redis.clients.jedis.Pipeline; import redis.clients.jedis.exceptions.JedisConnectionException; public class JedisUtils { private static JedisPool pool = null; private static Logger logger = LoggerFactory.getLogger(JedisUtils.class); private static Lock lock = new ReentrantLock(); public static int retryNum = 3;//jedis重连次数 // 每个线程共享一个连接 private static ThreadLocal<Jedis> connWrapper = new ThreadLocal<Jedis>(); // 每个线程共享一个连接 private static ThreadLocal<Pipeline> pipelineWrapper = new ThreadLocal<Pipeline>(); @Autowired private static RedisConfig redisConfig; public static Jedis jedis() { JedisPool pool = getJedisPool(); Jedis jedis = null; try{ jedis = pool.getResource(); }catch(JedisConnectionException e){ if(jedis != null){ jedis.close(); } logger.error("",e); return null; } return jedis; } /**返回一个线程共用一个redis连接 * @return */ public static Jedis myClient() { int count = 0; Jedis jedis = null; do { try { jedis = connWrapper.get(); if (jedis == null) { jedis= jedis(); connWrapper.set(jedis); } } catch (JedisConnectionException e) { // 出现链接异常的时候归还jedis if(jedis != null){ jedis.close(); } logger.error("",e); try { Thread.sleep(2000); //休眠2秒 } catch (InterruptedException e1) { e1.printStackTrace(); } } } while (jedis == null && count < retryNum); return jedis; } /**返回一个线程共用一个redis连接 * @return */ public static Pipeline myPipeline() { int count = 0; Pipeline pipeline = null; do { try { pipeline = pipelineWrapper.get(); if (pipeline == null) { pipeline= myClient().pipelined(); pipelineWrapper.set(pipeline); } } catch (Exception e) { // 出现链接异常的时候归还jedis if(pipeline != null){ pipeline.sync(); myClient().close(); } logger.error("",e); try { Thread.sleep(2000); //休眠2秒 } catch (InterruptedException e1) { e1.printStackTrace(); } } } while (pipeline == null && count < retryNum); return pipeline; } /** * 刷新管道数据,释放单线程redis管道连接 */ public static void closePipelineConnection() { Pipeline pipeline=pipelineWrapper.get(); if(pipeline!=null){ pipeline.sync(); } Jedis jedis = connWrapper.get(); if (jedis != null) { try { jedis.close(); } catch (Exception e) { logger.error("Can not close database connection", e); } // 释放掉保存的对象 pipelineWrapper.remove(); connWrapper.remove(); } } public static JedisPool getJedisPool() { if (pool == null) { lock.lock(); try{ if(pool == null){ JedisPoolConfig config = new JedisPoolConfig(); config.setMaxTotal(Integer.valueOf(redisConfig.getMaxactive())); if(StringUtils.isEmpty(redisConfig.getPassword())){ pool = new JedisPool(config, redisConfig.getHost(), Integer.valueOf(redisConfig.getPort())); }else{ pool = new JedisPool(config,redisConfig.getHost(), Integer.valueOf(redisConfig.getPort()), Integer.valueOf(redisConfig.getTimeout()),redisConfig.getPassword()); } } }catch(Exception e){ logger.error("",e); }finally{ lock.unlock(); } } return pool; } public static long del(String[] keys){ Jedis jedis = null; long ret = 0l; try{ jedis = JedisUtils.jedis(); if(jedis != null){ long start = System.currentTimeMillis(); ret = jedis.del(keys); logger.info("delete from redis cost:"+(System.currentTimeMillis()-start)+"ms ,key:"+StringUtils.join(keys,",")); } }catch(Exception e){ logger.error("删除缓存失败,key:"+StringUtils.join(keys,",")); logger.error("删除缓存失败",e); if(jedis != null){ jedis.close();//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close(); } }catch(Exception e){ logger.error("",e); } } return ret; } public static List<String> mget(String[] keys){ Jedis jedis = null; List<String> ret = new LinkedList<String>(); try{ jedis = JedisUtils.jedis(); if(jedis != null){ long start = System.currentTimeMillis(); ret = jedis.mget(keys); logger.info("get from redis cost:"+(System.currentTimeMillis()-start)+"ms ,key:"+keys); } }catch(Exception e){ logger.error("获取缓存失败,key:"+keys); logger.error("保存缓存失败",e); if(jedis != null){ jedis.close();//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close(); } }catch(Exception e){ logger.error("",e); } } return ret; } private static final String REDIS_POOL_JMX_PREFIX = "org.apache.commons.pool2:type=GenericObjectPool,name=pool1"; private static final String[] REDIS_POOL_JMX_ATTRS = {"NumActive","NumIdle","NumWaiters"}; /** * key不存在则存储,存在不覆盖 */ public static long setnx(String key,String value){ Jedis jedis = null; long ret = 0; try{ jedis = JedisUtils.jedis(); if(jedis != null){ ret = jedis.setnx(key, value); } }catch(Exception e){ logger.error("获取setnx数据失败,key:"+key,e); if(jedis != null){ jedis.close();//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close(); } }catch(Exception e){ logger.error("",e); } } return ret; } /** * key不存在则存储,存在时不覆盖,使用时请特别注意: * 此处的设置key和设置超时并不是一个统一的原子操作 * (如小概率事件:设置key成功了,但是设置超时失败了,此时会返回了成功: * 因为如果返回了失败,由于key不会过期,所以后续调用setnx会一直失败),因此, * 需要其它的辅佐手段检查key是否过期(如:key-value的value字段存放当前日期, * 锁定时,检查日期字段,看锁是否过期等) * @date 2015-11-25 * @author wenxiang_xu */ public static long setnx(String key,String value,int timeoutSeconds){ Jedis jedis = null; long ret = 0; try{ jedis = JedisUtils.jedis(); if(jedis != null){ ret = jedis.setnx(key, value); if(ret==1){ jedis.expire(key, timeoutSeconds); } } }catch(Exception e){ logger.error("获取setnx数据失败,key:"+key,e); if(jedis != null){ jedis.close();//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close(); } }catch(Exception e){ logger.error("",e); } } return ret; } /** * 删除key */ public static long del(String key){ Jedis jedis = null; long ret = 0l; try{ jedis = JedisUtils.jedis(); if(jedis != null){ ret = jedis.del(key); } }catch(Exception e){ logger.error("删除缓存失败,key:"+key); logger.error("删除缓存失败",e); if(jedis != null){ jedis.close();;//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close();; } }catch(Exception e){ logger.error("",e); } } return ret; } /** * 与getFromCache一致 */ public static String get(String key){ Jedis jedis = null; String ret = null; try{ jedis = JedisUtils.jedis(); if(jedis != null){ ret = jedis.get(key); } }catch(Exception e){ logger.error("获取get数据失败,key:"+key,e); if(jedis != null){ jedis.close();;//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close();; } }catch(Exception e){ logger.error("",e); } } return ret; } /** * 从redis队列尾部取值 * @param key * @param start: 起始index,可以为负值,如果是负值,则是从尾部开始计算,-1为尾部第一个 * @param end: 结束的index,可以为负值,如果是负值,则是从尾部开始计算 * @return */ public static List<String> lrange(String key,int start,int end){ Jedis jedis = null; List<String> ret = new ArrayList<String>(); try{ jedis = JedisUtils.jedis(); if(jedis != null){ ret = jedis.lrange(key,start,end); } }catch(Exception e){ logger.error("lrange失败",e); if(jedis != null){ jedis.close();//将连接失效,避免重用返回错误的对象 jedis = null; } }finally{ try{ if(jedis != null){ jedis.close(); } }catch(Exception e){ logger.error("",e); } } return ret; } }