Berkeley DB
最近用BDB写点东西，写了挺多个测试工程。列下表，也理清楚最近的思路
1.测试BDB程序，包括打开增加记录，查询记录，获取所有记录。将数据转存mysql
程序的不足，增加记录仅仅只有key和value，查询记录仅仅是简单的根据key或者同时制定key和value来查，只获取所有记录，没有获取部分记录，转存mysql消耗比较大
2.mysql和BDB插入和选择数据比较。
在java虚拟机内存相同的条件下，mysql插入数据量比较大时会报内存溢出，BDB 插入数据的量可以比mysql大。而且速度也快。查询所有记录mysql非常快，BDB相对来说比较慢，hbase只测试了下插入记录，插入一千条记录都慢得要死。单个数据选择BDB相对于mysql来说有绝对的优势。
3.修改1中的数据转存mysql。通过两台电脑，两个BDB+mysql进行。
大致流程是一个一台电脑A上有BDB，另外一台电脑B上有BDB和mysql。A上采集数据存储在BDB中。然后自己写一个文件发送的程序把BDB在硬盘上的.jdb文件发送到另外一台电脑。另外一台电脑通过BDB读取.jdb文件写进mysql里面。运行效果比较好，但是时间比直接插入一千万条记录进BDB要长。现在没看到应用领域。
4.序列化存储。实现用BDB存储学生信息
5.BDB游标的使用。对比getSearchKey,getSearchBoth,getSearchKeyRange,getSearchBothRange函数。最后写了个key的模糊查询。
以上基本上是一个星期学到的有关Berkeley DB的知识。
getSearchKey,getSearchBoth,getSearchKeyRange,getSearchBothRange对比程序
1 import java.io.File; 2 import java.io.UnsupportedEncodingException; 3 import java.util.ArrayList; 4 5 import com.sleepycat.je.Cursor; 6 import com.sleepycat.je.Database; 7 import com.sleepycat.je.DatabaseConfig; 8 import com.sleepycat.je.DatabaseEntry; 9 import com.sleepycat.je.Environment; 10 import com.sleepycat.je.EnvironmentConfig; 11 import com.sleepycat.je.LockMode; 12 import com.sleepycat.je.OperationStatus; 13 14 15 public class Query { 16 private Environment dbEnvironment = null; 17 private Database db = null; 18 19 20 private void DataConstruct(int t) throws UnsupportedEncodingException { 21 int Total = t; 22 long BDBTime = 0; 23 long start = System.currentTimeMillis(); 24 /* 25 * 26 */ 27 OpenBDB(); 28 while(Total!=0) 29 { 30 31 int k = Total; 32 int v = Total+1; 33 DatabaseEntry iKey = new DatabaseEntry(Integer.valueOf(k).toString().getBytes("UTF-8")); 34 DatabaseEntry iValue = new DatabaseEntry(Integer.valueOf(v).toString().getBytes("UTF-8")); 35 36 37 //System.out.println("Writing : Key= " + k + " Value = " + v + " count: " + Total); 38 39 40 db.put(null, iKey, iValue); 41 Total--; 42 } 43 long end = System.currentTimeMillis(); 44 BDBTime = end - start; 45 System.out.println(BDBTime); 46 47 48 49 } 50 51 //各种查询比较 52 53 public void QueryCompare(int t) throws UnsupportedEncodingException{ 54 55 DataConstruct(t); 56 //OpenBDB(); 57 58 Cursor cursor = null; 59 cursor = db.openCursor(null, null); 60 61 DatabaseEntry thekey = new DatabaseEntry(); 62 String key = "12345"; 63 thekey.setData(key.getBytes("utf-8")); 64 65 DatabaseEntry thedata = new DatabaseEntry(); 66 67 System.out.println(" >======== getSearchKey Start ========<"); 68 cursor.getSearchKey(thekey, thedata, LockMode.DEFAULT); 69 byte[] temp = thedata.getData(); 70 String res = new String(temp,"utf-8"); 71 System.out.println(" Key " + key + " Value " + res); 72 System.out.println(" >======== getSearchKey End ========<"); 73 System.out.println(); 74 75 System.out.println(" >======== getSearchKeyRange Start ========<"); 76 cursor.getSearchKeyRange(thekey, thedata, LockMode.DEFAULT); 77 temp = thedata.getData(); 78 res = new String(temp,"utf-8"); 79 System.out.println(" Key " + key + " Value " + res); 80 System.out.println(" >======== getSearchKeyRange End ========<"); 81 System.out.println(); 82 83 //如果要执行这个查询最后不用随机数，用随机数相查到结果不易 84 System.out.println(" >======== getSearchBoth Start ========<"); 85 thedata.setData("12346".getBytes("utf-8")); 86 OperationStatus status = cursor.getSearchBoth(thekey, thedata, LockMode.DEFAULT); 87 if(status == OperationStatus.SUCCESS){ 88 temp = thedata.getData(); 89 res = new String(temp,"utf-8"); 90 System.out.println(" Key " + key + " Value " + res); 91 } 92 else 93 System.out.println(" Not Such Record"); 94 95 System.out.println(" >======== getSearchKeyBoth End ========<"); 96 System.out.println(); 97 98 System.out.println(" >======== getSearchBothRange Start ========<"); 99 thedata.setData("123451".getBytes("utf-8")); 100 status = cursor.getSearchBoth(thekey, thedata, LockMode.DEFAULT); 101 if(status == OperationStatus.SUCCESS){ 102 temp = thedata.getData(); 103 res = new String(temp,"utf-8"); 104 System.out.println(" Key " + key + " Value " + res); 105 } 106 else 107 System.out.println(" Not Such Record"); 108 System.out.println(" RecordCount: " + cursor.count()); 109 110 System.out.println(" >======== getSearchKeyBothRange End ========<"); 111 System.out.println(); 112 113 /* 114 * getSearchKeyRange和getSearchKey都是将游标定位到匹配key值相同的地方，如果再用Cursor.getNext无论如何都是能得到游标下一个。区别是当用到B树的时候 115 * 根据getSearchKeyRange和getSearchKey和Cursor.getNext的输出结果的规律可以看出，BDB内部存储采用树形结构，类似字符数 116 * getSearchBoth同时匹配Key和Value才有结果，才能返回OperationStatus.SUCCESS才算操作成功 117 * 118 */ 119 /* 120 * 实现模糊查询，用getSearchKey配合getNext即可 121 * 122 */ 123 //单条件模糊查询测试代码,匹配12345,结束应该是12346 124 125 System.out.println(" >======== Simple Pattern key = 12345 getSearchKey Start ========<"); 126 thedata = new DatabaseEntry();//单条件查询，把之前的thedata设置为空 127 cursor.getSearchKeyRange(thekey, thedata, LockMode.DEFAULT); 128 temp = thedata.getData(); // 129 res = new String(temp,"utf-8"); 130 System.out.println(" Key " + key + " Value " + res); 131 thekey = new DatabaseEntry(); 132 while(cursor.getNext(thekey, thedata, LockMode.DEFAULT) != OperationStatus.NOTFOUND){ //结束条件是OperationStatus.NOTFOUND 133 temp = thedata.getData(); 134 byte[] tempkey = thekey.getData(); 135 res = new String(temp,"utf-8"); 136 String res1 = new String(tempkey,"utf-8"); 137 if(res1.equals("12346")) 138 break; 139 System.out.println(" Key " + res1 + " Value " + res); 140 } 141 System.out.println(" >======== Simple getSearchKey End ========<"); 142 System.out.println(); 143 144 //多条件模糊查询，如果没有相应的机制只能自己编写来查了, 145 //失败 146 147 System.out.println(" >======== multiplication Pattern Key=12345 Value = 123451 getSearchBothRange Start ========<"); 148 thedata.setData("123451".getBytes("utf-8")); 149 thekey.setData(key.getBytes("utf-8")); 150 status = cursor.getSearchBothRange(thekey, thedata, LockMode.DEFAULT); 151 //首先会去匹配12345 123456 152 if(status==OperationStatus.SUCCESS){ 153 temp = thedata.getData(); // 154 res = new String(temp,"utf-8"); 155 System.out.println(" Key " + key + " Value " + res); 156 while(cursor.getNext(thekey, thedata, LockMode.DEFAULT) != OperationStatus.NOTFOUND){ //结束条件是OperationStatus.NOTFOUND 157 temp = thedata.getData(); 158 byte[] tempkey = thekey.getData(); 159 res = new String(temp,"utf-8"); 160 String res1 = new String(tempkey,"utf-8"); 161 if(res1.equals("12346")) 162 break; 163 System.out.println(" Key " + res1 + " Value " + res); 164 } 165 } 166 else 167 System.out.println(" Not Such Record"); 168 169 170 System.out.println(" >======== multiplication getSearchBothRange End ========<"); 171 System.out.println(); 172 cursor.close(); 173 CloseBDB(); 174 175 } 176 177 //创建数据库 178 public void OpenBDB(){ 179 EnvironmentConfig envConfig = new EnvironmentConfig(); 180 envConfig.setAllowCreate(true); 181 dbEnvironment = new Environment( new File("D:/dbEnv"), envConfig ); 182 DatabaseConfig dbConfig = new DatabaseConfig(); 183 184 dbConfig.setAllowCreate(true); 185 dbConfig.setSortedDuplicates(true); 186 db = dbEnvironment.openDatabase(null,"BDB", dbConfig); 187 System.out.println("打开数据库成功"); 188 } 189 //关闭数据库 190 public void CloseBDB(){ 191 if(db != null){ 192 db.close(); 193 } 194 if(dbEnvironment != null){ 195 dbEnvironment.close(); 196 } 197 } 198 199 200 //获取数据库中所有记录， 201 public ArrayList<String> getEveryItem() throws UnsupportedEncodingException { 202 Cursor cursor = null; 203 OpenBDB(); 204 cursor= db.openCursor(null,null); 205 DatabaseEntry theKey=null; 206 DatabaseEntry theData=null; 207 theKey = new DatabaseEntry(); 208 theData = new DatabaseEntry(); 209 long total = 0; 210 ArrayList<String> list = new ArrayList<String>(); 211 212 while (cursor.getNext(theKey, theData, LockMode.DEFAULT) == OperationStatus.SUCCESS) { 213 //System.out.println("Reading: Key: " + new String( theKey.getData()) + "Value: " + new String(theData.getData())); 214 list.add(new String( theKey.getData()) ); 215 list.add(new String( theData.getData()) ); 216 total++; 217 } 218 cursor.close(); 219 System.out.println("total:" + total); 220 return list; 221 222 } 223 224 //getSearchKey 225 226 227 228 229 230 231 232 }
序列化创建学生信息
import java.io.File; import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.je.Cursor; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; public class SeriableDemo { public static void main(String args[]){ Environment dbEnv = null; Database db = null; Cursor cursor=null; try { EnvironmentConfig envconfig = new EnvironmentConfig(); envconfig.setAllowCreate(true); dbEnv = new Environment(new File("D:/dbEnv/"),envconfig); DatabaseConfig dbconfig = new DatabaseConfig(); dbconfig.setAllowCreate(true); dbconfig.setReadOnly(false); dbconfig.setSortedDuplicates(false); //data db = dbEnv.openDatabase(null, "Student_Data", dbconfig); //class Database myclassdb=dbEnv.openDatabase(null, "Student_class", dbconfig); //cataglog存储类信息 StoredClassCatalog classCatalog =new StoredClassCatalog(myclassdb); //创建绑定对象 EntryBinding dataBinging=new SerialBinding(classCatalog,Student.class); DatabaseEntry thekey=new DatabaseEntry("1200340233".getBytes("utf-8")); DatabaseEntry thedata=new DatabaseEntry(); Student st=new Student(); st.setStudentId(1200340233); st.setStudentName("邹颖玖"); st.setStudentAge(20); st.setStudentDept("计算机科学与工程"); //绑定数据 dataBinging.objectToEntry(st, thedata); //插入 db.put(null, thekey, thedata); DatabaseEntry data = new DatabaseEntry(); //DatabaseEntry key = new DatabaseEntry(); DatabaseEntry key = new DatabaseEntry("1200340233".getBytes("utf-8")); Student st1 = new Student(); st1.setStudentId(1200340233); dataBinging.objectToEntry(st1, data); //获取 OperationStatus status=db.get(null, key, data, LockMode.DEFAULT); if(status==OperationStatus.SUCCESS){ Student s=(Student) dataBinging.entryToObject(thedata); System.out.println(s.getStudentId()+" "+s.getStudentName()+" "+ s.getStudentAge() + " " + s.getStudentDept()); } }catch(Exception e){ e.printStackTrace(); } } }
上面代码用到学生类。
import java.io.Serializable; public class Student implements Serializable { private long StudentId; private String StudentName; private int StudentAge; private String StudentDept; public long getStudentId() { return StudentId; } public void setStudentId(long studentId) { StudentId = studentId; } public String getStudentName() { return StudentName; } public void setStudentName(String studentName) { StudentName = studentName; } public int getStudentAge() { return StudentAge; } public void setStudentAge(int studentAge) { StudentAge = studentAge; } public String getStudentDept() { return StudentDept; } public void setStudentDept(String studentDept) { StudentDept = studentDept; } }
以上是老师将来项目需要，估计以后我也有这种需要。没有时间，还是想自己实践弄一套音乐推荐系统出来。
