大数据小测——hbase+javaweb河北需求征集
这几天进行了紧急的hbase数据库完成web测试，就是把之前的mysql换成了hbase
hbase(2.4.10)的运行需要配置hadoop集群（3.1.3）、zookeeper(3.5.7)，安装好之后在虚拟机上面可以使用jps查看运行端口，
hbase的使用和mysql区别比较大，我在写项目的过程中，遇到的最大问题就是代码冗余，这个与我在最开始写MySQL测试时很像，目前还没有找到解决方法，hbase的数据操做并不像MySQL那样简单。
下面是我的项目结构（非常简单，只是测试使用）：
我使用的maven，进行导包很方便，如果使用普通的项目，结构配置会很麻烦<dependencies>
<!---必须和自己的hbase版本相同-->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>2.4.10</version>
<exclusions>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
</exclusion>
</exclusions>
</dependency>
<!---运行过程中可能报错，排除干扰-->
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
<version>3.0.1-b06</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>3.1.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.59</version>
</dependency>
</dependencies>
HbaseConnection，连接类
package com.util; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import java.io.IOException; public class HbaseConnection { //多线程连接 public static Connection connection = null; static { // 创建连接 //默认使用同步链接 try { //读取本地文件形式添加参数 hbase-site.xml(在虚拟机hbase/conf下) connection = ConnectionFactory.createConnection(); } catch (IOException e) { e.printStackTrace(); } } public static void closeConn(){ if(connection != null){ try { connection.close(); } catch (IOException e) { e.printStackTrace(); } } } }
我的hbase-site.xml
<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ --> <configuration> <!-- The following properties are set for running HBase as a single process on a developer workstation. With this configuration, HBase is running in "stand-alone" mode and without a distributed file system. In this mode, and without further configuration, HBase and ZooKeeper data are stored on the local filesystem, in a path under the value configured for `hbase.tmp.dir`. This value is overridden from its default value of `/tmp` because many systems clean `/tmp` on a regular basis. Instead, it points to a path within this HBase installation directory. Running against the `LocalFileSystem`, as opposed to a distributed filesystem, runs the risk of data integrity issues and data loss. Normally HBase will refuse to run in such an environment. Setting `hbase.unsafe.stream.capability.enforce` to `false` overrides this behavior, permitting operation. This configuration is for the developer workstation only and __should not be used in production!__ See also https://hbase.apache.org/book.html#standalone_dist --> <property> <name>hbase.zookeeper.quorum</name> <value>hadoop102,hadoop103,hadoop104</value> <description>The directory shared by RegionServers</description> </property> </configuration>
最重要的数据操作方法，需要熟悉hbase API
package com.dao.impl; import com.dao.DemandDao; import com.pojo.Demand; import com.util.HbaseConnection; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.util.Bytes; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Scanner; public class DemandImpl implements DemandDao { private Connection connection = HbaseConnection.connection;
//数据添加 @Override public void putCell(Demand demand) { try { Table table = connection.getTable(TableName.valueOf("testuse", "demand")); Put put = new Put(Bytes.toBytes(demand.getRowkey())); put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("organize_name"), Bytes.toBytes(demand.getOrganize_name())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("organize_manage"), Bytes.toBytes(demand.getOrganize_manage())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("postal_address"), Bytes.toBytes(demand.getPostal_address())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("location"), Bytes.toBytes(demand.getLocation())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("website"), Bytes.toBytes(demand.getWebsite())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("email"), Bytes.toBytes(demand.getEmail())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("legal"), Bytes.toBytes(demand.getLegal())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("postcode"), Bytes.toBytes(demand.getPostcode())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("relation_people"), Bytes.toBytes(demand.getRelation_people())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("phone"), Bytes.toBytes(demand.getPhone())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("organize_attribute"), Bytes.toBytes(demand.getOrganize_attribute())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("introduce"), Bytes.toBytes(demand.getIntroduce())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("demand_name"), Bytes.toBytes(demand.getDemand_name())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("demand_time"), Bytes.toBytes(demand.getDemand_time())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("moneySum"), Bytes.toBytes(demand.getMoneySum())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("solution"), Bytes.toBytes(demand.getSolution())) .addColumn(Bytes.toBytes("info"), Bytes.toBytes("activity_type"), Bytes.toBytes(demand.getActivity_type())); table.put(put); table.close(); } catch (IOException e) { e.printStackTrace(); } }
//扫描全部数据 @Override public List<Demand> scanRows() { List<Demand> demandList = new ArrayList<Demand>(); try { Table table = connection.getTable(TableName.valueOf("testuse", "demand")); //创建scan Scan scan = new Scan(); ResultScanner scanner = table.getScanner(scan); Iterator<Result> iterator = scanner.iterator(); while (iterator.hasNext()){ Result result = iterator.next(); String rowkey = Bytes.toString(result.getRow()); Cell corganize_name = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("organize_name")); Cell corganize_manage = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("organize_manage")); Cell cpostal_address = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("postal_address")); Cell clocation = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("location")); Cell cwebsite = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("website")); Cell cemail = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("email")); Cell clegal = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("legal")); Cell cpostcode = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("postcode")); Cell crelation_people = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("relation_people")); Cell cphone = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("phone")); Cell corganize_attribute = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("organize_attribute")); Cell cintroduce = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("introduce")); Cell cdemand_name = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("demand_name")); Cell cdemand_time = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("demand_time")); Cell cmoneySum = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("moneySum")); Cell csolution = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("solution")); Cell cactivity_type = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("activity_type")); String organize_name = Bytes.toString(CellUtil.cloneValue(corganize_name)); String organize_manage = Bytes.toString(CellUtil.cloneValue(corganize_manage)); String postal_address = Bytes.toString(CellUtil.cloneValue(cpostal_address)); String location = Bytes.toString(CellUtil.cloneValue(clocation)); String website = Bytes.toString(CellUtil.cloneValue(cwebsite)); String email = Bytes.toString(CellUtil.cloneValue(cemail)); String legal = Bytes.toString(CellUtil.cloneValue(clegal)); String postcode = Bytes.toString(CellUtil.cloneValue(cpostcode)); String relation_people = Bytes.toString(CellUtil.cloneValue(crelation_people)); String phone = Bytes.toString(CellUtil.cloneValue(cphone)); String organize_attribute = Bytes.toString(CellUtil.cloneValue(corganize_attribute)); String introduce = Bytes.toString(CellUtil.cloneValue(cintroduce)); String demand_name = Bytes.toString(CellUtil.cloneValue(cdemand_name)); String demand_time = Bytes.toString(CellUtil.cloneValue(cdemand_time)); int moneySum = Bytes.toInt(CellUtil.cloneValue(cmoneySum)); String solution = Bytes.toString(CellUtil.cloneValue(csolution)); String activity_type = Bytes.toString(CellUtil.cloneValue(cactivity_type)); Demand demand = new Demand(rowkey,organize_name,organize_manage,postal_address,location,website,email, legal,postcode,relation_people,phone,organize_attribute,introduce,demand_name,demand_time,moneySum,solution,activity_type); demandList.add(demand); } table.close(); } catch (IOException e) { e.printStackTrace(); } return demandList; }
//条件查询 @Override public List<Demand> filterByOName(Demand demand) { List<Demand> demandList = new ArrayList<Demand>(); try { Table table = connection.getTable(TableName.valueOf("testuse", "demand")); Scan scan = new Scan(); //添加过滤 FilterList filterList = new FilterList(); //创建过滤器 SingleColumnValueFilter valueFilter = new SingleColumnValueFilter( //列族名 Bytes.toBytes("info"), //列名 Bytes.toBytes("organize_name"), //比较关系 CompareOperator.EQUAL, //值 Bytes.toBytes(demand.getOrganize_name()) ); filterList.addFilter(valueFilter); scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); Iterator<Result> iterator = scanner.iterator(); while (iterator.hasNext()){ Result result = iterator.next(); String rowkey = Bytes.toString(result.getRow()); Cell corganize_name = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("organize_name")); Cell corganize_manage = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("organize_manage")); Cell cpostal_address = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("postal_address")); Cell clocation = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("location")); Cell cwebsite = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("website")); Cell cemail = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("email")); Cell clegal = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("legal")); Cell cpostcode = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("postcode")); Cell crelation_people = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("relation_people")); Cell cphone = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("phone")); Cell corganize_attribute = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("organize_attribute")); Cell cintroduce = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("introduce")); Cell cdemand_name = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("demand_name")); Cell cdemand_time = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("demand_time")); Cell cmoneySum = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("moneySum")); Cell csolution = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("solution")); Cell cactivity_type = result.getColumnLatestCell(Bytes.toBytes("info"),Bytes.toBytes("activity_type")); String organize_name = Bytes.toString(CellUtil.cloneValue(corganize_name)); String organize_manage = Bytes.toString(CellUtil.cloneValue(corganize_manage)); String postal_address = Bytes.toString(CellUtil.cloneValue(cpostal_address)); String location = Bytes.toString(CellUtil.cloneValue(clocation)); String website = Bytes.toString(CellUtil.cloneValue(cwebsite)); String email = Bytes.toString(CellUtil.cloneValue(cemail)); String legal = Bytes.toString(CellUtil.cloneValue(clegal)); String postcode = Bytes.toString(CellUtil.cloneValue(cpostcode)); String relation_people = Bytes.toString(CellUtil.cloneValue(crelation_people)); String phone = Bytes.toString(CellUtil.cloneValue(cphone)); String organize_attribute = Bytes.toString(CellUtil.cloneValue(corganize_attribute)); String introduce = Bytes.toString(CellUtil.cloneValue(cintroduce)); String demand_name = Bytes.toString(CellUtil.cloneValue(cdemand_name)); String demand_time = Bytes.toString(CellUtil.cloneValue(cdemand_time)); int moneySum = Bytes.toInt(CellUtil.cloneValue(cmoneySum)); String solution = Bytes.toString(CellUtil.cloneValue(csolution)); String activity_type = Bytes.toString(CellUtil.cloneValue(cactivity_type)); Demand demand1 = new Demand(rowkey,organize_name,organize_manage,postal_address,location,website,email, legal,postcode,relation_people,phone,organize_attribute,introduce,demand_name,demand_time,moneySum,solution,activity_type); demandList.add(demand1); } table.close(); } catch (IOException e) { e.printStackTrace(); } return demandList; } }
数据操作和mysql大同小异，
前端和servlet就不再上传了。
