大数据实验(Mysql、hbase、redis、MongoDBjava客户端连接)
1.MySQL
启动：虚拟机输入mysql -u root -p
输入密码: hadoop（黑马的mysql密码是hadoop）
pom.xml需要引入mysql
<properties> <maven.compiler.source>8</maven.compiler.source> <maven.compiler.target>8</maven.compiler.target> </properties> <dependencies> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.17</version> </dependency> </dependencies>
本机Navicat for MySQL已经和虚拟机中的MySQL连接成功的前提下：
package MysqlTest; import java.sql.*; public class Test { public static void main(String[] args) throws ClassNotFoundException, SQLException { //链接Mysql，本地Navicat已经和虚拟机中mysql链接起来了 Class.forName("com.mysql.jdbc.Driver"); Connection conn = DriverManager.getConnection("jdbc:mysql://node1:3306/test", "root", "hadoop"); //创建mysql执行语句 Statement statement = conn.createStatement(); String addsql = "INSERT into test.student VALUES('scofield',45,89,100);"; statement.executeUpdate(addsql); String getsql="select * from test.student where NAME='scofield';"; ResultSet rs = statement.executeQuery(getsql); while(rs.next()){ System.out.println("scofield 的 English成绩为: " + rs.getString("English")); } statement.close(); conn.close(); } }
2.HBase数据库
启动：zookeeper、hadoop、hbase，然后输入hbase shell
（zkServer.sh start、start-all.sh、start-hbase.sh）zookeeper需要三台虚拟机都打开（我的需要分开启动）
pom.xml需要引入的依赖
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>org.example</groupId> <artifactId>hw_connecthbase</artifactId> <version>1.0-SNAPSHOT</version> <name>hw_connecthbase</name> <!-- FIXME change it to the project's website --> <url>http://www.example.com</url> <repositories><!-- 代码库 --> <repository> <id>aliyun</id> <url>http://maven.aliyun.com/nexus/content/groups/public/</url> <releases> <enabled>true</enabled> </releases> <snapshots> <enabled>false</enabled> <updatePolicy>never</updatePolicy> </snapshots> </repository> </repositories> <dependencies> <!-- HBase的Java客户端 --> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>2.1.0</version> </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> <version>2.6</version> </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> <scope>test</scope> </dependency> <dependency> <groupId>org.testng</groupId> <artifactId>testng</artifactId> <version>6.14.3</version> <scope>test</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>3.1</version> <configuration> <target>1.8</target> <source>1.8</source> </configuration> </plugin> </plugins> </build> </project>
实现连接：参考：
大数据之Hadoop学习（十）HBase Java API编程_addrecord(string tablename, string row, string[] f-CSDN博客
大数据之HBase API 完整使用 (第四章)_第四章 hbase的api和工具类_小坏讲微服务的博客-CSDN博客
不导入文件连接方式如下：
public class TestApi { private static Connection connection = null; private static Admin admin = null; static { try { //获取配置信息 Configuration configuration = HBaseConfiguration.create(); configuration.set("hbase.zookeeper.quorum", "node1,node2,node3"); //创建链接对象 connection = ConnectionFactory.createConnection(configuration); //创建Admin的对象 admin = connection.getAdmin(); } catch (IOException e) { e.printStackTrace(); } } }
我的连接是建立在项目已经导入这三个文件的前提下：
package HBaseTest; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import java.io.IOException; public class Test { public static Configuration configuration; public static Connection connection; public static Admin admin; public static void main(String[] args) throws IOException { System.out.println("-----添加数据-----"); //添加数据 String[] field={"score:English","score:Math","score:Computer"}; String[] value={"45","89","100"}; // insert("student","scofield",field,value); System.out.println("-----获取指定数据-----"); //获取指定数据 getData("student","scofield","score","English"); System.out.println("-----获取指定列簇数据-----"); //获取指定列簇数据 // scanColumn("student","score"); } public static void insert(String tableName,String row,String[] field,String[] value) throws IOException { init(); Table table = connection.getTable(TableName.valueOf(tableName)); for (int i = 0; i != field.length; i++) { Put put = new Put(row.getBytes()); String[] cols = field[i].split(":"); put.addColumn(cols[0].getBytes(), cols[1].getBytes(), value[i].getBytes()); table.put(put); } table.close(); close(); } //浏览指定数据 public static void getData(String tableName, String rowKey, String columnFamily, String column) throws IOException { init(); //1、获取表对象 Table table = connection.getTable(TableName.valueOf(tableName)); //2、创建Get对象 Get get = new Get(Bytes.toBytes(rowKey)); //2.1指定获取列族 // get.addFamily(Bytes.toBytes(cf)); //2.2设置查询的列族和列的条件 get.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column)); //3、获取对象 Result result = table.get(get); System.out.println("Name: "+rowKey); //4、解析数据 for (Cell cell : result.rawCells()) { //打印数据 // System.out.println("columnFamily: " + Bytes.toString(CellUtil.cloneFamily(cell))); System.out.println("English: " + Bytes.toString(CellUtil.cloneQualifier(cell))); System.out.println("value: " + Bytes.toString(CellUtil.cloneValue(cell))); } table.close(); close(); } //浏览指定列簇数据 public static void scanColumn(String tableName,String column) throws IOException { init(); Table table = connection.getTable(TableName.valueOf(tableName)); Scan scan = new Scan(); //指定列族 scan.addFamily(Bytes.toBytes(column)); ResultScanner scanner=table.getScanner(scan); for(Result result = scanner.next();result!=null;result=scanner.next()){ showCell(result); } table.close(); close(); } public static void showCell(Result result){ Cell[] cells = result.rawCells(); for (Cell cell:cells ) { System.out.println("RowName:" + new String(CellUtil.cloneRow(cell)) + " "); System.out.println("Timetamp:" + cell.getTimestamp() + " "); System.out.println("column Family:" + new String(CellUtil.cloneFamily(cell)) + " "); System.out.println("row Name:" + new String(CellUtil.cloneQualifier(cell)) + " "); System.out.println("value:" + new String(CellUtil.cloneValue(cell)) + " "); } } public static void init() { Configuration configuration = HBaseConfiguration.create(); try{ connection = ConnectionFactory.createConnection(configuration); admin = connection.getAdmin(); }catch (Exception ex){ ex.printStackTrace(); } } public static void close() { try{ if(admin!=null){ admin.close(); } if(connection!=null){ connection.close(); } }catch (Exception ex){ ex.printStackTrace(); } } }
3.redis
启动：先cd /export/server/redis-5.0.7/
bin/redis-server redis.conf
后打开客户端(打开一个新窗口)： redis-cli
没安装的先安装
1.先安装
//安装 cd /export/server wget http://download.redis.io/releases/redis-5.0.7.tar.gz //解压 tar -zvxf redis-5.0.7.tar.gz //编译：进入到 redis目录 使用make命令 cd /export/server/redis-5.0.7 //如果报错，可能是没安装gcc，安装一下 //安装gcc命令：yum install gcc-c++ //安装成功后先输入make，如果报错就输入make distclean再执行make命令 make //安装 make PREFIX=/export/server/redis-5.0.7 install
cd /export/server/redis-5.0.7/src
make install //进入redis目录，对redis.conf文件更改参数
cd /export/server/redis-5.0.7 vim redis.conf 1.将bind 127.0.0.1改为 #bind 127.0.0.1 2.protected-mode yes 改成 protected-mode no //启动服务端 bin/redis-server redis.conf //退出服务端 ctrl+C
//启动客户端
redis-cli
//退出客户端
exit
如果启动客户端失败，-bash: redis-cli: 未找到命令，说明没有install
cd /export/server/redis-5.0.7/src
make install
2.添加数据：
3.查看zhangsan、lisi的信息
4.zhangsan的computer成绩
5.修改lisi的Math成绩
6.编程实现
添加scofield信息以及查询scofield的English信息
package RedisTest; import redis.clients.jedis.Jedis; import java.util.Map; public class Test { public static void main(String[] args){ Jedis jedis = new Jedis("192.168.88.151",6379); // // 添加学生信息 jedis.hset("student.scofield", "English","45"); jedis.hset("student.scofield", "Math","89"); jedis.hset("student.scofield", "Computer","100"); Map<String,String> value = jedis.hgetAll("student.scofield"); for(Map.Entry<String, String> entry:value.entrySet()) { //输出scofield的成绩信息 System.out.println(entry.getKey()+":"+entry.getValue()); } //获取学生scofield的English成绩 String scoinfo=jedis.hget("student.scofield", "English"); //输出scofield的英语成绩信息 System.out.println("scofield's English score is: "+scoinfo); jedis.close(); } }
pom.xml需要的依赖
<!--用于单元测试的包--> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> </dependency> <!--mysql的jdbc包--> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>8.0.18</version> </dependency> <!--redis数据库连接包--> <dependency> <groupId>redis.clients</groupId> <artifactId>jedis</artifactId> <version>2.9.0</version> </dependency>
4.Mongodb
虚拟机安装：
//先去到需要安装的目录下 cd /export/server //安装 wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-4.0.7.tgz //解压 tar -zxvf mongodb-linux-x86_64-4.0.7.tgz //在 mongodb-linux-x86_64-4.0.7 下创建两个目录 //存放数据以及日志信息 cd mongodb-linux-x86_64-4.0.7/ mkdir -p logs mkdir -p datas //创建一个mongodb.conf vim mongodb.conf //写入以下信息 1、dbpath = /export/server/mongodb-linux-x86_64-4.0.7/datas //数据文件 2、logpath =/export/server/mongodb-linux-x86_64-4.0.7/logs/mongodb.log //日记文件 3、port = 27017 //端口号 默认 4、fork = true //后台启动 //启动:需要注意一下路径，不让启动会失败，找不到文件或者目录 cd /export/server/mongodb-linux-x86_64-4.0.7/bin ./mongod --config ../mongodb.conf //查看进程 ps aux | grep mongodb
//启动mongo ./mongo
关闭，我不是很清楚怎么关闭的
1.设计表结构并且插入数据
1.（2）find()输出两个学生的信息
1.（3）find()方法查询zhangsan的所有成绩
1.（4）修改lisi的Math成绩为95
二、javaapi编程
2.(1)添加数据
2.（2）获取数据
pom.xml需要引入
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver</artifactId>
<version>3.2.2</version>
</dependency>
代码
package mongodbTest; import com.mongodb.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import org.bson.Document; import java.util.ArrayList; import java.util.List; public class Test { public static void main(String[] args) { //连接mongodb MongoClient mongoClient = new MongoClient("192.168.88.151", 27017); //数据库 MongoDatabase database = mongoClient.getDatabase("Stu"); //数据库中的某个集合 MongoCollection<Document> collection = database.getCollection("student"); //添加数据 Document document = new Document("name","scofield") .append("score",new Document("English",45) .append("Math",89).append("Computer",100)); List<Document> documents = new ArrayList<Document>(); documents.add(document); collection.insertMany(documents); //浏览数据 MongoCursor<Document> cursor=collection.find( new Document("name","lisi")). projection(new Document("score",1).append("_id", 0)).iterator(); while(cursor.hasNext()) System.out.println(cursor.next().toJson()); } }
可能会出现的问题：
在mongodb.conf加一条（该文件在安装目录下）
