linux安装Sqoop

前提

 已经安装了hadoop、mysql

将sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz上传到服务器

 # 解压
tar -zxvf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
# 移动
mv /home/sqoop-1.4.6.bin__hadoop-2.0.4-alpha/* /usr/local/software/sqoop-1.4.6
 
# 配置
vi /etc/profile
# 配置如下
export SQOOP_HOME=/usr/local/software/sqoop-1.4.6
export PATH=$SQOOP_HOME/bin:$PATH
# 生效
source /etc/profile
 
# 复制
cp /usr/local/software/sqoop-1.4.6/conf/sqoop-env-template.sh /usr/local/software/sqoop-1.4.6/conf/sqoop-env.sh
# 编辑
vi /usr/local/software/sqoop-1.4.6/conf/sqoop-env.sh
# 配置如下：必须配置hadoop，非必要配置hbase、hive、zookeeper
#Set path to where bin/hadoop is available
export HADOOP_COMMON_HOME=/usr/local/software/hadoop-2.9.2
#Set path to where hadoop-*-core.jar is available
export HADOOP_MAPRED_HOME=/usr/local/software/hadoop-2.9.2
 
# 移动mysql驱动
mv mysql-connector-java-8.0.30.jar /usr/local/software/sqoop-1.4.6/lib

测试

 # 启动mysql
mysql.server start
# 启动hadoop
start-dfs.sh
start-yarn.sh
 
cd /usr/local/software/sqoop-1.4.6/
# 测试
[root@master sqoop-1.4.6]# bin/sqoop list-databases --connect jdbc:mysql://master:3306/ --username root --password 123456
Warning: /usr/local/software/sqoop-1.4.6/../hbase does not exist! HBase imports will fail.
Please set $HBASE_HOME to the root of your HBase installation.
Warning: /usr/local/software/sqoop-1.4.6/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/software/sqoop-1.4.6/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/12/21 23:15:33 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6
23/12/21 23:15:33 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
23/12/21 23:15:33 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary.
mysql
information_schema
performance_schema
sys
hive_metastore
db_test
 
# 进入如下目录
cd /usr/local/software/sqoop-1.4.6/
# 执行如下
bin/sqoop import \
--driver com.mysql.cj.jdbc.Driver \
--connect jdbc:mysql://master:3306/db_test \
--username root \
--password 123456 \
--delete-target-dir \
--target-dir /sqoopresult001 \
--table student \
--m 1
 
# 查看hadoop执行目录下生成的文件
hdfs dfs -ls /sqoopresult001

查看详情

执行

 [root@master sqoop-1.4.6]# bin/sqoop import \
> --driver com.mysql.cj.jdbc.Driver \
> --connect jdbc:mysql://master:3306/db_test \
> --username root \
> --password 123456 \
> --delete-target-dir \
> --target-dir /sqoopresult001 \
> --table student \
> --m 1
Warning: /usr/local/software/sqoop-1.4.6/../hbase does not exist! HBase imports will fail.
Please set $HBASE_HOME to the root of your HBase installation.
Warning: /usr/local/software/sqoop-1.4.6/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/software/sqoop-1.4.6/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/12/22 00:00:31 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6
23/12/22 00:00:31 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
23/12/22 00:00:32 WARN sqoop.ConnFactory: Parameter --driver is set to an explicit driver however appropriate connection manager is not being set (via --connection-manager). Sqoop is going to fall back to org.apache.sqoop.manager.GenericJdbcManager. Please specify explicitly which connection manager should be used next time.
23/12/22 00:00:32 INFO manager.SqlManager: Using default fetchSize of 1000
23/12/22 00:00:32 INFO tool.CodeGenTool: Beginning code generation
23/12/22 00:00:32 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
23/12/22 00:00:32 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
23/12/22 00:00:32 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/local/software/hadoop-2.9.2
注: /tmp/sqoop-root/compile/1e6acb49375d5ccc8d258f91db944575/student.java使用或覆盖了已过时的 API。
注: 有关详细信息, 请使用 -Xlint:deprecation 重新编译。
23/12/22 00:00:34 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-root/compile/1e6acb49375d5ccc8d258f91db944575/student.jar
23/12/22 00:00:35 INFO tool.ImportTool: Destination directory /sqoopresult001 is not present, hence not deleting.
23/12/22 00:00:35 INFO mapreduce.ImportJobBase: Beginning import of student
23/12/22 00:00:35 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar
23/12/22 00:00:35 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
23/12/22 00:00:35 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps
23/12/22 00:00:35 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
23/12/22 00:00:45 INFO db.DBInputFormat: Using read commited transaction isolation
23/12/22 00:00:45 INFO mapreduce.JobSubmitter: number of splits:1
23/12/22 00:00:45 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled
23/12/22 00:00:45 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1703174394235_0001
23/12/22 00:00:46 INFO impl.YarnClientImpl: Submitted application application_1703174394235_0001
23/12/22 00:00:46 INFO mapreduce.Job: The url to track the job: http://master:8088/proxy/application_1703174394235_0001/
23/12/22 00:00:46 INFO mapreduce.Job: Running job: job_1703174394235_0001
23/12/22 00:00:55 INFO mapreduce.Job: Job job_1703174394235_0001 running in uber mode : false
23/12/22 00:00:55 INFO mapreduce.Job:  map 0% reduce 0%
23/12/22 00:01:02 INFO mapreduce.Job:  map 100% reduce 0%
23/12/22 00:01:02 INFO mapreduce.Job: Job job_1703174394235_0001 completed successfully
23/12/22 00:01:02 INFO mapreduce.Job: Counters: 30
	File System Counters
		FILE: Number of bytes read=0
		FILE: Number of bytes written=206715
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
		HDFS: Number of bytes read=87
		HDFS: Number of bytes written=192
		HDFS: Number of read operations=4
		HDFS: Number of large read operations=0
		HDFS: Number of write operations=2
	Job Counters 
		Launched map tasks=1
		Other local map tasks=1
		Total time spent by all maps in occupied slots (ms)=4185
		Total time spent by all reduces in occupied slots (ms)=0
		Total time spent by all map tasks (ms)=4185
		Total vcore-milliseconds taken by all map tasks=4185
		Total megabyte-milliseconds taken by all map tasks=8570880
	Map-Reduce Framework
		Map input records=8
		Map output records=8
		Input split bytes=87
		Spilled Records=0
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=81
		CPU time spent (ms)=1060
		Physical memory (bytes) snapshot=154923008
		Virtual memory (bytes) snapshot=2088615936
		Total committed heap usage (bytes)=60882944
	File Input Format Counters 
		Bytes Read=0
	File Output Format Counters 
		Bytes Written=192
23/12/22 00:01:02 INFO mapreduce.ImportJobBase: Transferred 192 bytes in 27.0684 seconds (7.0931 bytes/sec)
23/12/22 00:01:02 INFO mapreduce.ImportJobBase: Retrieved 8 records.

查看

 [root@master ~]# hdfs dfs -ls /sqoopresult001
Found 2 items
-rw-r--r--   3 root supergroup          0 2023-12-22 00:01 /sqoopresult001/_SUCCESS
-rw-r--r--   3 root supergroup        192 2023-12-22 00:01 /sqoopresult001/part-m-00000
[root@master ~]# hdfs dfs -cat /sqoopresult001/part-m-00000
1,赵雷,1990-01-01,男
2,钱电,1990-12-21,男
3,孙风,1990-05-20,男
4,李云,1990-08-06,男
5,周梅,1991-12-01,女
6,吴兰,1992-03-01,女
7,郑竹,1989-07-01,女
8,王菊,1990-01-20,女

posted @ 2023-12-13 19:50 DogLeftover 阅读(93) 评论(0) 编辑收藏举报

刷新页面返回顶部

登录后才能查看或发表评论，立即登录或者逛逛博客园首页

相关博文：

· sqoop导入oracle数据到hive

· 脚本开发思路

· Sqoop 组件安装与配置

· sqoop--mysql导出到hdfs

· Hadoop全分布部署

阅读排行：
· 阿里巴巴 QwQ-32B真的超越了 DeepSeek R-1吗？
· 10年+ .NET Coder 心语 ── 封装的思维：从隐藏、稳定开始理解其本质意义
· 【设计模式】告别冗长if-else语句：使用策略模式优化代码结构
· 字符编码：从基础到乱码解决
· 提示词工程——AI应用必不可少的技术

历史上的今天：
2021-12-13 使用命令构建vue项目
2021-12-13 vue项目入口main.js配置

linux安装Sqoop

我的标签

合集 (4)

随笔分类 (1350)

阅读排行榜

推荐排行榜

	# 解压
	tar -zxvf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
	# 移动
	mv /home/sqoop-1.4.6.bin__hadoop-2.0.4-alpha/* /usr/local/software/sqoop-1.4.6

	# 配置
	vi /etc/profile
	# 配置如下
	export SQOOP_HOME=/usr/local/software/sqoop-1.4.6
	export PATH=$SQOOP_HOME/bin:$PATH
	# 生效
	source /etc/profile

	# 复制
	cp /usr/local/software/sqoop-1.4.6/conf/sqoop-env-template.sh /usr/local/software/sqoop-1.4.6/conf/sqoop-env.sh
	# 编辑
	vi /usr/local/software/sqoop-1.4.6/conf/sqoop-env.sh
	# 配置如下：必须配置hadoop，非必要配置hbase、hive、zookeeper
	#Set path to where bin/hadoop is available
	export HADOOP_COMMON_HOME=/usr/local/software/hadoop-2.9.2
	#Set path to where hadoop-*-core.jar is available
	export HADOOP_MAPRED_HOME=/usr/local/software/hadoop-2.9.2

	# 移动mysql驱动
	mv mysql-connector-java-8.0.30.jar /usr/local/software/sqoop-1.4.6/lib

	# 启动mysql
	mysql.server start
	# 启动hadoop
	start-dfs.sh
	start-yarn.sh

	cd /usr/local/software/sqoop-1.4.6/
	# 测试
	[root@master sqoop-1.4.6]# bin/sqoop list-databases --connect jdbc:mysql://master:3306/ --username root --password 123456
	Warning: /usr/local/software/sqoop-1.4.6/../hbase does not exist! HBase imports will fail.
	Please set $HBASE_HOME to the root of your HBase installation.
	Warning: /usr/local/software/sqoop-1.4.6/../hcatalog does not exist! HCatalog jobs will fail.
	Please set $HCAT_HOME to the root of your HCatalog installation.
	Warning: /usr/local/software/sqoop-1.4.6/../accumulo does not exist! Accumulo imports will fail.
	Please set $ACCUMULO_HOME to the root of your Accumulo installation.
	23/12/21 23:15:33 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6
	23/12/21 23:15:33 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
	23/12/21 23:15:33 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
	Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary.
	mysql
	information_schema
	performance_schema
	sys
	hive_metastore
	db_test

	# 进入如下目录
	cd /usr/local/software/sqoop-1.4.6/
	# 执行如下
	bin/sqoop import \
	--driver com.mysql.cj.jdbc.Driver \
	--connect jdbc:mysql://master:3306/db_test \
	--username root \
	--password 123456 \
	--delete-target-dir \
	--target-dir /sqoopresult001 \
	--table student \
	--m 1

	# 查看hadoop执行目录下生成的文件
	hdfs dfs -ls /sqoopresult001

	[root@master sqoop-1.4.6]# bin/sqoop import \
	> --driver com.mysql.cj.jdbc.Driver \
	> --connect jdbc:mysql://master:3306/db_test \
	> --username root \
	> --password 123456 \
	> --delete-target-dir \
	> --target-dir /sqoopresult001 \
	> --table student \
	> --m 1
	Warning: /usr/local/software/sqoop-1.4.6/../hbase does not exist! HBase imports will fail.
	Please set $HBASE_HOME to the root of your HBase installation.
	Warning: /usr/local/software/sqoop-1.4.6/../hcatalog does not exist! HCatalog jobs will fail.
	Please set $HCAT_HOME to the root of your HCatalog installation.
	Warning: /usr/local/software/sqoop-1.4.6/../accumulo does not exist! Accumulo imports will fail.
	Please set $ACCUMULO_HOME to the root of your Accumulo installation.
	23/12/22 00:00:31 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6
	23/12/22 00:00:31 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
	23/12/22 00:00:32 WARN sqoop.ConnFactory: Parameter --driver is set to an explicit driver however appropriate connection manager is not being set (via --connection-manager). Sqoop is going to fall back to org.apache.sqoop.manager.GenericJdbcManager. Please specify explicitly which connection manager should be used next time.
	23/12/22 00:00:32 INFO manager.SqlManager: Using default fetchSize of 1000
	23/12/22 00:00:32 INFO tool.CodeGenTool: Beginning code generation
	23/12/22 00:00:32 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
	23/12/22 00:00:32 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
	23/12/22 00:00:32 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/local/software/hadoop-2.9.2
	注: /tmp/sqoop-root/compile/1e6acb49375d5ccc8d258f91db944575/student.java使用或覆盖了已过时的 API。
	注: 有关详细信息, 请使用 -Xlint:deprecation 重新编译。
	23/12/22 00:00:34 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-root/compile/1e6acb49375d5ccc8d258f91db944575/student.jar
	23/12/22 00:00:35 INFO tool.ImportTool: Destination directory /sqoopresult001 is not present, hence not deleting.
	23/12/22 00:00:35 INFO mapreduce.ImportJobBase: Beginning import of student
	23/12/22 00:00:35 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar
	23/12/22 00:00:35 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
	23/12/22 00:00:35 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps
	23/12/22 00:00:35 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
	23/12/22 00:00:45 INFO db.DBInputFormat: Using read commited transaction isolation
	23/12/22 00:00:45 INFO mapreduce.JobSubmitter: number of splits:1
	23/12/22 00:00:45 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled
	23/12/22 00:00:45 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1703174394235_0001
	23/12/22 00:00:46 INFO impl.YarnClientImpl: Submitted application application_1703174394235_0001
	23/12/22 00:00:46 INFO mapreduce.Job: The url to track the job: http://master:8088/proxy/application_1703174394235_0001/
	23/12/22 00:00:46 INFO mapreduce.Job: Running job: job_1703174394235_0001
	23/12/22 00:00:55 INFO mapreduce.Job: Job job_1703174394235_0001 running in uber mode : false
	23/12/22 00:00:55 INFO mapreduce.Job: map 0% reduce 0%
	23/12/22 00:01:02 INFO mapreduce.Job: map 100% reduce 0%
	23/12/22 00:01:02 INFO mapreduce.Job: Job job_1703174394235_0001 completed successfully
	23/12/22 00:01:02 INFO mapreduce.Job: Counters: 30
	File System Counters
	FILE: Number of bytes read=0
	FILE: Number of bytes written=206715
	FILE: Number of read operations=0
	FILE: Number of large read operations=0
	FILE: Number of write operations=0
	HDFS: Number of bytes read=87
	HDFS: Number of bytes written=192
	HDFS: Number of read operations=4
	HDFS: Number of large read operations=0
	HDFS: Number of write operations=2
	Job Counters
	Launched map tasks=1
	Other local map tasks=1
	Total time spent by all maps in occupied slots (ms)=4185
	Total time spent by all reduces in occupied slots (ms)=0
	Total time spent by all map tasks (ms)=4185
	Total vcore-milliseconds taken by all map tasks=4185
	Total megabyte-milliseconds taken by all map tasks=8570880
	Map-Reduce Framework
	Map input records=8
	Map output records=8
	Input split bytes=87
	Spilled Records=0
	Failed Shuffles=0
	Merged Map outputs=0
	GC time elapsed (ms)=81
	CPU time spent (ms)=1060
	Physical memory (bytes) snapshot=154923008
	Virtual memory (bytes) snapshot=2088615936
	Total committed heap usage (bytes)=60882944
	File Input Format Counters
	Bytes Read=0
	File Output Format Counters
	Bytes Written=192
	23/12/22 00:01:02 INFO mapreduce.ImportJobBase: Transferred 192 bytes in 27.0684 seconds (7.0931 bytes/sec)
	23/12/22 00:01:02 INFO mapreduce.ImportJobBase: Retrieved 8 records.

	[root@master ~]# hdfs dfs -ls /sqoopresult001
	Found 2 items
	-rw-r--r-- 3 root supergroup 0 2023-12-22 00:01 /sqoopresult001/_SUCCESS
	-rw-r--r-- 3 root supergroup 192 2023-12-22 00:01 /sqoopresult001/part-m-00000
	[root@master ~]# hdfs dfs -cat /sqoopresult001/part-m-00000
	1,赵雷,1990-01-01,男
	2,钱电,1990-12-21,男
	3,孙风,1990-05-20,男
	4,李云,1990-08-06,男
	5,周梅,1991-12-01,女
	6,吴兰,1992-03-01,女
	7,郑竹,1989-07-01,女
	8,王菊,1990-01-20,女