已经安装了hadoop、mysql
- 将sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz上传到服务器
# 解压
tar -zxvf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
# 移动
mv /home/sqoop-1.4.6.bin__hadoop-2.0.4-alpha/* /usr/local/software/sqoop-1.4.6
# 配置
vi /etc/profile
# 配置如下
export SQOOP_HOME=/usr/local/software/sqoop-1.4.6
export PATH=$SQOOP_HOME/bin:$PATH
# 生效
source /etc/profile
# 复制
cp /usr/local/software/sqoop-1.4.6/conf/sqoop-env-template.sh /usr/local/software/sqoop-1.4.6/conf/sqoop-env.sh
# 编辑
vi /usr/local/software/sqoop-1.4.6/conf/sqoop-env.sh
# 配置如下：必须配置hadoop，非必要配置hbase、hive、zookeeper
#Set path to where bin/hadoop is available
export HADOOP_COMMON_HOME=/usr/local/software/hadoop-2.9.2
#Set path to where hadoop-*-core.jar is available
export HADOOP_MAPRED_HOME=/usr/local/software/hadoop-2.9.2
# 移动mysql驱动
mv mysql-connector-java-8.0.30.jar /usr/local/software/sqoop-1.4.6/lib
# 启动mysql
mysql.server start
# 启动hadoop
start-dfs.sh
start-yarn.sh
cd /usr/local/software/sqoop-1.4.6/
# 测试
[root@master sqoop-1.4.6]# bin/sqoop list-databases --connect jdbc:mysql://master:3306/ --username root --password 123456
Warning: /usr/local/software/sqoop-1.4.6/../hbase does not exist! HBase imports will fail.
Please set $HBASE_HOME to the root of your HBase installation.
Warning: /usr/local/software/sqoop-1.4.6/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/software/sqoop-1.4.6/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/12/21 23:15:33 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6
23/12/21 23:15:33 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
23/12/21 23:15:33 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary.
mysql
information_schema
performance_schema
sys
hive_metastore
db_test
# 进入如下目录
cd /usr/local/software/sqoop-1.4.6/
# 执行如下
bin/sqoop import \
--driver com.mysql.cj.jdbc.Driver \
--connect jdbc:mysql://master:3306/db_test \
--username root \
--password 123456 \
--delete-target-dir \
--target-dir /sqoopresult001 \
--table student \
--m 1
# 查看hadoop执行目录下生成的文件
hdfs dfs -ls /sqoopresult001
查看详情
[root@master sqoop-1.4.6]# bin/sqoop import \
> --driver com.mysql.cj.jdbc.Driver \
> --connect jdbc:mysql://master:3306/db_test \
> --username root \
> --password 123456 \
> --delete-target-dir \
> --target-dir /sqoopresult001 \
> --table student \
> --m 1
Warning: /usr/local/software/sqoop-1.4.6/../hbase does not exist! HBase imports will fail.
Please set $HBASE_HOME to the root of your HBase installation.
Warning: /usr/local/software/sqoop-1.4.6/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/software/sqoop-1.4.6/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/12/22 00:00:31 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6
23/12/22 00:00:31 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
23/12/22 00:00:32 WARN sqoop.ConnFactory: Parameter --driver is set to an explicit driver however appropriate connection manager is not being set (via --connection-manager). Sqoop is going to fall back to org.apache.sqoop.manager.GenericJdbcManager. Please specify explicitly which connection manager should be used next time.
23/12/22 00:00:32 INFO manager.SqlManager: Using default fetchSize of 1000
23/12/22 00:00:32 INFO tool.CodeGenTool: Beginning code generation
23/12/22 00:00:32 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
23/12/22 00:00:32 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
23/12/22 00:00:32 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /usr/local/software/hadoop-2.9.2
注: /tmp/sqoop-root/compile/1e6acb49375d5ccc8d258f91db944575/student.java使用或覆盖了已过时的 API。
注: 有关详细信息, 请使用 -Xlint:deprecation 重新编译。
23/12/22 00:00:34 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-root/compile/1e6acb49375d5ccc8d258f91db944575/student.jar
23/12/22 00:00:35 INFO tool.ImportTool: Destination directory /sqoopresult001 is not present, hence not deleting.
23/12/22 00:00:35 INFO mapreduce.ImportJobBase: Beginning import of student
23/12/22 00:00:35 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar
23/12/22 00:00:35 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM student AS t WHERE 1=0
23/12/22 00:00:35 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps
23/12/22 00:00:35 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
23/12/22 00:00:45 INFO db.DBInputFormat: Using read commited transaction isolation
23/12/22 00:00:45 INFO mapreduce.JobSubmitter: number of splits:1
23/12/22 00:00:45 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled
23/12/22 00:00:45 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1703174394235_0001
23/12/22 00:00:46 INFO impl.YarnClientImpl: Submitted application application_1703174394235_0001
23/12/22 00:00:46 INFO mapreduce.Job: The url to track the job: http://master:8088/proxy/application_1703174394235_0001/
23/12/22 00:00:46 INFO mapreduce.Job: Running job: job_1703174394235_0001
23/12/22 00:00:55 INFO mapreduce.Job: Job job_1703174394235_0001 running in uber mode : false
23/12/22 00:00:55 INFO mapreduce.Job: map 0% reduce 0%
23/12/22 00:01:02 INFO mapreduce.Job: map 100% reduce 0%
23/12/22 00:01:02 INFO mapreduce.Job: Job job_1703174394235_0001 completed successfully
23/12/22 00:01:02 INFO mapreduce.Job: Counters: 30
File System Counters
FILE: Number of bytes read=0
FILE: Number of bytes written=206715
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=87
HDFS: Number of bytes written=192
HDFS: Number of read operations=4
HDFS: Number of large read operations=0
HDFS: Number of write operations=2
Job Counters
Launched map tasks=1
Other local map tasks=1
Total time spent by all maps in occupied slots (ms)=4185
Total time spent by all reduces in occupied slots (ms)=0
Total time spent by all map tasks (ms)=4185
Total vcore-milliseconds taken by all map tasks=4185
Total megabyte-milliseconds taken by all map tasks=8570880
Map-Reduce Framework
Map input records=8
Map output records=8
Input split bytes=87
Spilled Records=0
Failed Shuffles=0
Merged Map outputs=0
GC time elapsed (ms)=81
CPU time spent (ms)=1060
Physical memory (bytes) snapshot=154923008
Virtual memory (bytes) snapshot=2088615936
Total committed heap usage (bytes)=60882944
File Input Format Counters
Bytes Read=0
File Output Format Counters
Bytes Written=192
23/12/22 00:01:02 INFO mapreduce.ImportJobBase: Transferred 192 bytes in 27.0684 seconds (7.0931 bytes/sec)
23/12/22 00:01:02 INFO mapreduce.ImportJobBase: Retrieved 8 records.
[root@master ~]# hdfs dfs -ls /sqoopresult001
Found 2 items
-rw-r--r-- 3 root supergroup 0 2023-12-22 00:01 /sqoopresult001/_SUCCESS
-rw-r--r-- 3 root supergroup 192 2023-12-22 00:01 /sqoopresult001/part-m-00000
[root@master ~]# hdfs dfs -cat /sqoopresult001/part-m-00000
1,赵雷,1990-01-01,男
2,钱电,1990-12-21,男
3,孙风,1990-05-20,男
4,李云,1990-08-06,男
5,周梅,1991-12-01,女
6,吴兰,1992-03-01,女
7,郑竹,1989-07-01,女
8,王菊,1990-01-20,女
