sqoop 1.4.7
tar -zxvf /usr/tools/sqoop-1.4.7.tar.gz
環境變量配置
vim /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_211-amd64
export PATH=$PATH:JAVA_HOME/bin
export HADOOP_HOME=/home/hadoop-3.2.0
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export HIVE_HOME=/usr/local/hive-3.1.1
export PATH=$PATH:$HIVE_HOME/bin
export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.14
export PATH=$PATH:$ZOOKEEPER_HOME/bin
#export SQOOP_HOME=/usr/local/sqoop-1.99.7
export SQOOP_HOME=/usr/local/sqoop-1.4.7
export PATH=$PATH:$SQOOP_HOME/bin
source /etc/profile
設置HADOOP_COMMON_HOME
[root@node3 sqoop-1.4.7]# cd conf
[root@node3 conf]# ls
oraoop-site-template.xml sqoop-env-template.cmd sqoop-env-template.sh sqoop-site-template.xml
[root@node3 conf]# cp sqoop-env-template.sh sqoop-env.sh
[root@node3 conf]# vi sqoop-env.sh
#Set path to where bin/hadoop is available
export HADOOP_COMMON_HOME=/home/hadoop-3.2.0
#Set path to where hadoop-*-core.jar is available
export HADOOP_MAPRED_HOME=/home/hadoop-3.2.0
#set the path to where bin/hbase is available
export HBASE_HOME=/usr/local/hbase-2.0.5
#Set the path to where bin/hive is available
export HIVE_HOME=/usr/local/hive-3.1.1
#Set the path for where zookeper config dir is
export ZOOCFGDIR=/usr/local/zookeeper-3.4.14
驗證是否安裝完成
[root@node3 conf]# sqoop help
Warning: /usr/local/sqoop-1.4.7/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/sqoop-1.4.7/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
Warning: /usr/local/sqoop-1.4.7/../zookeeper does not exist! Accumulo imports will fail.
Please set $ZOOKEEPER_HOME to the root of your Zookeeper installation.
Error: Could not find or load main class org.apache.sqoop.Sqoop
解決方法：修改sqoop安裝目錄bin文件夾下的configure-sqoop文件，找到如下位置並注釋
## Moved to be a runtime check in sqoop.
#if [ ! -d "${HCAT_HOME}" ]; then
# echo "Warning: $HCAT_HOME does not exist! HCatalog jobs will fail."
# echo 'Please set $HCAT_HOME to the root of your HCatalog installation.'
#fi
#if [ ! -d "${ACCUMULO_HOME}" ]; then
# echo "Warning: $ACCUMULO_HOME does not exist! Accumulo imports will fail."
# echo 'Please set $ACCUMULO_HOME to the root of your Accumulo installation.'
#fi
[root@node3 bin]# sqoop help
Error: Could not find or load main class org.apache.sqoop.Sqoop
缺少jar包：
在http://central.maven.org/maven2/org/apache/sqoop/sqoop/1.4.7/下載jar包，並將jar包放入到sqoop的lib目錄下
將mysql的jdbc jar包放入到sqoop的lib目錄下
[root@node3 lib]# cp /usr/tools/mysql-connector-java-5.1.47.jar ./
連接數據庫
[root@node3 lib]# sqoop list-tables --connect jdbc:mysql://node1:3306/ecs --username sqoop --password sqoop
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/hadoop-3.2.0/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-2.0.5/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
2019-05-30 09:45:33,362 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
2019-05-30 09:45:33,618 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/avro/LogicalType
at org.apache.sqoop.manager.DefaultManagerFactory.accept(DefaultManagerFactory.java:67)
解決方法
在 https://jar-download.com/artifacts/org.apache.avro/avro/1.8.1/source-code下載avro-1.8.1.jar並放到sqoop的lib目錄下
[root@node3 lib]# sqoop list-tables --connect jdbc:mysql://node1:3306/ecs --username sqoop --password sqoop
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/hadoop-3.2.0/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-2.0.5/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
2019-05-30 09:56:09,508 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
2019-05-30 09:56:09,738 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
2019-05-30 09:56:10,069 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Thu May 30 09:56:10 HKT 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
role
users
使用sqoop從mysql導入到hdfs
sqoop import --connect jdbc:mysql://node1:3306/ecs --username sqoop --password sqoop --table role --target-dir /user/data/role
[root@node3 ~]# sqoop import-all-tables --connect jdbc:mysql://node1:3306/ecs --username sqoop --password sqoop \
--warehouse-dir /user/hive/warehouse/ecs