Sqoop优化

1.加参数-m 和切分字段--split-by 增加map数

2.分库分表数据接入拉取使用union all优化

 

e.g.1:

sqoop import "-Dorg.apache.sqoop.splitter.allow_text_splitter=true"  --connect $connector --null-string '\\N' --null-non-string '\\N' --query "select  CARD_ID,MEMBER_ID,MOBILE,TYPE,PASSWORD

,MEMBER_NAME,USER_ID,TEMP_ID,DEPARTMENT,REGISTER_CODE,COMPANY_CODE,CHANNEL,STATUS,IS_FREEZE,CUSTOMER_CODE,CREATE_DATE,LASTUPDATE from member_hqm_${k}_prd.gome_card_info_$i where  \$CONDITIONS" --fields-terminated-by '\001' --lines-terminated-by '\n' 

--split-by CARD_ID  -m 10 --hive-drop-import-delims --compression-codec "com.hadoop.compression.lzo.LzopCodec" --target-dir $path 

 

e.g.2:

myTempSql="select  ID,LASTUPDATE from mytbl"${k}"_test.mytbl_0 union all select  ID,LASTUPDATE from mytbl"${k}"_test.mytbl_1 union all select  ID,LASTUPDATE from mytbl"${k}"_test.mytbl_2 where  \$CONDITIONS"

sqoop import  --connect $connector --null-string '\\N' --null-non-string '\\N' --query "${myTempSql}" --fields-terminated-by '\001' --lines-terminated-by '\n' -m 1 --hive-drop-import-delims --compression-codec "com.hadoop.compression.lzo.LzopCodec" --target-dir $path