#!/bin/bash
######################################################################
# Download map data files from xxx Maps website:
# 1. download the fileDownloadable.xml which contains downloading info
# 2. parse fileDownloadable.xml to get the filenames, urls and md5sum
# 3. carry out the file downloading
# 4. make md5sum against downloaded files and save result to file
# 5. check md5sum to make sure downloaded files valid
######################################################################
set -e
## Expected env variables xxx_USERNAME and xxx_PASSWORD
if [[ ! ($xxx_USERNAME && $xxx_PASSWORD) ]]; then
echo "===> Error: Expected env variable xxx_USERNAME and xxx_PASSWORD!"
exit 1
fi
## install wget if not already installed in the docker container
apt-get update && apt-get -y install wget
wget --version | awk 'NR==1 {print}'
md5sum --version | grep 'GNU'
DIR_DOWNLOADED="downloaded"
FILES_DOWNLOADABLE=filesDownloadable.xml
FILES_DOWNLOADABLE_URI="https://xxx.xxx.com/control/navt/login?username=$xxx_USERNAME&password=$xxx_PASSWORD&action=authenticate&accountID=xxxxxx&nextURL=%2Fcontrol%2Fnavt%2FfilesDownloadable%3Faction%3Dxml%26limitDays%3D10"
echo "===> Cleaning up any existing files and folders"
if [[ -d $DIR_DOWNLOADED ]]; then
rm -rf $DIR_DOWNLOADED
echo "===> Info: deleted existing folder $DIR_DOWNLOADED"
fi
if [[ -f verifymd5.tmp ]]; then
rm verifymd5.tmp
echo "===> Info: deleted existing file verifymd5.tmp"
fi
if [[ -f verifymd5.txt ]]; then
rm verifymd5.txt
echo "===> Info: deleted existing file verifymd5.txt"
fi
echo "===> xxx maps username: $xxx_USERNAME, xxx maps password: $xxx_PASSWORD"
echo "===> Downloding $FILES_DOWNLOADABLE: url=$FILES_DOWNLOADABLE_URI"
wget -q -L -O $FILES_DOWNLOADABLE "$FILES_DOWNLOADABLE_URI"
if ! [[ -e $FILES_DOWNLOADABLE && -s $FILES_DOWNLOADABLE ]]; then
echo "===> Error: $FILES_DOWNLOADABLE was not downloaded or empty!"
exit 1
fi
if [[ $IS_DEV == "true" ]]; then
echo "===> Info: Print filesDownloadable.xml"
cat $FILES_DOWNLOADABLE
fi;
awk 'BEGIN {filename=""; md5=""; link=""}
/<FileName>/ {filename=$0; gsub("<FileName><!\\[CDATA\\[", "", filename); gsub("\\]\\]></FileName>", "", filename); gsub("[ \t]", "", filename)}
/<DownloadLink>/ {link=$0; gsub("<DownloadLink><!\\[CDATA\\[", "", link); gsub("\\]\\]></DownloadLink>", "", link); gsub("[ \t]", "", link); print "wget -L -O \"" filename "\" \"" link"\"" > "dcmd.sh"}
/<MD5CheckSum>/ {md5=$0; gsub("<MD5CheckSum><!\\[CDATA\\[", "", md5); gsub("\\]\\]></MD5CheckSum>", "", md5); gsub("[ \t]", "", md5); print md5 " " filename > "md5.txt"}
' $FILES_DOWNLOADABLE
if ! [[ -e dcmd.sh && -s dcmd.sh && -e md5.txt && -s md5.txt ]]; then
echo "===> Warn: dcmd.sh and/or md5.txt was not created or empty! There might be no files available for downloading."
exit 0
fi
if [[ $IS_DEV == "true" ]]; then
cat dcmd.sh | awk 'NR<3 {print > "dcmd.sh"}'
echo "===> Info: Print reduced wget command list in dcmd.sh"
cat dcmd.sh
cat md5.txt | awk 'NR<3 {print > "md5.txt"}'
echo "===> Info: Print reduced hashes in md5.txt"
cat md5.txt
fi
# Allow 60s to manually remove large files from download cmd - comment out in production
#echo "===> Sleeping for 60 seconds... you can manully remove unwanted files now..."
#sleep 60
mkdir -p $DIR_DOWNLOADED
cd $DIR_DOWNLOADED
echo "===> Downloading xxx Maps data files to folder $DIR_DOWNLOADED ... this may take a while..."
source ../dcmd.sh
echo "===> Creating md5sum for downloaded files"
for afile in *
do
if [[ -f $afile ]]; then
md5sum $afile >> ../verifymd5.tmp
fi
done
cd ..
awk '{gsub("\*",""); print $1 " " $2 > "verifymd5.txt"}' verifymd5.tmp
echo "===> Verify verifiymd5.txt against md5.txt ..."
while read -r line
do
if grep -q "$line" md5.txt; then
echo "===> Info: md5sum checked OK: $line"
else
echo "===> Error: md5sum does not match: $line"
exit 1
fi
done < verifymd5.txt
# Get s3 version from pipeline input
S3_VERSION=$(cat version/number)
tar -czvf mapload/mapload-${S3_VERSION}.tgz $DIR_DOWNLOADED
echo "===> Info: zipped up files to upload. s3 version number: $S3_VERSION"
ls -l $DIR_DOWNLOADED
ls -l mapload
echo "===> xxx Maps data downloaded successfully"
exit 0
相关推荐
rootroot@rootroot-E400:~/wyb/ap6212a0_a33_sc3817r/lichee$ ./build.sh config Welcome to mkscript setup progress All available chips: 0. sun8iw5p1 Choice: 0 All available platforms: 0. android 1. ...
rootroot@rootroot-E400:~/wyb/ap6212a0_a33_sc3817r/lichee$ ./build.sh config Welcome to mkscript setup progress All available chips: 0. sun8iw5p1 Choice: 0 All available platforms: 0. android 1. ...
Jenkins作为一个流行的持续集成/持续部署(CI/CD)工具,能够帮助自动化软件开发流程,包括构建、测试和部署。本篇文章将深入探讨如何在Kubernetes集群上成功部署Jenkins及其相关的scripts。 首先,我们需要准备...
Python参考手册,官方正式版参考手册,chm版。以下摘取部分内容:Navigation ...PEP 338: Executing Modules as Scripts PEP 341: Unified try/except/finally PEP 342: New Generator Features PEP 343: The ‘with...
data_type rdval; printk("****wyb %s:%d/%s()! \n", __FILE__, __LINE__, __func__); LOG_ERR_RET(sensor_read(sd, 0x300a, &rdval;)) printk("****wyb %s:%d/%s()! 0x300a rdval=0xx\n", __FILE__, __LINE__,...