mapdata/ci/scripts/mapDownload.sh

#!/bin/bash

 

######################################################################

# Download map data files from xxx Maps website: 

# 1. download the fileDownloadable.xml which contains downloading info

# 2. parse fileDownloadable.xml to get the filenames, urls and md5sum

# 3. carry out the file downloading

# 4. make md5sum against downloaded files and save result to file

# 5. check md5sum to make sure downloaded files valid

######################################################################

 

set -e

 

## Expected env variables xxx_USERNAME and xxx_PASSWORD

if [[ ! ($xxx_USERNAME && $xxx_PASSWORD) ]]; then

  echo "===> Error: Expected env variable xxx_USERNAME and xxx_PASSWORD!"

  exit 1

fi

 

## install wget if not already installed in the docker container

apt-get update && apt-get -y install wget

wget --version | awk 'NR==1 {print}'

md5sum --version | grep 'GNU'

 

DIR_DOWNLOADED="downloaded"

FILES_DOWNLOADABLE=filesDownloadable.xml

FILES_DOWNLOADABLE_URI="https://xxx.xxx.com/control/navt/login?username=$xxx_USERNAME&password=$xxx_PASSWORD&action=authenticate&accountID=xxxxxx&nextURL=%2Fcontrol%2Fnavt%2FfilesDownloadable%3Faction%3Dxml%26limitDays%3D10"

 

echo "===> Cleaning up any existing files and folders"

if [[ -d $DIR_DOWNLOADED ]]; then

rm -rf $DIR_DOWNLOADED

echo "===> Info: deleted existing folder $DIR_DOWNLOADED"

fi

 

if [[ -f verifymd5.tmp ]]; then

rm verifymd5.tmp

echo "===> Info: deleted existing file verifymd5.tmp"

fi

 

if [[ -f verifymd5.txt ]]; then

rm verifymd5.txt

echo "===> Info: deleted existing file verifymd5.txt"

fi

 

echo "===> xxx maps username: $xxx_USERNAME, xxx maps password: $xxx_PASSWORD"

echo "===> Downloding $FILES_DOWNLOADABLE: url=$FILES_DOWNLOADABLE_URI"

 

wget -q -L -O $FILES_DOWNLOADABLE "$FILES_DOWNLOADABLE_URI"

 

if ! [[ -e $FILES_DOWNLOADABLE && -s $FILES_DOWNLOADABLE ]]; then

  echo "===> Error: $FILES_DOWNLOADABLE was not downloaded or empty!"

  exit 1

fi

 

if [[ $IS_DEV == "true" ]]; then

  echo "===> Info: Print filesDownloadable.xml"

  cat $FILES_DOWNLOADABLE

fi;

 

awk 'BEGIN {filename=""; md5=""; link=""}

     /<FileName>/ {filename=$0; gsub("<FileName><!\\[CDATA\\[", "", filename); gsub("\\]\\]></FileName>", "", filename); gsub("[ \t]", "", filename)}

     /<DownloadLink>/ {link=$0; gsub("<DownloadLink><!\\[CDATA\\[", "", link); gsub("\\]\\]></DownloadLink>", "", link); gsub("[ \t]", "", link); print "wget -L -O \"" filename "\" \"" link"\"" > "dcmd.sh"}

     /<MD5CheckSum>/ {md5=$0; gsub("<MD5CheckSum><!\\[CDATA\\[", "", md5); gsub("\\]\\]></MD5CheckSum>", "", md5); gsub("[ \t]", "", md5); print md5 " " filename > "md5.txt"}

' $FILES_DOWNLOADABLE

 

if ! [[ -e dcmd.sh && -s dcmd.sh && -e md5.txt && -s md5.txt ]]; then

  echo "===> Warn: dcmd.sh and/or md5.txt was not created or empty! There might be no files available for downloading."

  exit 0

fi

 

if [[ $IS_DEV == "true" ]]; then

  cat dcmd.sh | awk 'NR<3 {print > "dcmd.sh"}'

  echo "===> Info: Print reduced wget command list in dcmd.sh"

  cat dcmd.sh

  

  cat md5.txt | awk 'NR<3 {print > "md5.txt"}'

  echo "===> Info: Print reduced hashes in md5.txt"

  cat md5.txt

fi

 

# Allow 60s to manually remove large files from download cmd - comment out in production

#echo "===> Sleeping for 60 seconds... you can manully remove unwanted files now..."

#sleep 60

 

mkdir -p $DIR_DOWNLOADED

cd $DIR_DOWNLOADED

 

echo "===> Downloading xxx Maps data files to folder $DIR_DOWNLOADED ... this may take a while..."

source ../dcmd.sh

 

echo "===> Creating md5sum for downloaded files"

for afile in *

do

  if [[ -f $afile ]]; then

    md5sum $afile >> ../verifymd5.tmp

  fi

done

 

cd ..

awk '{gsub("\*",""); print $1 " " $2 > "verifymd5.txt"}' verifymd5.tmp 

 

echo "===> Verify verifiymd5.txt against md5.txt ..."

while read -r line

do 

  if grep -q "$line" md5.txt; then

    echo "===> Info: md5sum checked OK: $line"

  else

    echo "===> Error: md5sum does not match: $line"

    exit 1

  fi

done < verifymd5.txt

 

# Get s3 version from pipeline input

S3_VERSION=$(cat version/number)

tar -czvf mapload/mapload-${S3_VERSION}.tgz $DIR_DOWNLOADED

echo "===> Info: zipped up files to upload. s3 version number: $S3_VERSION"

 

ls -l $DIR_DOWNLOADED

ls -l mapload

 

echo "===> xxx Maps data downloaded successfully"

 

exit 0