Bash Script for automatically downloading, copying and updating LatLong geo location files from Digital Element

Problem this snippet solves:

This is a script that automatically downloads the latest geolocation database file from Digital Element, copies it to a list of previously defined LTMs and runs the geoip_update_data command.

You need to place this script in a directory and also create a simple text file where the script will read the LTM name and ip information from. The format of the file should follow the simple rule "ltm_name ltm_ip", with spaces or tabs delimiting the two columns, like the below example. The script is DNS agnostic so it will use the names as reference and the IPs to connect to.

la01ltm1 10.10.1.7

The script collects some information prior to executing and validates reachability of all the devices on file. It uses sshpass to connect to but you can change to SSH keys if preferred.

Code :

#!/bin/bash
echo "

Starting GeoLocation Update Script...
"
#read
function check_update {
#checks installed versions on F5 and compares with most recent file
echo "
Checking for updates and comparing with LTM $LTM_CHECKED...
"
GEOFILENAME_ONLINE_DATE=(`echo $GEOFILENAME | sed -n 's/.*-\([^.]*\).*/\1/p'`)
GEOFILENAME_LOCAL_DATE=(`sshpass -p '$LTM_PASSWORD' ssh -o StrictHostKeyChecking=no $LTM_USERNAME@$LTM_CHECKED geoip_lookup -f /shared/GeoIP/F5GeoIPCity2.dat 65.61.115.197 | grep version | sed -n 's/.* \(.*\) Build.*/\1/p'`)
echo "Current version date = $GEOFILENAME_LOCAL_DATE
Available version date = $GEOFILENAME_ONLINE_DATE
"
if [ $GEOFILENAME_ONLINE_DATE -gt $GEOFILENAME_LOCAL_DATE ]; then
echo "Update required. Hit enter to run script...
"
read
else
echo "Update not required. Exiting...
"
exit
fi
}
function default_option {
    #Presents the user a menu with the preconfigured options and gives them the option to change parameters
DIRECTORY=Scripts/GeoUpdate/Images
DEST_DIRECTORY=/shared/images
date=*
USERNAME=root
DEVICE_FILE=devices_adv_geo.txt
   echo -ne "
These are the default configs for the script. 
1 - Source Dir: $DIRECTORY
2 - Dest. Dir: $DEST_DIRECTORY
3 - Date: Most Recent File
4 - Username: $USERNAME
5 - Source device file: $DEVICE_FILE

Do you want to change? [Y/N - ENTER for NO] "
read -n 1 opt # waits for a y or n from the user and stores in $opt - "-n 1" means wait for one digit and don't wait for enter
#opt=n
if [ "$opt" = "y" ]; then #result if $opt is y
#assigns variable values or defaults
if [ -z $1 ]; then # if no parameters have been put in the command line then run the questions
echo -ne "
Enter source directory (Enter for default: $DIRECTORY): "
read DIRECTORY # reads source directory and stores in $DIRECTORY
if [ -z $DIRECTORY ]; then #if no input from user then set $DIRECTORY to default
DIRECTORY=Scripts/GeoUpdate/Images
fi
echo -ne "Enter destination directory (Enter for default: $DEST_DIRECTORY): "
read DEST_DIRECTORY
if [ -z $DEST_DIRECTORY ]; then 
DEST_DIRECTORY=/shared/images
fi
echo -ne "Enter date for Geo Files in format YYYYMMDD (Hit ENTER to use most recent file copied to $DIRECTORY): "
read date
if [ -z $date ]; then 
date=*
fi
echo -ne "Enter username (default $USERNAME): "
read USERNAME
if [ -z $USERNAME ]; then 
USERNAME=root
else 
echo -ne "Enter password for $USERNAME: "
read PASSWORD
fi
echo -ne "Enter device file (default $DEVICE_FILE). Device file must contain server name and ip, separated by a space, one server per line: "
read DEVICE_FILE
if [ -z $DEVICE_FILE ]; then 
DEVICE_FILE=devices_adv_geo.txt
fi
fi
elif [[ "$opt" = "n" || "$opt" = "" ]]; then #in case user doesn't want to change parameters, set all variables to default
echo "

Running Script with Default configs...
"
else #in case user types anything other than y or n, presents an error message and prints the menu again
echo "
Bad Option. Type y or n"
#read
#printf "\033c" 
default_option
fi
}
function read_usernames {
echo "
Collecting credentials...
"
echo -ne "Enter Digital Element username [default blank]: "
read DE_USERNAME
if [ -z $DE_USERNAME ]; then 
DE_USERNAME=""
fi
echo -ne "Enter Digital Element password [default blank]: "
read -s DE_PASSWORD
if [ -z $DE_PASSWORD ]; then 
DE_PASSWORD=""
fi
echo -ne "
Enter LTM username [default blank]: "
read LTM_USERNAME
if [ -z $LTM_USERNAME ]; then
LTM_USERNAME=""
fi
echo -ne "Enter LTM password [default blank]: "
read -s LTM_PASSWORD
if [ -z $LTM_PASSWORD ]; then 
LTM_PASSWORD=""
fi
LTM_CHECKED=(`head -1 $DEVICE_FILE | awk '{print $1}'`)
echo -ne "
Enter LTM to check with [default first device on file $DEVICE_FILE - $LTM_CHECKED]: "
read LTM_CHECKED
if [ -z $LTM_CHECKED ]; then 
LTM_CHECKED=(`head -1 $DEVICE_FILE | awk '{print $1}'`)
fi
}
default_option
read_usernames
GEOFILENAME=(`wget -qO- --user=$DE_USERNAME --http-password=$DE_PASSWORD https://portal.digitalelement.com/downloads/F5Edge/files/ | grep City2 | sed -n 's/[^"]*"\.\/\([^"]*\).*/\1/p'`)
check_update
echo "Downloading Advanced Geo File from Digital Element..."
function download_geo_file {
  if [ -s "$DIRECTORY/$GEOFILENAME" ]; then
LOCALGEOFILESIZE=(`ls -la $DIRECTORY/$GEOFILENAME | awk {'print $5'}`)
LOCALGEOFILESIZE=(`echo $[LOCALGEOFILESIZE/1024/1024]`)
REMOTEGEOFILESIZE=(`wget -qO- --user=$DE_USERNAME --http-password=$DE_PASSWORD https://portal.digitalelement.com/downloads/F5Edge/files/ | grep City2 | sed -n 's/.*[^\.]'$GEOFILENAME'[^(]*(\([0-9]*\)M).*/\1/p'`)
echo -ne "

File with same name exists on target directory.

File name: $GEOFILENAME

File size on local directory is $LOCALGEOFILESIZE MB.
File size on Digital Element is $REMOTEGEOFILESIZE MB

Do you want to re-download and overwrite it [y/n - ENTER for YES]?"
read -n 1 opt
if [[ "$opt" = "y"  || "$opt" = "Y" || "$opt" = "" ]]; then 
wget -q --show-progress --user=$DE_USERNAME --http-password=$DE_PASSWORD https://portal.digitalelement.com/downloads/F5Edge/files/$GEOFILENAME -O "$DIRECTORY/$GEOFILENAME"
echo "
done
"
else 
echo "
Skypping file download...
"
return
fi
  fi
}
download_geo_file

FILE_ID=(City2) #creates an array with the 5 types of files to be transferred/updated on the F5 appliances
deviceip=(`cat $DEVICE_FILE | awk '{print $2}'`) #reads device file and prints only second column to array $deviceip
devicename=(`cat $DEVICE_FILE | awk '{print $1}'`) #reads device file and prints only first column to array $devicename
len=${#deviceip[*]} #creates variable $len with total number of lines read on file
fil=${#FILE_ID[*]} #creates variable $fil with total number of file types existing - always 5 unless otherwise stated
function display {
 # displays the list of devices and IPs and does a small sanity check (ping)
 i=0 #auxiliar variable to while command
l=1 #will transform array zero in 'item number 1'
echo "
LIST OF DEVICES:

Checking hosts availability...
"
echo "+----------+-------------------------------+----------------+------------+"
echo "| Device   |  IP Address                   | Name           | Available? |"
echo "|----------+-------------------------------+----------------+------------+"
while [ $i -lt $len ]; do #executes while auxiliar variable is lower than the number of devices read previously
ping -c 2 -t 1 -i 0.2 ${deviceip[$i]} > /dev/null #ping on device
avail=$? #throws ping status ($?) into variable $avail
echo -ne "| $l  " $'\t' "  |  ${deviceip[$i]} " $'\t' $'\t' "  | ${devicename[$i]}"; if [ $avail == 0 ]; then #prints number of device (array position 0 will be device 1, then prints the ip then prints the name then
echo -ne $'\t' "   |  "; echo -ne '\033[1;32m' "UP" '\033[0m'; echo "      |"                         # if status comes out 0 means that host is UP, prints UP in green, DOWN in red
else echo -ne $'\t' "   |  "; echo -ne '\033[1;31m' "DOWN" '\033[0m'; echo "    |"
fi
let i++
let l++
done
echo "+----------+-------------------------------+----------------+------------+"
echo "
LIST OF FILES:
"
j=0 #auxiliar variable to
 # runs a ls sorted by time on the source directory, looking for one of the four file types. If user provided a specific date, also looks for it
 # otherwise it will display the most recent file. head 1 prints only one file and xargs basename will strip out the directory from the ls output
while [ $j -lt $fil ]; do 
filename=(`ls -t $DIRECTORY/*${FILE_ID[$j]}*$date* | head -1 | xargs -n1 basename`)                   
echo "${FILE_ID[$j]} File: $filename "
let j++
done
}
function filecopy {
 #copies the file via scp to the host
 filename=(`ls -t $DIRECTORY/*$1*$date* | head -1 | xargs -n1 basename`) #runs the same ls as before to assign the filename to the variable. $1 is the parameter passed whe the function was called
echo -ne '\033[1;36m' "
*** Copying $1 to ${devicename[$i]} ***
" '\033[0m'; #informs the user about the type of file being copied and to which host
echo -ne '\033[1;19m' "
Executing Command: scp $DIRECTORY/$filename $LTM_USERNAME@${deviceip[$i]}:$DEST_DIRECTORY/$filename
" '\033[0m'; #informs the user the specific command
sshpass -p '$LTM_PASSWORD' scp $DIRECTORY/$filename $LTM_USERNAME@${deviceip[$i]}:$DEST_DIRECTORY/$filename #copies using source directory specified in beginning, filename just recently listed and to device ip sent as a parameter when the function was called
status=${?} #saves copy status in a variable
if [ $status != 0 ]; then 
echo "Transfer failed" #error message in case the status is different then zero
fi
}
function run_update {
filename=(`ls -t $DIRECTORY/*$1*$date* | head -1 | xargs -n1 basename`)
echo -ne '\033[1;36m' "

*** Executing $1 update on ${devicename[$i]} ***" '\033[0m';
        LOGDIR=(`echo $DIRECTORY | sed -n 's/\/Images/\/log/p'`)
        mkdir -p $LOGDIR
sshpass -p '$LTM_PASSWORD' ssh -o StrictHostKeyChecking=no -v -E $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt $LTM_USERNAME@${deviceip[$i]} geoip_update_data -f $DEST_DIRECTORY/$filename
status=${?}
if [ $status == 0 ]; then
printcmd="ssh -v -E /tmp/${deviceip[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt $LTM_USERNAME@${deviceip[$i]} geoip_update_data -f $DEST_DIRECTORY/$filename" #saves the command -v verbose, -E creates a file log
echo -ne '\033[1;19m' "

Running:"; echo $printcmd | awk {'print $1 " " $5 " " $6 " " $7 " " $8'}
echo -ne "
" '\033[0m'
awk '{ $1=""; print $0 }' $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt | grep Connecting | sort -u
awk '{ $1=""; print $0 }' $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt | grep Connection | sort -u
awk '{ $1=""; print $0 }' $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt | grep 'Sending command' | sort -u
echo " Command Executed Successfully"
GEOFILENAME_TESTED_DATE=(`sshpass -p '$LTM_PASSWORD' ssh $LTM_USERNAME@${deviceip[$i]} geoip_lookup -f /shared/GeoIP/F5GeoIPCity2.dat 65.61.115.197 | grep version | sed -n 's/.* \(.*\) Build.*/\1/p'`)
if [ $GEOFILENAME_ONLINE_DATE -eq $GEOFILENAME_TESTED_DATE ]; then
VERIFY_PASS="UP TO DATE"
else
VERIFY_PASS="OUTDATED FILE!!!!"
fi
echo " 

Verifying database date installed on ${devicename[$i]} : $VERIFY_PASS"
rm $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt
else echo "Command failed - Check Log file: $LOGDIR/${devicename[$i]}_file_$1_copy.txt"
fi
}
display
echo "
Hit Enter to start Script, Ctrl-C to stop"
read
i=0 # not a simple auxiliar variable. this "i" will be called by the two functions filecopy and run_update. it will determine IN THE FUNCTION what is the array position the command should be called, although it's not passed as parameter, it can be used anytime in the script.
while [ $i -lt $len ]; do #it will run for as many IPs are in the file
k=0
while [ $k -lt $fil ]; do #it will run for as many types of files exist. (Remember, so far this number is fixed in 4) This internal while will run 4 times for each ip, and the number of times for each ip will be determined by the outer while
echo -ne '\033[1;31m' "

============================  Processing file ${FILE_ID[$k]} on ${devicename[$i]}  =========================================================
" '\033[0m';
filecopy ${FILE_ID[$k]} #pass the type of file as a parameter. function will see ${FILE_ID[$k]} as $1. Will use i to determine what IP will be treated this time
run_update ${FILE_ID[$k]} 
let k++
done
let i++
done
echo -ne '\033[1;19m'"

*************************** Script complete. For full details check log file *******************************************

" '\033[0m'

Tested this on version: