Microsoft Office 365 IP intelligence
Problem this snippet solves: This snippet adds Microsoft Office 365 IP intelligence. This snippet parses the O365IPAddresses.xml file that Microsoft supplies to help identify Microsoft URLs and IP address ranges. With this snippet you can check if an IP address belongs to Microsoft and let the BIG-IP decide to allow or deny traffic. For more info see: Office 365 URLs and IP address ranges DISCLAIMER: This code has never been tested outside a lab environment. Consider this snippet to be a proof-of-concept. How to use this snippet: Prepare BIG-IP Create LX Workspace: office365_ipi Add iRule: office365_ipi_irule Add Extension: office365_ipi_extension Add LX Plugin: office365_ipi_plugin -> From Workspace: office365_ipi Install node.js modules # cd /var/ilx/workspaces/Common/office365_ipi/extensions/office365_ipi_extension # npm install xml2js https repeat lokijs ip-range-check --save office365_ipi_irule ### ### Name : office365_ipi_irule ### Author : Niels van Sluis, (niels@van-sluis.nl) ### Version: 0.1 ### Date : 2017-07-25 ### when RULE_INIT { # set table timeout to 1 hour set static::office365_ipi_timeout 3600 set static::office365_ipi_lifetime 3600 } when CLIENT_ACCEPTED { # Valid product names are: # o365, LYO, Planner, Teams, ProPlus, OneNote, Yammer, # EXO, Identity, Office365Video, WAC, SPO, RCA, Sway, # EX-Fed, OfficeMobile, CRLs, OfficeiPad, EOP # # Use 'any' to match an IP address in all products. set productName "o365" set ipAddress [IP::client_addr] set key $productName:$ipAddress set verdict [table lookup -notouch $key] if { $verdict eq "" } { log local0. "Need to retrieve verdict via iruleslx" set rpc_handle [ILX::init office365_ipi_plugin office365_ipi_extension] if {[catch {ILX::call $rpc_handle checkProductIP $productName $ipAddress} verdict]} { log local0.error "Client - [IP::client_addr], ILX failure: $verdict" return } log local0. "The verdict for $ipAddress: $verdict"; # cache verdict table set $key $verdict $static::office365_ipi_timeout $static::office365_ipi_lifetime } # verdict is 0 (reject) or 1 (allow) if { !($verdict) } { log local0. "rejected IP address: $ipAddress" reject } } Code : /** *** Name : office365_ipi_extension *** Author : Niels van Sluis, *** Version: 0.1 *** Date : 2017-07-25 **/ 'use strict'; // Import the f5-nodejs module and others. var f5 = require('f5-nodejs'); var parseString = require('xml2js').parseString; var https = require('https'); var repeat = require('repeat'); var loki = require('lokijs'); var ipRangeCheck = require('ip-range-check'); // Create (in-memory) LokiJS database. var db = new loki('db.json'); var products = db.addCollection('products'); // Create a new rpc server for listening to TCL iRule calls. var ilx = new f5.ILXServer(); // URL to Microsoft Office 365 XML file. var url = "https://support.content.office.net/en-us/static/O365IPAddresses.xml"; // Function to get XML file and convert to JSON object. function xmlToJson(url, callback) { var req = https.get(url, function(res) { var xml = ''; res.on('data', function(chunk) { xml += chunk; }); res.on('error', function(e) { callback(e, null); }); res.on('timeout', function(e) { callback(e, null); }); res.on('end', function() { if(res.statusCode == 200) { parseString(xml, function(err, result) { callback(null, result); }); } }); }); } // Function that uses the data in the XML file to create a database // that can be used to perform IP address and URL lookups. function getOffice365() { xmlToJson(url, function(err,data) { if(err) { console.log("Error: xmlToJson failed"); return; } // if xml happens to be empty due to an error, do not continue. if(!data) { console.log("Error: No data in XML file"); return; } // Get date updated: 7/13/2017 var productsUpdated = data.products.$.updated; // Only update if version changed var versionCheck = products.findObject({'name':'any'}); if(versionCheck && versionCheck.version == productsUpdated) { console.log("Info: product version didn't changed; No update required"); return; } var allIpAdresses = []; var allUrls = []; data.products.product.forEach(function (product) { var ipAddresses = []; var urls = []; product.addresslist.forEach(function (addresslist) { if(addresslist.$.type == "IPv4" || addresslist.$.type == "IPv6") { if ( typeof addresslist.address !== 'undefined' && addresslist.address ) { addresslist.address.forEach(function (address) { ipAddresses.push(address); allIpAdresses.push(address); }); } } else if(addresslist.$.type == "URL") { if ( typeof addresslist.address !== 'undefined' && addresslist.address ) { addresslist.address.forEach(function (address) { urls.push(address); allUrls.push(address); }); } } }); var p = products.findObject({'name':product.$.name.toLowerCase()}); if(!p) { products.insert({ name: product.$.name.toLowerCase(), ipAddresses: ipAddresses, urls: urls, version: productsUpdated }); } else { p.ipAddresses = ipAddresses; p.urls = urls; p.version = productsUpdated; } }); var p = products.findObject({'name':'any'}); if(!p) { products.insert({ name: 'any', ipAddresses: allIpAdresses, urls: allUrls, version: productsUpdated }); } else { p.ipAddresses = Array.from(new Set(allIpAdresses)); p.urls = Array.from(new Set(allUrls)); p.version = productsUpdated; } console.log("Info: update finished; " + products.data.length + " product records in database."); }); } // refresh Microsoft Office 365 XML every hour repeat(getOffice365).every(1, 'hour').start.now(); // ILX::call to check if an IP address is part of Office365 ilx.addMethod('checkProductIP', function(objArgs, objResponse) { var productName = objArgs.params()[0]; var ipAddress = objArgs.params()[1]; // fail-open = true, fail-close = false var verdict = true; var req = products.findObject( { 'name':productName.toLowerCase()}); if(req) { verdict = ipRangeCheck(ipAddress, req.ipAddresses); } // return AuthnRequest to Tcl iRule objResponse.reply(verdict); }); // Start listening for ILX::call and ILX::notify events. ilx.listen(); Tested this on version: 13.0
DDoS IPI List - Whitelist Update Domains
Problem this snippet solves: Legitimate IP address ranges and Domain Names of valid update servers. Additional info can be found: https://github.com/c2theg/DDoS_lists How to use this snippet: Add to IPI feeds. Security >> Network Firewall >> IP Intelligence : Feed Lists Create new list: DDoS_Feeds Add rule. Give a good name, IE: whitelist_update_servers List Type: whitelist Update frequency: 3600 Default Blacklist Category: (Create new one) Whitelisted_Source Admin / Password: <Leave Blank> Tested this on version: 13.0DDoS IPI List - Whitelist NTP Servers
Problem this snippet solves: Legitimate IP address ranges of valid NTP Servers. Additional info can be found: https://github.com/c2theg/DDoS_lists How to use this snippet: Add to IPI feeds. Security >> Network Firewall >> IP Intelligence : Feed Lists Create new list: DDoS_Feeds Add rule. Give a good name, IE: whitelist_ntp_servers List Type: whitelist Update frequency: 3600 Default Blacklist Category: (Create new one) Whitelisted_Source Admin / Password: <Leave Blank> Tested this on version: 13.0DDoS IPI List - Whitelist DNS Servers
Problem this snippet solves: Legitimate IP address ranges of valid DNS Servers Additional info can be found: https://github.com/c2theg/DDoS_lists How to use this snippet: Add to IPI feeds. Security >> Network Firewall >> IP Intelligence : Feed Lists Create new list: DDoS_Feeds Add rule. Give a good name, IE: whitelist_dns_servers List Type: whitelist Update frequency: 3600 Default Blacklist Category: (Create new one) Whitelisted_Source Admin / Password: <Leave Blank> Tested this on version: 13.0DDoS IPI List - Bogons
Problem this snippet solves: Bogon IP address ranges to block traffic from Additional info can be found: https://github.com/c2theg/DDoS_lists How to use this snippet: Add to IPI feeds. Security >> Network Firewall >> IP Intelligence : Feed Lists Create new list: DDoS_Feeds Add rule. Give a good name, IE: blacklist_bogon List Type: blacklist Update frequency: 432000 Default Blacklist Category: (Create new one) Blacklisted_Source Admin / Password: <Leave Blank> Tested this on version: 13.0
Rid of bots
Problem this snippet solves: This snippet gets rid of those annoying bot pests How to use this snippet: Copy and share Code : # ---- String data group ---- 02 ltm data-group internal /Common/bots { 03 records { 04 fast { } 05 googlebot { } 06 msnbot { } 07 scooter { } 08 slurp { } 09 teoma { } 10 } 11 type string 12 } 13 14 15 # ---- iRule ---- 16 when HTTP_REQUEST { 17 if { [class match [string tolower [HTTP::header User-Agent]] contains bots] } { 18 pool slow_webbot_pool 19 } else { 20 pool default_pool 21 } 22 }
Bash Script for automatically downloading, copying and updating LatLong geo location files from Digital Element
Problem this snippet solves: This is a script that automatically downloads the latest geolocation database file from Digital Element, copies it to a list of previously defined LTMs and runs the geoip_update_data command. You need to place this script in a directory and also create a simple text file where the script will read the LTM name and ip information from. The format of the file should follow the simple rule "ltm_name ltm_ip", with spaces or tabs delimiting the two columns, like the below example. The script is DNS agnostic so it will use the names as reference and the IPs to connect to. la01ltm1 10.10.1.7 The script collects some information prior to executing and validates reachability of all the devices on file. It uses sshpass to connect to but you can change to SSH keys if preferred. Code : #!/bin/bash echo " Starting GeoLocation Update Script... " #read function check_update { #checks installed versions on F5 and compares with most recent file echo " Checking for updates and comparing with LTM $LTM_CHECKED... " GEOFILENAME_ONLINE_DATE=(`echo $GEOFILENAME | sed -n 's/.*-\([^.]*\).*/\1/p'`) GEOFILENAME_LOCAL_DATE=(`sshpass -p '$LTM_PASSWORD' ssh -o StrictHostKeyChecking=no $LTM_USERNAME@$LTM_CHECKED geoip_lookup -f /shared/GeoIP/F5GeoIPCity2.dat 65.61.115.197 | grep version | sed -n 's/.* \(.*\) Build.*/\1/p'`) echo "Current version date = $GEOFILENAME_LOCAL_DATE Available version date = $GEOFILENAME_ONLINE_DATE " if [ $GEOFILENAME_ONLINE_DATE -gt $GEOFILENAME_LOCAL_DATE ]; then echo "Update required. Hit enter to run script... " read else echo "Update not required. Exiting... " exit fi } function default_option { #Presents the user a menu with the preconfigured options and gives them the option to change parameters DIRECTORY=Scripts/GeoUpdate/Images DEST_DIRECTORY=/shared/images date=* USERNAME=root DEVICE_FILE=devices_adv_geo.txt echo -ne " These are the default configs for the script. 1 - Source Dir: $DIRECTORY 2 - Dest. Dir: $DEST_DIRECTORY 3 - Date: Most Recent File 4 - Username: $USERNAME 5 - Source device file: $DEVICE_FILE Do you want to change? [Y/N - ENTER for NO] " read -n 1 opt # waits for a y or n from the user and stores in $opt - "-n 1" means wait for one digit and don't wait for enter #opt=n if [ "$opt" = "y" ]; then #result if $opt is y #assigns variable values or defaults if [ -z $1 ]; then # if no parameters have been put in the command line then run the questions echo -ne " Enter source directory (Enter for default: $DIRECTORY): " read DIRECTORY # reads source directory and stores in $DIRECTORY if [ -z $DIRECTORY ]; then #if no input from user then set $DIRECTORY to default DIRECTORY=Scripts/GeoUpdate/Images fi echo -ne "Enter destination directory (Enter for default: $DEST_DIRECTORY): " read DEST_DIRECTORY if [ -z $DEST_DIRECTORY ]; then DEST_DIRECTORY=/shared/images fi echo -ne "Enter date for Geo Files in format YYYYMMDD (Hit ENTER to use most recent file copied to $DIRECTORY): " read date if [ -z $date ]; then date=* fi echo -ne "Enter username (default $USERNAME): " read USERNAME if [ -z $USERNAME ]; then USERNAME=root else echo -ne "Enter password for $USERNAME: " read PASSWORD fi echo -ne "Enter device file (default $DEVICE_FILE). Device file must contain server name and ip, separated by a space, one server per line: " read DEVICE_FILE if [ -z $DEVICE_FILE ]; then DEVICE_FILE=devices_adv_geo.txt fi fi elif [[ "$opt" = "n" || "$opt" = "" ]]; then #in case user doesn't want to change parameters, set all variables to default echo " Running Script with Default configs... " else #in case user types anything other than y or n, presents an error message and prints the menu again echo " Bad Option. Type y or n" #read #printf "\033c" default_option fi } function read_usernames { echo " Collecting credentials... " echo -ne "Enter Digital Element username [default blank]: " read DE_USERNAME if [ -z $DE_USERNAME ]; then DE_USERNAME="" fi echo -ne "Enter Digital Element password [default blank]: " read -s DE_PASSWORD if [ -z $DE_PASSWORD ]; then DE_PASSWORD="" fi echo -ne " Enter LTM username [default blank]: " read LTM_USERNAME if [ -z $LTM_USERNAME ]; then LTM_USERNAME="" fi echo -ne "Enter LTM password [default blank]: " read -s LTM_PASSWORD if [ -z $LTM_PASSWORD ]; then LTM_PASSWORD="" fi LTM_CHECKED=(`head -1 $DEVICE_FILE | awk '{print $1}'`) echo -ne " Enter LTM to check with [default first device on file $DEVICE_FILE - $LTM_CHECKED]: " read LTM_CHECKED if [ -z $LTM_CHECKED ]; then LTM_CHECKED=(`head -1 $DEVICE_FILE | awk '{print $1}'`) fi } default_option read_usernames GEOFILENAME=(`wget -qO- --user=$DE_USERNAME --http-password=$DE_PASSWORD https://portal.digitalelement.com/downloads/F5Edge/files/ | grep City2 | sed -n 's/[^"]*"\.\/\([^"]*\).*/\1/p'`) check_update echo "Downloading Advanced Geo File from Digital Element..." function download_geo_file { if [ -s "$DIRECTORY/$GEOFILENAME" ]; then LOCALGEOFILESIZE=(`ls -la $DIRECTORY/$GEOFILENAME | awk {'print $5'}`) LOCALGEOFILESIZE=(`echo $[LOCALGEOFILESIZE/1024/1024]`) REMOTEGEOFILESIZE=(`wget -qO- --user=$DE_USERNAME --http-password=$DE_PASSWORD https://portal.digitalelement.com/downloads/F5Edge/files/ | grep City2 | sed -n 's/.*[^\.]'$GEOFILENAME'[^(]*(\([0-9]*\)M).*/\1/p'`) echo -ne " File with same name exists on target directory. File name: $GEOFILENAME File size on local directory is $LOCALGEOFILESIZE MB. File size on Digital Element is $REMOTEGEOFILESIZE MB Do you want to re-download and overwrite it [y/n - ENTER for YES]?" read -n 1 opt if [[ "$opt" = "y" || "$opt" = "Y" || "$opt" = "" ]]; then wget -q --show-progress --user=$DE_USERNAME --http-password=$DE_PASSWORD https://portal.digitalelement.com/downloads/F5Edge/files/$GEOFILENAME -O "$DIRECTORY/$GEOFILENAME" echo " done " else echo " Skypping file download... " return fi fi } download_geo_file FILE_ID=(City2) #creates an array with the 5 types of files to be transferred/updated on the F5 appliances deviceip=(`cat $DEVICE_FILE | awk '{print $2}'`) #reads device file and prints only second column to array $deviceip devicename=(`cat $DEVICE_FILE | awk '{print $1}'`) #reads device file and prints only first column to array $devicename len=${#deviceip[*]} #creates variable $len with total number of lines read on file fil=${#FILE_ID[*]} #creates variable $fil with total number of file types existing - always 5 unless otherwise stated function display { # displays the list of devices and IPs and does a small sanity check (ping) i=0 #auxiliar variable to while command l=1 #will transform array zero in 'item number 1' echo " LIST OF DEVICES: Checking hosts availability... " echo "+----------+-------------------------------+----------------+------------+" echo "| Device | IP Address | Name | Available? |" echo "|----------+-------------------------------+----------------+------------+" while [ $i -lt $len ]; do #executes while auxiliar variable is lower than the number of devices read previously ping -c 2 -t 1 -i 0.2 ${deviceip[$i]} > /dev/null #ping on device avail=$? #throws ping status ($?) into variable $avail echo -ne "| $l " $'\t' " | ${deviceip[$i]} " $'\t' $'\t' " | ${devicename[$i]}"; if [ $avail == 0 ]; then #prints number of device (array position 0 will be device 1, then prints the ip then prints the name then echo -ne $'\t' " | "; echo -ne '\033[1;32m' "UP" '\033[0m'; echo " |" # if status comes out 0 means that host is UP, prints UP in green, DOWN in red else echo -ne $'\t' " | "; echo -ne '\033[1;31m' "DOWN" '\033[0m'; echo " |" fi let i++ let l++ done echo "+----------+-------------------------------+----------------+------------+" echo " LIST OF FILES: " j=0 #auxiliar variable to # runs a ls sorted by time on the source directory, looking for one of the four file types. If user provided a specific date, also looks for it # otherwise it will display the most recent file. head 1 prints only one file and xargs basename will strip out the directory from the ls output while [ $j -lt $fil ]; do filename=(`ls -t $DIRECTORY/*${FILE_ID[$j]}*$date* | head -1 | xargs -n1 basename`) echo "${FILE_ID[$j]} File: $filename " let j++ done } function filecopy { #copies the file via scp to the host filename=(`ls -t $DIRECTORY/*$1*$date* | head -1 | xargs -n1 basename`) #runs the same ls as before to assign the filename to the variable. $1 is the parameter passed whe the function was called echo -ne '\033[1;36m' " *** Copying $1 to ${devicename[$i]} *** " '\033[0m'; #informs the user about the type of file being copied and to which host echo -ne '\033[1;19m' " Executing Command: scp $DIRECTORY/$filename $LTM_USERNAME@${deviceip[$i]}:$DEST_DIRECTORY/$filename " '\033[0m'; #informs the user the specific command sshpass -p '$LTM_PASSWORD' scp $DIRECTORY/$filename $LTM_USERNAME@${deviceip[$i]}:$DEST_DIRECTORY/$filename #copies using source directory specified in beginning, filename just recently listed and to device ip sent as a parameter when the function was called status=${?} #saves copy status in a variable if [ $status != 0 ]; then echo "Transfer failed" #error message in case the status is different then zero fi } function run_update { filename=(`ls -t $DIRECTORY/*$1*$date* | head -1 | xargs -n1 basename`) echo -ne '\033[1;36m' " *** Executing $1 update on ${devicename[$i]} ***" '\033[0m'; LOGDIR=(`echo $DIRECTORY | sed -n 's/\/Images/\/log/p'`) mkdir -p $LOGDIR sshpass -p '$LTM_PASSWORD' ssh -o StrictHostKeyChecking=no -v -E $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt $LTM_USERNAME@${deviceip[$i]} geoip_update_data -f $DEST_DIRECTORY/$filename status=${?} if [ $status == 0 ]; then printcmd="ssh -v -E /tmp/${deviceip[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt $LTM_USERNAME@${deviceip[$i]} geoip_update_data -f $DEST_DIRECTORY/$filename" #saves the command -v verbose, -E creates a file log echo -ne '\033[1;19m' " Running:"; echo $printcmd | awk {'print $1 " " $5 " " $6 " " $7 " " $8'} echo -ne " " '\033[0m' awk '{ $1=""; print $0 }' $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt | grep Connecting | sort -u awk '{ $1=""; print $0 }' $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt | grep Connection | sort -u awk '{ $1=""; print $0 }' $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt | grep 'Sending command' | sort -u echo " Command Executed Successfully" GEOFILENAME_TESTED_DATE=(`sshpass -p '$LTM_PASSWORD' ssh $LTM_USERNAME@${deviceip[$i]} geoip_lookup -f /shared/GeoIP/F5GeoIPCity2.dat 65.61.115.197 | grep version | sed -n 's/.* \(.*\) Build.*/\1/p'`) if [ $GEOFILENAME_ONLINE_DATE -eq $GEOFILENAME_TESTED_DATE ]; then VERIFY_PASS="UP TO DATE" else VERIFY_PASS="OUTDATED FILE!!!!" fi echo " Verifying database date installed on ${devicename[$i]} : $VERIFY_PASS" rm $LOGDIR/${devicename[$i]}_file_${filename}_copy_$(date +"%m_%d_%Y").txt else echo "Command failed - Check Log file: $LOGDIR/${devicename[$i]}_file_$1_copy.txt" fi } display echo " Hit Enter to start Script, Ctrl-C to stop" read i=0 # not a simple auxiliar variable. this "i" will be called by the two functions filecopy and run_update. it will determine IN THE FUNCTION what is the array position the command should be called, although it's not passed as parameter, it can be used anytime in the script. while [ $i -lt $len ]; do #it will run for as many IPs are in the file k=0 while [ $k -lt $fil ]; do #it will run for as many types of files exist. (Remember, so far this number is fixed in 4) This internal while will run 4 times for each ip, and the number of times for each ip will be determined by the outer while echo -ne '\033[1;31m' " ============================ Processing file ${FILE_ID[$k]} on ${devicename[$i]} ========================================================= " '\033[0m'; filecopy ${FILE_ID[$k]} #pass the type of file as a parameter. function will see ${FILE_ID[$k]} as $1. Will use i to determine what IP will be treated this time run_update ${FILE_ID[$k]} let k++ done let i++ done echo -ne '\033[1;19m'" *************************** Script complete. For full details check log file ******************************************* " '\033[0m' Tested this on version: 11.6
