Python script that clusters two BIG-IPs

[Gregory_Robinso]

Problem this snippet solves:

This script automates the clustering of two BIG-IPs and performs some pre-clustering validation (NTP, CM device configuration, matching TMOS)

How to use this snippet:

Configure your BIG-IPs hostnames, CM device characteristics, NTP, etc and execute the script.

Code :

#!/usr/bin/env python3

# Import the proper python modules we'll need
import datetime
import time
import requests
import sys
import json

from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

#
# Declare functions
#

def abort_script(reason):
    print('*** Aborting script execution! ***')
    if len(str(reason)) > 0:
        print('ERROR: ' + str(reason))
    sys.exit(2)

def icontrol_get(host,username,password,path):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm' + path
    try:
        apiResponse = apiCall.get(apiUri,verify=False)
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    return(apiResponse.text)

def icontrol_post(host,username,password,path,api_payload):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm' + path
    try:
        apiResponse = apiCall.post(apiUri,verify=False,data=json.dumps(api_payload))
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    return(apiResponse.text)

def icontrol_put(host,username,password,path,api_payload):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm' + path
    try:
        apiResponse = apiCall.put(apiUri,verify=False,data=json.dumps(api_payload))
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    return(apiResponse.text)

def icontrol_patch(host,username,password,path,api_payload):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm' + path
    try:
        apiResponse = apiCall.patch(apiUri,verify=False,data=json.dumps(api_payload))
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    return(apiResponse.text)

def icontrol_delete(host,username,password,path,object):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm' + path + object
    try:
        apiResponse = apiCall.delete(apiUri,verify=False)
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    return(apiResponse.text)

def icontrol_test_connection(host,username,password):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm/sys/clock'
    try:
        apiResponse = apiCall.get(apiUri,verify=False)
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    if '"kind"' in apiResponse.text:
        testresult = True
    else:
        testresult = False
    return(testresult)

def icontrol_save_config(host,username,password):
    apiCall = requests.session()
    apiCall.headers.update({'Content-type':'application/json'})
    apiCall.auth = (username,password)
    apiUri = 'https://' + host + '/mgmt/tm/sys/config'
    try:
        apiResponse = apiCall.post(apiUri,verify=False,data=json.dumps({'command':'save'}))
    except requests.exceptions.RequestException as e:
        abort_script(str(e))
    return(apiResponse.text)

#
# We will save variables in the primary and standby variables, which we'll define now
#

primary = {}
standby = {}

###############################
# Pre-clustering Verification
###############################

#
# Ask for the cluster member mgmt IP addresses, usernames and passwords
#

primary['mgmt_ip'] = input('Enter the management IP address of the primary node: ')
primary['username'] = input('Enter the username for the primary node: ')
primary['password'] = input('Password: ')
standby['mgmt_ip'] = input('Enter the management IP address of the secondary node: ')
standby['username'] = input('Enter the username for the secondary node: ')
standby['password'] = input('Password: ')
print('Enter the desired device group name on the BIG-IPs')
cluster_group_name = input('[If you\'re unsure, enter failover-bigip]: ')

#
# Test iControl REST connectivity
#

print('Testing API connectivity to the primary node')
primary_test = icontrol_test_connection(primary['mgmt_ip'],primary['username'],primary['password'])
if primary_test == True:
    print('  Successfully connected to the primary node')
else:
    abort_script('Could not connect to the primary node during API check')

print('Testing API connectivity to the secondary node')
secondary_test = icontrol_test_connection(standby['mgmt_ip'],standby['username'],standby['password'])
if secondary_test == True:
    print('  Successfully connected to the secondary node')
else:
    abort_script('Could not connect to the secondary node during API check')

#
# Gather details about each node
#

apiResponse = icontrol_get(primary['mgmt_ip'],primary['username'],primary['password'],'/sys/global-settings/')
primary['hostname'] = json.loads(apiResponse)['hostname']

apiResponse = icontrol_get(standby['mgmt_ip'],standby['username'],standby['password'],'/sys/global-settings/')
standby['hostname'] = json.loads(apiResponse)['hostname']

apiResponse = icontrol_get(primary['mgmt_ip'],primary['username'],primary['password'],'/cm/device/')
primary['cm_devices'] = json.loads(apiResponse)['items']
for current_cm_device in primary['cm_devices']:
    if current_cm_device['selfDevice'] == 'true':
        primary['cm_properties'] = current_cm_device
        primary['cm_name'] = current_cm_device['name']
        primary['sw_version'] = current_cm_device['version']
        primary['sw_build'] = current_cm_device['build']

apiResponse = icontrol_get(standby['mgmt_ip'],standby['username'],standby['password'],'/cm/device/')
standby['cm_devices'] = json.loads(apiResponse)['items']
for current_cm_device in standby['cm_devices']:
    if current_cm_device['selfDevice'] == 'true':
        standby['cm_properties'] = current_cm_device
        standby['cm_name'] = current_cm_device['name']
        standby['sw_version'] = current_cm_device['version']
        standby['sw_build'] = current_cm_device['build']

#
# Verify that the TMOS versions match between nodes
#

if not (primary['sw_version'] == standby['sw_version']) and (primary['sw_build'] == standby['sw_build']):
    abort_script('Software mismatch on cluster members!')

#
# Verify that the CM device has been configured from default
#

if primary['cm_name'] == 'bigip1' or standby['cm_name'] == 'bigip1':
    abort_script('A cluster member was found to have the default CM device name. Please run the configuration script.')

#
# Verify NTP is configured
#

print('Checking for NTP server configuration on primary')
apiResponse = icontrol_get(primary['mgmt_ip'],primary['username'],primary['password'],'/sys/ntp')
if 'servers' in apiResponse:
    primary['ntp_servers'] = json.loads(apiResponse)['servers']
    primary['timezone'] = json.loads(apiResponse)['timezone']
    print('  Success!')
else:
    abort_script('NTP not configured on cluster member!')

print('Checking for NTP server configuration on secondary')
apiResponse = icontrol_get(standby['mgmt_ip'],standby['username'],standby['password'],'/sys/ntp')
if 'servers' in apiResponse:
    standby['ntp_servers'] = json.loads(apiResponse)['servers']
    standby['timezone'] = json.loads(apiResponse)['timezone']
    print('  Success!')
else:
    abort_script('NTP not configured on cluster member!')

#
# Verify host clocks are in sync
#
print('Verifying device clock sync')
apiResponse = icontrol_get(primary['mgmt_ip'],primary['username'],primary['password'],'/sys/clock')
primary['clock_timestamp']=json.loads(apiResponse)['entries']['https://localhost/mgmt/tm/sys/clock/0']['nestedStats']['entries']['fullDate']['description'][:16]

apiResponse = icontrol_get(standby['mgmt_ip'],standby['username'],standby['password'],'/sys/clock')
standby['clock_timestamp'] = json.loads(apiResponse)['entries']['https://localhost/mgmt/tm/sys/clock/0']['nestedStats']['entries']['fullDate']['description'][:16]

if primary['clock_timestamp'] != standby['clock_timestamp']:
    abort_script('Device clocks are out of sync!')
else:
    print('  Success!')

#
# Show details about each node and ask user for final verification to continue
#
print('\nPrimary details')
print(' - Mgmt IP: ' + primary['mgmt_ip'])
print(' - Hostname: ' + primary['hostname'])
print(' - Cluster Device Name: ' + primary['cm_name'])
print(' - Version ' + primary['sw_version'] + ', build ' + primary['sw_build'])
print(' - Time zone: ' + primary['timezone'])
print(' - NTP servers: ' + str(primary['ntp_servers']))
print('\nSecondary details')
print(' - Mgmt IP: ' + standby['mgmt_ip'])
print(' - Hostname: ' + standby['hostname'])
print(' - Cluster Device Name: ' + standby['cm_name'])
print(' - Version ' + standby['sw_version'] + ', build ' + standby['sw_build'])
print(' - Time zone: ' + standby['timezone'])
print(' - NTP servers: ' + str(standby['ntp_servers']))
print('\nThis is the last chance to abort before issuing the clustering commands!\n')
confirmation = input('Enter YES to continue ')
if not confirmation == 'YES':
    abort_script('User cancelled at final confirmation!')

#######################
# Clustering Commands
#######################

#
# Save the config on each device
#

print('Saving the device configuration on the primary')
save_result = icontrol_save_config(primary['mgmt_ip'],primary['username'],primary['password'])
if '"kind"' in save_result:
    print("  Configuration saved successfully.")
else:
    abort_script("Configuration could not be successfully saved! " + save_result)

print('Saving the device configuration on the secondary')
save_result = icontrol_save_config(standby['mgmt_ip'],standby['username'],standby['password'])
if '"kind"' in save_result:
    print("  Configuration saved successfully.")
else:
    abort_script("Configuration could not be successfully saved! " + save_result)


#
# Save a pre-clustering UCS on each box for rollback
#

ucs_filename = 'pre-cluster-script-{:%Y-%m-%d-%H%M%S}.ucs'.format(datetime.datetime.utcnow())
print('Saving UCS to /var/local/ucs/' + ucs_filename + ' on primary device')
ucs_create_result = icontrol_post(primary['mgmt_ip'],primary['username'],primary['password'],'/sys/ucs',{'command':'save','name':ucs_filename})

print('Saving UCS to /var/local/ucs/' + ucs_filename + ' on secondary device')
ucs_create_result = icontrol_post(standby['mgmt_ip'],standby['username'],standby['password'],'/sys/ucs',{'command':'save','name':ucs_filename})

#
# Add the secondary to the primary as a peer
#

print('Adding the nodes to the trust domain')
apiResponse = icontrol_post(primary['mgmt_ip'],primary['username'],primary['password'],'/cm/add-to-trust',{
    'command':'run',
    'name':'Root',
    'caDevice':True,
    'device':standby['mgmt_ip'],
    'deviceName':standby['cm_name'],
    'username':standby['username'],
    'password':standby['password']
})
if '"kind":"tm:cm:add-to-trust:runstate"' in apiResponse:
    print('  Command issued successfully but clustering must be verified!')
else:
    abort_script('Error! ' + apiResponse)

#
# Verify status of members
#

print('Verifying trust sync status on the primary')
apiResponse = icontrol_get(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/sync-status')
if '"mode":{"description":"trust-only"}' in apiResponse:
    print('  Devices are in sync!')
else:
    print('Sleeping 60 seconds to allow devices to discover and sync')
    time.sleep(60)
    apiResponse = icontrol_get(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/sync-status')
    if '"mode":{"description":"trust-only"}' in apiResponse:
        print('  Devices are in sync!')
    else:
        abort_script('Error! Devices not syncing! ' + apiResponse)

print('Verifying trust sync status on the secondary')
apiResponse = icontrol_get(standby['mgmt_ip'], standby['username'], standby['password'], '/cm/sync-status')
if '"mode":{"description":"trust-only"}' in apiResponse:
    print('  Devices are in sync!')
else:
    print('Sleeping 60 seconds to allow devices to discover and sync')
    time.sleep(60)
    apiResponse = icontrol_get(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/sync-status')
    if '"mode":{"description":"trust-only"}' in apiResponse:
        print('  Devices are in sync!')
    else:
        abort_script('Error! Devices not syncing!' + apiResponse)

#
# Create the device group
#

print('Creating the ' + cluster_group_name + ' device group')
apiResponse = icontrol_post(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/device-group',{
    'name':cluster_group_name,
    'type':'sync-failover'
})
if '"kind":"tm:cm:device-group:device-groupstate"' in apiResponse:
    print('  Success!')
elif '"01020066:3:' in apiResponse:
    print('  WARNING! Device group already created! Ignoring and continuing script execution!')
else:
    abort_script('Error! ' + apiResponse)

#
# Add both devices to the device group
#

print('Adding the primary node to the device group')
apiResponse = icontrol_post(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/device-group/' + cluster_group_name + '/devices',{
    'name':primary['cm_name']
})
if '"kind":"tm:cm:device-group:devices:devicesstate"' in apiResponse:
    print('  Success!')
elif '"code":409,"message":"01020037:3:' in apiResponse:
    print('  WARNING! Device is already a member of the device group! Ignoring and continuing script execution!')
else:
    abort_script('Error! ' + apiResponse)

print('Adding the secondary node to the device group')
apiResponse = icontrol_post(standby['mgmt_ip'], standby['username'], standby['password'], '/cm/device-group/' + cluster_group_name + '/devices',{
    'name':standby['cm_name']
})
if '"kind":"tm:cm:device-group:devices:devicesstate"' in apiResponse:
    print('  Success!')
elif '"code":409,"message":"01020037:3:' in apiResponse:
    print('  WARNING! Device is already a member of the device group! Ignoring and continuing script execution!')
else:
    abort_script('Error! ' + apiResponse)

#
# Enable auto-sync on both members
#

print('Enabling auto-sync on the primary member')
apiResponse = icontrol_patch(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/device-group/' + cluster_group_name,{
    'autoSync':'enabled'
})
if '"kind":"tm:cm:device-group:device-groupstate"' and '"autoSync":"enabled"' in apiResponse:
    print('  Success!')
else:
    abort_script('Error! ' + apiResponse)

print('Enabling auto-sync on the secondary member')
apiResponse = icontrol_patch(standby['mgmt_ip'], standby['username'], standby['password'], '/cm/device-group/' + cluster_group_name,{
    'autoSync':'enabled'
})
if '"kind":"tm:cm:device-group:device-groupstate"' and '"autoSync":"enabled"' in apiResponse:
    print('  Success!')
else:
    abort_script('Error! ' + apiResponse)

#
# Save the config on each device
#

print('Saving the device configuration on the primary')
save_result = icontrol_save_config(primary['mgmt_ip'],primary['username'],primary['password'])
if '"kind"' in save_result:
    print("  Configuration saved successfully.")
else:
    abort_script("Configuration could not be successfully saved! " + save_result)

print('Saving the device configuration on the secondary')
save_result = icontrol_save_config(standby['mgmt_ip'],standby['username'],standby['password'])
if '"kind"' in save_result:
    print("  Configuration saved successfully.")
else:
    abort_script("Configuration could not be successfully saved! " + save_result)


###############################
# Post-clustering steps
###############################

#
# Force the initial sync
#

print('Forcing configuration sync between peers')
apiResponse = icontrol_post(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/config-sync',{
    'command':'run',
    'utilCmdArgs':'to-group ' + cluster_group_name
})
if '"kind":"tm:cm:config-sync:runstate"' in apiResponse:
    print('  Command issued successfully but sync status must be verified!')
else:
    abort_script('Error! ' + apiResponse)

#
# Verify sync status of members
#

print('Verifying trust sync status on the primary')
apiResponse = icontrol_get(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/sync-status')
if '"description":"failover-bigip (In Sync): All devices in the device group are in sync"':
    print('  Success!')
else:
    print('Sleeping 30 seconds to allow devices to fully sync')
    time.sleep(30)
    if '"description":"failover-bigip (In Sync): All devices in the device group are in sync"':
        print('  Devices are in sync!')
    else:
        abort_script('Error! Devices not syncing!')

print('Verifying trust sync status on the primary')
apiResponse = icontrol_get(primary['mgmt_ip'], primary['username'], primary['password'], '/cm/sync-status')
if '"description":"failover-bigip (In Sync): All devices in the device group are in sync"':
    print('  Success!')
else:
    print('Sleeping 30 seconds to allow devices to fully sync')
    time.sleep(30)
    if '"description":"failover-bigip (In Sync): All devices in the device group are in sync"':
        print('  Devices are in sync!')
    else:
        abort_script('Error! Devices not syncing!')

#
# Force the primary to active state, if necessary
#

print('Foricng the primary node to active status')
apiResponse = icontrol_post(standby['mgmt_ip'], standby['username'], standby['password'], '/sys/failover',{
    'command':'run',
    'standby':True,
    'traffic-group':'traffic-group-1'
})
if '"kind":"tm:sys:failover:runstate"' in apiResponse:
    print('  Success!')
else:
    abort_script('Error! ' + apiResponse)

################
# End of Script
################

print('\nEnd of execution. Script has completed.')
SystemExit()

Tested this on version:

12.1