Connection list via iControlREST API

JCohen · ‎29-Dec-2015

Problem this snippet solves:

This python script will retrieve a filtered list of active connections via the iControl REST API. It supports both the default connection list as well as the detailed list you get by specifying 'all-properties'.

It has options to generate output on STDOUT as displayed by TMSH (raw), as a JSON formatted dictionary, or written to an Excel spreadsheet.

Currently tested in python 2.7 and python 3.5 against BIGIP 11.6.0.

How to use this snippet:

$ ./conn-list.py -h
usage: conn-list.py [-h] [-a] [-x FILE | -j | -r] -f P=V host

positional arguments:
  host                  Host name to connect. Specified as
                        [<username>@]<hostname>

optional arguments:
  -h, --help            show this help message and exit
  -a, --all-properties  Get detailed connection information
  -f P=V, --filter P=V  You must have at least one filter argument, but may
                        have multiple.

Output Type:
  -x FILE, --xlout FILE
                        Excel workbook to be created.
  -j, --json            JSON formatted output to STDOUT
  -r, --raw             RAW ouptut from API request (default)

P=V: P = Connection Property (below), V = Value to match.
 Multiple -f options are joined as logical AND.

    age             Specifies the age, in seconds, of a connection
    cs-client-addr  Specifies the clientside remote address of the active connections
    cs-client-port  Specifies the clientside remote port of the active connections
    cs-server-addr  Specifies the clientside local address of the active connections
    cs-server-port  Specifies the clientside local port of the active connections
    protocol        Specifies the protocol used for specified connections (for example: tcp, udp)
    ss-client-addr  Specifies the serverside local address of the active connections
    ss-client-port  Specifies the serverside local port of the active connections
    ss-server-addr  Specifies the serverside remote address of the active connections
    ss-server-port  Specifies the serverside remote port of the active connections
    type            Specifies the connnection type used for specified connections (for example: any, mirror, self)

$ ./conn-list.py -r -f cs-server-port=80 admin@192.0.2.45
Password:
Sys::Connections
192.0.2.31:55345  192.0.2.20:80  192.0.2.31:55345  198.51.100.66:80  tcp  6  (tmm: 1)  none
Total records returned: 1

$ ./conn-list.py -j -f cs-server-port=80 admin@192.0.62.45
Password:
[{"acceleration": "none", "cs-server": "192.0.2.20:80", "protocol": "tcp", "cs-client": "192.0.2.31:55613", "idle": 1, "ss-server": "198.51.100.66:80", "tmm": 1, "ss-client": "192.0.2.31:55613"}]

Code :

#!/usr/bin/env python
"""
This script will use F5's iControl REST API to collect current connection data.  It enforces the use a at least
one filter criteria.

This has only been tested on 11.6.0 so far.  Should be extendable to 11.5.x and 12.x by adding to or
duplicating Record Definitions and RE sections.
"""


import json
import sys
import getpass
import argparse
import re
try:
    # Py3
    from urllib.parse import urlparse, parse_qs
except ImportError:
    # Py2
    from urlparse import urlparse, parse_qs

import requests
from openpyxl import Workbook


RE_LAST = re.compile('(\w+)$')
RE_LAST_TWO = re.compile('([\w/:\.]+)\s+([\w/:\.]+)$')
DETAILED_DEF = {
    '11.6.0':
        {'rec-sep': '---',  # actually second line in record.
         'fields':
             {'Slot': {'re': RE_LAST, 'id': ['slot']},
              'TMM': {'re': RE_LAST, 'id': ['tmm']},
              'Acceleration': {'re': RE_LAST, 'id': ['acceleration']},
              'Protocol': {'re': RE_LAST, 'id': ['protocol']},
              'Idle Time ': {'re': RE_LAST, 'id': ['idle']},
              'Idle Timeout': {'re': RE_LAST, 'id': ['idle_timeout']},
              'Lasthop': {'re': RE_LAST_TWO, 'id': ['lasthop-vlan', 'lasthop-mac']},
              'Client Addr': {'re': RE_LAST_TWO, 'id': ['cs-client', 'ss-client']},
              'Server Addr': {'re': RE_LAST_TWO, 'id': ['cs-server', 'ss-server']},
              'Bits In': {'re': RE_LAST_TWO, 'id': ['cs-bits-in', 'ss-bits-in']},
              'Bits Out': {'re': RE_LAST_TWO, 'id': ['cs-bits-out', 'ss-bits-out']},
              'Packets In': {'re': RE_LAST_TWO, 'id': ['cs-packets-in', 'ss-packets-in']},
              'Packets Out': {'re': RE_LAST_TWO, 'id': ['cs-packets-out', 'ss-packets-out']},
              }
         }
}
SHRT_RE_AP = \
    re.compile(
        '([\w:\.]+)\s+([\w:\.]+)\s+([\w:\.]+)\s+([\w:\.]+)\s+(\w+)\s+(\d+)\s+\(tmm: (\d+)\)\s+(\w+)')
SHRT_RE_CH = \
    re.compile(
        '([\w:\.]+)\s+([\w:\.]+)\s+([\w:\.]+)\s+([\w:\.]+)\s+(\w+)\s+(\d+)\s+\(slot/tmm: (\d+)/(\d+)\)\s+(\w+)')
SHORT_DEF = {
    '11.6.0':
        {'chs-indc': 'slot',
         'fields':
             {'appliance': {'re': SHRT_RE_AP, 'id': ['cs-client', 'cs-server', 'ss-client',
                                                     'ss-server', 'protocol', 'idle', 'tmm',
                                                     'acceleration']},
              'chassis': {'re': SHRT_RE_CH, 'id': ['cs-client', 'cs-server', 'ss-client',
                                                   'ss-server', 'protocol', 'idle', 'slot', 'tmm',
                                                   'acceleration']},
              }
         }
}

requests.packages.urllib3.disable_warnings()
try:
    dict.iteritems
except AttributeError:
    # Py3
    def itervalues(d):
        return iter(d.values())

    def iteritems(d):
        return iter(d.items())
else:
    # Py2
    def itervalues(d):
        return d.itervalues()

    def iteritems(d):
        return d.iteritems()


def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-a', '--all-properties', dest='detail', action='store_true',
                        help='Get detailed connection information')
    outype = parser.add_argument_group("Output Type")
    outputs = outype.add_mutually_exclusive_group()
    outputs.add_argument('-x', '--xlout', action='store', metavar="FILE",
                         help='Excel workbook to be created.')
    outputs.add_argument('-j', '--json', action='store_true',
                         help='JSON formatted output to STDOUT')
    outputs.add_argument('-r', '--raw', action='store_true',
                         help='RAW ouptut from API request (default)')
    parser.add_argument('-f', '--filter', required=True, action='append', metavar='P=V',
                        help='You must have at least one filter argument, but may have multiple.')
    parser.add_argument('host',
                        help='Host name to connect.  Specified as [@]')
    parser.epilog = """
P=V: P = Connection Property (below), V = Value to match.
 Multiple -f options are joined as logical AND.

    age             Specifies the age, in seconds, of a connection
    cs-client-addr  Specifies the clientside remote address of the active connections
    cs-client-port  Specifies the clientside remote port of the active connections
    cs-server-addr  Specifies the clientside local address of the active connections
    cs-server-port  Specifies the clientside local port of the active connections
    protocol        Specifies the protocol used for specified connections (for example: tcp, udp)
    ss-client-addr  Specifies the serverside local address of the active connections
    ss-client-port  Specifies the serverside local port of the active connections
    ss-server-addr  Specifies the serverside remote address of the active connections
    ss-server-port  Specifies the serverside remote port of the active connections
    type            Specifies the connnection type used for specified connections (for example: any, mirror, self)
"""
    args = parser.parse_args()

    (username, unused, host) = args.host.rpartition('@')
    if not username:
        username = raw_input('Username: ')
    password = getpass.getpass('Password: ')
    filter_prop = '+'.join(args.filter)
    filter_prop = filter_prop.replace('=', '+')
    (ctext, ver) = get_conn_list(host, username, password, filter_prop, args.detail)
    if not ctext:
        print('No connection list was returned from {}.'.format(host))
        quit()
    if not args.xlout and not args.json:
        raw_output(ctext)
    else:
        if args.detail:
            conns, patterns = process_detailed_conns(ctext, ver)
        else:
            conns, patterns = process_short_conns(ctext, ver)
        if args.xlout:
            excel_output(conns, args.xlout, patterns)
        elif args.json:
            json_output(conns)


# noinspection PyBroadException
def get_conn_list(host, uname, pw, options, detail):
    conntext = []
    version = ''
    b = requests.session()
    b.auth = (uname, pw)
    b.verify = False
    b.headers.update({'Content-Type': 'application/json'})
    b_url = 'https://{}/mgmt/tm'.format(host)
    if detail:
        options += "+all-properties"
    try:
        resp = b.get(b_url + '/sys/connection/?options=' + options, timeout=6.05)
        if resp.status_code == requests.codes.ok:
            j = json.loads(resp.text)
            version = parse_qs(urlparse(j['selfLink']).query)['ver'][0]
            conntext = j['apiRawValues']['apiAnonymous'].splitlines()
        else:
            sys.stderr.write(
                'Error: {} status returned from: {}\n       {}\n'.format(resp.status_code, host, resp.reason))
    except:
        sys.stderr.write('Error: Could not get data from {}: {}\n'.format(host, sys.exc_info()[0]))
    return conntext, version


def process_detailed_conns(conntext, ver):
    connlist = []
    patterns = {}
    detailed_record_length = 0
    more_than_one_record = False
    for ndx, val in enumerate(conntext):
        if val.startswith(DETAILED_DEF[ver]['rec-sep']):
            if detailed_record_length == 0:
                detailed_record_length = ndx
            else:
                detailed_record_length = ndx - detailed_record_length
                more_than_one_record = True
                break
        else:
            for p in DETAILED_DEF[ver]['fields']:
                if val.lstrip().startswith(p):
                    patterns[ndx - 1] = DETAILED_DEF[ver]['fields'][p]
                    break
    if not more_than_one_record:  # only one record returned
        detailed_record_length = ndx - 1

    ndx = 0
    while ndx < len(conntext) - 1:
        if ndx % detailed_record_length == 1:
            obj = {}
            for offset, data_def in iteritems(patterns):
                match = data_def['re'].search(conntext[ndx + offset])
                for ndx2, col_heading in enumerate(data_def['id']):
                    grouping = match.group(ndx2 + 1)
                    obj[col_heading] = int(grouping) if grouping.isnumeric() else grouping
            connlist.append(obj)
            ndx += detailed_record_length
        else:
            ndx += 1
    return connlist, patterns


def process_short_conns(ctext, ver):
    connlist = []
    patterns = {}
    if len(ctext) > 2:
        if SHORT_DEF[ver]['chs-indc'] in ctext[1]:
            patterns = SHORT_DEF[ver]['fields']['chassis']
        else:
            patterns = SHORT_DEF[ver]['fields']['appliance']

    for row in ctext:
        obj = {}
        match = patterns['re'].search(row)
        if match:
            for ndx, col_heading in enumerate(patterns['id']):
                grouping = match.group(ndx + 1)
                if grouping:
                    obj[col_heading] = int(grouping) if grouping.isnumeric() else grouping
            connlist.append(obj)
    return connlist, {1: patterns}


def excel_output(conns, fname, patterns):
    wb = Workbook()
    ws = wb.active
    header = []
    for h in patterns.values():
        header += h['id']
    ws.append(header)
    for r in conns:
        row = []
        for h in header:
            row.append(r[h])
        ws.append(row)
    if not fname.endswith('.xlsx'):
        fname += '.xlsx'
    wb.save(fname)


def json_output(conns):
    print(json.dumps(conns))


def raw_output(data):
    for l in data:
        print(l)


if __name__ == '__main__':
    main()

Tested this on version:

11.6

JRahm · ‎06-Jan-2016

great stuff, Jason! Looks like the json format output isn't 12.0 compatible yet: FLD-ML-RAHM:downloads rahm$ python cl.py -j -f protocol=tcp admin@172.16.44.15 Password: Traceback (most recent call last): File "cl.py", line 255, in main() File "cl.py", line 138, in main conns, patterns = process_short_conns(ctext, ver) File "cl.py", line 211, in process_short_conns if SHORT_DEF[ver]['chs-indc'] in ctext[1]: KeyError: u'12.0.0' I'll debug when I get a chance.

DevCentral

Connection list via iControlREST API