walk F5 Stats

Problem this snippet solves:

walkF5Stats is a medium sized perl application that runs in a loop, and every 20 seconds (Configurable on the command line) polls a cluster of BigIP devices for statistics (Both system and those associated with the current configuration). Statistics are gathered via both iControl and SNMP.

The walkF5Stats application updates both the database and the RRD’s used to keep historical stats. The stats gathered include the areas of Pools, PoolMembers, Virtual Servers, Networks & Interfaces, CPU & Memory, Profiles (SSL), HA and configuration sync status.

Note: walkF5Stats is specific to F5 BIGIP appliances

Pools - Gather pool availability, traffic and connection stats
PoolMembers - Gather poolmember availability, traffic and connection stats
Virtual Servers - Gather virtual server availability, traffic and connection stats
CPU & memory - Dependent on the BigIP version, gather memory and CPU counters for the host and individual CPU’s. 
SSL Profiles - Gather statistics (counters) for SSL profiles for hardware of software offloading of SSL connections 
Networks & Interfaces - Gather Byte counts and availability of networks and interfaces. Including VLAN’s and aggregated links
HA & Config Sync Status - Gather the HA status (Active or Standby) and configuration sync status (In sync, local or remote changed) for saving in the DB

Note

Check out the companion article for more information Dynamic-Intelligent-Application-Delivery-in-a-Distributed-EnvironmentndashPart-2.aspx

Code :

#!/usr/local/bin/perl

#!/du/appl/perl-5.8.8/bin/perl

#*******************************************************************
#** walkF5Stats                                                   **
#**                                                               **
#** Copyright (C) 2007, Hamish Marson  **
#**                                                               **
#** This program is free software; you can redistribute it and/or **
#** modify it under the terms of the GNU General Public License   **
#** version 3 as published by the Free Software Foundation.       **
#**                                                               **
#** This program is distributed in the hope that it will be       **
#** useful, but WITHOUT ANY WARRANTY; without even the implied    **
#** warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR       **
#** PURPOSE. See the GNU General Public License for more details. **
#**                                                               **
#** You should have received a copy of the GNU General Public     **
#** License along with this program; if not, write to the Free    **
#** Software Foundation, Inc., 59 Temple Place, Suite 330,        **
#** Boston, MA   02111-1307   USA                                 **
#**                                                               **
#*******************************************************************


#
# use the iControl interface to gather stats & save them
#
#
#

use HTTP::Cookies;
#use SOAP::Lite + trace => qw(method debug);
use SOAP::Lite;

use iControlTypeCast;

use Getopt::Std;

use Data::Dumper; 

use DBI;

use Net::DNS;
use Net::SNMP;

use lib "/opt/local/rrdtool-1.4.3/lib/perl";

use RRDs;

use Time::HiRes qw( gettimeofday tv_interval );;

#use Time::gmtime;
use POSIX; # qw(strftime);

my $saveBIGINT=1;

my $clusterName;
my $progname="walkF5Stats";
my $SNMPcommunity="public";

my $debug=0;
my $icUser="";
my $icPass="";
my $sProtocol = "https";                                                            # Always https...
my $rrdpath="/tmp";

my $adb;

my $debugLevel=0;

my $cookieJar=HTTP::Cookies->new(ignore_discard => 1);

%sql_clause = (
  "sel_vr_4stat" => "select * from f5_virtual where device_id=? and stats='Y'",
  "sel_pl_4stat" => "select * from f5_pool where device_id=? and stats='Y'",
  "sel_nt_4stat" => "select * from f5_interface where devid=? and stats='Y'",

  "sel_st_parm"  => "select * from statistic where statname=?",
  "sel_dbusrid"  => "select user_id from user where loginid=?",
  "sel_mgmtusr"  => "select authen.username,authen.password from authen where authen.id=? and access='R'",
  "sel_dbdvsnc"  => "select * from f5deviceSync where id=?",
  "sel_clstmbrs" => "select device.deviceid,device.fqdn,device.mgmtaddr,device.mgmtport,device.state,device.rrdpath from device,cluster_device,cluster where device.deviceid=cluster_device.deviceid and cluster_device.clusterid=cluster.clusterid and cluster.name=?",
  "sel_f5clstrs" => "select cluster.clusterid,cluster.name,cluster.description,cluster.state from cluster where cluster.type like 'F%'",  "sel_statents" => "select * from statistic",
  "sel_apvers"   => "select version from device where deviceid=?",



  "ins_dbdvsnc"  => "insert into f5deviceSync values ( ?, ?, ?, ?, ?, ?)",

  "upd_knownip"  => "update device set knownip=? where id=?",
  "upd_rm_vsrv"  => "update f5_virtual set operation=?,stats=? where name=?",
  "upd_rm_pool"  => "update f5_pool set operation=?,stats=? where name=?",

);

%sql_query = ();

sub dbprint {
 my($message)=@_;
# my $debug=1;

 if($debugLevel>0) {
   my $thisiso=strftime("%Y%m%dT%H%M%SZ", gmtime(time));

   print "DEBUG: $$ : $thisiso : [$message]\n";

 }
}

sub build_queries {
  my ($dbh)=(@_);

  foreach $query (keys %sql_clause) {
    dbprint "build_queries: Preparing [$query] => [$sql_clause{$query}]\n";
    $sql_query{$query}=$dbh->prepare($sql_clause{$query});
  }
  dbprint "build_queries: done";
}

#
#
#
sub log_action {
  use Time::gmtime();

  my ($elapsed, $code, $action, $description)=(@_);
  my $lt=strftime("%Y/%m/%d %H:%M:%S", gmtime(time));
  my $tracefile="/var/log/$progname/$clusterName.trace";

  open(LOGFILE, ">>$tracefile");
  printf(LOGFILE "$lt : %s : %lu : %10.6f : %lu: %s : %s\n", $progname, $$, $elapsed, $code, $action, $description);
  close(LOGFILE);
}

#----------------------------------------------------------------------------
# sleepuntil
#----------------------------------------------------------------------------
sub sleepuntil {
  use Time::gmtime;

  # Sleep until a multiple of $_[0] seconds
  my($dbh, $sleeptime)=(@_);
  my($lt)=gmtime();

  my($random)=0;
 
  my $mydelay=$sleeptime-($lt->sec % $sleeptime)+$random;

  sleep $mydelay;
}

#----------------------------------------------------------------------------
# Database
#----------------------------------------------------------------------------
sub getDBUserID {
  my ($dbh, $username) = (@_);
  my $userid;
  my $t0=[gettimeofday];

  dbprint "getDBUserID: $username\n";
  
  #my $query=$dbh->prepare("select user_id from user where loginid='$username'");
  #my $query=$dbh->prepare($sql_clause{sel_dbusrid});
  #$query->execute($username);
  $sql_query{'sel_dbusrid'}->execute($username);

  ### Retrieve the returned rows of data
  #while ( my @row = $query->fetchrow_array(  ) ) {
  while ( my @row = $sql_query{'sel_dbusrid'}->fetchrow_array(  ) ) {
    dbprint "getDBUserID: row - @row\n";
    $userid=$row[0];

  }
  $sql_query{'sel_dbusrid'}->finish();

  dbprint "getDBUserID: result $userid\n";

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getDBUserID", "select user_id from user where loginid='$username' => $userid");
  return $userid;
}

sub saveValue64 {
 my ($dbh, $rrdpath, $statName, $statFile, $statValue, $time_stamp)=(@_);
 my $t0=[gettimeofday];

 dbprint "saveValue64: $statName, $statFile, value $statValue, stamp $time_stamp\n";

 my $rrd_file="$rrdpath/$statFile.rrd";
 if ( ! -f $rrd_file ) {
  dbprint "saveValue64: SQL [$sql_clause{sel_st_parm} [$statName]]\n";
  $sql_query{sel_st_parm}->execute($statName);

  my $result=$sql_query{sel_st_parm}->fetchrow_hashref();

  my $p1=$result->{period};
  my $r1=$result->{retention};
  my $p2=$result->{lw_period};
  my $r2=$result->{lw_retain};
  my $p3=$result->{ar_period};
  my $r3=$result->{ar_retain};

  my $datatype=$result->{type};
  my $stepsize=$result->{stepsize};
  $sql_query{sel_st_parm}->finish();

  dbprint "saveValue64: create $rrd_file stepsize $stepsize\n";
  dbprint "saveValue64:  RRA:AVERAGE:0.1:$p1:$r1 RRA:MAX:0.1:$p1:$r1\n";
  dbprint "saveValue64:  RRA:AVERAGE:0.1:$p2:$r2 RRA:MAX:0.1:$p2:$r2\n";
  dbprint "saveValue64:  RRA:AVERAGE:0.1:$p3:$r3 RRA:MAX:0.1:$p3:$r3\n";

  RRDs::create("$rrd_file", "-s 60", "DS:count:$datatype:$stepsize:0:U", "RRA:AVERAGE:0.1:$p1:$r1","RRA:MAX:0.1:$p1:$r1", "RRA:AVERAGE:0.1:$p2:$r2","RRA:MAX:0.1:$p2:$r2", "RRA:AVERAGE:0.1:$p3:$r3","RRA:MAX:0.1:$p3:$r3");

  my $ERR=RRDs::error;
  printf "saveValue64: ERROR while creating $rrd_file: $ERR\n" if $ERR;
 }
 RRDs::updatev("$rrd_file", "$time_stamp:$statValue");
 my $el2=tv_interval ( $t1);

 log_action($elapsed, 0, "saveValue64", "$statName, $statFile, $statValue");
 dbprint "saveValue64: Done\n";
}


#----------------------------------------------------------------------------
# saveValue
#----------------------------------------------------------------------------
sub saveValue {
  my ($dbh, $rrdpath, $statName, $statFile, $statValue, $time_stamp)=(@_);
  my $t0=[gettimeofday];

  dbprint "saveValue: $statName, $statFile, value $statValue, stamp $time_stamp\n";

  my $rrd_file="$rrdpath/$statFile.rrd";
  if ( ! -f $rrd_file ) {
    #
    # Grab the settings for this statistic

    dbprint "saveValue: SQL [$sql_clause{sel_st_parm} [$statName]]\n";
    $sql_query{sel_st_parm}->execute($statName);

    my $result=$sql_query{sel_st_parm}->fetchrow_hashref();

    my $p1=$result->{period};
    my $r1=$result->{retention};
    my $p2=$result->{lw_period};
    my $r2=$result->{lw_retain};
    my $p3=$result->{ar_period};
    my $r3=$result->{ar_retain};

    my $datatype=$result->{type};
    my $stepsize=$result->{stepsize};
    $sql_query{sel_st_parm}->finish();

    dbprint "saveValue: create $rrd_file stepsize $stepsize\n";
    dbprint "saveValue:  RRA:AVERAGE:0.1:$p1:$r1 RRA:MAX:0.1:$p1:$r1\n";
    dbprint "saveValue:  RRA:AVERAGE:0.1:$p2:$r2 RRA:MAX:0.1:$p2:$r2\n";
    dbprint "saveValue:  RRA:AVERAGE:0.1:$p3:$r3 RRA:MAX:0.1:$p3:$r3\n";

    RRDs::create("$rrd_file", "-s 60", "DS:count:$datatype:$stepsize:0:U", "RRA:AVERAGE:0.1:$p1:$r1","RRA:MAX:0.1:$p1:$r1", "RRA:AVERAGE:0.1:$p2:$r2","RRA:MAX:0.1:$p2:$r2", "RRA:AVERAGE:0.1:$p3:$r3","RRA:MAX:0.1:$p3:$r3");

    my $ERR=RRDs::error;
    print "saveValue: ERROR while creating $rrd_file: $ERR\n" if $ERR;
  }
  my $t1=[gettimeofday];
  my $quick64=($statValue->{high}*4294967296)+($statValue->{low}<0?(4294967296+$statValue->{low}):$statValue->{low});
  my $el1=tv_interval ( $t1);
  RRDs::updatev("$rrd_file", "$time_stamp:$quick64");
  my $el2=tv_interval ( $t1);

  printf "saveValue: $rrd_file Calc %12.10f save %12.10f Quick64 check high %10d low %10d => 64 %10d\n", $el1, $el2, $statValue->{high}, $statValue->{low}, $quick64;
  log_action($elapsed, 0, "saveValue", "$statName, $statFile, $statValue");

  dbprint "saveValue: Done\n";
}

#----------------------------------------------------------------------------
# Transport Information
#----------------------------------------------------------------------------
sub SOAP::Transport::HTTP::Client::get_basic_credentials {
  dbprint "SOAP::Transport::HTTP::Client::get_basic_credentials($icUser, $icPass)\n";
  return "$icUser" => "$icPass";
}

#----------------------------------------------------------------------------
# checkResponse
#----------------------------------------------------------------------------
sub checkResponse {
  my ($soapResponse) = (@_);
  my $t0=[gettimeofday];

  my($status, $text)=(1, "OK");

  if( $soapResponse->fault ) {
    print "checkResponse: ", $soapResponse->faultcode, " ", $soapResponse->faultstring, "\n";
    $status=$soapResponse->faultcode;
    $text=$soapResponse->faultstring;
  }
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "checkResponse", "$status $text");

  dbprint "checkResponse: returning [$status] [$text]\n";
  return ($status, $text);
}

sub getMgmtUser {
  my($dbh, $devid)=(@_);
  my $t0=[gettimeofday];

  dbprint "getMgmtUser: [$sql_clause{sel_mgmtusr}] [$devid]\n";

  $sql_query{sel_mgmtusr}->execute($devid);
  my @row = $sql_query{sel_mgmtusr}->fetchrow_array(  );
  dbprint "getMgmtUser: @row\n";

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getMgmtUser", "$sql_clause{sel_mgmtusr} ==> $row[0], xxx");
  my ($userName, $userPass)=($row[0], $row[1]);
  $sql_query{sel_mgmtusr}->finish();

  return($userName, $userPass);
}

sub getDBDeviceSync {
  my $dbh=$_[0];
  my $devID=$_[1];

  my $t0=[gettimeofday];

  my %f5dS=();

  dbprint "getDBDeviceSync: SQL [$sql_clause{querystr}]\n";
  $sql_query{sel_dbdvsnc}->execute($devID);

  if(my @row = $sql_query{sel_dbdvsnc}->fetchrow_array(  )) {
    # Got a row... So pull the info info %f5dS and return it
    dbprint "getDBDeviceSync: result is @row\n";
    %f5DS=(id => $row[0], autodetect => $row[1], localcfgtime => $row[2], peercfgtime => $row[3], localsynctime => $row[4], state => $row[5] );
  } else {
    # No row found... Set %f5DS to blank, insert a row & return %f5dS
    print "getDBDeviceSync: No result. Creating a blank record\n";
    %f5DS=( id => $devID, autodetect => 0, localcfgtime => "", peercfgtime => "", localsynctime => "", state => "Initialising Record" );

    dbprint "getDBDeviceSync: SQL [$sql_clause{ins_dbdvsnc}]\n";
    $sql_query{ins_dbdvsnc}->execute( $devID, 0, '', '', '', 'Initialising Record');
    $sql_query{ins_dbdvsnc}->finish();
  }
  $sql_query{sel_dbdvsnc}->finish();

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getDBDeviceSync", "$querystr");
    
  return \%f5dS;
}

#---------------------------------------------------------------------------- 
# getConfigSyncStatus 
#---------------------------------------------------------------------------- 
sub getConfigSyncStatus { 
  my $dbh=$_[0];
  my $dev=$_[1];
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  @dbKeys = ( 
      "configsync.autodetect", 
      "configsync.localconfigtime", 
      "configsync.peerconfigtime", 
      "configsync.localsyncedtime", 
      "configsync.state"); 
  (@dbValues) = &getDBVariable($dev, @dbKeys); 
  
  my $autodetect;

  dbprint "check " , gmtime($dbValues[1]);

  print "CONFIG SYNC STATUS --\n"; 
  if ( "disable" eq $dbValues[0] ) { 
    #Standalone 
    dbprint "    Status:  disabled\n"; 
    dbprint "    Last change (Self):  ", strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[1])), "\n"; 
    $autodetect=0;
  } else { 
    #HA Pair 
    $autodetect=1;
    dbprint "    Status:  $dbValues[4]\n"; 
    dbprint "    Last change (Self):  ", strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[1])), "\n"; 
    dbprint "    Last change (Peer):  ", strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[2])), "\n"; 
    dbprint "    Last Configsync:     ", strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[3])), "\n"; 
  } 
  # get the current f5deviceSync information. This creates blank info if one doesn't exist already...
  my $f5dS=getDBDeviceSync($dbh, $devID);

  # Update the information... If it's changed. And set an alert or warning (In DB only?)...
  my $localcfgtime=strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[1]));
  my $peercfgtime=strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[2]));
  my $localsynctime=strftime("%Y%m%dT%H%M%SZ", gmtime($dbValues[3]));

  my ($statusCode)=($dbValues[4] =~ /(\d+) \-/);

  dbprint "getConfigSyncStatus: Status code of $statusCode from $dbValues[4]\n";

  if(($localcfgtime ne $f5dS->{localcfgtime})||($peercfgtime ne $f5dS->{peercfgtime})||($localsynctime ne $f5dS->{localsynctime})) {
    dbprint "  Status Change. Updating DB\n";
    my $updatestr="update f5deviceSync set autodetect=$autodetect,localcfgtime='$localcfgtime',peercfgtime='$peercfgtime',localsynctime='$localcfgtime',state='$dbValues[4]' where id=$devID";
    dbprint "getConfigSyncStatus: SQL [$updatestr]\n";

    my $update=$dbh->prepare($updatestr);

    
    $update->execute;

   
  }
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getConfigSyncStatus", "");

  $t0=[gettimeofday];

  # Not very useful... This overwrites the last one... Needs fixing...
  my $updatestr="update cluster,cluster_device set insync=$statusCode where cluster.clusterid=cluster_device.clusterid and cluster_device.deviceid=$devID";
  dbprint "getConfigSyncStatus: SQL [$updatestr]\n";

  my $update=$dbh->prepare($updatestr);
  $update->execute;

  $updatestr="update device set sync=$statusCode where id=$devID";
  $update=$dbh->prepare($updatestr);
  $update->execute;

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getConfigSyncStatus", "$updatestr");

} 

sub getFailoverState {
  my ($dbh, $dev)=(@_);
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  #dbprint "getFailoverState: cookiejar (Before)\n";
  #print Dumper($cookieJar);
  my $Failover =  SOAP::Lite
        -> uri('urn:iControl:System/Failover')
        -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);
  $Failover->transport->cookie_jar($cookieJar);

  dbprint "getFailoverState: Getting failover state for $devFQDN\n";

  my $soapResponse = $Failover->get_failover_state();

  &checkResponse($soapResponse);
  my $failoverState=$soapResponse->result;

  dbprint "getFailoverState: state=$failoverState\n";

  my $updatestr="update device set fostate='$failoverState' where deviceid=$devID";
  dbprint "getFailoverState: SQL [$updatestr]\n";
  my $update=$dbh->prepare($updatestr);
  $update->execute;

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getFailoverState", "$updatestr");

}

#---------------------------------------------------------------------------- 
# getDBVariable 
#---------------------------------------------------------------------------- 
sub getDBVariable() { 
  ($dev, @dbKeys) = @_; 
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  my $DBVariable = SOAP::Lite 
     -> uri('urn:iControl:Management/DBVariable') 
     -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);

  $soapResponse = $DBVariable->query(SOAP::Data->name(variables => [@dbKeys])); 
  &checkResponse($soapResponse); 
  
  @VariableNameValueList = @{$soapResponse->result}; 
  
  my @ValueArray=(); 
  
  foreach $VariableNameValue (@VariableNameValueList) { 
    $name = $VariableNameValue->{"name"}; 
    $value = $VariableNameValue->{"value"}; 
    push @ValueArray, $value; 
  } 
 
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getDBVariable", "$updatestr");

  return @ValueArray; 
} 

sub processPools {
  my $dbh=$_[0];
  my $dev=$_[1];
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  my($now)       = time();

  my @pList=();

  dbprint "processPools: $devID $devFQDN\n";

  # Pools
  my $Pool = SOAP::Lite
       -> uri('urn:iControl:LocalLB/Pool')
       -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);

  dbprint "processPools: SQL [$sql_clause{sel_pl_4stat}]\n";

  $sql_query{sel_pl_4stat}->execute($devID);
  while ( my $row = $sql_query{sel_pl_4stat}->fetchrow_hashref()) {
    dbprint "processPools: adding pool $row->{name}\n";
    push(@pList, $row->{name});
  }
  $sql_query{sel_pl_4stat}->finish();

  # Get poolmember dynamic ratios...
  {
    my $PoolMember = SOAP::Lite
       -> uri('urn:iControl:LocalLB/PoolMember')
       -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);
    my $soapResponse = $PoolMember->get_dynamic_ratio(SOAP::Data->name(pool_names => [@pList]));

    my ($status, $text)=&checkResponse($soapResponse);

    my $PStatList = $soapResponse->result;

    print "processPools: NEW: get_dynamic_ratio(@pList)\n";
    my $pIndex=0;
    print "Processing poolMember dynamic_ratio information\n";

    foreach $poolName (@pList) {
      print "poolMember DynamicRatio $poolName [pIndex=$pIndex]\n";

      $DynamicRatioList=@$PStatList[$pIndex];

      $pIndex++;
    }
  }

  my $soapResponse = $Pool->get_statistics(SOAP::Data->name(pool_names => [@pList]));
  my ($status, $text)=&checkResponse($soapResponse);
  if(($status != 1)||($text ne "OK")) {
    my $vsName;

    # Got an error of some sort... Try & determine what it was...
    # Commonly it can be trying to get stats on a non-existent object...
    if(($plName)=($text =~ /pool \((.*)\) was not found/)) {
      print "processPools: PL $plName doesn't exist... Mark as gone...\n";

      $sql_query{upd_rm_pool}->execute('R', 'N', $plName);

    } else {
      print "processPools: Unknown error...\n";
    }
  }

  $PStatList = $soapResponse->result;

  @PStatEntries=@{$PStatList->{statistics}};

  foreach $entry (@PStatEntries) {
    my $poolName=$entry->{pool_name};

    foreach $stat (@{$entry->{statistics}}) {
      my $statname=$stat->{type};

      dbprint "processPools: $poolName type $statname ";
      if(defined($statistics{$statname})) {
        dbprint "Saving\n";
        my $defentry=$statistics{$statname}->{ENTRY};

        saveValue($dbh, $dev->{rrdpath}, $statname, "$devFQDN.POOL.$poolName.$statname", $stat->{value}, $now); 

      } else {
        dbprint "Skipping\n";
      }
    }
  }

  foreach $poolName (@pList) {
    dbprint "processPools: Checking poolmember for $poolName\n";
  }

  my $PoolMember = SOAP::Lite
       -> uri('urn:iControl:LocalLB/PoolMember')
       -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);

  dbprint "processPools: Getting poolMember stats\n";

  my $soapResponse = $PoolMember->get_all_statistics(SOAP::Data->name(pool_names => [@pList]));

  dbprint "processPools: Checking PoolMember->get_all_statistics() response\n";

  my ($status, $text)=&checkResponse($soapResponse);

  my @MemberStatistics=@{$soapResponse->result};

  my $pIndex=0;

  dbprint "Processing poolMember statistics\n";

  foreach $poolName (@pList) {
    dbprint "poolMemberStat $poolName\n";

    $MemberStatList=@MemberStatistics[$pIndex];
    
    dbprint "poolMemberStat $poolName keys " . keys(%$MemberStatList) . "\n";

    foreach $entry (@{$MemberStatList->{statistics}}) {

      my $memberIP=$entry->{member}->{address};
      my $memberPort=$entry->{member}->{port};

      dbprint "poolMemberStat $devFQDN.POOLMEMBER.$poolName.$memberIP.$memberPort\n";

      foreach $stat (@{$entry->{statistics}}) {
        my $statname=$stat->{type};

        dbprint "poolMemberStat $devFQDN.POOLMEMBER.$poolName.$memberIP.$memberPort $name type $statname ";
        if(defined($statistics{$statname})) {
          dbprint "Saving\n";
          my $defentry=$statistics{$statname}->{ENTRY};

          saveValue($dbh, $dev->{rrdpath}, $statname, "$devFQDN.POOLMEMBER.$poolName.$memberIP.$memberPort.$statname", $stat->{value}, $now); #, $defentry->{period}, $defentry->{retention}, $defentry->{type});

        } else {
          dbprint "Skipping\n";
        }
      }
    }

    $pIndex++;
  }  
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "processPools", "$updatestr");

}

sub processVirtuals {
  my $dbh=$_[0];
  my $dev=$_[1];
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  my($now)       = time();
  my @vlist      = ();


  dbprint "processVirtuals: $devID $devFQDN\n";

  # Virtual Servers...
  my $VirtualServer = SOAP::Lite
       -> uri('urn:iControl:LocalLB/VirtualServer')
       -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);

  dbprint "processVirtuals: building vlist with $sql_clause{sel_vr_4stat}\n";

  $sql_query{sel_vr_4stat}->execute($devID);

  while ( my $row = $sql_query{sel_vr_4stat}->fetchrow_hashref()) {
    dbprint "processVirtuals: adding virtual $row->{name}\n";
    push(@vlist, $row->{name});
  }

  dbprint "processVirtuals: Fetching stats...\n";
  my $soapResponse = $VirtualServer->get_statistics(SOAP::Data->name(virtual_servers => [@vlist]));
  my ($status, $text)=&checkResponse($soapResponse);
  if(($status != 1)||($text ne "OK")) {
    my $vsName;

    # Got an error of some sort... Try & determine what it was...
    # Commonly it can be trying to get stats on a non-existent object...
    if(($vsName)=($text =~ /virtual server \((.*)\) was not found/)) {
      print "processVirtuals: VS $vsName doesn't exist ($text)... Mark as gone...\n";
      
    } else {
      print "processVirtuals: Unknown error...\n";
    }
  }

  $VStatList = $soapResponse->result;

  @VStatEntries=@{$VStatList->{statistics}};
  foreach $entry (@VStatEntries) {
    my $virtualName=$entry->{virtual_server}->{name};

    foreach $stat (@{$entry->{statistics}}) {
      my $statname=$stat->{type};

      dbprint "processVirtuals: $name type $statname ";
      if(defined($statistics{$statname})) {
        dbprint "Saving\n";
        my $defentry=$statistics{$statname}->{ENTRY};

        saveValue($dbh, $dev->{rrdpath}, $statname, "$devFQDN.VIRTUAL.$virtualName.$statname", $stat->{value}, $now); #, $defentry->{period}, $defentry->{retention}, $defentry->{type});

      } else {
        dbprint "Skipping\n";
      }
    }
  }
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "processVirtuals", "");
}

#----------------------------------------------------------------------------
# processNetworkList
#----------------------------------------------------------------------------
sub processNetworkList {
  my ($dbh, $dev, $now, $StatList)=(@_);
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  dbprint "processNetworkList:\n";

  my $hostList=$StatList->{statistics};

  foreach $hostStatEntry (@{$hostList}) {
    my $interfaceName=$hostStatEntry->{'interface_name'};

    dbprint "processNetworkList: $devID/$devFQDN interface $hostStatEntry{'interface_name'}\n";

    my $statistics=$hostStatEntry->{statistics};

    foreach $stat (@{$statistics}) {
      my $statname=$stat->{type};

      dbprint "processNetworkList:  type $statname ";
  
      if(defined($statistics{$statname})) {
        dbprint "Saving\n";
        my $defentry=$statistics{$statname}->{ENTRY};

        saveValue($dbh, $dev->{rrdpath}, $statname, "$devFQDN.NET.$interfaceName.$statname", $stat->{value}, $now); 
 
      } else {
        dbprint "Skipping $statname\n";
      }
    }
  }
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "processNetworkList", "");
}

#----------------------------------------------------------------------------
# processStatList
#----------------------------------------------------------------------------
sub processStatList {
  my ($dbh, $dev, $now, $StatList)=(@_);
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  my $statistics=$StatList->{statistics};
  foreach $stat (@{$statistics}) {
    my $statname=$stat->{type};

    dbprint "getSystemStats: $devID/$devFQDN type $statname ";
    if(defined($statistics{$statname})) {
      dbprint "Saving\n";
      my $defentry=$statistics{$statname}->{ENTRY};

      saveValue($dbh, $dev->{rrdpath}, $statname, "$devFQDN.SYSTEM.$statname", $stat->{value}, $now); 

    } else {
      dbprint "Skipping\n";
    }
  }
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "processStatList", "");
}


#----------------------------------------------------------------------------
# getNetworkStats
#----------------------------------------------------------------------------
sub getNetworkStats {
  my ($dbh, $dev)=(@_);
  my $t0=[gettimeofday];
  my @nList=();

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  dbprint "getNetworkStats: $devID $devFQDN\n";
  my $Interface = SOAP::Lite
       -> uri('urn:iControl:Networking/Interfaces')
       -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);

  my($now)       = time();

  ##HNM##
  $sql_query{sel_nt_4stat}->execute($devID);
  while ( my $row = $sql_query{sel_nt_4stat}->fetchrow_hashref()) {
    dbprint "getNetworkStats: adding interface $row->{name}\n";
    push(@nList, $row->{name});
  }

  my $soapResponse = $Interface->get_statistics(SOAP::Data->name(interfaces => [@nList]));
  my ($status, $text)=&checkResponse($soapResponse);
  $StatList = $soapResponse->result;

  dbprint "getNetworkStats: dumping\n";
  dbprint "getNetworkStats: nList [@nList]\n";

  processNetworkList($dbh, $dev, $now, $StatList);

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getNetworkStats", "");

}

sub lookupHostVersion {
 my ($dbh, $dev)=(@_);
 my $t0=[gettimeofday];
 my $devID=$dev->{ID};
 my $devFQDN=$dev->{FQDN};
 my $hostversion="";

 dbprint "lookupHostVersion:\n";
 dbprint "lookupHostVersion: SQL [$sql_clause{sel_apvers}] [$devID]\n";
 $sql_query{sel_apvers}->execute($devID);
 my $row = $sql_query{sel_apvers}->fetchrow_hashref();
 if(defined($row)) {
  dbprint "lookupHostVersion: got $row->{version}\n";
  $hostVersion=$row->{version};
 } else {
  print "lookupHostVersion: Couldn't get host version from DB\n";
 }
 $sql_query{sel_apvers}->finish();

 return $hostVersion;
}

#----------------------------------------------------------------------------
# getHostSNMPCPUStats
#----------------------------------------------------------------------------
sub getHostSNMPCPUStats {
  my ($dbh, $dev)=(@_);
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  my $oid_enterprises=".1.3.6.1.4.1";
  my $oid_f5="$oid_enterprises.3375";
  my $oid_bigipTrafficMgmt="$oid_f5.2";
  my $oid_bigipSystem="$oid_bigipTrafficMgmt.1";
  my $oid_sysHostInfoStat="$oid_bigipSystem.7";

  my $oid_sysHostMemory="oid_sysHostInfoStat.1";
  my $oid_sysHostMemoryTotal="$oid_sysHostMemory.1";
  my $oid_sysHostMemoryUsed="$oid_sysHostMemory.2";

  my $oid_sysHostCpu="$oid_sysHostInfoStat.2";
  my $oid_sysHostCpuNumber="$oid_sysHostCpu.1.0";
  my $oid_sysHostCpuTable="$oid_sysHostCpu.2";


  my $hostVersion=lookupHostVersion($dbh, $dev);
  dbprint "getHostSNMPCPUStats: Host version is $hostVersion\n";
  my($majVer, $minVer, $pttVer)=split('\.', $hostVersion);

  if(!(($majVer>=9)and($minVer>=3))) {
   print "getHostSNMPCPUStats: Appliance is version $verDecimal [$hostVersion == $majVer $minVer $pttVer]\n";
   print "getHostSNMPCPUStats: Need at least v9.3.0 for SNMP CPU Stats\n";
   return;
  }
  my ($snmp_session, $snmp_err) = Net::SNMP->session(-hostname => $icServer, -community => $SNMPcommunity, -version => 2);

  my($now)       = time();

  dbprint "getHostSNMPCPUStats: Grabbing oid_sysHostCpuNumber with $oid_sysHostCpuNumber\n";
  my $result=$snmp_session->get_request(-varbindlist => [$oid_sysHostCpuNumber,
                                                         $oid_sysHostMemoryTotal, $oid_sysHostMemoryUsed]);

  # Save out the host memory stats first...
  saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_HOST_MEMORY_TOTAL", "$devFQDN.SYSTEM.STATISTIC_HOST_MEMORY_TOTAL", $result->{$oid_sysHostMemoryTotal}, $now);
  saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_HOST_MEMORY_USED",  "$devFQDN.SYSTEM.STATISTIC_HOST_MEMORY_USED",  $result->{$oid_sysHostMemoryUsed},  $now);
  
  # Now do the CPU stats...
  my $hostCpuCount=$result->{$oid_sysHostCpuNumber};
  dbprint "getHostSNMPCPUStats: System $devFQDN has $hostCpuCount Host CPU's\n";

  dbprint "getHostSNMPCPUStats: Getting table \@ $oid_sysHostCpuTable with community $SNMPcommunity\n";
  my $result=$snmp_session->get_table(-baseoid        => $oid_sysHostCpuTable);
  if(defined($result)) {
   my $snmpCPUNum;
   my $oid_sysHostCpuEntry="$oid_sysHostCpuTable.1";
 
   for ($snmpCPUNum=1; $snmpCPUNum<=$hostCpuCount; $snmpCPUNum++) {
   
    my $oid_sysHostCpuId="$oid_sysHostCpuEntry.2.$snmpCPUNum";
    my $oid_sysHostCpuUser="$oid_sysHostCpuEntry.3.$snmpCPUNum";
    my $oid_sysHostCpuNice="$oid_sysHostCpuEntry.4.$snmpCPUNum";
    my $oid_sysHostCpuSystem="$oid_sysHostCpuEntry.5.$snmpCPUNum";
    my $oid_sysHostCpuIdle="$oid_sysHostCpuEntry.6.$snmpCPUNum";
    my $oid_sysHostCpuIrq="$oid_sysHostCpuEntry.7.$snmpCPUNum";
    my $oid_sysHostCpuSoftirq="$oid_sysHostCpuEntry.8.$snmpCPUNum";
    my $oid_sysHostCpuIowait="$oid_sysHostCpuEntry.9.$snmpCPUNum";

    dbprint "getHostSNMPCPUStats: $devFQDN oid_sysHostCpuUser   [$oid_sysHostCpuUser] => $result->{$oid_sysHostCpuUser}\n";
    dbprint "getHostSNMPCPUStats: $devFQDN oid_sysHostCpuNice   [$oid_sysHostCpuNice] => $result->{$oid_sysHostCpuNice}\n";
    dbprint "getHostSNMPCPUStats: $devFQDN oid_sysHostCpuSystem [$oid_sysHostCpuSystem] => $result->{$oid_sysHostCpuSystem}\n";
    dbprint "getHostSNMPCPUStats: $devFQDN oid_sysHostCpuIdle   [$oid_sysHostCpuIdle] => $result->{$oid_sysHostCpuIdle}\n";
    dbprint "getHostSNMPCPUStats: $devFQDN oid_sysHostCpuIowait [$oid_sysHostCpuIowait] => $result->{$oid_sysHostCpuIowait}\n";

    # SNMP numbers the CPU's from 1... So we need to subtract 1 to get the raw (Or real) CPU number, counting from 0...
    # (Because everything else numbers from 0... e.g. top etc)
    my $rawCPUNum=$snmpCPUNum-1;

    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_USER",    "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_USER.$rawCPUNum",    $result->{$oid_sysHostCpuUser},    $now);
    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_NICED",   "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_NICED.$rawCPUNum",   $result->{$oid_sysHostCpuNice},    $now);
    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_SYSTEM",  "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_SYSTEM.$rawCPUNum",  $result->{$oid_sysHostCpuSystem},  $now);
    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_IDLE",    "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_IDLE.$rawCPUNum",    $result->{$oid_sysHostCpuIdle},    $now);
    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_IRQ",     "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_IRQ.$rawCPUNum",     $result->{$oid_sysHostCpuIrq},     $now);
    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_SOFTIRQ", "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_SOFTIRQ.$rawCPUNum", $result->{$oid_sysHostCpuSoftirq}, $now);
    saveValue64($dbh, $dev->{rrdpath}, "STATISTIC_CPU_INFO_IOWAIT",  "$devFQDN.SYSTEM.STATISTIC_HOSTCPU_IOWAIT.$rawCPUNum",  $result->{$oid_sysHostCpuIowait},  $now);

    {
     my $cpuUser=$result->{$oid_sysHostCpuUser};
     my $cpuSystem=$result->{$oid_sysHostCpuSystem};
     my $cpuIdle=$result->{$oid_sysHostCpuIdle};
     my $cpuIowait=$result->{$oid_sysHostCpuIowait};

     dbprint "getHostSNMPCPUStats: $icServer Processor $rawCPUNum idle $cpuIdle user $cpuUser system $cpuSystem iowait $cpuIowait\n";
    }
   }
 
  }
  $snmp_session->close();
}

#----------------------------------------------------------------------------
# getSystemStats
#----------------------------------------------------------------------------
sub getSystemStats {
  my ($dbh, $dev)=(@_);
  my $t0=[gettimeofday];

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  # System Statistics...
  my $System = SOAP::Lite
       -> uri('urn:iControl:System/Statistics')
       -> proxy("$sProtocol://$icUser:$icPass\@$icServer:$icPort/iControl/iControlPortal.cgi", cookie_jar => $cookieJar);

  my($now)       = time();

  my $soapResponse = $System->get_global_statistics();
  my ($status, $text)=&checkResponse($soapResponse);
  $GlobalStatList = $soapResponse->result;
  my $soapResponse = $System->get_http_statistics();
  my ($status, $text)=&checkResponse($soapResponse);
  $HTTPStatList = $soapResponse->result;
  my $soapResponse = $System->get_client_ssl_statistics();
  my ($status, $text)=&checkResponse($soapResponse);
  $clSSLStatList = $soapResponse->result;

  getHostSNMPCPUStats($dbh, $dev);

  processStatList($dbh, $dev, $now, $GlobalStatList);
  processStatList($dbh, $dev, $now, $HTTPStatList);

  processStatList($dbh, $dev, $now, $clSSLStatList);

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "getNetworkStats", "");

}

#----------------------------------------------------------------------------
# walkDevice
#----------------------------------------------------------------------------
sub walkDevice {
  my $dbh=$_[0];
  my $dev=$_[1];
  my $t0=[gettimeofday];

  my $status=0;

  my $devID=$dev->{ID};
  my $devFQDN=$dev->{FQDN};
  my $icServer=$dev->{MgmtAddr};
  my $icPort=$dev->{MgmtPort};

  dbprint "WalkDevice: $devID $devFQDN\n";
  ($icUser, $icPass)=getMgmtUser($dbh, $devID);
  if(!defined($icUser) || !defined($icPass)) {

    return $status;
  }

  if( ! -d $dev{rrdpath} ) {
    mkdir($dev{rrdpath});
  }

  # Check cluster information... 
  # need to get info for
  #   Are we in sync?
  #   number of devices up/down
  #getConfigSyncStatus($dbh, $dev);

  #
  # Get failover status. Active/Standby
  getFailoverState($dbh, $dev);


  # Resolve the management address. If it doesn't resolve, mark it & return...
  my $res   = Net::DNS::Resolver->new;
  my $query = $res->search($icServer);
  if ($query) {
    foreach my $rr ($query->answer) {
      next unless $rr->type eq "A";

      $ipaddr=$rr->address;
      #$updatestr="update device set knownip='$ipaddr' where id=$devID";
      dbprint "WalkDevice: SQL [$sql_clause{upd_knownip}]\n";

      #my $update;
      #$update=$dbh->prepare($updatestr);
      $sql_query{upd_knownip}->execute($ipaddr, $devID);

    }
  } else {
    print "WalkDevice: query failed: ", $res->errorstring, "\n";
    setDevStatus($dbh, $dev, 5, "No resolvable management address");
    return $status;
  }
 
  $status=1;
  getSystemStats($dbh, $dev);

  getNetworkStats($dbh, $dev);

  processVirtuals($dbh, $dev);
  processPools($dbh, $dev);

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "WalkDevice", "");

  return $status;
}

#----------------------------------------------------------------------------
# processCluster
#----------------------------------------------------------------------------
sub processCluster {
  my($dbh, $clusterName)=(@_);
  my $t0=[gettimeofday];

  dbprint "processCluster: $clusterName\n";

  # Grab cluster members...
  $sql_query{sel_clstmbrs}->execute($clusterName);

  my $nodesDown=0;
  my $nodesUp=0;

  ### Retrieve the returned rows of data
  while ( my @devrow = $sql_query{sel_clstmbrs}->fetchrow_array(  ) ) {
    my %dev=();

    dbprint "devRow: @devrow\n";
    $dev{ID}=$devrow[0];
    $dev{FQDN}=$devrow[1];
    $dev{MgmtAddr}=$devrow[2];
    $dev{MgmtPort}=$devrow[3];
    $dev{state}=$devrow[4];
    $dev{rrdpath}=$devrow[5];

    $status=walkDevice($dbh, \%dev);
    if($status!=0) {
      dbprint "readClusters: Device Up...\n";
      $nodesUp++;
    } else {
      $nodesDown++;
    }
  }

  dbprint "readClusters: Cluster. nodesUp $nodesUp nodesDown $nodesDown\n";
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "readClusters", "cluster $clusterName nodesUp $nodesUp nodesDown $nodesDown");

}

#----------------------------------------------------------------------------
# readClusters
#----------------------------------------------------------------------------
sub readClusters {
  my($dbh)=(@_);
  my $t0=[gettimeofday];

  dbprint "readClusters\n";
  $sql_query{sel_f5clstrs}->execute();

  ### Retrieve the returned rows of data
  while ( my @row = $sql_query{sel_f5clstrs}->fetchrow_array(  ) ) {
    dbprint "Row: @row\n";

    my $clusterName=$row[1];

    processCluster($dbh, $clusterName);
  }

  
  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "readClusters", "$sql_clause{sel_f5clstrs}");
}

sub readStatEntries {
  my ($dbh)=(@_);
  my $t0=[gettimeofday];

  $sql_query{sel_statents}->execute();

  while(my $stat=$sql_query{sel_statents}->fetchrow_hashref(  )) {
    my $statname=$stat->{statname};

    dbprint "readStatEntries: $statname\n";

    $statistics{$stat->{statname}}->{ENTRY}=$stat;
  }
  
  # Check
  my $statname;

  foreach $statname (keys(%statistics)) {
    my ($checkname, $checkmode, $checkres, $checktmo);

    $checkname=$statistics{$statname}->{ENTRY}->{statname};
    $checkmode=$statistics{$statname}->{ENTRY}->{mode};
    $checkres=$statistics{$statname}->{ENTRY}->{period};
    $checktmo=$statistics{$statname}->{ENTRY}->{retention};

    dbprint "readStatEntries: Check $statname => $checkname, $checkmode, $checkres, $checktmo\n";
  }

  my $elapsed=tv_interval ( $t0 );
  log_action($elapsed, 0, "readStatEntries", "$querystr");

}


#----------------------------------------------------------------------------
# Validate Arguments
#----------------------------------------------------------------------------
#
# -h host
# -D username
# -w passwd
getopts('a:c:d:h:m:r:t:l:');

$dbhostname="127.0.0.1";               $dbhostname=$opt_h if(defined $opt_h);        # Where the DB resides...
$audithost="127.0.0.1";                $audithost=$opt_a if(defined $opt_a);        # Where the DB resides...
$clusterName="-";                      $clusterName=$opt_c if(defined $opt_c);       # Cluster. Default is ALL of them
$loopTime=30;                          $loopTime=$opt_t if(defined $opt_t);          # How many seconds to loop on. 
$loopCount=86400/$loopTime;            $loopCount=$opt_l if (defined $opt_l);
       $rrdpath=$opt_r if(defined $opt_r);
       $SNMPcommunity=$opt_m if(defined $opt_m);
$debugLevel=0;                         $debugLevel=$opt_d if(defined $opt_d);

print "walkF5Stats: cluster $clusterName loop $loopTime count $loopCount\n";

my $dbh = DBI->connect( "dbi:mysql:monitor:$dbhostname", "f5monitor", "f5monitor" , {
  AutoCommit => 0,
  RaiseError => 1,

} );
$adb = DBI->connect( "dbi:mysql:audit:$audithost", "f5monitor", "f5monitor" , {
  AutoCommit => 1,
  RaiseError => 1,
} );

build_queries($dbh);

my $thisCount=0;

while($thisCount!=$loopCount) {
  print "walkF5Stats: Spin $thisCount of $loopCount\n";

  $username=getpwuid($<);
  $userid=getDBUserID($dbh, $username);
  if($userid==0) {
    die("User $username not found");
  }

  # Get the wanted stats...
  readStatEntries($dbh);

  if($clusterName eq "-") {
    # process All the clusters
    readClusters($dbh);
  } else {
   processCluster($dbh, $clusterName);
  }

  sleepuntil($dbh, $loopTime);

  $thisCount++;
}

$dbh->disconnect();