TransparentBotProtection

Problem this snippet solves:

This Solution will act as a Web Form Application protection layer against malicious entities such as automated bots.

This is split into two iRules, the first will perform the actual protection while the second will server an admin console to allow you to look at the internals of the data groups and session tables. The solution attempts to solve the following requirements:

Uniquely identify users
Verify that requests follow the correct application flow (ie, the user can’t jump direction to a form submission without first going through the actual form first).
Incorporate rate limiting/blocking so that valid users (or bots that have worked their way around #2) can’t issue repetitive valid requests in an abusive manner.
Provide an administration console to be able to monitor the status of requests.

For more details, see the accompanying article coming to DevCentral in the near future.

Code :

# iRule: bot_protection

priority 50
when HTTP_REQUEST {

  set STOP_PROCESSING 0;
  set DEBUG 0;
  
  if { $DEBUG } { log local0. "-----------------------------------"; }
  if { $DEBUG } { log local0. "CLIENTLESS BOT PROTECTION"; }

  # Move to static variables later on
  # form request timeout
  set lifetime 60;
  set repeattime [expr {int(rand()*5)} + 2]

  # The class contiaining the form flows. The format is with the key
  # being the form request uri and the value being the form submit uri.
  # "uri1:uri2" := ""
  set flowclass "flow_class";

  # The table containing the validated requests.  The Key is the client flow id
  # and the value is the form request uri.
  # client_id:uri1
  set validationtable "flow_validation";

  # The table containing the timeout before a subsequent request can occur for
  # a given URI from a client flow
  # client_id:uri1
  set retrytable "bot_retries";

  set lpath [string tolower [HTTP::path]];
  set lquery [string tolower [HTTP::query]];
  set luri [string tolower [HTTP::uri]];

  if { $DEBUG } { log local0. "URI: $luri"; }
  if { $DEBUG } { log local0. "PATH: $lpath"; }
  if { $DEBUG } { log local0. "QUERY: $lquery"; }
  if { $DEBUG } { log local0. "METHOD: [HTTP::method];" }

  # Create unique client identifier.
  set client_id "[IP::client_addr]:[TCP::client_port]:[IP::local_addr]:[TCP::local_port]";

  # First lookup to see if this URI is the target of a form submission.  
  # This has to be done by looking up the values in the flowtable table.

  set entry [class search -name $flowclass ends_with ":${lpath}"];
  if { "" ne $entry } {

    set tokens [split $entry ":"];
    set uri1 [lindex $tokens 0];
    set uri2 [lindex $tokens 1];

    if { $DEBUG } { log local0. "URI Path defined as a submission '$lpath' -> '$entry'"; }

    # Found current URI in the submit section of the flow class.  
    # Determine whether this is a form submission or a page request
    if { ([HTTP::method] eq "POST") || ([HTTP::query] ne "") } {

      if { $DEBUG } { log local0. "VALID FORM SUBMISSION"; }
      # Form submission

      # First line of defense: check referer header
      if { [HTTP::header "Referer"] ne "http://[HTTP::host]$uri1" } {

        if { $DEBUG } { log local0. "REFERER HEADER '[HTTP::header Referer]' does not match request URI 'http://[HTTP::host]$uri1'"; }

        # No referer header.  Say hasta
        HTTP::respond 200 Content "NICE TRYERROR: INVALID REFERAL!";
        set STOP_PROCESSING 1;
        return;

      } else {

        if { $DEBUG } { log local0. "REFERER HEADER is VALID"; }

        # Referer is valid.  Now lookup validation record.
        set is_valid [table lookup -notouch -subtable $validationtable "$client_id:$uri1"];
        if { $is_valid ne "" } { 

          if { $DEBUG } { log local0. "VALIDATION RECORD - VALID"; }

          if { $DEBUG } { log local0. "DELETING VALIDATION RECORD '$client_id:$uri1'"; }

          # valid request
          # cleanup entry in validation table and allow through
          table delete -subtable $validationtable "$client_id:$uri1";

          if { $DEBUG } { log local0. "INSERTING BOT TIMEOUT RECORD '$client_id:$uri1' for timeout $repeattime"; }

          # Insert record into bot table
          table set -subtable $retrytable "$client_id:$uri1" 1 $repeattime indefinite;

          # Allow Through
          return;

        } else {

          if { $DEBUG } { log local0. "VALIDATION RECORD NOT PRESENT '$client_id:$uri1'"; }

          # invalid request
          HTTP::respond 200 Content "NICE TRYERROR: NO VALIDATION RECORD! Try Again";
          set STOP_PROCESSING 1;
          return;
        }
      }
    } else {
      # URI is in the submit field but isn't a form submission.
      # Test again to see if it's a request.
      if { $DEBUG } { log local0. "NOT VALID FORM SUBMISSION"; }
    }
  }  


  # Current request isn't a form submission
  # check to see if it's a configured form request.

  set entry [class search -name $flowclass starts_with "${lpath}:"];
  if { "" ne $entry } {

    set tokens [split $entry ":"];
    set uri1 [lindex $tokens 0];
    set uri2 [lindex $tokens 1];

    if { $DEBUG } { log local0. "URI Path defined as a request '$lpath' -> '$entry'"; }

    # Check bot table
    set entry [table lookup -notouch -subtable $retrytable "$client_id:$uri1"];
    if { "" eq $entry } {

      # no bot entry, allow through
      if { $DEBUG } { log local0. "Adding Valdiation Record '$client_id:$uri1' for $lifetime s."; }

      # Found current request in flowclass.  Assume this is a form submission
      table set -subtable $validationtable "$client_id:$uri1" [crc32 "$client_id:$uri1"] $lifetime indefinite;

      return;

    } else {

      if { $DEBUG } { log local0. "ERROR: BOT ATTEMPT found for '$client_id:$uri1'."; }

      # user retried within request window.  Reject.
      HTTP::respond 200 Content "NICE TRYERROR: BOT ATTEMPT! Try Again";
      set STOP_PROCESSING 1;
      return;
    }

  } else {

    # Not either a form submission or request from the flow table
    # Allow Through
    if { $DEBUG } { log local0. "NOT FORM REQUEST OR SUBMISSION - passing through..."; }

    return;

  }
}

# iRule: bot_protection_admin

priority 75
when HTTP_REQUEST {
  switch -glob $lpath {
    "/cmd/clean" {
      if { $DEBUG } { log local0. "COMMAND REQUEST: clean.  Cleaning validation table"; }
      table delete -subtable $validationtable -all;
      table delete -subtable $retrytable -all;
      HTTP::redirect "/cmd/info";
      set STOP_PROCESSING 1;
      return;
    }
    "/cmd/info" {

      set metarefresh "";
      if { [HTTP::query] contains "refresh" } {
        set refreshtime 5;
        set t [URI::query [HTTP::uri] "refresh"]
        if { "" ne $t } { set refreshtime $t; }
        set metarefresh "";
      }

      set css ""

      set dt [clock format [clock seconds] -format "%m/%d/%Y - %H:%M:%S"]
      set refresh "\[Refresh : 1,";
      append refresh "3,5,";
      append refresh "10\]";
      set clean "\[Clean Table\]";

      set keys [class names $flowclass];
      set ft "\n";
      append ft "\n";
      append ft "\n";
      foreach  key $keys {
        set tokens [split $key ":"];
        append ft "\n";
      }
      append ft "Form Flow Table
Form Request Form Post
[lindex $tokens 0] -> [lindex $tokens 1]\n";

      set keys [table keys -notouch -subtable $validationtable]
      set vt "\n";
      append vt "\n";
      append vt "";
      append vt "\n";
      foreach  key $keys {
        append vt "";
        append vt "";
        append vt "";
        append vt "";
        append vt "";
        append vt "\n";
      }
      append vt "Validation Table
Key Value Timeout   
$key [table lookup -notouch -subtable $validationtable $key] [table timeout -subtable $validationtable -remaining $key] \[X\]\n";

      set keys [table keys -notouch -subtable $retrytable]
      set bt "\n";
      append bt "\n";
      append bt "\n";
      foreach  key $keys {
        append bt "";
        append bt "";
        append bt "";
        append bt "";
        append bt "\n";
      }
      append bt "Bot Replay Window Table
Key Value Timeout
$key [table lookup -notouch -subtable $retrytable $key] [table timeout -subtable $retrytable -remaining $key]\n";
      
      set page "Bot Protection$metarefresh$css";
      append page "$dt$ft$vt$bt$clean | $refresh";

      HTTP::respond 200 Content $page;
      set STOP_PROCESSING 1;
    }
    "/cmd/removekey/*" {
      set key [string map {/cmd/removekey/ ""} $luri];

      if { "" ne $key } {
        if { $DEBUG } { log local0. "MANUALLY DELETING KEY $key"; }

        table delete -subtable $validationtable $key;
      }
      HTTP::redirect "/cmd/info";
      set STOP_PROCESSING 1;
      break;
    }
  }
}