Here's where I am with my rate limiting iRule, though I havnt even checked to see if it'll parse yet:
when RULE_INIT {
array set ::active_crawlers { }
set ::min_interval 1
set ::rate_limit_message "You've been rate limited for sending more than 1 request every $::min_interval seconds."
}
when HTTP_REQUEST {
set user_agent [string tolower [HTTP::header "User-Agent"]]
if { [matchclass [$user_agent contains $::Crawlers] }
Throttle crawlers.
set curr_time [clock seconds]
if { [info exists ::active_crawlers($user_agent)] } {
if { [ $::active_crawlers($user_agent) < $curr_time ] } {
set ::active_crawlers($user_agent) [expr {$curr_time + $::min_interval}]
} else {
block it somehow
HTTP::respond 500 content $::rate_limit_message
}
} else {
set ::active_crawlers($user_agent) [expr {$curr_time + $::min_interval}]
}
}
}