Log large HTTP payloads in chunks locally and remotely
Problem this snippet solves: Log HTTP POST request payloads remotely via High Speed Logging (HSL) to a syslog server and locally. Code : # Log POST request payloads remotely via HSL to a syslog server and locally. # Based on Steve Hillier's example and the HTTP::collect wiki page # https://devcentral.f5.com/s/wiki/iRules.http__collect.ashx # Note that although any size payload can theoretically be collected, the maximum size of a Tcl variable in v9 and v10 is 4MB # with a smaller functional maximum after charset expansion of approximately 1Mb. # In v11, the maximum variable size was increased to 32Mb. when RULE_INIT { # Log debug to /var/log/ltm? 1=yes, 0=no set static::payload_dbg 1 # Limit payload collection to 5Mb set static::max_collect_len 5242880 # HSL pool name set static::hsl_pool "my_hsl_tcp_pool" # Max characters to log locally (must be less than 1024 bytes) # https://devcentral.f5.com/s/wiki/iRules.log.ashx set static::max_chars 900 } when HTTP_REQUEST { # Only collect POST request payloads if {[HTTP::method] equals "POST"}{ if {$static::payload_dbg}{log local0. "POST request"} # Open HSL connection set hsl [HSL::open -proto TCP -pool $static::hsl_pool] # Get the content length so we can request the data to be processed in the HTTP_REQUEST_DATA event. if {[HTTP::header exists "Content-Length"]}{ set content_length [HTTP::header "Content-Length"] } else { set content_length 0 } # content_length of 0 indicates chunked data (of unknown size) if {$content_length > 0 && $content_length < $static::max_collect_len}{ set collect_length $content_length } else { set collect_length $static::max_collect_len } if {$static::payload_dbg}{log local0. "Content-Length: $content_length, Collect length: $collect_length"} } } when HTTP_REQUEST_DATA { # Log the bytes collected if {$static::payload_dbg}{log local0. "Collected [HTTP::payload length] bytes"} # Send all the collected payload to the remote syslog server HSL::send $hsl "<190>[HTTP::payload]\n" # Log the payload locally if {[HTTP::payload length] < $static::max_chars}{ log local0. "Payload=[HTTP::payload]" } else { # Initialize variables set remaining $payload set position 0 set count 1 set bytes_logged 0 # Loop through and log each chunk of the payload while {[string length $remaining] > $static::max_chars}{ # Get the current chunk to log (subtract 1 from the end as string range is 0 indexed) set current [string range $remaining $position [expr {$position + $static::max_chars -1}]] log local0. "chunk $count=$current" # Add the length of the current chunk to the position for the next chunk incr position [string length $current] # Get the next chunk to log set remaining [string range $remaining $position end] incr count incr bytes_logged $position log local0. "remaining bytes=[string length $remaining], \$position=$position, \$count=$count, \$bytes_logged=$bytes_logged" } if {[string length $remaining]}{ log local0. "chunk $count=$current" incr bytes_logged [string length $remaining] } log local0. "Logged $count chunks for a total of $bytes_logged bytes" } }