Log HTTP Request and Response Payload via HSL and Locally

Problem this snippet solves:

To log full HTTP Request data, to include Headers and Payload.

How to use this snippet:

To use this code, you will need to setup an HSL pool.

Though there is the option to log locally, I would highly recommend only logging off-box, as this could very easily fill your log files or, depending on how much HTTP traffic you pass through your LTM, could produce I/O errors while attempting to write to disk.

Code :

when RULE_INIT {

# Log debug to /var/log/ltm? 1=yes, 0=no
set static::payload_dbg 1

set static::log_local 0

# Limit payload collection to 5Mb
set static::max_collect_len 5368709120

# HSL pool name
set static::hsl_pool "hsl_pool"

# Max characters to log locally (must be less than 1024 bytes)
# https://devcentral.f5.com/s/wiki/iRules.log.ashx
set static::max_chars 900

}

when CLIENT_ACCEPTED {
set hsl [HSL::open -proto UDP -pool $static::hsl_pool]
}
when CLIENTSSL_HANDSHAKE {
# Identify the Client and negotiated cipher.
if {$static::payload_dbg}{log local0.debug "Connection from Client: [IP::client_addr] with Cipher: [SSL::cipher name] and SSL Version: [SSL::cipher version]"}
}

when SERVERSSL_HANDSHAKE {
# Identify the connected server and negotiated cipher.
if {$static::payload_dbg}{log local0.debug "Connection to Remote Server: [IP::server_addr] with Cipher: [SSL::cipher name] and SSL Version: [SSL::cipher version]"}
}

when HTTP_REQUEST {

set LogString "Client [IP::client_addr]:[TCP::client_port] -> [HTTP::host][HTTP::uri]"
if {$static::payload_dbg}{log local0.debug "============================================="   }
if {$static::payload_dbg}{log local0.debug "$LogString (request)"}

# log each Header.
foreach aHeader [HTTP::header names] {
if {$static::payload_dbg}{log local0.debug "$aHeader: [HTTP::header value $aHeader]"}
}
if {$static::payload_dbg}{log local0.debug "============================================="}

if {[HTTP::header "Content-Length"] ne "" && [HTTP::header "Content-Length"] <= 1048000} {
HTTP::collect [HTTP::header "Content-Length"]
} else {
HTTP::collect 1048000
}

# Prevent the server from sending a compressed response
# remove the compression offerings from the client
HTTP::header remove "Accept-Encoding"

# Don't allow response data to be chunked
if { [HTTP::version] eq "1.1" } {

# Force downgrade to HTTP 1.0, but still allow keep-alive connections.
# Since HTTP 1.1 is keep-alive by default, and 1.0 isn't,
# we need make sure the headers reflect the keep-alive status.

# Check if this is a keep alive connection
if { [HTTP::header is_keepalive] } {

 # Replace the connection header value with "Keep-Alive"
 HTTP::header replace "Connection" "Keep-Alive"
}

# Set server side request version to 1.0
# This forces the server to respond without chunking
HTTP::version "1.0"
}

}

when HTTP_REQUEST_DATA {

# Log the bytes collected
if {$static::payload_dbg}{log local0.debug "Collected [HTTP::payload length] bytes"}

# Log the payload locally 
if {[HTTP::payload length] < $static::max_chars}{
if {$static::payload_dbg}{log local0.debug "Payload=[HTTP::payload]"}
} else {
set payloadlength [HTTP::payload length]

# Initialize variables
set remaining [HTTP::payload length]
set position 0
set count 1
set bytes_logged 0
set current 0

# Loop through and log each chunk of the payload
if {$static::payload_dbg}{log local0.debug "remaining=$remaining, static_max_chars=$static::max_chars"}
while {$remaining > $static::max_chars}{

# Get the current chunk to log (subtract 1 from the end as string range is 0 indexed)
if {$static::payload_dbg}{log local0.debug "position + static::max_chars -1 == [expr {$position + $static::max_chars -1}]"}
set current [expr {$position + $static::max_chars -1}]

if {$static::payload_dbg}{log local0.debug "chunk $count=$current"}

# Log the chunk of HTTP Payload locally.
if {$static::log_local}{log local0.debug "[string range "[HTTP::payload]" $position $current]"}

# Send all the collected payload to the remote syslog server
HSL::send $hsl "<190>[string range "[HTTP::payload]" $position $current]\n"

# Add the length of the current chunk to the position for the next chunk
# incr position $static::max_chars
set position $current

# Get the next chunk to log
set remaining [expr {$remaining - $static::max_chars}]
incr count
incr bytes_logged $static::max_chars
if {$static::payload_dbg}{log local0.debug "remaining bytes=$remaining, \$position=$position, \$count=$count, \$bytes_logged=$bytes_logged"}
}
if {$remaining < $static::max_chars}{
if {$static::payload_dbg}{log local0.debug "chunk $count=$current"}
incr bytes_logged $remaining
}
if {$static::payload_dbg}{log local0.debug "Logged $count chunks for a total of $bytes_logged bytes"}
}

}

when HTTP_RESPONSE {

# Log the response headers.
if {$static::payload_dbg}{log local0.debug "============================================="}
if {$static::payload_dbg}{log local0.debug "$LogString (response) - status: [HTTP::status]"}
foreach aHeader [HTTP::header names] {
if {$static::payload_dbg}{log local0.debug "$aHeader: [HTTP::header value $aHeader]"}
}
if {$static::payload_dbg}{log local0.debug "============================================="   }

switch -glob -- "[HTTP::header Content-Type]" {

"*image*" -
"*png*" {return}
default {

if {[HTTP::header "Content-Length"] ne "" && [HTTP::header "Content-Length"] <= 1048000} {
HTTP::collect [HTTP::header "Content-Length"]
} else {
HTTP::collect 1048000
}
}
}

}

when HTTP_RESPONSE_DATA {

# Log the bytes collected
if {$static::payload_dbg}{log local0.debug "Collected [HTTP::payload length] bytes"}

# Log the payload locally 
if {[HTTP::payload length] < $static::max_chars}{
if {$static::payload_dbg}{log local0.debug "Payload=[HTTP::payload]"}
} else {
set payloadlength [HTTP::payload length]

# Initialize variables
set remaining [HTTP::payload length]
set position 0
set count 1
set bytes_logged 0
set current 0

# Loop through and log each chunk of the payload
if {$static::payload_dbg}{log local0.debug "remaining=$remaining, static_max_chars=$static::max_chars"}
while {$remaining > $static::max_chars}{

# Get the current chunk to log (subtract 1 from the end as string range is 0 indexed)
if {$static::payload_dbg}{log local0.debug "position + static::max_chars -1 == [expr {$position + $static::max_chars -1}]"}
set current [expr {$position + $static::max_chars -1}]

if {$static::payload_dbg}{log local0.debug "chunk $count=$current"}

# Log the chunk of HTTP Payload locally.
if {$static::log_local}{log local0.debug "[string range "[HTTP::payload]" $position $current]"}

# Send all the collected payload to the remote syslog server
HSL::send $hsl "<190>[string range "[HTTP::payload]" $position $current]\n"

# Add the length of the current chunk to the position for the next chunk
# incr position $static::max_chars
set position $current

# Get the next chunk to log
set remaining [expr {$remaining - $static::max_chars}]
incr count
incr bytes_logged $static::max_chars
if {$static::payload_dbg}{log local0.debug "remaining bytes=$remaining, \$position=$position, \$count=$count, \$bytes_logged=$bytes_logged"}
}
if {$remaining < $static::max_chars}{
if {$static::payload_dbg}{log local0.debug "chunk $count=$current"}
incr bytes_logged $remaining
}
if {$static::payload_dbg}{log local0.debug "Logged $count chunks for a total of $bytes_logged bytes"}
}

}