DNS domain blocking using UDP payload

are you using double back slashes in data group?

this is mine.

 config

root@(ve11a)(cfg-sync In Sync)(Active)(/Common)(tmos) list ltm virtual bar
ltm virtual bar {
    destination 172.28.24.10:53
    ip-protocol udp
    mask 255.255.255.255
    pool foo
    profiles {
        udp_gtm_dns { }
    }
    rules {
        qux
    }
    source 0.0.0.0/0
    source-address-translation {
        type automap
    }
    vs-index 58
}
root@(ve11a)(cfg-sync In Sync)(Active)(/Common)(tmos) list ltm rule qux
ltm rule qux {
    when CLIENT_ACCEPTED  {
  binary scan [UDP::payload] H4@12A*@12H* id dname question
  set dname [string tolower [getfield $dname \x00 1]]
  if {[class match -- $dname contains blackhole_domain]} {
    log local0. "drop"
    drop
    return
  }
  log local0. "not drop"
}
}
root@(ve11a)(cfg-sync In Sync)(Active)(/Common)(tmos) list ltm data-group internal blackhole_domain
ltm data-group internal blackhole_domain {
    records {
        \\x08doohotok\\x03com { }
    }
    type string
}

 trace

[root@ve11a:Active:In Sync] tmp  tcpdump -nni 0.0 -s0 port 53
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on 0.0, link-type EN10MB (Ethernet), capture size 65535 bytes
07:38:00.143874 IP 172.28.24.1.44967 > 172.28.24.10.53:  62524+ A? cmz.www.doohotok.com. (38) in slot1/tmm0 lis=

 /var/log/ltm

[root@ve11a:Active:In Sync] tmp  cat /var/log/ltm
Jul  8 07:38:00 ve11a info tmm[29362]: Rule /Common/qux : drop