Fortigate IPSEC VPN - One Way Traffic issues

[bksup]

I have 2 Fortigate 201F set up with a tunnel and I'm seeing some strange traffic (mostly missing) from site A to site B.  Site A has 2 active WAN interfaces with the 2nd interface static DF route at a higher priority. My tunnel is assigned to the interface with the lower routing priority.  I'm not able to ping devices on Site B, yet I'm able to access the internal web interface of the Fortigate of Site B.
Site B only has one WAN interface and I'm able to ping from devices within Site B to devices on Site A, but I'm not able to connect to NTP, DNS or LDAP from the Fortigate B to Site A.

Both tunnels were created as custom pointing to the IP address of the outside interface of the opposite.  I have phase 2 selectors set for each of the VLANs from Site A and all show active.  
Both sites are set with incoming/outgoing policies using the named tunnel and allowing ALL traffic and stated VLANs.

Both sites have static routes set to use the tunnel interface for traffic.  Site A has a single route and Site B has 3 routes(for each of the VLANs).
I've tried reducing both sides to a single network and get the same results.  I kind of feel like the issue resides on Site A where I have the active/active failover WANs but I don't know how to resolve.  I'm out of ideas and would appreciate any help.

Site A

config system interface
edit "port16"
set vdom "root"
set ip 199.x.x.x
set type physical
set description "Outside Interface A"
set alias "Wan1"
set monitor-bandwidth enable
set role wan
set snmp-index 19
edit "HQKC"
set vdom "root"
set allowaccess ping
set type tunnel
set snmp-index 10
set interface "port16"

config firewall policy
edit 90
set name "HQtoKC"
set srcintf "x1"
set dstintf "HQKC"
set action accept
set srcaddr "obj-East-Servers" "obj-EastMobile" "obj-HQGeneralVLAN"
set dstaddr "kctest"
set schedule "always"
set service "ALL"
set logtraffic all
next
edit 91
set name "KCtoHQ"
set srcintf "HQKC"
set dstintf "x1"
set action accept
set srcaddr "kctest"
set dstaddr "obj-East-Servers" "obj-EastMobile" "obj-HQGeneralVLAN"
set schedule "always"
set service "ALL"
set logtraffic all

config vpn ipsec phase1-interface
edit "HQKC"
set interface "port16"
set ike-version 2
set local-gw 199.x.x.x
set keylife 28800
set peertype any
set net-device disable
set proposal aes128-sha256
set dhgrp 5
set remote-gw 24.2x.x.x

config vpn ipsec phase2-interface
edit "HQKC"
set phase1name "HQKC"
set proposal aes256-sha256
set dhgrp 5
set auto-negotiate enable
set keylifeseconds 3600
set src-subnet 10.253.10.0 255.255.255.0
set dst-subnet 10.253.25.0 255.255.255.0
next
edit "HQKC1"
set phase1name "HQKC"
set proposal aes256-sha256
set dhgrp 5
set auto-negotiate enable
set keylifeseconds 3600
set src-subnet 10.253.252.0 255.255.254.0
set dst-subnet 10.253.25.0 255.255.255.0
next
edit "HQKC2"
set phase1name "HQKC"
set proposal aes256-sha256
set dhgrp 5
set auto-negotiate enable
set keylifeseconds 3600
set src-subnet 10.0.0.0 255.255.240.0
set dst-subnet 10.253.25.0 255.255.255.0

config router static
edit 20
set dst 10.253.25.0 255.255.255.0
set device "HQKC"

Site B

config system interface
edit "port16"
set vdom "root"
set ip 24.2x.x.x
set type physical
set description "Outside Interface"
set alias "Wan1"
set monitor-bandwidth enable
set role wan
set snmp-index 18
next
edit "KCHQ"
set vdom "root"
set type tunnel
set snmp-index 37
set interface "port16"

config firewall policy
edit 16
set name "KCtoHQ"
set srcintf "port1"
set dstintf "KCHQ"
set action accept
set srcaddr "kctest"
set dstaddr "EastServerVlan" "EastMobileVlan" "EastGeneralVlan"
set schedule "always"
set service "ALL"
set logtraffic all
next
edit 17
set name "HQtoKC"
set srcintf "KCHQ"
set dstintf "port1"
set action accept
set srcaddr "EastServerVlan" "EastMobileVlan" "EastGeneralVlan"
set dstaddr "kctest"
set schedule "always"
set service "ALL"
set logtraffic all

config vpn ipsec phase1-interface
edit "KCHQ"
set interface "port16"
set ike-version 2
set local-gw 24.2x.x.x
set keylife 28800
set peertype any
set net-device disable
set proposal aes128-sha256
set dhgrp 5
set remote-gw 199.x.x.x

config vpn ipsec phase2-interface
edit "KCHQ"
set phase1name "KCHQ"
set proposal aes256-sha256
set dhgrp 5
set auto-negotiate enable
set keylifeseconds 3600
set src-subnet 10.253.25.0 255.255.255.0
set dst-subnet 10.253.10.0 255.255.255.0
next
edit "KCHQ1"
set phase1name "KCHQ"
set proposal aes256-sha256
set auto-negotiate enable
set keylifeseconds 3600
set src-subnet 10.253.25.0 255.255.255.0
set dst-subnet 10.253.252.0 255.255.254.0
next
edit "KCHQ2"
set phase1name "KCHQ"
set proposal aes256-sha256
set dhgrp 5
set auto-negotiate enable
set keylifeseconds 3600
set src-subnet 10.253.25.0 255.255.255.0
set dst-subnet 10.0.0.0 255.255.240.0

config router static
edit 10
set dst 10.253.10.0 255.255.255.0
set device "KCHQ"
next
edit 11
set dst 10.253.252.0 255.255.254.0
set device "KCHQ"
next
edit 12
set dst 10.0.0.0 255.255.240.0
set device "KCHQ"

[Toshi_Esumi]

Since your have:
config vpn ipsec phase1-interface
  edit "HQKC"
    set interface "port16"
configured, the IPsec wouldn't fail-over to another port even you have a backup port for the internet. So as long as you don't lost the primary default route to port16, the tunnel traffic always go out from port16 on SiteA side.
If something specific destinations, you mentioned NTP, DNS or LDAP, are not reachable from SiteB, you should run regular troubleshooting methods to narrow down the breaking point and find out why.

The first thing is traceroute from source device on B-side toward those destinations. I'm assuming you can't get response back from the A-side FGT. Then run sniffing on the tunnel interface on the A-side to see if it's receiving those packets. If receiving but not going further, you need to run flow debugging to figure out why it's dropping. If it's not even receiving, the problem is on the B-side then run sniffing/flow debugging on the B-side to see why it's not sending into the tunnel.
You likely need to disable ASIC offloading at least at those policies "set auto-asic-offload disable" to see those debug output.

Toshi 

[dingjerry_FTNT]

Hi @bksup ,

We need to run the debug flow commands to see why it is not working.

1) Run "set auto-asic-offload disable" in all relevant firewall policies.

2) Run the following commands:

diag debug flow show iprope enable

diag debug flow filter proto 1

diag debug flow filter addr x.x.x.x    //  This is the IP you are Pinging

diag debug flow trace start 100

diag debug enable

Then reproduce this issue with Ping.  Please do not run continuous Ping.

3) For DNS traffic:

diag debug flow show iprope enable

diag debug flow filter clear

diag debug flow filter addr x.x.x.x    //  This is the DNS server IP

diag debug flow filter port 53

diag debug flow trace start 1000

diag debug enable

Then reproduce the issue with DNS traffic.

[bksup]

Here are the ping debugs from Site A and B

SiteA #

E-SRV-WR5-ASA1 # diag debug flow show iprope enable
show trace messages about iprope
E-SRV-WR5-ASA1 # diag debug flow filter proto 1
E-SRV-WR5-ASA1 # diag debug flow filter addr 10.253.25.4
E-SRV-WR5-ASA1 # diag debug flow trace start 100
E-SRV-WR5-ASA1 # diag debug enable

E-SRV-WR5-ASA1 # id=65308 trace_id=101 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.253.252.123:1->10.253.25.4:2048) tun_id=0.0.0.0 from x1. type=8, code=0, id=1, seq=63971."
id=65308 trace_id=101 func=init_ip_session_common line=6070 msg="allocate a new session-1d8b1252"
id=65308 trace_id=101 func=iprope_dnat_check line=5472 msg="in-[x1], out-[]"
id=65308 trace_id=101 func=iprope_dnat_tree_check line=834 msg="len=0"
id=65308 trace_id=101 func=iprope_dnat_check line=5497 msg="result: skb_flags-02000000, vid-0, ret-no-match, act-accept, flag-00000000"
id=65308 trace_id=101 func=vf_ip_route_input_common line=2612 msg="find a route: flag=00000000 gw-24.223.107.122 via HQKC"
id=65308 trace_id=101 func=__iprope_fwd_check line=807 msg="in-[x1], out-[HQKC], skb_flags-02000000, vid-0, app_id: 0, url_cat_id: 0"
id=65308 trace_id=101 func=__iprope_tree_check line=539 msg="gnum-100004, use addr/intf hash, len=14"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-114, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-99, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-10054, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-146, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-10082, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-168, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-10084, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-163, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-10099, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-10106, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-85, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-90, ret-matched, act-accept"
id=65308 trace_id=101 func=__iprope_user_identity_check line=1894 msg="ret-matched"
id=65308 trace_id=101 func=__iprope_check line=2395 msg="gnum-4e21, check-ffffffffa002f300"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check line=2412 msg="gnum-4e21 check result: ret-no-match, act-accept, flag-00000000, flag2-00000000"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2365 msg="policy-90 is matched, act-accept"
id=65308 trace_id=101 func=__iprope_fwd_check line=844 msg="after iprope_captive_check(): is_captive-0, ret-matched, act-accept, idx-90"
id=65308 trace_id=101 func=iprope_fwd_auth_check line=873 msg="after iprope_captive_check(): is_captive-0, ret-matched, act-accept, idx-90"
id=65308 trace_id=101 func=iprope_reverse_dnat_check line=1344 msg="in-[x1], out-[HQKC], skb_flags-02000000, vid-0"
id=65308 trace_id=101 func=iprope_reverse_dnat_tree_check line=926 msg="len=0"
id=65308 trace_id=101 func=iprope_central_nat_check line=1367 msg="in-[x1], out-[HQKC], skb_flags-02000000, vid-0"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-1, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-4, ret-no-match, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-0, ret-matched, act-accept"
id=65308 trace_id=101 func=__iprope_check_one_policy line=2365 msg="policy-0 is matched, act-accept"
id=65308 trace_id=101 func=fw_snat_check line=679 msg="NAT disabled by central SNAT policy!"
id=65308 trace_id=101 func=fw_forward_handler line=991 msg="Allowed by Policy-90:"
id=65308 trace_id=101 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface HQKC, tun_id=0.0.0.0"
id=65308 trace_id=101 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel HQKC, tun_id=24.223.107.122, vrf 0"
id=65308 trace_id=101 func=esp_output4 line=876 msg="IPsec encrypt/auth"
id=65308 trace_id=101 func=ipsec_output_finish line=666 msg="send to 199.200.72.1 via intf-port16"
id=65308 trace_id=102 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.253.252.123:1->10.253.25.4:2048) tun_id=0.0.0.0 from x1. type=8, code=0, id=1, seq=63972."
id=65308 trace_id=102 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-1d8b1252, original direction"
id=65308 trace_id=102 func=npu_handle_session44 line=1224 msg="Trying to offloading session from x1 to HQKC, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x01040001"
id=65308 trace_id=102 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=01040001"
id=65308 trace_id=102 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface HQKC, tun_id=0.0.0.0"
id=65308 trace_id=102 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel HQKC, tun_id=24.223.107.122, vrf 0"
id=65308 trace_id=102 func=esp_output4 line=876 msg="IPsec encrypt/auth"
id=65308 trace_id=102 func=ipsec_output_finish line=666 msg="send to 199.200.72.1 via intf-port16"
id=65308 trace_id=103 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.253.252.123:1->10.253.25.4:2048) tun_id=0.0.0.0 from x1. type=8, code=0, id=1, seq=63973."
id=65308 trace_id=103 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-1d8b1252, original direction"
id=65308 trace_id=103 func=npu_handle_session44 line=1224 msg="Trying to offloading session from x1 to HQKC, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x01040001"
id=65308 trace_id=103 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=01040001"
id=65308 trace_id=103 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface HQKC, tun_id=0.0.0.0"
id=65308 trace_id=103 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel HQKC, tun_id=24.223.107.122, vrf 0"
id=65308 trace_id=103 func=esp_output4 line=876 msg="IPsec encrypt/auth"
id=65308 trace_id=103 func=ipsec_output_finish line=666 msg="send to 199.200.72.1 via intf-port16"
id=65308 trace_id=104 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.253.252.123:1->10.253.25.4:2048) tun_id=0.0.0.0 from x1. type=8, code=0, id=1, seq=63974."
id=65308 trace_id=104 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-1d8b1252, original direction"
id=65308 trace_id=104 func=npu_handle_session44 line=1224 msg="Trying to offloading session from x1 to HQKC, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x01040001"
id=65308 trace_id=104 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=01040001"
id=65308 trace_id=104 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface HQKC, tun_id=0.0.0.0"
id=65308 trace_id=104 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel HQKC, tun_id=24.223.107.122, vrf 0"
id=65308 trace_id=104 func=esp_output4 line=876 msg="IPsec encrypt/auth"
id=65308 trace_id=104 func=ipsec_output_finish line=666 msg="send to 199.200.72.1 via intf-port16"
##########

SiteB #
K-SRV-WR1-ASA1 # diag debug flow filter proto 1
K-SRV-WR1-ASA1 # diag debug flow filter addr 10.253.25.4
K-SRV-WR1-ASA1 # diag debug flow trace start 100
K-SRV-WR1-ASA1 # diag debug enable

K-SRV-WR1-ASA1 # id=65308 trace_id=1011 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.0.13.14:1->10.253.25.4:2048) tun_id=199.200.72.2 from KCHQ. type=8, code=0, id=1, seq=64140."
id=65308 trace_id=1011 func=ipsec_spoofed4 line=245 msg="src ip 10.0.13.14 match selector 0 range 10.0.0.0-10.0.15.255"
id=65308 trace_id=1011 func=init_ip_session_common line=6070 msg="allocate a new session-0002db39"
id=65308 trace_id=1011 func=iprope_dnat_check line=5472 msg="in-[KCHQ], out-[]"
id=65308 trace_id=1011 func=iprope_dnat_tree_check line=834 msg="len=0"
id=65308 trace_id=1011 func=iprope_dnat_check line=5497 msg="result: skb_flags-02000008, vid-0, ret-no-match, act-accept, flag-00000000"
id=65308 trace_id=1011 func=vf_ip_route_input_common line=2612 msg="find a route: flag=00000000 gw-10.253.25.4 via port1"
id=65308 trace_id=1011 func=__iprope_fwd_check line=807 msg="in-[KCHQ], out-[port1], skb_flags-02000008, vid-0, app_id: 0, url_cat_id: 0"
id=65308 trace_id=1011 func=__iprope_tree_check line=528 msg="gnum-100004, use int hash, slot=7, len=2"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-17, ret-matched, act-accept"
id=65308 trace_id=1011 func=__iprope_user_identity_check line=1894 msg="ret-matched"
id=65308 trace_id=1011 func=__iprope_check line=2395 msg="gnum-4e21, check-ffffffffa002f300"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=1011 func=__iprope_check line=2412 msg="gnum-4e21 check result: ret-no-match, act-accept, flag-00000000, flag2-00000000"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2365 msg="policy-17 is matched, act-accept"
id=65308 trace_id=1011 func=__iprope_fwd_check line=844 msg="after iprope_captive_check(): is_captive-0, ret-matched, act-accept, idx-17"
id=65308 trace_id=1011 func=iprope_fwd_auth_check line=873 msg="after iprope_captive_check(): is_captive-0, ret-matched, act-accept, idx-17"
id=65308 trace_id=1011 func=iprope_reverse_dnat_check line=1344 msg="in-[KCHQ], out-[port1], skb_flags-02000008, vid-0"
id=65308 trace_id=1011 func=iprope_reverse_dnat_tree_check line=926 msg="len=0"
id=65308 trace_id=1011 func=iprope_central_nat_check line=1367 msg="in-[KCHQ], out-[port1], skb_flags-02000008, vid-0"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-1, ret-no-match, act-accept"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-0, ret-matched, act-accept"
id=65308 trace_id=1011 func=__iprope_check_one_policy line=2365 msg="policy-0 is matched, act-accept"
id=65308 trace_id=1011 func=fw_snat_check line=679 msg="NAT disabled by central SNAT policy!"
id=65308 trace_id=1011 func=fw_forward_handler line=991 msg="Allowed by Policy-17:"
id=65308 trace_id=1012 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.0.13.14:1->10.253.25.4:2048) tun_id=199.200.72.2 from KCHQ. type=8, code=0, id=1, seq=64141."
id=65308 trace_id=1012 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002db39, original direction"
id=65308 trace_id=1012 func=ipsec_spoofed4 line=245 msg="src ip 10.0.13.14 match selector 0 range 10.0.0.0-10.0.15.255"
id=65308 trace_id=1012 func=npu_handle_session44 line=1224 msg="Trying to offloading session from KCHQ to port1, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x00000001"
id=65308 trace_id=1012 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=00000001"
id=65308 trace_id=1013 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.0.13.14:1->10.253.25.4:2048) tun_id=199.200.72.2 from KCHQ. type=8, code=0, id=1, seq=64142."
id=65308 trace_id=1013 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002db39, original direction"
id=65308 trace_id=1013 func=ipsec_spoofed4 line=245 msg="src ip 10.0.13.14 match selector 0 range 10.0.0.0-10.0.15.255"
id=65308 trace_id=1013 func=npu_handle_session44 line=1224 msg="Trying to offloading session from KCHQ to port1, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x00000001"
id=65308 trace_id=1013 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=00000001"
id=65308 trace_id=1014 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.0.13.14:1->10.253.25.4:2048) tun_id=199.200.72.2 from KCHQ. type=8, code=0, id=1, seq=64143."
id=65308 trace_id=1014 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002db39, original direction"
id=65308 trace_id=1014 func=ipsec_spoofed4 line=245 msg="src ip 10.0.13.14 match selector 0 range 10.0.0.0-10.0.15.255"
id=65308 trace_id=1014 func=npu_handle_session44 line=1224 msg="Trying to offloading session from KCHQ to port1, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x00000001"
id=65308 trace_id=1014 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=00000001"

[dingjerry_FTNT · Accepted solution]

Hi @bksup ,

1) Ping issue:

I can see the Ping packets entering the IPSec VPN tunnel on Site A and coming out of the IPSec VPN tunnel on Site B:

id=65308 trace_id=1011 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=1, 10.0.13.14:1->10.253.25.4:2048) tun_id=199.200.72.2 from KCHQ. type=8, code=0, id=1, seq=64140."

ICMP type 8 code 0 is ICMP Echo Request.

id=65308 trace_id=1011 func=vf_ip_route_input_common line=2612 msg="find a route: flag=00000000 gw-10.253.25.4 via port1"

And it found a route via port1.

id=65308 trace_id=1011 func=fw_snat_check line=679 msg="NAT disabled by central SNAT policy!"
id=65308 trace_id=1011 func=fw_forward_handler line=991 msg="Allowed by Policy-17:"

And this Ping echo request packet was allowed by policy 17. 

Then FGT never received any Ping echo reply packets.

You need to double check on host 10.253.25.4 whether it received the Ping or not.  If yes, did it send the Ping echo reply?

2) DNS issue:

id=65308 trace_id=11 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "

"from local"  means the DNS traffic is not passing through traffic.  It is local-out DNS traffic.  That means it is the FGT originated the DNS traffic.   

id=65308 trace_id=11 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=11 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=11 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"

It tried to enter the IPSec VPN, however, the source IP was 192.168.1.99, it does not match to any IPSec selector, so it got dropped.

Solution:

You need to specify a source IP that matches any of the IPSec selectors in the "config system dns."

[dingjerry_FTNT]

And the same thing for NTP & LDAP settings, you need to specify a source IP for them.

[bksup]

I was missing the source IP and got that put in place which fixed things up.  Thank you for your help.

[dingjerry_FTNT]

I am glad that your issue is fixed.

[bksup]

Here is the DNS debug from Site B

K-SRV-WR1-ASA1 # diag debug flow show iprope enable
show trace messages about iprope
K-SRV-WR1-ASA1 # diag debug flow filter clear
K-SRV-WR1-ASA1 # diag debug flow filter addr 10.253.10.84
K-SRV-WR1-ASA1 # diag debug flow filter port 53
K-SRV-WR1-ASA1 # diag debug flow trace start 1000

K-SRV-WR1-ASA1 # id=65308 trace_id=11 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=11 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=11 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=11 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=11 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=11 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=12 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=12 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=12 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=12 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=12 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=12 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=13 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=13 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=13 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=13 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=13 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=13 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=14 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=14 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=14 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=14 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=14 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=14 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=15 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=15 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=15 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=15 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=15 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=15 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=16 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=16 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=16 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=16 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=16 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=16 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=17 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=17 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=17 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=17 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=17 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=17 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=18 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=18 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=18 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=18 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=18 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=18 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=19 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=19 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=19 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=19 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=19 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=19 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=20 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=20 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=20 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=20 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=20 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=20 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=21 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=21 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=21 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=21 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=21 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=21 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=22 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=22 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=22 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=22 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=22 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=22 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=23 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=23 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=23 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=23 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=23 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=23 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=24 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=24 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=24 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=24 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=24 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=24 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=25 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=25 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=25 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=25 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=25 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=25 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=26 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=26 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=26 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=26 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=26 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=26 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
diag debug enaid=65308 trace_id=27 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=27 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=27 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=27 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=27 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=27 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
ble

K-SRV-WR1-ASA1 # id=65308 trace_id=28 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=28 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=28 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=28 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=28 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=28 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=29 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 10.253.25.4:53009->10.253.10.84:53) tun_id=0.0.0.0 from port1. "
id=65308 trace_id=29 func=init_ip_session_common line=6070 msg="allocate a new session-0002d70e"
id=65308 trace_id=29 func=iprope_dnat_check line=5472 msg="in-[port1], out-[]"
id=65308 trace_id=29 func=iprope_dnat_tree_check line=834 msg="len=0"
id=65308 trace_id=29 func=iprope_dnat_check line=5497 msg="result: skb_flags-02000000, vid-0, ret-no-match, act-accept, flag-00000000"
id=65308 trace_id=29 func=vf_ip_route_input_common line=2612 msg="find a route: flag=04000000 gw-199.200.72.2 via KCHQ"
id=65308 trace_id=29 func=__iprope_fwd_check line=807 msg="in-[port1], out-[KCHQ], skb_flags-02000000, vid-0, app_id: 0, url_cat_id: 0"
id=65308 trace_id=29 func=__iprope_tree_check line=528 msg="gnum-100004, use int hash, slot=104, len=2"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2131 msg="checked gnum-100004 policy-16, ret-matched, act-accept"
id=65308 trace_id=29 func=__iprope_user_identity_check line=1894 msg="ret-matched"
id=65308 trace_id=29 func=__iprope_check line=2395 msg="gnum-4e21, check-ffffffffa002f300"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2131 msg="checked gnum-4e21 policy-6, ret-no-match, act-accept"
id=65308 trace_id=29 func=__iprope_check line=2412 msg="gnum-4e21 check result: ret-no-match, act-accept, flag-00000000, flag2-00000000"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2365 msg="policy-16 is matched, act-accept"
id=65308 trace_id=29 func=__iprope_fwd_check line=844 msg="after iprope_captive_check(): is_captive-0, ret-matched, act-accept, idx-16"
id=65308 trace_id=29 func=iprope_fwd_auth_check line=873 msg="after iprope_captive_check(): is_captive-0, ret-matched, act-accept, idx-16"
id=65308 trace_id=29 func=iprope_reverse_dnat_check line=1344 msg="in-[port1], out-[KCHQ], skb_flags-02000000, vid-0"
id=65308 trace_id=29 func=iprope_reverse_dnat_tree_check line=926 msg="len=0"
id=65308 trace_id=29 func=iprope_central_nat_check line=1367 msg="in-[port1], out-[KCHQ], skb_flags-02000000, vid-0"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-1, ret-no-match, act-accept"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2131 msg="checked gnum-10000d policy-0, ret-matched, act-accept"
id=65308 trace_id=29 func=__iprope_check_one_policy line=2365 msg="policy-0 is matched, act-accept"
id=65308 trace_id=29 func=fw_snat_check line=679 msg="NAT disabled by central SNAT policy!"
id=65308 trace_id=29 func=fw_forward_handler line=991 msg="Allowed by Policy-16:"
id=65308 trace_id=29 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=29 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=29 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=29 func=esp_output4 line=876 msg="IPsec encrypt/auth"
id=65308 trace_id=29 func=ipsec_output_finish line=666 msg="send to 24.223.107.126 via intf-port16"
id=65308 trace_id=30 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 10.253.25.4:53009->10.253.10.84:53) tun_id=0.0.0.0 from port1. "
id=65308 trace_id=30 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d70e, original direction"
id=65308 trace_id=30 func=npu_handle_session44 line=1224 msg="Trying to offloading session from port1 to KCHQ, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x01040001"
id=65308 trace_id=30 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=01040001"
id=65308 trace_id=30 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=30 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=30 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=30 func=esp_output4 line=876 msg="IPsec encrypt/auth"
id=65308 trace_id=30 func=ipsec_output_finish line=666 msg="send to 24.223.107.126 via intf-port16"
id=65308 trace_id=31 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 10.253.10.84:53->10.253.25.4:53009) tun_id=199.200.72.2 from KCHQ. "
id=65308 trace_id=31 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d70e, reply direction"
id=65308 trace_id=31 func=ipsec_spoofed4 line=245 msg="src ip 10.253.10.84 match selector 0 range 10.253.10.0-10.253.10.255"
id=65308 trace_id=31 func=vf_ip_route_input_common line=2612 msg="find a route: flag=00000000 gw-10.253.25.4 via port1"
id=65308 trace_id=31 func=npu_handle_session44 line=1224 msg="Trying to offloading session from KCHQ to port1, skb.npu_flag=00000000 ses.state=00000204 ses.npu_state=0x01040001"
id=65308 trace_id=31 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=01040001"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=31 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=31 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=reply)"
id=65308 trace_id=32 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 10.253.10.84:53->10.253.25.4:53009) tun_id=199.200.72.2 from KCHQ. "
id=65308 trace_id=32 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d70e, reply direction"
id=65308 trace_id=32 func=ipsec_spoofed4 line=245 msg="src ip 10.253.10.84 match selector 0 range 10.253.10.0-10.253.10.255"
id=65308 trace_id=32 func=npu_handle_session44 line=1224 msg="Trying to offloading session from KCHQ to port1, skb.npu_flag=00000400 ses.state=00000204 ses.npu_state=0x01040001"
id=65308 trace_id=32 func=fw_forward_dirty_handler line=443 msg="state=00000204, state2=00000001, npu_state=01040001"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__iprope_check line=2395 msg="gnum-100008, check-ffffffffa002f0e0"
id=65308 trace_id=32 func=iprope_policy_group_check line=4892 msg="after check: ret-no-match, act-drop, flag-00000000, flag2-00000000"
id=65308 trace_id=32 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=reply)"
id=65308 trace_id=33 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=33 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=33 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=33 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=33 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=33 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=34 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=34 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=34 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=34 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=34 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=34 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=35 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=35 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=35 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=35 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=35 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=35 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=36 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=36 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=36 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=36 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=36 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=36 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=37 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=37 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=37 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=37 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=37 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=37 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=38 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=38 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=38 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=38 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=38 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=38 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=39 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=39 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=39 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=39 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=39 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=39 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=40 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=40 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=40 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=40 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=40 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=40 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=41 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=41 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=41 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=41 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=41 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=41 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=42 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=42 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=42 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=42 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=42 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=42 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=43 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=43 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=43 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=43 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=43 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=43 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=44 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=44 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=44 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=44 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=44 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=44 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=45 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=45 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=45 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=45 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=45 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=45 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=46 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=46 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=46 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=46 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=46 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=46 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=47 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=47 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=47 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=47 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=47 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=47 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=48 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=48 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=48 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=48 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=48 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=48 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=49 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=49 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=49 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=49 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=49 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=49 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"
id=65308 trace_id=50 func=print_pkt_detail line=5879 msg="vd-root:0 received a packet(proto=17, 192.168.1.99:3600->10.253.10.84:53) tun_id=0.0.0.0 from local. "
id=65308 trace_id=50 func=resolve_ip_tuple_fast line=5974 msg="Find an existing session, id-0002d6e8, original direction"
id=65308 trace_id=50 func=__ip_session_run_tuple line=3487 msg="run helper-dns-udp(dir=original)"
id=65308 trace_id=50 func=ipsecdev_hard_start_xmit line=669 msg="enter IPSec interface KCHQ, tun_id=0.0.0.0"
id=65308 trace_id=50 func=_do_ipsecdev_hard_start_xmit line=229 msg="output to IPSec tunnel KCHQ, tun_id=199.200.72.2, vrf 0"
id=65308 trace_id=50 func=ipsec_common_output4 line=917 msg="No matching IPsec selector, drop"