FortiGate Hub/Spoke topology (wrong behavior for routing via Hub)

[POMA]

Hi,
trying to find out what's going wrong concerning typical 1x Hub - 2x Spoke setup with internal BGP.
First setup is the setup with IBGP. This works fine, the spokes can communicate with the hub.

Below is the design and the relevant info. The Hub is a FGT100F, the spokes are 60F. All in R7.4.9
Under this info and code, I'll explain my issue when I want to do spoke1 > spoke2 communication via the Hub (no ADVPN wanted)

Hub

• wan1 = 94.104.146.35
• lan, IP 192.168.200.1/24
• Lo_BGP, loopback, 192.168.255.1/32
• IPsec: tnl_Spokes, ike V2, dynamic
• SDWAN zone Internet, contains wan1 for internet access
• SDWAN zone RemoteSites, contains "tnl_Spokes"
• SDWAN rules: 1x towards Spokes, 1x towards Internet
• policies:
  • LAN to Internet (all/all)
  • LAN to RemoteSites (all/all)
  • RemotSites to LAN (all/all)
  • RemoteSites to Lo_BGP (loopback addresses of spokes > Lo_BGP)

Spoke1

• wan1, IP obtained via DHCP
• internal (lan), IP 192.168.10.1/24
• Lo_BGP, loopback, 192.168.255.10/32
• IPsec: tnl_Hub, ike V2, dialup to public IP of hub
• SDWAN zone Internet, contains wan1 for internet access
• SDWAN zone Hub, contains "tnl_Hub"
  SDWAN rules: 1x towards Hub, 1x towards Internet
• policies:
  •  lan (internal) > Internet (all/all)
  •  lan (internal) > Hub (all/all)
  •  Hub (internal) > internal(lan) (all/all)

Spoke2

• wan1, IP obtained via DHCP
• internal (lan), IP 192.168.20.1/24
• Lo_BGP, loopback, 192.168.255.20/32
• IPsec: tnl_Hub, ike V2, dialup to public IP of hub
• SDWAN zone Internet, contains wan1 for internet access
• SDWAN zone Hub, contains "tnl_Hub"
• SDWAN rules: 1x towards Hub, 1x towards Internet
• policies:
  •  lan (internal) > Internet (all/all)
  •  lan (internal) > Hub (all/all)
  • Hub > internal(lan) (all/all)

Code on the HUB (interfaces, ipsec, sdwan, policies, bgp)

INTERFACES CONFIG
-----------------

config system interface
    edit "lan"
        set ip 192.168.200.1 255.255.255.0
        set allowaccess ping https ssh fabric
        set type hard-switch
        set alias "LAN"
        set role lan
    next
    edit "Lo_BGP"
        set ip 192.168.255.1 255.255.255.255
        set allowaccess ping
        set type loopback
        set role lan
    next
    edit "tnl_Spokes"
        set vdom "root"
        set type tunnel
        set interface "wan1"
    next
end

IPSEC CONFIG
------------
config vpn ipsec phase1-interface
    edit "tnl_Spokes"
        set type dynamic
        set interface "wan1"
        set ike-version 2
        set peertype one
        set net-device disable
        set exchange-interface-ip enable
        set exchange-ip-addr4 192.168.255.1
        set proposal aes256-sha256
        set add-route disable
        set dpd on-idle
        set dhgrp 14
        set peerid "Hub"
        set psksecret mysecretpassword
        set dpd-retryinterval 60
    next
end
config vpn ipsec phase2-interface
    edit "tnl_Spokes"
        set phase1name "tnl_Spokes"
        set proposal aes256-sha256
        set dhgrp 14
        set keepalive enable
        set route-overlap allow
    next
end


SDWAN config
-------------
config system sdwan
    set status enable
    config zone
        edit "virtual-wan-link"
        next
        edit "Internet"
        next
        edit "RemoteSites"
        next
    end
    config members
        edit 1
            set interface "wan1"
            set zone "Internet"
        next
        edit 2
            set interface "tnl_Spokes"
            set zone "RemoteSites"
        next
    end
    config service
        edit 1
            set name "To_Spokes"
            set dst "LAN_Spoke1" "LAN_Spoke2"
            set src "all"
            set priority-members 2
        next
        edit 2
            set name "To_Internet"
            set dst "all"
            set src "all"
            set priority-members 1
        next
    end
end




POLICIES
--------
config firewall policy
    edit 1
        set name "To Internet"
        set srcintf "lan"
        set dstintf "Internet"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set nat enable
    next
    edit 2
        set name "LAN > Spokes"
        set srcintf "lan"
        set dstintf "RemoteSites"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
    next
    edit 3
        set name "Spokes > LAN"
        set srcintf "RemoteSites"
        set dstintf "lan"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
    next
    edit 4
        set name "Spokes > BGP peering"
        set srcintf "RemoteSites"
        set dstintf "Lo_BGP"
        set action accept
        set srcaddr "Lo_BGP_Spoke1" "Lo_BGP_Spoke2"
        set dstaddr "Lo_BGP"
        set schedule "always"
        set service "BGP"
    next
end




BGP config
----------
config router bgp
    set as 65200
    set router-id 192.168.255.1
    set ebgp-multipath enable
    set ibgp-multipath enable
    set network-import-check disable
    set recursive-inherit-priority enable
    set graceful-restart enable
    config neighbor-group
        edit "RemoteSites"
            set capability-graceful-restart enable
            set next-hop-self enable
            set soft-reconfiguration enable
            set remote-as 65200
            set update-source "Lo_BGP"
        next
    end
    config neighbor-range
        edit 1
            set prefix 192.168.255.0 255.255.255.0
            set max-neighbor-num 100
            set neighbor-group "RemoteSites"
        next
    end
    config network
        edit 1
            set prefix 192.168.255.0 255.255.255.0
        next
        edit 2
            set prefix 192.168.200.0 255.255.255.0
        next
    end
    config redistribute "connected"
        set status enable
    end
end

Code on Spoke1 (interfaces, ipsec, sdwan, policies, bgp)

INTERFACES CONFIG

config system interface   
    edit "internal"
        set vdom "root"
        set ip 192.168.10.1 255.255.255.0
        set allowaccess ping https ssh
        set type hard-switch
        set alias "LAN"
        set role lan
    next
    edit "Lo_BGP"
        set vdom "root"
        set ip 192.168.255.10 255.255.255.255
        set allowaccess ping
        set type loopback
        set role lan
    next
    edit "tnl_Hub"
        set vdom "root"
        set type tunnel
        set interface "wan1"
    next
end


IPSEC CONFIG
------------
config vpn ipsec phase1-interface
    edit "tnl_Hub"
        set interface "wan1"
        set ike-version 2
        set peertype any
        set net-device disable
        set exchange-interface-ip enable
        set exchange-ip-addr4 192.168.255.10
        set proposal aes256-sha256
        set localid "Hub"
        set dhgrp 14
        set remote-gw 94.104.146.35
        set psksecret mysecretpassword
    next
end
config vpn ipsec phase2-interface
    edit "tnl_Hub"
        set phase1name "tnl_Hub"
        set proposal aes256-sha256
        set dhgrp 14
        set auto-negotiate enable
    next
end





SDWAN config
-------------
config system sdwan
    set status enable
    config zone
        edit "virtual-wan-link"
        next
        edit "Internet"
        next
        edit "Hub"
        next
    end
    config members
        edit 1
            set interface "wan1"
            set zone "Internet"
        next
        edit 2
            set interface "tnl_Hub"
            set zone "Hub"
        next
    end
    config service
        edit 1
            set name "To_Hub"
            set dst "LAN_Hub" "Lo_Hub" "LAN_Spoke1" "LAN_Spoke2"
            set src "all"
            set priority-members 2
        next
        edit 2
            set name "To_Internet"
            set dst "all"
            set src "all"
            set priority-members 1
        next
    end
end



POLICIES
--------
config firewall policy
    edit 1
        set name "LAN > HUB"
        set srcintf "internal"
        set dstintf "Hub"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set logtraffic disable
    next
    edit 2
        set name "HUB > LAN"
        set srcintf "Hub"
        set dstintf "internal"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set logtraffic disable
    next
    edit 3
        set name "LAN > internet"
        set srcintf "internal"
        set dstintf "Internet"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set nat enable
    next
end

Spoke1 (policy) #


BGP config
----------
config router bgp
    set as 65200
    set router-id 192.168.255.10
    set ibgp-multipath enable
    set network-import-check disable
    config neighbor
        edit "192.168.255.1"
            set capability-graceful-restart enable
            set soft-reconfiguration enable
            set remote-as 65200
            set update-source "Lo_BGP"
        next
    end
    config network
        edit 1
            set prefix 192.168.10.0 255.255.255.0
        next
    end
    config redistribute "connected"
        set status enable
    end
end

Code on Spoke2 (interfaces, ipsec, sdwan, policies, bgp)

INTERFACES CONFIG

config system interface   
    edit "internal"
        set vdom "root"
        set ip 192.168.20.1 255.255.255.0
        set allowaccess ping https ssh
        set type hard-switch
        set role lan
    next
    edit "Lo_BGP"
        set vdom "root"
        set ip 192.168.255.20 255.255.255.255
        set allowaccess ping
        set type loopback
        set role lan
    next
    edit "tnl_Hub"
        set vdom "root"
        set type tunnel
        set interface "wan1"
    next
end


IPSEC CONFIG
------------
config vpn ipsec phase1-interface
    edit "tnl_Hub"
        set interface "wan1"
        set ike-version 2
        set peertype any
        set net-device disable
        set exchange-interface-ip enable
        set exchange-ip-addr4 192.168.255.20
        set proposal aes256-sha256
        set localid "Hub"
        set dhgrp 14
        set remote-gw 94.104.146.35
        set psksecret mysecretpassword
    next
end
config vpn ipsec phase2-interface
    edit "tnl_Hub"
        set phase1name "tnl_Hub"
        set proposal aes256-sha256
        set dhgrp 14
        set auto-negotiate enable
    next
end




SDWAN config
-------------
config system sdwan
    set status enable
    config zone
        edit "virtual-wan-link"
        next
        edit "Internet"
        next
        edit "Hub"
        next
    end
    config members
        edit 1
            set interface "wan1"
            set zone "Internet"
        next
        edit 2
            set interface "tnl_Hub"
            set zone "Hub"
        next
    end
    config service
        edit 1
            set name "To_Hub"
            set dst "LAN_Hub" "Lo_Hub" "LAN_Spoke1" "LAN_Spoke2"
            set src "all"
            set priority-members 2
        next
        edit 2
            set name "To_Internet"
            set dst "all"
            set src "all"
            set priority-members 1
        next
    end
end



POLICIES
--------
config firewall policy
    edit 1
        set name "LAN > HUB"
        set srcintf "internal"
        set dstintf "Hub"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set logtraffic disable
    next
    edit 2
        set name "HUB > LAN"
        set srcintf "Hub"
        set dstintf "internal"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set logtraffic disable
    next
    edit 3
        set name "LAN > internet"
        set srcintf "internal"
        set dstintf "Internet"
        set action accept
        set srcaddr "all"
        set dstaddr "all"
        set schedule "always"
        set service "ALL"
        set nat enable
    next
end



BGP config
----------
config router bgp
    set as 65200
    set router-id 192.168.255.20
    set ibgp-multipath enable
    set network-import-check disable
    config neighbor
        edit "192.168.255.1"
            set capability-graceful-restart enable
            set soft-reconfiguration enable
            set remote-as 65200
            set update-source "Lo_BGP"
        next
    end
    config network
        edit 1
            set prefix 192.168.20.0 255.255.255.0
        next
    end
    config redistribute "connected"
        set status enable
    end
   
end

So far, so good.
On spoke1 + spoke2 I see BGP peering with the hub (neighbor 192.168.255.1) and the routes are exchanged.
Ping from spoke1 + spoke2 towards the hub is fine...

Now, I want communication between spoke1 and spoke2 via the HUB (not using ADVP). So I changed the config:

Hub:
- policy: RemoteSites > RemoteSites (all / all)
- BGP: route-reflector-client enable (on the neighbor-group)

Spoke1+Spoke2
- added SDWAN rule: src all > dst Spoke1_lan + Spoke2_lan via the ipsec
- policy is already ok since lan > hub is all/all

Problem:
get router info routing-table all on spoke1 shows
BGP route 192.168.200.0/24 via tnl1_Hub (= ok, this is the LAN of the hub)
on spoke1: route 192.168.20.0/24, via wan1 . --> this is wrong, should be the ipsec to route it via the hub
on spoke2: route 192.168.10.0/24, via wan1 --> this is wrong, should be the ipsec to route it via the hub

So, when spoke1 tries to ping spoke2 (192.168.20.1), the traffic is being sent over the WAN interface instead over the tunnel.

Any ideas or tips what I'm doing wrong?

[funkylicious]

i think, set exchange-interface-ip should be disabled.

[POMA]

Hi, thank you for the feedback and the time you took to check this...

I think the exchange-interface-ip is needed so the IPsec can come online and the BGP loopback addresses can communicate with each other for the ibgp peering...

But, after this post I was doing some extra tests in my setup and I think I got it working.

Although, I changed quite a lot and still need to figure out what's really needed or not, I think the following did the trick...

Hub bgp config

added next-hop-self enable

added next-hop-self-rr enable

also added link-down-failover enable (but this won't be needed I guess - have to check exactly what this does)

config router bgp
...
    config neighbor-group
        edit "RemoteSites"
            ...
            set next-hop-self enable
            set next-hop-self-rr enable
            set route-reflector-client enable
        next
    end
    ...

However, I will do some extra testing & post extra feedback on this topic.
But this seemed to do the trick...

On Spoke1 when I check my routing  table, I see the subnets of the Hub + Spoke2 - going via the tunnel (I also forgot a policy on my hub, from RemoteSites to RemoteSites - but this is just to allow it or not)

[funkylicious]

it is when you do ADVPN w/ BGP on loopback according to this - https://community.fortinet.com/t5/FortiGate/Technical-Tip-How-to-use-exchange-interface-IP-in-IPsec-... 

in your case, the hub should be the RR and handle all the traffic and all next-hops should point to it's ip address like you already managed to do.