Mirrors/juanfont.headscale
1
0
Fork 0
mirror of https://github.com/juanfont/headscale.git synced 2025-05-27 01:18:48 +02:00
Code Issues Projects Releases Wiki Activity

fix HA subnet routers

Browse Source 
Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
This commit is contained in:
Kristoffer Dalby 2025-03-17 09:37:10 +01:00
parent dea7b9c617
commit 0ca68ff61b
No known key found for this signature in database
6 changed files with 586 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hscontrol/mapper/tail.go
View File

@ -2,11 +2,8 @@ package mapper
import (
"fmt"
"net/netip"
"time"
"slices"
"github.com/juanfont/headscale/hscontrol/policy"
"github.com/juanfont/headscale/hscontrol/routes"
"github.com/juanfont/headscale/hscontrol/types"
@ -51,13 +48,6 @@ func tailNode(
) (*tailcfg.Node, error) {
addrs := node.Prefixes()
// we append the node own IP, as it is required by the clients
allowedIPs := slices.Clone(node.Prefixes())
for _, route := range node.SubnetRoutes() {
allowedIPs = append(allowedIPs, netip.Prefix(route))
}
var derp int
// TODO(kradalby): legacyDERP was removed in tailscale/tailscale@2fc4455e6dd9ab7f879d4e2f7cffc2be81f14077
@ -105,7 +95,7 @@ func tailNode(
DiscoKey: node.DiscoKey,
Addresses: addrs,
PrimaryRoutes: primary.PrimaryRoutes(node.ID),
AllowedIPs: allowedIPs,
AllowedIPs: allowed,
Endpoints: node.Endpoints,
HomeDERP: derp,
LegacyDERPString: legacyDERP,

21
hscontrol/routes/primary.go
View File

@ -74,18 +74,12 @@ func (pr *PrimaryRoutes) updatePrimaryLocked() bool {
// If the current primary is not available, select a new one.
for prefix, nodes := range allPrimaries {
if node, ok := pr.primaries[prefix]; ok {
if len(nodes) < 2 {
delete(pr.primaries, prefix)
changed = true
continue
}
// If the current primary is still available, continue.
if slices.Contains(nodes, node) {
continue
}
}
if len(nodes) >= 2 {
if len(nodes) >= 1 {
pr.primaries[prefix] = nodes[0]
changed = true
}
@ -121,12 +115,17 @@ func (pr *PrimaryRoutes) SetRoutes(node types.NodeID, prefix ...netip.Prefix) bo
return false
}
if _, ok := pr.routes[node]; !ok {
pr.routes[node] = make(set.Set[netip.Prefix], len(prefix))
rs := make(set.Set[netip.Prefix], len(prefixes))
for _, prefix := range prefixes {
if !tsaddr.IsExitRoute(prefix) {
rs.Add(prefix)
}
}
for _, p := range prefix {
pr.routes[node].Add(p)
if rs.Len() != 0 {
pr.routes[node] = rs
} else {
delete(pr.routes, node)
}
return pr.updatePrimaryLocked()

333
hscontrol/routes/primary_test.go
View File

@ -6,8 +6,10 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
"tailscale.com/util/set"
)
// mp is a helper function that wraps netip.MustParsePrefix.
@ -19,18 +21,32 @@ func TestPrimaryRoutes(t *testing.T) {
tests := []struct {
name string
operations func(pr *PrimaryRoutes) bool
nodeID types.NodeID
expectedRoutes []netip.Prefix
expectedRoutes map[types.NodeID]set.Set[netip.Prefix]
expectedPrimaries map[netip.Prefix]types.NodeID
expectedIsPrimary map[types.NodeID]bool
expectedChange bool
// primaries is a map of prefixes to the node that is the primary for that prefix.
primaries map[netip.Prefix]types.NodeID
isPrimary map[types.NodeID]bool
}{
{
name: "single-node-registers-single-route",
operations: func(pr *PrimaryRoutes) bool {
return pr.SetRoutes(1, mp("192.168.1.0/24"))
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 1,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
},
expectedChange: true,
},
{
name: "multiple-nodes-register-different-routes",
@ -38,19 +54,45 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(1, mp("192.168.1.0/24"))
return pr.SetRoutes(2, mp("192.168.2.0/24"))
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.2.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 1,
mp("192.168.2.0/24"): 2,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
2: true,
},
expectedChange: true,
},
{
name: "multiple-nodes-register-overlapping-routes",
operations: func(pr *PrimaryRoutes) bool {
pr.SetRoutes(1, mp("192.168.1.0/24")) // false
return pr.SetRoutes(2, mp("192.168.1.0/24")) // true
pr.SetRoutes(1, mp("192.168.1.0/24")) // true
return pr.SetRoutes(2, mp("192.168.1.0/24")) // false
},
nodeID: 1,
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
expectedChange: true,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 1,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
},
expectedChange: false,
},
{
name: "node-deregisters-a-route",
@ -58,9 +100,10 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(1, mp("192.168.1.0/24"))
return pr.SetRoutes(1) // Deregister by setting no routes
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
expectedPrimaries: nil,
expectedIsPrimary: nil,
expectedChange: true,
},
{
name: "node-deregisters-one-of-multiple-routes",
@ -68,9 +111,18 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(1, mp("192.168.1.0/24"), mp("192.168.2.0/24"))
return pr.SetRoutes(1, mp("192.168.2.0/24")) // Deregister one route by setting the remaining route
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.2.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.2.0/24"): 1,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
},
expectedChange: true,
},
{
name: "node-registers-and-deregisters-routes-in-sequence",
@ -80,18 +132,23 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(1) // Deregister by setting no routes
return pr.SetRoutes(1, mp("192.168.3.0/24"))
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.3.0/24"): {},
},
{
name: "no-change-in-primary-routes",
operations: func(pr *PrimaryRoutes) bool {
return pr.SetRoutes(1, mp("192.168.1.0/24"))
2: {
mp("192.168.2.0/24"): {},
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.2.0/24"): 2,
mp("192.168.3.0/24"): 1,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
2: true,
},
expectedChange: true,
},
{
name: "multiple-nodes-register-same-route",
@ -100,22 +157,25 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(2, mp("192.168.1.0/24")) // true
return pr.SetRoutes(3, mp("192.168.1.0/24")) // false
},
nodeID: 1,
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.1.0/24"): {},
},
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 1,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
},
expectedChange: false,
},
{
name: "register-multiple-routes-shift-primary-check-old-primary",
operations: func(pr *PrimaryRoutes) bool {
pr.SetRoutes(1, mp("192.168.1.0/24")) // false
pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary
pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary
return pr.SetRoutes(1) // true, 2 primary
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: true,
},
{
name: "register-multiple-routes-shift-primary-check-primary",
operations: func(pr *PrimaryRoutes) bool {
@ -124,20 +184,20 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary
return pr.SetRoutes(1) // true, 2 primary
},
nodeID: 2,
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
expectedChange: true,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
2: {
mp("192.168.1.0/24"): {},
},
{
name: "register-multiple-routes-shift-primary-check-non-primary",
operations: func(pr *PrimaryRoutes) bool {
pr.SetRoutes(1, mp("192.168.1.0/24")) // false
pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary
pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary
return pr.SetRoutes(1) // true, 2 primary
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 2,
},
expectedIsPrimary: map[types.NodeID]bool{
2: true,
},
nodeID: 3,
expectedRoutes: nil,
expectedChange: true,
},
{
@ -150,8 +210,17 @@ func TestPrimaryRoutes(t *testing.T) {
return pr.SetRoutes(2) // true, no primary
},
nodeID: 2,
expectedRoutes: nil,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 3,
},
expectedIsPrimary: map[types.NodeID]bool{
3: true,
},
expectedChange: true,
},
{
@ -165,9 +234,7 @@ func TestPrimaryRoutes(t *testing.T) {
return pr.SetRoutes(3) // false, no primary
},
nodeID: 2,
expectedRoutes: nil,
expectedChange: false,
expectedChange: true,
},
{
name: "primary-route-map-is-cleared-up",
@ -179,8 +246,17 @@ func TestPrimaryRoutes(t *testing.T) {
return pr.SetRoutes(2) // true, no primary
},
nodeID: 2,
expectedRoutes: nil,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 3,
},
expectedIsPrimary: map[types.NodeID]bool{
3: true,
},
expectedChange: true,
},
{
@ -193,8 +269,23 @@ func TestPrimaryRoutes(t *testing.T) {
return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 2 primary
},
nodeID: 2,
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.1.0/24"): {},
},
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 2,
},
expectedIsPrimary: map[types.NodeID]bool{
2: true,
},
expectedChange: false,
},
{
@ -207,8 +298,23 @@ func TestPrimaryRoutes(t *testing.T) {
return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 2 primary
},
nodeID: 1,
expectedRoutes: nil,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.1.0/24"): {},
},
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 2,
},
expectedIsPrimary: map[types.NodeID]bool{
2: true,
},
expectedChange: false,
},
{
@ -218,15 +324,30 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary
pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary
pr.SetRoutes(1) // true, 2 primary
pr.SetRoutes(2) // true, no primary
pr.SetRoutes(1, mp("192.168.1.0/24")) // true, 1 primary
pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary
pr.SetRoutes(1) // true, 2 primary
pr.SetRoutes(2) // true, 3 primary
pr.SetRoutes(1, mp("192.168.1.0/24")) // true, 3 primary
pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 3 primary
pr.SetRoutes(1) // true, 3 primary
return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 2 primary
return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 3 primary
},
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.1.0/24"): {},
},
3: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 3,
},
expectedIsPrimary: map[types.NodeID]bool{
3: true,
},
nodeID: 2,
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
expectedChange: false,
},
{
@ -235,16 +356,28 @@ func TestPrimaryRoutes(t *testing.T) {
pr.SetRoutes(1, mp("0.0.0.0/0"), mp("192.168.1.0/24"))
return pr.SetRoutes(2, mp("192.168.1.0/24"))
},
nodeID: 1,
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
expectedChange: true,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
2: {
mp("192.168.1.0/24"): {},
},
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 1,
mp("0.0.0.0/0"): 1,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
},
expectedChange: false,
},
{
name: "deregister-non-existent-route",
operations: func(pr *PrimaryRoutes) bool {
return pr.SetRoutes(1) // Deregister by setting no routes
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
},
@ -253,16 +386,6 @@ func TestPrimaryRoutes(t *testing.T) {
operations: func(pr *PrimaryRoutes) bool {
return pr.SetRoutes(1)
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
},
{
name: "deregister-empty-prefix-list",
operations: func(pr *PrimaryRoutes) bool {
return pr.SetRoutes(1)
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
},
@ -284,19 +407,23 @@ func TestPrimaryRoutes(t *testing.T) {
return change1 || change2
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{
1: {
mp("192.168.1.0/24"): {},
},
{
name: "no-routes-registered",
operations: func(pr *PrimaryRoutes) bool {
// No operations
return false
2: {
mp("192.168.2.0/24"): {},
},
nodeID: 1,
expectedRoutes: nil,
expectedChange: false,
},
expectedPrimaries: map[netip.Prefix]types.NodeID{
mp("192.168.1.0/24"): 1,
mp("192.168.2.0/24"): 2,
},
expectedIsPrimary: map[types.NodeID]bool{
1: true,
2: true,
},
expectedChange: true,
},
}
@ -307,9 +434,15 @@ func TestPrimaryRoutes(t *testing.T) {
if change != tt.expectedChange {
t.Errorf("change = %v, want %v", change, tt.expectedChange)
}
routes := pr.PrimaryRoutes(tt.nodeID)
if diff := cmp.Diff(tt.expectedRoutes, routes, util.Comparers...); diff != "" {
t.Errorf("PrimaryRoutes() mismatch (-want +got):\n%s", diff)
comps := append(util.Comparers, cmpopts.EquateEmpty())
if diff := cmp.Diff(tt.expectedRoutes, pr.routes, comps...); diff != "" {
t.Errorf("routes mismatch (-want +got):\n%s", diff)
}
if diff := cmp.Diff(tt.expectedPrimaries, pr.primaries, comps...); diff != "" {
t.Errorf("primaries mismatch (-want +got):\n%s", diff)
}
if diff := cmp.Diff(tt.expectedIsPrimary, pr.isPrimary, comps...); diff != "" {
t.Errorf("isPrimary mismatch (-want +got):\n%s", diff)
}
})
}

2
hscontrol/types/node.go
View File

@ -213,7 +213,7 @@ func (node *Node) RequestTags() []string {
}
func (node *Node) Prefixes() []netip.Prefix {
addrs := []netip.Prefix{}
var addrs []netip.Prefix
for _, nodeAddress := range node.IPs() {
ip := netip.PrefixFrom(nodeAddress, nodeAddress.BitLen())
addrs = append(addrs, ip)

402
integration/route_test.go
View File

@ -39,7 +39,9 @@ func TestEnablingRoutes(t *testing.T) {
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("clienableroute"))
err = scenario.CreateHeadscaleEnv(
[]tsic.Option{tsic.WithAcceptRoutes()},
hsic.WithTestName("clienableroute"))
assertNoErrHeadscaleEnv(t, err)
allClients, err := scenario.ListTailscaleClients()
@ -189,11 +191,11 @@ func TestEnablingRoutes(t *testing.T) {
assert.Nil(t, peerStatus.PrimaryRoutes)
if peerStatus.ID == "1" {
assertPeerSubnetRoutes(t, peerStatus, nil)
requirePeerSubnetRoutes(t, peerStatus, nil)
} else if peerStatus.ID == "2" {
assertPeerSubnetRoutes(t, peerStatus, nil)
requirePeerSubnetRoutes(t, peerStatus, nil)
} else {
assertPeerSubnetRoutes(t, peerStatus, []netip.Prefix{netip.MustParsePrefix("10.0.2.0/24")})
requirePeerSubnetRoutes(t, peerStatus, []netip.Prefix{netip.MustParsePrefix("10.0.2.0/24")})
}
}
}
@ -204,21 +206,26 @@ func TestHASubnetRouterFailover(t *testing.T) {
t.Parallel()
spec := ScenarioSpec{
NodesPerUser: 4,
Users: []string{"user1"},
NodesPerUser: 3,
Users: []string{"user1", "user2"},
Networks: map[string][]string{
"usernet1": {"user1"},
"usernet2": {"user2"},
},
ExtraService: map[string][]extraServiceFunc{
"usernet1": {Webservice},
},
// We build the head image with curl and traceroute, so only use
// that for this test.
Versions: []string{"head"},
}
scenario, err := NewScenario(spec)
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
// defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv([]tsic.Option{},
err = scenario.CreateHeadscaleEnv(
[]tsic.Option{tsic.WithAcceptRoutes()},
hsic.WithTestName("clienableroute"),
hsic.WithEmbeddedDERPServerOnly(),
hsic.WithTLS(),
@ -234,11 +241,22 @@ func TestHASubnetRouterFailover(t *testing.T) {
headscale, err := scenario.Headscale()
assertNoErrGetHeadscale(t, err)
expectedRoutes := map[string]string{
"1": "10.0.0.0/24",
"2": "10.0.0.0/24",
"3": "10.0.0.0/24",
}
prefp, err := scenario.SubnetOfNetwork("usernet1")
require.NoError(t, err)
pref := *prefp
t.Logf("usernet1 prefix: %s", pref.String())
usernet1, err := scenario.Network("usernet1")
require.NoError(t, err)
services, err := scenario.Services("usernet1")
require.NoError(t, err)
require.Len(t, services, 1)
web := services[0]
webip := netip.MustParseAddr(web.GetIPInNetwork(usernet1))
weburl := fmt.Sprintf("http://%s/etc/hostname", webip)
t.Logf("webservice: %s, %s", webip.String(), weburl)
// Sort nodes by ID
sort.SliceStable(allClients, func(i, j int) bool {
@ -248,6 +266,9 @@ func TestHASubnetRouterFailover(t *testing.T) {
return statusI.Self.ID < statusJ.Self.ID
})
// This is ok because the scenario makes users in order, so the three first
// nodes, which are subnet routes, will be created first, and the last user
// will be created with the second.
subRouter1 := allClients[0]
subRouter2 := allClients[1]
subRouter3 := allClients[2]
@ -260,28 +281,23 @@ func TestHASubnetRouterFailover(t *testing.T) {
// ID 2 will be standby
// ID 3 will be standby
for _, client := range allClients[:3] {
status, err := client.Status()
require.NoError(t, err)
if route, ok := expectedRoutes[string(status.Self.ID)]; ok {
command := []string{
"tailscale",
"set",
"--advertise-routes=" + route,
"--advertise-routes=" + pref.String(),
}
_, _, err = client.Execute(command)
require.NoErrorf(t, err, "failed to advertise route: %s", err)
} else {
t.Fatalf("failed to find route for Node %s (id: %s)", status.Self.HostName, status.Self.ID)
}
}
err = scenario.WaitForTailscaleSync()
assertNoErrSync(t, err)
time.Sleep(3 * time.Second)
nodes, err := headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 4)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 0, 0)
assertNodeRouteCount(t, nodes[1], 1, 0, 0)
@ -297,28 +313,30 @@ func TestHASubnetRouterFailover(t *testing.T) {
peerStatus := status.Peer[peerKey]
assert.Nil(t, peerStatus.PrimaryRoutes)
assertPeerSubnetRoutes(t, peerStatus, nil)
requirePeerSubnetRoutes(t, peerStatus, nil)
}
}
// Enable all routes
for _, node := range nodes {
_, err := headscale.ApproveRoutes(
node.GetId(),
util.MustStringsToPrefixes(node.GetAvailableRoutes()),
// Enable route on node 1
t.Logf("Enabling route on subnet router 1, no HA")
_, err = headscale.ApproveRoutes(
1,
[]netip.Prefix{pref},
)
require.NoError(t, err)
}
time.Sleep(3 * time.Second)
nodes, err = headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 4)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 1, 1)
assertNodeRouteCount(t, nodes[1], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 1, 1)
assertNodeRouteCount(t, nodes[1], 1, 0, 0)
assertNodeRouteCount(t, nodes[2], 1, 0, 0)
// Verify that the client has routes from the primary machine
// Verify that the client has routes from the primary machine and can access
// the webservice.
srs1 := subRouter1.MustStatus()
srs2 := subRouter2.MustStatus()
srs3 := subRouter3.MustStatus()
@ -336,11 +354,135 @@ func TestHASubnetRouterFailover(t *testing.T) {
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
require.NotNil(t, srs1PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref})
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, nil)
t.Logf("got list: %v, want in: %v", srs1PeerStatus.PrimaryRoutes.AsSlice(), pref)
assert.Contains(t,
srs1PeerStatus.PrimaryRoutes.AsSlice(),
netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]),
pref,
)
t.Logf("Validating access via subnetrouter(%s) to %s, no HA", subRouter1.MustIPv4().String(), webip.String())
result, err := client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err := client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter1.MustIPv4())
// Enable route on node 2, now we will have a HA subnet router
t.Logf("Enabling route on subnet router 2, now HA, subnetrouter 1 is primary, 2 is standby")
_, err = headscale.ApproveRoutes(
2,
[]netip.Prefix{pref},
)
require.NoError(t, err)
time.Sleep(3 * time.Second)
nodes, err = headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 1, 1)
assertNodeRouteCount(t, nodes[1], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 0, 0)
// Verify that the client has routes from the primary machine
srs1 = subRouter1.MustStatus()
srs2 = subRouter2.MustStatus()
srs3 = subRouter3.MustStatus()
clientStatus = client.MustStatus()
srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey]
srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey]
srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey]
assert.True(t, srs1PeerStatus.Online, "r1 up, r2 up")
assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up")
assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up")
assert.Nil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
require.NotNil(t, srs1PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref})
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, nil)
t.Logf("got list: %v, want in: %v", srs1PeerStatus.PrimaryRoutes.AsSlice(), pref)
assert.Contains(t,
srs1PeerStatus.PrimaryRoutes.AsSlice(),
pref,
)
t.Logf("Validating access via subnetrouter(%s) to %s, 2 is standby", subRouter1.MustIPv4().String(), webip.String())
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter1.MustIPv4())
// Enable route on node 3, now we will have a second standby and all will
// be enabled.
t.Logf("Enabling route on subnet router 3, now HA, subnetrouter 1 is primary, 2 and 3 is standby")
_, err = headscale.ApproveRoutes(
3,
[]netip.Prefix{pref},
)
require.NoError(t, err)
time.Sleep(3 * time.Second)
nodes, err = headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 1, 1)
assertNodeRouteCount(t, nodes[1], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 1, 1)
// Verify that the client has routes from the primary machine
srs1 = subRouter1.MustStatus()
srs2 = subRouter2.MustStatus()
srs3 = subRouter3.MustStatus()
clientStatus = client.MustStatus()
srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey]
srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey]
srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey]
assert.True(t, srs1PeerStatus.Online, "r1 up, r2 up")
assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up")
assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up")
assert.Nil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
require.NotNil(t, srs1PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref})
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, nil)
t.Logf("got list: %v, want in: %v", srs1PeerStatus.PrimaryRoutes.AsSlice(), pref)
assert.Contains(t,
srs1PeerStatus.PrimaryRoutes.AsSlice(),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter1.MustIPv4())
// Take down the current primary
t.Logf("taking down subnet router r1 (%s)", subRouter1.Hostname())
t.Logf("expecting r2 (%s) to take over as primary", subRouter2.Hostname())
@ -364,12 +506,24 @@ func TestHASubnetRouterFailover(t *testing.T) {
require.NotNil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, nil)
requirePeerSubnetRoutes(t, srs2PeerStatus, []netip.Prefix{pref})
requirePeerSubnetRoutes(t, srs3PeerStatus, nil)
assert.Contains(
t,
srs2PeerStatus.PrimaryRoutes.AsSlice(),
netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter2.MustIPv4())
// Take down subnet router 2, leaving none available
t.Logf("taking down subnet router r2 (%s)", subRouter2.Hostname())
t.Logf("expecting no primary, r3 available, but no HA so no primary")
@ -395,7 +549,19 @@ func TestHASubnetRouterFailover(t *testing.T) {
assert.Nil(t, srs1PeerStatus.PrimaryRoutes)
assert.Nil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
require.NotNil(t, srs3PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, nil)
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, []netip.Prefix{pref})
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter3.MustIPv4())
// Bring up subnet router 1, making the route available from there.
t.Logf("bringing up subnet router r1 (%s)", subRouter1.Hostname())
@ -417,16 +583,28 @@ func TestHASubnetRouterFailover(t *testing.T) {
assert.False(t, srs2PeerStatus.Online, "r1 is back up, r2 down")
assert.True(t, srs3PeerStatus.Online, "r1 is back up, r3 available")
assert.NotNil(t, srs1PeerStatus.PrimaryRoutes)
assert.Nil(t, srs1PeerStatus.PrimaryRoutes)
assert.Nil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
require.NotNil(t, srs3PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, nil)
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, []netip.Prefix{pref})
assert.Contains(
t,
srs1PeerStatus.PrimaryRoutes.AsSlice(),
netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]),
srs3PeerStatus.PrimaryRoutes.AsSlice(),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter3.MustIPv4())
// Bring up subnet router 2, should result in no change.
t.Logf("bringing up subnet router r2 (%s)", subRouter2.Hostname())
t.Logf("all online, expecting r1 (%s) to still be primary (no flapping)", subRouter1.Hostname())
@ -447,30 +625,86 @@ func TestHASubnetRouterFailover(t *testing.T) {
assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up")
assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up")
assert.Nil(t, srs1PeerStatus.PrimaryRoutes)
assert.Nil(t, srs2PeerStatus.PrimaryRoutes)
require.NotNil(t, srs3PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, nil)
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, []netip.Prefix{pref})
assert.Contains(
t,
srs3PeerStatus.PrimaryRoutes.AsSlice(),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter3.MustIPv4())
t.Logf("disabling route in subnet router r3 (%s)", subRouter3.Hostname())
t.Logf("expecting route to failover to r1 (%s), which is still available with r2", subRouter1.Hostname())
_, err = headscale.ApproveRoutes(nodes[2].GetId(), []netip.Prefix{})
time.Sleep(5 * time.Second)
nodes, err = headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 1, 1)
assertNodeRouteCount(t, nodes[1], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 0, 0)
// Verify that the route is announced from subnet router 1
clientStatus, err = client.Status()
require.NoError(t, err)
srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey]
srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey]
srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey]
require.NotNil(t, srs1PeerStatus.PrimaryRoutes)
assert.Nil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref})
requirePeerSubnetRoutes(t, srs2PeerStatus, nil)
requirePeerSubnetRoutes(t, srs3PeerStatus, nil)
assert.Contains(
t,
srs1PeerStatus.PrimaryRoutes.AsSlice(),
netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter1.MustIPv4())
// Disable the route of subnet router 1, making it failover to 2
t.Logf("disabling route in subnet router r1 (%s)", subRouter1.Hostname())
t.Logf("expecting route to failover to r2 (%s), which is still available with r3", subRouter2.Hostname())
t.Logf("expecting route to failover to r2 (%s)", subRouter2.Hostname())
_, err = headscale.ApproveRoutes(nodes[0].GetId(), []netip.Prefix{})
time.Sleep(5 * time.Second)
nodes, err = headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 4)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 0, 0)
assertNodeRouteCount(t, nodes[1], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 0, 0)
// Verify that the route is announced from subnet router 1
clientStatus, err = client.Status()
@ -481,15 +715,27 @@ func TestHASubnetRouterFailover(t *testing.T) {
srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey]
assert.Nil(t, srs1PeerStatus.PrimaryRoutes)
assert.NotNil(t, srs2PeerStatus.PrimaryRoutes)
require.NotNil(t, srs2PeerStatus.PrimaryRoutes)
assert.Nil(t, srs3PeerStatus.PrimaryRoutes)
requirePeerSubnetRoutes(t, srs1PeerStatus, nil)
requirePeerSubnetRoutes(t, srs2PeerStatus, []netip.Prefix{pref})
requirePeerSubnetRoutes(t, srs3PeerStatus, nil)
assert.Contains(
t,
srs2PeerStatus.PrimaryRoutes.AsSlice(),
netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter2.MustIPv4())
// enable the route of subnet router 1, no change expected
t.Logf("enabling route in subnet router 1 (%s)", subRouter1.Hostname())
t.Logf("both online, expecting r2 (%s) to still be primary (no flapping)", subRouter2.Hostname())
@ -502,11 +748,11 @@ func TestHASubnetRouterFailover(t *testing.T) {
nodes, err = headscale.ListNodes()
require.NoError(t, err)
assert.Len(t, nodes, 4)
assert.Len(t, nodes, 6)
assertNodeRouteCount(t, nodes[0], 1, 1, 1)
assertNodeRouteCount(t, nodes[1], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 1, 1)
assertNodeRouteCount(t, nodes[2], 1, 0, 0)
// Verify that the route is announced from subnet router 1
clientStatus, err = client.Status()
@ -523,8 +769,16 @@ func TestHASubnetRouterFailover(t *testing.T) {
assert.Contains(
t,
srs2PeerStatus.PrimaryRoutes.AsSlice(),
netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]),
pref,
)
result, err = client.Curl(weburl)
require.NoError(t, err)
assert.Len(t, result, 13)
tr, err = client.Traceroute(webip)
require.NoError(t, err)
assertTracerouteViaIP(t, tr, subRouter2.MustIPv4())
}
func TestEnableDisableAutoApprovedRoute(t *testing.T) {
@ -542,7 +796,10 @@ func TestEnableDisableAutoApprovedRoute(t *testing.T) {
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv([]tsic.Option{tsic.WithTags([]string{"tag:approve"})}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy(
err = scenario.CreateHeadscaleEnv([]tsic.Option{
tsic.WithTags([]string{"tag:approve"}),
tsic.WithAcceptRoutes(),
}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy(
&policyv1.ACLPolicy{
ACLs: []policyv1.ACL{
{
@ -640,7 +897,10 @@ func TestAutoApprovedSubRoute2068(t *testing.T) {
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv([]tsic.Option{tsic.WithTags([]string{"tag:approve"})},
err = scenario.CreateHeadscaleEnv([]tsic.Option{
tsic.WithTags([]string{"tag:approve"}),
tsic.WithAcceptRoutes(),
},
hsic.WithTestName("clienableroute"),
hsic.WithEmbeddedDERPServerOnly(),
hsic.WithTLS(),
@ -712,7 +972,9 @@ func TestSubnetRouteACL(t *testing.T) {
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy(
err = scenario.CreateHeadscaleEnv([]tsic.Option{
tsic.WithAcceptRoutes(),
}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy(
&policyv1.ACLPolicy{
Groups: policyv1.Groups{
"group:admins": {user},
@ -805,7 +1067,7 @@ func TestSubnetRouteACL(t *testing.T) {
peerStatus := status.Peer[peerKey]
assert.Nil(t, peerStatus.PrimaryRoutes)
assertPeerSubnetRoutes(t, peerStatus, nil)
requirePeerSubnetRoutes(t, peerStatus, nil)
}
}
@ -832,7 +1094,7 @@ func TestSubnetRouteACL(t *testing.T) {
srs1PeerStatus := clientStatus.Peer[srs1.Self.PublicKey]
assertPeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{netip.MustParsePrefix(expectedRoutes["1"])})
requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{netip.MustParsePrefix(expectedRoutes["1"])})
clientNm, err := client.Netmap()
require.NoError(t, err)
@ -1010,32 +1272,6 @@ func TestEnablingExitRoutes(t *testing.T) {
}
}
// assertPeerSubnetRoutes asserts that the peer has the expected subnet routes.
func assertPeerSubnetRoutes(t *testing.T, status *ipnstate.PeerStatus, expected []netip.Prefix) {
t.Helper()
if status.AllowedIPs.Len() <= 2 && len(expected) != 0 {
t.Errorf("peer %s (%s) has no subnet routes, expected %v", status.HostName, status.ID, expected)
return
}
if len(expected) == 0 {
expected = []netip.Prefix{}
}
got := status.AllowedIPs.AsSlice()[2:]
if diff := cmp.Diff(expected, got, util.PrefixComparer); diff != "" {
t.Errorf("peer %s (%s) subnet routes, unexpected result (-want +got):\n%s", status.HostName, status.ID, diff)
}
}
func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subnet int) {
t.Helper()
assert.Len(t, node.GetAvailableRoutes(), announced)
assert.Len(t, node.GetApprovedRoutes(), approved)
assert.Len(t, node.GetSubnetRoutes(), subnet)
}
// TestSubnetRouterMultiNetwork is an evolution of the subnet router test.
// This test will set up multiple docker networks and use two isolated tailscale
// clients and a service available in one of the networks to validate that a
@ -1117,7 +1353,7 @@ func TestSubnetRouterMultiNetwork(t *testing.T) {
peerStatus := status.Peer[peerKey]
assert.Nil(t, peerStatus.PrimaryRoutes)
assertPeerSubnetRoutes(t, peerStatus, nil)
requirePeerSubnetRoutes(t, peerStatus, nil)
}
// Enable route
@ -1141,7 +1377,7 @@ func TestSubnetRouterMultiNetwork(t *testing.T) {
for _, peerKey := range status.Peers() {
peerStatus := status.Peer[peerKey]
// TestSubnetRouterMultiNetworkExitNode
assert.Contains(t, peerStatus.PrimaryRoutes.AsSlice(), *pref)
func TestSubnetRouterMultiNetworkExitNode(t *testing.T) {
IntegrationSkip(t)
t.Parallel()
@ -1275,7 +1511,7 @@ func TestSubnetRouterMultiNetworkExitNode(t *testing.T) {
}
assert.Nil(t, peerStatus.PrimaryRoutes)
assertPeerSubnetRoutes(t, peerStatus, []netip.Prefix{*pref})
requirePeerSubnetRoutes(t, peerStatus, []netip.Prefix{*pref})
}
usernet1, err := scenario.Network("usernet1")

7
integration/scenario.go
View File

@ -134,6 +134,9 @@ type ScenarioSpec struct {
// typically dont run Tailscale, e.g. web service to test subnet router.
ExtraService map[string][]extraServiceFunc
// Versions is specific list of versions to use for the test.
Versions []string
// OIDCUsers, if populated, will start a Mock OIDC server and populate
// the user login stack with the given users.
// If the NodesPerUser is set, it should align with this list to ensure
@ -514,8 +517,12 @@ func (s *Scenario) CreateTailscaleNodesInUser(
for i := range count {
version := requestedVersion
if requestedVersion == "all" {
if s.spec.Versions != nil {
version = s.spec.Versions[i%len(s.spec.Versions)]
} else {
version = MustTestVersions[i%len(MustTestVersions)]
}
}
versions = append(versions, version)
headscale, err := s.Headscale()