diff --git a/hscontrol/mapper/tail.go b/hscontrol/mapper/tail.go index c8c61825..e5101f29 100644 --- a/hscontrol/mapper/tail.go +++ b/hscontrol/mapper/tail.go @@ -2,11 +2,8 @@ package mapper import ( "fmt" - "net/netip" "time" - "slices" - "github.com/juanfont/headscale/hscontrol/policy" "github.com/juanfont/headscale/hscontrol/routes" "github.com/juanfont/headscale/hscontrol/types" @@ -51,13 +48,6 @@ func tailNode( ) (*tailcfg.Node, error) { addrs := node.Prefixes() - // we append the node own IP, as it is required by the clients - allowedIPs := slices.Clone(node.Prefixes()) - - for _, route := range node.SubnetRoutes() { - allowedIPs = append(allowedIPs, netip.Prefix(route)) - } - var derp int // TODO(kradalby): legacyDERP was removed in tailscale/tailscale@2fc4455e6dd9ab7f879d4e2f7cffc2be81f14077 @@ -105,7 +95,7 @@ func tailNode( DiscoKey: node.DiscoKey, Addresses: addrs, PrimaryRoutes: primary.PrimaryRoutes(node.ID), - AllowedIPs: allowedIPs, + AllowedIPs: allowed, Endpoints: node.Endpoints, HomeDERP: derp, LegacyDERPString: legacyDERP, diff --git a/hscontrol/routes/primary.go b/hscontrol/routes/primary.go index 317bf450..1045b129 100644 --- a/hscontrol/routes/primary.go +++ b/hscontrol/routes/primary.go @@ -74,18 +74,12 @@ func (pr *PrimaryRoutes) updatePrimaryLocked() bool { // If the current primary is not available, select a new one. for prefix, nodes := range allPrimaries { if node, ok := pr.primaries[prefix]; ok { - if len(nodes) < 2 { - delete(pr.primaries, prefix) - changed = true - continue - } - // If the current primary is still available, continue. if slices.Contains(nodes, node) { continue } } - if len(nodes) >= 2 { + if len(nodes) >= 1 { pr.primaries[prefix] = nodes[0] changed = true } @@ -121,12 +115,17 @@ func (pr *PrimaryRoutes) SetRoutes(node types.NodeID, prefix ...netip.Prefix) bo return false } - if _, ok := pr.routes[node]; !ok { - pr.routes[node] = make(set.Set[netip.Prefix], len(prefix)) + rs := make(set.Set[netip.Prefix], len(prefixes)) + for _, prefix := range prefixes { + if !tsaddr.IsExitRoute(prefix) { + rs.Add(prefix) + } } - for _, p := range prefix { - pr.routes[node].Add(p) + if rs.Len() != 0 { + pr.routes[node] = rs + } else { + delete(pr.routes, node) } return pr.updatePrimaryLocked() diff --git a/hscontrol/routes/primary_test.go b/hscontrol/routes/primary_test.go index c58337c0..03ceadd5 100644 --- a/hscontrol/routes/primary_test.go +++ b/hscontrol/routes/primary_test.go @@ -6,8 +6,10 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/hscontrol/util" + "tailscale.com/util/set" ) // mp is a helper function that wraps netip.MustParsePrefix. @@ -17,20 +19,34 @@ func mp(prefix string) netip.Prefix { func TestPrimaryRoutes(t *testing.T) { tests := []struct { - name string - operations func(pr *PrimaryRoutes) bool - nodeID types.NodeID - expectedRoutes []netip.Prefix - expectedChange bool + name string + operations func(pr *PrimaryRoutes) bool + expectedRoutes map[types.NodeID]set.Set[netip.Prefix] + expectedPrimaries map[netip.Prefix]types.NodeID + expectedIsPrimary map[types.NodeID]bool + expectedChange bool + + // primaries is a map of prefixes to the node that is the primary for that prefix. + primaries map[netip.Prefix]types.NodeID + isPrimary map[types.NodeID]bool }{ { name: "single-node-registers-single-route", operations: func(pr *PrimaryRoutes) bool { return pr.SetRoutes(1, mp("192.168.1.0/24")) }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 1, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + }, + expectedChange: true, }, { name: "multiple-nodes-register-different-routes", @@ -38,19 +54,45 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(1, mp("192.168.1.0/24")) return pr.SetRoutes(2, mp("192.168.2.0/24")) }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.2.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 1, + mp("192.168.2.0/24"): 2, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + 2: true, + }, + expectedChange: true, }, { name: "multiple-nodes-register-overlapping-routes", operations: func(pr *PrimaryRoutes) bool { - pr.SetRoutes(1, mp("192.168.1.0/24")) // false - return pr.SetRoutes(2, mp("192.168.1.0/24")) // true + pr.SetRoutes(1, mp("192.168.1.0/24")) // true + return pr.SetRoutes(2, mp("192.168.1.0/24")) // false }, - nodeID: 1, - expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")}, - expectedChange: true, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 1, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + }, + expectedChange: false, }, { name: "node-deregisters-a-route", @@ -58,9 +100,10 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(1, mp("192.168.1.0/24")) return pr.SetRoutes(1) // Deregister by setting no routes }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, + expectedRoutes: nil, + expectedPrimaries: nil, + expectedIsPrimary: nil, + expectedChange: true, }, { name: "node-deregisters-one-of-multiple-routes", @@ -68,9 +111,18 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(1, mp("192.168.1.0/24"), mp("192.168.2.0/24")) return pr.SetRoutes(1, mp("192.168.2.0/24")) // Deregister one route by setting the remaining route }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.2.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.2.0/24"): 1, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + }, + expectedChange: true, }, { name: "node-registers-and-deregisters-routes-in-sequence", @@ -80,18 +132,23 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(1) // Deregister by setting no routes return pr.SetRoutes(1, mp("192.168.3.0/24")) }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, - }, - { - name: "no-change-in-primary-routes", - operations: func(pr *PrimaryRoutes) bool { - return pr.SetRoutes(1, mp("192.168.1.0/24")) + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.3.0/24"): {}, + }, + 2: { + mp("192.168.2.0/24"): {}, + }, }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.2.0/24"): 2, + mp("192.168.3.0/24"): 1, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + 2: true, + }, + expectedChange: true, }, { name: "multiple-nodes-register-same-route", @@ -100,21 +157,24 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(2, mp("192.168.1.0/24")) // true return pr.SetRoutes(3, mp("192.168.1.0/24")) // false }, - nodeID: 1, - expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")}, - expectedChange: false, - }, - { - name: "register-multiple-routes-shift-primary-check-old-primary", - operations: func(pr *PrimaryRoutes) bool { - pr.SetRoutes(1, mp("192.168.1.0/24")) // false - pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary - pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary - return pr.SetRoutes(1) // true, 2 primary + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.1.0/24"): {}, + }, + 3: { + mp("192.168.1.0/24"): {}, + }, }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: true, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 1, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + }, + expectedChange: false, }, { name: "register-multiple-routes-shift-primary-check-primary", @@ -124,20 +184,20 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary return pr.SetRoutes(1) // true, 2 primary }, - nodeID: 2, - expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")}, - expectedChange: true, - }, - { - name: "register-multiple-routes-shift-primary-check-non-primary", - operations: func(pr *PrimaryRoutes) bool { - pr.SetRoutes(1, mp("192.168.1.0/24")) // false - pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary - pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary - return pr.SetRoutes(1) // true, 2 primary + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 2: { + mp("192.168.1.0/24"): {}, + }, + 3: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 2, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 2: true, }, - nodeID: 3, - expectedRoutes: nil, expectedChange: true, }, { @@ -150,8 +210,17 @@ func TestPrimaryRoutes(t *testing.T) { return pr.SetRoutes(2) // true, no primary }, - nodeID: 2, - expectedRoutes: nil, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 3: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 3, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 3: true, + }, expectedChange: true, }, { @@ -165,9 +234,7 @@ func TestPrimaryRoutes(t *testing.T) { return pr.SetRoutes(3) // false, no primary }, - nodeID: 2, - expectedRoutes: nil, - expectedChange: false, + expectedChange: true, }, { name: "primary-route-map-is-cleared-up", @@ -179,8 +246,17 @@ func TestPrimaryRoutes(t *testing.T) { return pr.SetRoutes(2) // true, no primary }, - nodeID: 2, - expectedRoutes: nil, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 3: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 3, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 3: true, + }, expectedChange: true, }, { @@ -193,8 +269,23 @@ func TestPrimaryRoutes(t *testing.T) { return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 2 primary }, - nodeID: 2, - expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")}, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.1.0/24"): {}, + }, + 3: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 2, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 2: true, + }, expectedChange: false, }, { @@ -207,8 +298,23 @@ func TestPrimaryRoutes(t *testing.T) { return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 2 primary }, - nodeID: 1, - expectedRoutes: nil, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.1.0/24"): {}, + }, + 3: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 2, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 2: true, + }, expectedChange: false, }, { @@ -218,15 +324,30 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary pr.SetRoutes(3, mp("192.168.1.0/24")) // false, 1 primary pr.SetRoutes(1) // true, 2 primary - pr.SetRoutes(2) // true, no primary - pr.SetRoutes(1, mp("192.168.1.0/24")) // true, 1 primary - pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 1 primary - pr.SetRoutes(1) // true, 2 primary + pr.SetRoutes(2) // true, 3 primary + pr.SetRoutes(1, mp("192.168.1.0/24")) // true, 3 primary + pr.SetRoutes(2, mp("192.168.1.0/24")) // true, 3 primary + pr.SetRoutes(1) // true, 3 primary - return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 2 primary + return pr.SetRoutes(1, mp("192.168.1.0/24")) // false, 3 primary + }, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.1.0/24"): {}, + }, + 3: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 3, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 3: true, }, - nodeID: 2, - expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")}, expectedChange: false, }, { @@ -235,16 +356,28 @@ func TestPrimaryRoutes(t *testing.T) { pr.SetRoutes(1, mp("0.0.0.0/0"), mp("192.168.1.0/24")) return pr.SetRoutes(2, mp("192.168.1.0/24")) }, - nodeID: 1, - expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")}, - expectedChange: true, + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.1.0/24"): {}, + }, + }, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 1, + mp("0.0.0.0/0"): 1, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + }, + expectedChange: false, }, { name: "deregister-non-existent-route", operations: func(pr *PrimaryRoutes) bool { return pr.SetRoutes(1) // Deregister by setting no routes }, - nodeID: 1, expectedRoutes: nil, expectedChange: false, }, @@ -253,16 +386,6 @@ func TestPrimaryRoutes(t *testing.T) { operations: func(pr *PrimaryRoutes) bool { return pr.SetRoutes(1) }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, - }, - { - name: "deregister-empty-prefix-list", - operations: func(pr *PrimaryRoutes) bool { - return pr.SetRoutes(1) - }, - nodeID: 1, expectedRoutes: nil, expectedChange: false, }, @@ -284,19 +407,23 @@ func TestPrimaryRoutes(t *testing.T) { return change1 || change2 }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, - }, - { - name: "no-routes-registered", - operations: func(pr *PrimaryRoutes) bool { - // No operations - return false + expectedRoutes: map[types.NodeID]set.Set[netip.Prefix]{ + 1: { + mp("192.168.1.0/24"): {}, + }, + 2: { + mp("192.168.2.0/24"): {}, + }, }, - nodeID: 1, - expectedRoutes: nil, - expectedChange: false, + expectedPrimaries: map[netip.Prefix]types.NodeID{ + mp("192.168.1.0/24"): 1, + mp("192.168.2.0/24"): 2, + }, + expectedIsPrimary: map[types.NodeID]bool{ + 1: true, + 2: true, + }, + expectedChange: true, }, } @@ -307,9 +434,15 @@ func TestPrimaryRoutes(t *testing.T) { if change != tt.expectedChange { t.Errorf("change = %v, want %v", change, tt.expectedChange) } - routes := pr.PrimaryRoutes(tt.nodeID) - if diff := cmp.Diff(tt.expectedRoutes, routes, util.Comparers...); diff != "" { - t.Errorf("PrimaryRoutes() mismatch (-want +got):\n%s", diff) + comps := append(util.Comparers, cmpopts.EquateEmpty()) + if diff := cmp.Diff(tt.expectedRoutes, pr.routes, comps...); diff != "" { + t.Errorf("routes mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.expectedPrimaries, pr.primaries, comps...); diff != "" { + t.Errorf("primaries mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.expectedIsPrimary, pr.isPrimary, comps...); diff != "" { + t.Errorf("isPrimary mismatch (-want +got):\n%s", diff) } }) } diff --git a/hscontrol/types/node.go b/hscontrol/types/node.go index e506a2c5..0679e97f 100644 --- a/hscontrol/types/node.go +++ b/hscontrol/types/node.go @@ -213,7 +213,7 @@ func (node *Node) RequestTags() []string { } func (node *Node) Prefixes() []netip.Prefix { - addrs := []netip.Prefix{} + var addrs []netip.Prefix for _, nodeAddress := range node.IPs() { ip := netip.PrefixFrom(nodeAddress, nodeAddress.BitLen()) addrs = append(addrs, ip) diff --git a/integration/route_test.go b/integration/route_test.go index 403e311f..54d9aab7 100644 --- a/integration/route_test.go +++ b/integration/route_test.go @@ -39,7 +39,9 @@ func TestEnablingRoutes(t *testing.T) { require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) - err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("clienableroute")) + err = scenario.CreateHeadscaleEnv( + []tsic.Option{tsic.WithAcceptRoutes()}, + hsic.WithTestName("clienableroute")) assertNoErrHeadscaleEnv(t, err) allClients, err := scenario.ListTailscaleClients() @@ -189,11 +191,11 @@ func TestEnablingRoutes(t *testing.T) { assert.Nil(t, peerStatus.PrimaryRoutes) if peerStatus.ID == "1" { - assertPeerSubnetRoutes(t, peerStatus, nil) + requirePeerSubnetRoutes(t, peerStatus, nil) } else if peerStatus.ID == "2" { - assertPeerSubnetRoutes(t, peerStatus, nil) + requirePeerSubnetRoutes(t, peerStatus, nil) } else { - assertPeerSubnetRoutes(t, peerStatus, []netip.Prefix{netip.MustParsePrefix("10.0.2.0/24")}) + requirePeerSubnetRoutes(t, peerStatus, []netip.Prefix{netip.MustParsePrefix("10.0.2.0/24")}) } } } @@ -204,21 +206,26 @@ func TestHASubnetRouterFailover(t *testing.T) { t.Parallel() spec := ScenarioSpec{ - NodesPerUser: 4, - Users: []string{"user1"}, + NodesPerUser: 3, + Users: []string{"user1", "user2"}, Networks: map[string][]string{ "usernet1": {"user1"}, + "usernet2": {"user2"}, }, ExtraService: map[string][]extraServiceFunc{ "usernet1": {Webservice}, }, + // We build the head image with curl and traceroute, so only use + // that for this test. + Versions: []string{"head"}, } scenario, err := NewScenario(spec) require.NoErrorf(t, err, "failed to create scenario: %s", err) - defer scenario.ShutdownAssertNoPanics(t) + // defer scenario.ShutdownAssertNoPanics(t) - err = scenario.CreateHeadscaleEnv([]tsic.Option{}, + err = scenario.CreateHeadscaleEnv( + []tsic.Option{tsic.WithAcceptRoutes()}, hsic.WithTestName("clienableroute"), hsic.WithEmbeddedDERPServerOnly(), hsic.WithTLS(), @@ -234,11 +241,22 @@ func TestHASubnetRouterFailover(t *testing.T) { headscale, err := scenario.Headscale() assertNoErrGetHeadscale(t, err) - expectedRoutes := map[string]string{ - "1": "10.0.0.0/24", - "2": "10.0.0.0/24", - "3": "10.0.0.0/24", - } + prefp, err := scenario.SubnetOfNetwork("usernet1") + require.NoError(t, err) + pref := *prefp + t.Logf("usernet1 prefix: %s", pref.String()) + + usernet1, err := scenario.Network("usernet1") + require.NoError(t, err) + + services, err := scenario.Services("usernet1") + require.NoError(t, err) + require.Len(t, services, 1) + + web := services[0] + webip := netip.MustParseAddr(web.GetIPInNetwork(usernet1)) + weburl := fmt.Sprintf("http://%s/etc/hostname", webip) + t.Logf("webservice: %s, %s", webip.String(), weburl) // Sort nodes by ID sort.SliceStable(allClients, func(i, j int) bool { @@ -248,6 +266,9 @@ func TestHASubnetRouterFailover(t *testing.T) { return statusI.Self.ID < statusJ.Self.ID }) + // This is ok because the scenario makes users in order, so the three first + // nodes, which are subnet routes, will be created first, and the last user + // will be created with the second. subRouter1 := allClients[0] subRouter2 := allClients[1] subRouter3 := allClients[2] @@ -260,28 +281,23 @@ func TestHASubnetRouterFailover(t *testing.T) { // ID 2 will be standby // ID 3 will be standby for _, client := range allClients[:3] { - status, err := client.Status() - require.NoError(t, err) - - if route, ok := expectedRoutes[string(status.Self.ID)]; ok { - command := []string{ - "tailscale", - "set", - "--advertise-routes=" + route, - } - _, _, err = client.Execute(command) - require.NoErrorf(t, err, "failed to advertise route: %s", err) - } else { - t.Fatalf("failed to find route for Node %s (id: %s)", status.Self.HostName, status.Self.ID) + command := []string{ + "tailscale", + "set", + "--advertise-routes=" + pref.String(), } + _, _, err = client.Execute(command) + require.NoErrorf(t, err, "failed to advertise route: %s", err) } err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + time.Sleep(3 * time.Second) + nodes, err := headscale.ListNodes() require.NoError(t, err) - assert.Len(t, nodes, 4) + assert.Len(t, nodes, 6) assertNodeRouteCount(t, nodes[0], 1, 0, 0) assertNodeRouteCount(t, nodes[1], 1, 0, 0) @@ -297,28 +313,30 @@ func TestHASubnetRouterFailover(t *testing.T) { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) - assertPeerSubnetRoutes(t, peerStatus, nil) + requirePeerSubnetRoutes(t, peerStatus, nil) } } - // Enable all routes - for _, node := range nodes { - _, err := headscale.ApproveRoutes( - node.GetId(), - util.MustStringsToPrefixes(node.GetAvailableRoutes()), - ) - require.NoError(t, err) - } + // Enable route on node 1 + t.Logf("Enabling route on subnet router 1, no HA") + _, err = headscale.ApproveRoutes( + 1, + []netip.Prefix{pref}, + ) + require.NoError(t, err) + + time.Sleep(3 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) - assert.Len(t, nodes, 4) + assert.Len(t, nodes, 6) assertNodeRouteCount(t, nodes[0], 1, 1, 1) - assertNodeRouteCount(t, nodes[1], 1, 1, 1) - assertNodeRouteCount(t, nodes[2], 1, 1, 1) + assertNodeRouteCount(t, nodes[1], 1, 0, 0) + assertNodeRouteCount(t, nodes[2], 1, 0, 0) - // Verify that the client has routes from the primary machine + // Verify that the client has routes from the primary machine and can access + // the webservice. srs1 := subRouter1.MustStatus() srs2 := subRouter2.MustStatus() srs3 := subRouter3.MustStatus() @@ -336,11 +354,135 @@ func TestHASubnetRouterFailover(t *testing.T) { assert.Nil(t, srs3PeerStatus.PrimaryRoutes) require.NotNil(t, srs1PeerStatus.PrimaryRoutes) + requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref}) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, nil) + + t.Logf("got list: %v, want in: %v", srs1PeerStatus.PrimaryRoutes.AsSlice(), pref) assert.Contains(t, srs1PeerStatus.PrimaryRoutes.AsSlice(), - netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]), + pref, ) + t.Logf("Validating access via subnetrouter(%s) to %s, no HA", subRouter1.MustIPv4().String(), webip.String()) + result, err := client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err := client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter1.MustIPv4()) + + // Enable route on node 2, now we will have a HA subnet router + t.Logf("Enabling route on subnet router 2, now HA, subnetrouter 1 is primary, 2 is standby") + _, err = headscale.ApproveRoutes( + 2, + []netip.Prefix{pref}, + ) + require.NoError(t, err) + + time.Sleep(3 * time.Second) + + nodes, err = headscale.ListNodes() + require.NoError(t, err) + assert.Len(t, nodes, 6) + + assertNodeRouteCount(t, nodes[0], 1, 1, 1) + assertNodeRouteCount(t, nodes[1], 1, 1, 1) + assertNodeRouteCount(t, nodes[2], 1, 0, 0) + + // Verify that the client has routes from the primary machine + srs1 = subRouter1.MustStatus() + srs2 = subRouter2.MustStatus() + srs3 = subRouter3.MustStatus() + clientStatus = client.MustStatus() + + srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] + srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] + srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] + + assert.True(t, srs1PeerStatus.Online, "r1 up, r2 up") + assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up") + assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up") + + assert.Nil(t, srs2PeerStatus.PrimaryRoutes) + assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + require.NotNil(t, srs1PeerStatus.PrimaryRoutes) + + requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref}) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, nil) + + t.Logf("got list: %v, want in: %v", srs1PeerStatus.PrimaryRoutes.AsSlice(), pref) + assert.Contains(t, + srs1PeerStatus.PrimaryRoutes.AsSlice(), + pref, + ) + + t.Logf("Validating access via subnetrouter(%s) to %s, 2 is standby", subRouter1.MustIPv4().String(), webip.String()) + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter1.MustIPv4()) + + // Enable route on node 3, now we will have a second standby and all will + // be enabled. + t.Logf("Enabling route on subnet router 3, now HA, subnetrouter 1 is primary, 2 and 3 is standby") + _, err = headscale.ApproveRoutes( + 3, + []netip.Prefix{pref}, + ) + require.NoError(t, err) + + time.Sleep(3 * time.Second) + + nodes, err = headscale.ListNodes() + require.NoError(t, err) + assert.Len(t, nodes, 6) + + assertNodeRouteCount(t, nodes[0], 1, 1, 1) + assertNodeRouteCount(t, nodes[1], 1, 1, 1) + assertNodeRouteCount(t, nodes[2], 1, 1, 1) + + // Verify that the client has routes from the primary machine + srs1 = subRouter1.MustStatus() + srs2 = subRouter2.MustStatus() + srs3 = subRouter3.MustStatus() + clientStatus = client.MustStatus() + + srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] + srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] + srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] + + assert.True(t, srs1PeerStatus.Online, "r1 up, r2 up") + assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up") + assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up") + + assert.Nil(t, srs2PeerStatus.PrimaryRoutes) + assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + require.NotNil(t, srs1PeerStatus.PrimaryRoutes) + + requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref}) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, nil) + + t.Logf("got list: %v, want in: %v", srs1PeerStatus.PrimaryRoutes.AsSlice(), pref) + assert.Contains(t, + srs1PeerStatus.PrimaryRoutes.AsSlice(), + pref, + ) + + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter1.MustIPv4()) + // Take down the current primary t.Logf("taking down subnet router r1 (%s)", subRouter1.Hostname()) t.Logf("expecting r2 (%s) to take over as primary", subRouter2.Hostname()) @@ -364,12 +506,24 @@ func TestHASubnetRouterFailover(t *testing.T) { require.NotNil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + requirePeerSubnetRoutes(t, srs1PeerStatus, nil) + requirePeerSubnetRoutes(t, srs2PeerStatus, []netip.Prefix{pref}) + requirePeerSubnetRoutes(t, srs3PeerStatus, nil) + assert.Contains( t, srs2PeerStatus.PrimaryRoutes.AsSlice(), - netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]), + pref, ) + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter2.MustIPv4()) + // Take down subnet router 2, leaving none available t.Logf("taking down subnet router r2 (%s)", subRouter2.Hostname()) t.Logf("expecting no primary, r3 available, but no HA so no primary") @@ -395,7 +549,19 @@ func TestHASubnetRouterFailover(t *testing.T) { assert.Nil(t, srs1PeerStatus.PrimaryRoutes) assert.Nil(t, srs2PeerStatus.PrimaryRoutes) - assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + require.NotNil(t, srs3PeerStatus.PrimaryRoutes) + + requirePeerSubnetRoutes(t, srs1PeerStatus, nil) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, []netip.Prefix{pref}) + + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter3.MustIPv4()) // Bring up subnet router 1, making the route available from there. t.Logf("bringing up subnet router r1 (%s)", subRouter1.Hostname()) @@ -417,16 +583,28 @@ func TestHASubnetRouterFailover(t *testing.T) { assert.False(t, srs2PeerStatus.Online, "r1 is back up, r2 down") assert.True(t, srs3PeerStatus.Online, "r1 is back up, r3 available") - assert.NotNil(t, srs1PeerStatus.PrimaryRoutes) + assert.Nil(t, srs1PeerStatus.PrimaryRoutes) assert.Nil(t, srs2PeerStatus.PrimaryRoutes) - assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + require.NotNil(t, srs3PeerStatus.PrimaryRoutes) + + requirePeerSubnetRoutes(t, srs1PeerStatus, nil) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, []netip.Prefix{pref}) assert.Contains( t, - srs1PeerStatus.PrimaryRoutes.AsSlice(), - netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]), + srs3PeerStatus.PrimaryRoutes.AsSlice(), + pref, ) + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter3.MustIPv4()) + // Bring up subnet router 2, should result in no change. t.Logf("bringing up subnet router r2 (%s)", subRouter2.Hostname()) t.Logf("all online, expecting r1 (%s) to still be primary (no flapping)", subRouter1.Hostname()) @@ -447,30 +625,86 @@ func TestHASubnetRouterFailover(t *testing.T) { assert.True(t, srs2PeerStatus.Online, "r1 up, r2 up") assert.True(t, srs3PeerStatus.Online, "r1 up, r2 up") + assert.Nil(t, srs1PeerStatus.PrimaryRoutes) + assert.Nil(t, srs2PeerStatus.PrimaryRoutes) + require.NotNil(t, srs3PeerStatus.PrimaryRoutes) + + requirePeerSubnetRoutes(t, srs1PeerStatus, nil) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, []netip.Prefix{pref}) + + assert.Contains( + t, + srs3PeerStatus.PrimaryRoutes.AsSlice(), + pref, + ) + + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter3.MustIPv4()) + + t.Logf("disabling route in subnet router r3 (%s)", subRouter3.Hostname()) + t.Logf("expecting route to failover to r1 (%s), which is still available with r2", subRouter1.Hostname()) + _, err = headscale.ApproveRoutes(nodes[2].GetId(), []netip.Prefix{}) + + time.Sleep(5 * time.Second) + + nodes, err = headscale.ListNodes() + require.NoError(t, err) + assert.Len(t, nodes, 6) + + assertNodeRouteCount(t, nodes[0], 1, 1, 1) + assertNodeRouteCount(t, nodes[1], 1, 1, 1) + assertNodeRouteCount(t, nodes[2], 1, 0, 0) + + // Verify that the route is announced from subnet router 1 + clientStatus, err = client.Status() + require.NoError(t, err) + + srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] + srs2PeerStatus = clientStatus.Peer[srs2.Self.PublicKey] + srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] + require.NotNil(t, srs1PeerStatus.PrimaryRoutes) assert.Nil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{pref}) + requirePeerSubnetRoutes(t, srs2PeerStatus, nil) + requirePeerSubnetRoutes(t, srs3PeerStatus, nil) + assert.Contains( t, srs1PeerStatus.PrimaryRoutes.AsSlice(), - netip.MustParsePrefix(expectedRoutes[string(srs1.Self.ID)]), + pref, ) + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter1.MustIPv4()) + // Disable the route of subnet router 1, making it failover to 2 t.Logf("disabling route in subnet router r1 (%s)", subRouter1.Hostname()) - t.Logf("expecting route to failover to r2 (%s), which is still available with r3", subRouter2.Hostname()) + t.Logf("expecting route to failover to r2 (%s)", subRouter2.Hostname()) _, err = headscale.ApproveRoutes(nodes[0].GetId(), []netip.Prefix{}) time.Sleep(5 * time.Second) nodes, err = headscale.ListNodes() require.NoError(t, err) - assert.Len(t, nodes, 4) + assert.Len(t, nodes, 6) assertNodeRouteCount(t, nodes[0], 1, 0, 0) assertNodeRouteCount(t, nodes[1], 1, 1, 1) - assertNodeRouteCount(t, nodes[2], 1, 1, 1) + assertNodeRouteCount(t, nodes[2], 1, 0, 0) // Verify that the route is announced from subnet router 1 clientStatus, err = client.Status() @@ -481,15 +715,27 @@ func TestHASubnetRouterFailover(t *testing.T) { srs3PeerStatus = clientStatus.Peer[srs3.Self.PublicKey] assert.Nil(t, srs1PeerStatus.PrimaryRoutes) - assert.NotNil(t, srs2PeerStatus.PrimaryRoutes) + require.NotNil(t, srs2PeerStatus.PrimaryRoutes) assert.Nil(t, srs3PeerStatus.PrimaryRoutes) + requirePeerSubnetRoutes(t, srs1PeerStatus, nil) + requirePeerSubnetRoutes(t, srs2PeerStatus, []netip.Prefix{pref}) + requirePeerSubnetRoutes(t, srs3PeerStatus, nil) + assert.Contains( t, srs2PeerStatus.PrimaryRoutes.AsSlice(), - netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]), + pref, ) + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter2.MustIPv4()) + // enable the route of subnet router 1, no change expected t.Logf("enabling route in subnet router 1 (%s)", subRouter1.Hostname()) t.Logf("both online, expecting r2 (%s) to still be primary (no flapping)", subRouter2.Hostname()) @@ -502,11 +748,11 @@ func TestHASubnetRouterFailover(t *testing.T) { nodes, err = headscale.ListNodes() require.NoError(t, err) - assert.Len(t, nodes, 4) + assert.Len(t, nodes, 6) assertNodeRouteCount(t, nodes[0], 1, 1, 1) assertNodeRouteCount(t, nodes[1], 1, 1, 1) - assertNodeRouteCount(t, nodes[2], 1, 1, 1) + assertNodeRouteCount(t, nodes[2], 1, 0, 0) // Verify that the route is announced from subnet router 1 clientStatus, err = client.Status() @@ -523,8 +769,16 @@ func TestHASubnetRouterFailover(t *testing.T) { assert.Contains( t, srs2PeerStatus.PrimaryRoutes.AsSlice(), - netip.MustParsePrefix(expectedRoutes[string(srs2.Self.ID)]), + pref, ) + + result, err = client.Curl(weburl) + require.NoError(t, err) + assert.Len(t, result, 13) + + tr, err = client.Traceroute(webip) + require.NoError(t, err) + assertTracerouteViaIP(t, tr, subRouter2.MustIPv4()) } func TestEnableDisableAutoApprovedRoute(t *testing.T) { @@ -542,7 +796,10 @@ func TestEnableDisableAutoApprovedRoute(t *testing.T) { require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) - err = scenario.CreateHeadscaleEnv([]tsic.Option{tsic.WithTags([]string{"tag:approve"})}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy( + err = scenario.CreateHeadscaleEnv([]tsic.Option{ + tsic.WithTags([]string{"tag:approve"}), + tsic.WithAcceptRoutes(), + }, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy( &policyv1.ACLPolicy{ ACLs: []policyv1.ACL{ { @@ -640,7 +897,10 @@ func TestAutoApprovedSubRoute2068(t *testing.T) { require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) - err = scenario.CreateHeadscaleEnv([]tsic.Option{tsic.WithTags([]string{"tag:approve"})}, + err = scenario.CreateHeadscaleEnv([]tsic.Option{ + tsic.WithTags([]string{"tag:approve"}), + tsic.WithAcceptRoutes(), + }, hsic.WithTestName("clienableroute"), hsic.WithEmbeddedDERPServerOnly(), hsic.WithTLS(), @@ -712,7 +972,9 @@ func TestSubnetRouteACL(t *testing.T) { require.NoErrorf(t, err, "failed to create scenario: %s", err) defer scenario.ShutdownAssertNoPanics(t) - err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy( + err = scenario.CreateHeadscaleEnv([]tsic.Option{ + tsic.WithAcceptRoutes(), + }, hsic.WithTestName("clienableroute"), hsic.WithACLPolicy( &policyv1.ACLPolicy{ Groups: policyv1.Groups{ "group:admins": {user}, @@ -805,7 +1067,7 @@ func TestSubnetRouteACL(t *testing.T) { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) - assertPeerSubnetRoutes(t, peerStatus, nil) + requirePeerSubnetRoutes(t, peerStatus, nil) } } @@ -832,7 +1094,7 @@ func TestSubnetRouteACL(t *testing.T) { srs1PeerStatus := clientStatus.Peer[srs1.Self.PublicKey] - assertPeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{netip.MustParsePrefix(expectedRoutes["1"])}) + requirePeerSubnetRoutes(t, srs1PeerStatus, []netip.Prefix{netip.MustParsePrefix(expectedRoutes["1"])}) clientNm, err := client.Netmap() require.NoError(t, err) @@ -1010,32 +1272,6 @@ func TestEnablingExitRoutes(t *testing.T) { } } -// assertPeerSubnetRoutes asserts that the peer has the expected subnet routes. -func assertPeerSubnetRoutes(t *testing.T, status *ipnstate.PeerStatus, expected []netip.Prefix) { - t.Helper() - if status.AllowedIPs.Len() <= 2 && len(expected) != 0 { - t.Errorf("peer %s (%s) has no subnet routes, expected %v", status.HostName, status.ID, expected) - return - } - - if len(expected) == 0 { - expected = []netip.Prefix{} - } - - got := status.AllowedIPs.AsSlice()[2:] - - if diff := cmp.Diff(expected, got, util.PrefixComparer); diff != "" { - t.Errorf("peer %s (%s) subnet routes, unexpected result (-want +got):\n%s", status.HostName, status.ID, diff) - } -} - -func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subnet int) { - t.Helper() - assert.Len(t, node.GetAvailableRoutes(), announced) - assert.Len(t, node.GetApprovedRoutes(), approved) - assert.Len(t, node.GetSubnetRoutes(), subnet) -} - // TestSubnetRouterMultiNetwork is an evolution of the subnet router test. // This test will set up multiple docker networks and use two isolated tailscale // clients and a service available in one of the networks to validate that a @@ -1117,7 +1353,7 @@ func TestSubnetRouterMultiNetwork(t *testing.T) { peerStatus := status.Peer[peerKey] assert.Nil(t, peerStatus.PrimaryRoutes) - assertPeerSubnetRoutes(t, peerStatus, nil) + requirePeerSubnetRoutes(t, peerStatus, nil) } // Enable route @@ -1141,7 +1377,7 @@ func TestSubnetRouterMultiNetwork(t *testing.T) { for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] -// TestSubnetRouterMultiNetworkExitNode + assert.Contains(t, peerStatus.PrimaryRoutes.AsSlice(), *pref) func TestSubnetRouterMultiNetworkExitNode(t *testing.T) { IntegrationSkip(t) t.Parallel() @@ -1275,7 +1511,7 @@ func TestSubnetRouterMultiNetworkExitNode(t *testing.T) { } assert.Nil(t, peerStatus.PrimaryRoutes) - assertPeerSubnetRoutes(t, peerStatus, []netip.Prefix{*pref}) + requirePeerSubnetRoutes(t, peerStatus, []netip.Prefix{*pref}) } usernet1, err := scenario.Network("usernet1") diff --git a/integration/scenario.go b/integration/scenario.go index b1cbbe34..e0cbdc21 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -134,6 +134,9 @@ type ScenarioSpec struct { // typically dont run Tailscale, e.g. web service to test subnet router. ExtraService map[string][]extraServiceFunc + // Versions is specific list of versions to use for the test. + Versions []string + // OIDCUsers, if populated, will start a Mock OIDC server and populate // the user login stack with the given users. // If the NodesPerUser is set, it should align with this list to ensure @@ -514,7 +517,11 @@ func (s *Scenario) CreateTailscaleNodesInUser( for i := range count { version := requestedVersion if requestedVersion == "all" { - version = MustTestVersions[i%len(MustTestVersions)] + if s.spec.Versions != nil { + version = s.spec.Versions[i%len(s.spec.Versions)] + } else { + version = MustTestVersions[i%len(MustTestVersions)] + } } versions = append(versions, version)