From 02a3cb0eff21e5a68ea572cb04d4dae54201e645 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 27 Aug 2025 16:11:36 +0200 Subject: [PATCH] integration: validate expected online status in ping Signed-off-by: Kristoffer Dalby --- integration/control.go | 3 ++ integration/general_test.go | 80 ++++++++++++++++++++++++++++++++++++- integration/hsic/hsic.go | 29 +++++++++++--- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/integration/control.go b/integration/control.go index df1d5d13..e3cb17bd 100644 --- a/integration/control.go +++ b/integration/control.go @@ -5,7 +5,9 @@ import ( v1 "github.com/juanfont/headscale/gen/go/headscale/v1" policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2" + "github.com/juanfont/headscale/hscontrol/types" "github.com/ory/dockertest/v3" + "tailscale.com/tailcfg" ) type ControlServer interface { @@ -29,4 +31,5 @@ type ControlServer interface { GetCert() []byte GetHostname() string SetPolicy(*policyv2.Policy) error + GetAllMapReponses() (map[types.NodeID][]tailcfg.MapResponse, error) } diff --git a/integration/general_test.go b/integration/general_test.go index 4e250854..4bf36567 100644 --- a/integration/general_test.go +++ b/integration/general_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "tailscale.com/client/tailscale/apitype" + "tailscale.com/tailcfg" "tailscale.com/types/key" ) @@ -55,6 +56,17 @@ func TestPingAllByIP(t *testing.T) { err = scenario.WaitForTailscaleSync() assertNoErrSync(t, err) + hs, err := scenario.Headscale() + require.NoError(t, err) + + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + all, err := hs.GetAllMapReponses() + assert.NoError(ct, err) + + onlineMap := buildExpectedOnlineMap(all) + assertExpectedOnlineMapAllOnline(ct, len(allClients)-1, onlineMap) + }, 30*time.Second, 2*time.Second) + // assertClientsState(t, allClients) allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { @@ -940,6 +952,9 @@ func TestPingAllByIPManyUpDown(t *testing.T) { ) assertNoErrHeadscaleEnv(t, err) + hs, err := scenario.Headscale() + require.NoError(t, err) + allClients, err := scenario.ListTailscaleClients() assertNoErrListClients(t, err) @@ -961,7 +976,7 @@ func TestPingAllByIPManyUpDown(t *testing.T) { wg, _ := errgroup.WithContext(context.Background()) for run := range 3 { - t.Logf("Starting DownUpPing run %d", run+1) + t.Logf("Starting DownUpPing run %d at %s", run+1, time.Now().Format("2006-01-02T15-04-05.999999999")) for _, client := range allClients { c := client @@ -974,6 +989,7 @@ func TestPingAllByIPManyUpDown(t *testing.T) { if err := wg.Wait(); err != nil { t.Fatalf("failed to take down all nodes: %s", err) } + t.Logf("All nodes taken down at %s", time.Now().Format("2006-01-02T15-04-05.999999999")) for _, client := range allClients { c := client @@ -984,13 +1000,24 @@ func TestPingAllByIPManyUpDown(t *testing.T) { } if err := wg.Wait(); err != nil { - t.Fatalf("failed to take down all nodes: %s", err) + t.Fatalf("failed to bring up all nodes: %s", err) } + t.Logf("All nodes brought up at %s", time.Now().Format("2006-01-02T15-04-05.999999999")) // Wait for sync and successful pings after nodes come back up err = scenario.WaitForTailscaleSync() assert.NoError(t, err) + t.Logf("All nodes synced up %s", time.Now().Format("2006-01-02T15-04-05.999999999")) + + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + all, err := hs.GetAllMapReponses() + assert.NoError(ct, err) + + onlineMap := buildExpectedOnlineMap(all) + assertExpectedOnlineMapAllOnline(ct, len(allClients)-1, onlineMap) + }, 60*time.Second, 2*time.Second) + success := pingAllHelper(t, allClients, allAddrs) assert.Equalf(t, len(allClients)*len(allIps), success, "%d successful pings out of %d", success, len(allClients)*len(allIps)) } @@ -1103,3 +1130,52 @@ func Test2118DeletingOnlineNodePanics(t *testing.T) { assert.True(t, nodeListAfter[0].GetOnline()) assert.Equal(t, nodeList[1].GetId(), nodeListAfter[0].GetId()) } + +func buildExpectedOnlineMap(all map[types.NodeID][]tailcfg.MapResponse) map[types.NodeID]map[types.NodeID]bool { + res := make(map[types.NodeID]map[types.NodeID]bool) + for nid, mrs := range all { + res[nid] = make(map[types.NodeID]bool) + for _, mr := range mrs { + for _, peer := range mr.Peers { + if peer.Online != nil { + res[nid][types.NodeID(peer.ID)] = *peer.Online + } + } + + for _, peer := range mr.PeersChanged { + if peer.Online != nil { + res[nid][types.NodeID(peer.ID)] = *peer.Online + } + } + + for _, peer := range mr.PeersChangedPatch { + if peer.Online != nil { + res[nid][types.NodeID(peer.NodeID)] = *peer.Online + } + } + } + } + return res +} + +func assertExpectedOnlineMapAllOnline(t *assert.CollectT, expectedPeerCount int, onlineMap map[types.NodeID]map[types.NodeID]bool) { + for nid, peers := range onlineMap { + onlineCount := 0 + for _, online := range peers { + if online { + onlineCount++ + } + } + assert.Equalf(t, expectedPeerCount, len(peers), "node:%d had an unexpected number of peers in online map", nid) + if expectedPeerCount != onlineCount { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("Not all of node:%d peers where online:\n", nid)) + for pid, online := range peers { + sb.WriteString(fmt.Sprintf("\tPeer node:%d online: %t\n", pid, online)) + } + sb.WriteString("timestamp: " + time.Now().Format("2006-01-02T15-04-05.999999999") + "\n") + sb.WriteString("expected all peers to be online.") + t.Errorf("%s", sb.String()) + } + } +} diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index fbbd9cb8..22250eb4 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -622,7 +622,7 @@ func extractTarToDirectory(tarData []byte, targetDir string) error { } tarReader := tar.NewReader(bytes.NewReader(tarData)) - + // Find the top-level directory to strip var topLevelDir string firstPass := tar.NewReader(bytes.NewReader(tarData)) @@ -634,13 +634,13 @@ func extractTarToDirectory(tarData []byte, targetDir string) error { if err != nil { return fmt.Errorf("failed to read tar header: %w", err) } - + if header.Typeflag == tar.TypeDir && topLevelDir == "" { topLevelDir = strings.TrimSuffix(header.Name, "/") break } } - + // Second pass: extract files, stripping the top-level directory tarReader = tar.NewReader(bytes.NewReader(tarData)) for { @@ -665,7 +665,7 @@ func extractTarToDirectory(tarData []byte, targetDir string) error { // Skip the top-level directory itself continue } - + // Skip empty paths after stripping if cleanName == "" { continue @@ -684,7 +684,7 @@ func extractTarToDirectory(tarData []byte, targetDir string) error { if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { return fmt.Errorf("failed to create parent directories for %s: %w", targetPath, err) } - + // Create file outFile, err := os.Create(targetPath) if err != nil { @@ -1282,3 +1282,22 @@ func (t *HeadscaleInContainer) SendInterrupt() error { return nil } + +func (t *HeadscaleInContainer) GetAllMapReponses() (map[types.NodeID][]tailcfg.MapResponse, error) { + // Execute curl inside the container to access the debug endpoint locally + command := []string{ + "curl", "-s", "-H", "Accept: application/json", "http://localhost:9090/debug/mapresponses", + } + + result, err := t.Execute(command) + if err != nil { + return nil, fmt.Errorf("fetching mapresponses from debug endpoint: %w", err) + } + + var res map[types.NodeID][]tailcfg.MapResponse + if err := json.Unmarshal([]byte(result), &res); err != nil { + return nil, fmt.Errorf("decoding routes response: %w", err) + } + + return res, nil +}