diff --git a/integration/route_test.go b/integration/route_test.go index bd582235..479a6aaf 100644 --- a/integration/route_test.go +++ b/integration/route_test.go @@ -205,6 +205,12 @@ func TestHASubnetRouterFailover(t *testing.T) { spec := ScenarioSpec{ NodesPerUser: 4, Users: []string{"user1"}, + Networks: map[string][]string{ + "usernet1": {"user1"}, + }, + ExtraService: map[string][]extraServiceFunc{ + "usernet1": {Webservice}, + }, } scenario, err := NewScenario(spec) @@ -1028,3 +1034,76 @@ func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subn assert.Len(t, node.GetApprovedRoutes(), approved) assert.Len(t, node.GetSubnetRoutes(), subnet) } + +func TestHASubnetRouterFailover2(t *testing.T) { + IntegrationSkip(t) + t.Parallel() + + spec := ScenarioSpec{ + NodesPerUser: 4, + Users: []string{"user1"}, + Networks: map[string][]string{ + "usernet1": {"user1"}, + }, + ExtraService: map[string][]extraServiceFunc{ + "usernet1": {Webservice}, + }, + } + + scenario, err := NewScenario(spec) + require.NoErrorf(t, err, "failed to create scenario: %s", err) + defer scenario.ShutdownAssertNoPanics(t) + + err = scenario.CreateHeadscaleEnv([]tsic.Option{}, + hsic.WithTestName("clienableroute"), + hsic.WithEmbeddedDERPServerOnly(), + hsic.WithTLS(), + ) + assertNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + assertNoErrListClients(t, err) + + err = scenario.WaitForTailscaleSync() + assertNoErrSync(t, err) + + headscale, err := scenario.Headscale() + assertNoErrGetHeadscale(t, err) +} + +// requirePeerSubnetRoutes asserts that the peer has the expected subnet routes. +func requirePeerSubnetRoutes(t *testing.T, status *ipnstate.PeerStatus, expected []netip.Prefix) { + t.Helper() + if status.AllowedIPs.Len() <= 2 && len(expected) != 0 { + t.Fatalf("peer %s (%s) has no subnet routes, expected %v", status.HostName, status.ID, expected) + return + } + + if len(expected) == 0 { + expected = []netip.Prefix{} + } + + got := slicesx.Filter(nil, status.AllowedIPs.AsSlice(), func(p netip.Prefix) bool { + if tsaddr.IsExitRoute(p) { + return true + } + for _, ip := range status.TailscaleIPs { + if p.Contains(ip) { + return false + } + } + + return true + }) + + if diff := cmp.Diff(expected, got, util.PrefixComparer, cmpopts.EquateEmpty()); diff != "" { + t.Fatalf("peer %s (%s) subnet routes, unexpected result (-want +got):\n%s", status.HostName, status.ID, diff) + } +} + +func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subnet int) { + t.Helper() + assert.Len(t, node.GetAvailableRoutes(), announced) + assert.Len(t, node.GetApprovedRoutes(), approved) + assert.Len(t, node.GetSubnetRoutes(), subnet) +} diff --git a/integration/scenario.go b/integration/scenario.go index d3816e10..d1303e45 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -100,9 +100,10 @@ type Scenario struct { users map[string]*User - pool *dockertest.Pool - networks map[string]*dockertest.Network - mockOIDC scenarioOIDC + pool *dockertest.Pool + networks map[string]*dockertest.Network + mockOIDC scenarioOIDC + extraServices map[string][]*dockertest.Resource mu sync.Mutex @@ -120,7 +121,7 @@ type ScenarioSpec struct { // NodesPerUser is how many nodes should be attached to each user. NodesPerUser int - // Networks, if set, is the deparate Docker networks that should be + // Networks, if set, is the seperate Docker networks that should be // created and a list of the users that should be placed in those networks. // If not set, a single network will be created and all users+nodes will be // added there. @@ -128,6 +129,11 @@ type ScenarioSpec struct { // connections between them might fall back to DERP. Networks map[string][]string + // ExtraService, if set, is additional a map of network to additional + // container services that should be set up. These container services + // typically dont run Tailscale, e.g. web service to test subnet router. + ExtraService map[string][]extraServiceFunc + // OIDCUsers, if populated, will start a Mock OIDC server and populate // the user login stack with the given users. // If the NodesPerUser is set, it should align with this list to ensure @@ -189,6 +195,16 @@ func NewScenario(spec ScenarioSpec) (*Scenario, error) { } } + for network, extras := range spec.ExtraService { + for _, extra := range extras { + svc, err := extra(s, network) + if err != nil { + return nil, err + } + s.extraServices[TestHashPrefix+"-"+network] = append(s.extraServices[TestHashPrefix+"-"+network], svc) + } + } + s.userToNetwork = userToNetwork if spec.OIDCUsers != nil && len(spec.OIDCUsers) != 0 { @@ -282,6 +298,13 @@ func (s *Scenario) ShutdownAssertNoPanics(t *testing.T) { } } + for _, svc := range s.extraServices { + err := svc.Close() + if err != nil { + log.Printf("failed to tear down service %q: %s", svc.Container.Name, err) + } + } + if s.mockOIDC.r != nil { s.mockOIDC.r.Close() if err := s.mockOIDC.r.Close(); err != nil { @@ -1088,3 +1111,78 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse return nil } + +type extraServiceFunc func(*Scenario, string) (*dockertest.Resource, error) + +func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) { + // port, err := dockertestutil.RandomFreeHostPort() + // if err != nil { + // log.Fatalf("could not find an open port: %s", err) + // } + // portNotation := fmt.Sprintf("%d/tcp", port) + + hash := util.MustGenerateRandomStringDNSSafe(hsicOIDCMockHashLength) + + hostname := fmt.Sprintf("hs-webservice-%s", hash) + + network, ok := s.networks[TestHashPrefix+"-"+networkName] + if !ok { + return nil, fmt.Errorf("network does not exist: %s", networkName) + } + + webOpts := &dockertest.RunOptions{ + Name: hostname, + Cmd: []string{"/bin/sh", "-c", "python3 -m http.server --bind :: 80"}, + // ExposedPorts: []string{portNotation}, + // PortBindings: map[docker.Port][]docker.PortBinding{ + // docker.Port(portNotation): {{HostPort: strconv.Itoa(port)}}, + // }, + Networks: []*dockertest.Network{network}, + Env: []string{}, + } + + webBOpts := &dockertest.BuildOptions{ + Dockerfile: hsic.IntegrationTestDockerFileName, + ContextDir: dockerContextPath, + } + + web, err := s.pool.BuildAndRunWithBuildOptions( + webBOpts, + webOpts, + dockertestutil.DockerRestartPolicy) + if err != nil { + return nil, err + } + + // headscale needs to set up the provider with a specific + // IP addr to ensure we get the correct config from the well-known + // endpoint. + // ipAddr := web.GetIPInNetwork(network) + + // log.Println("Waiting for headscale mock oidc to be ready for tests") + // hostEndpoint := net.JoinHostPort(ipAddr, strconv.Itoa(port)) + + // if err := s.pool.Retry(func() error { + // oidcConfigURL := fmt.Sprintf("http://%s/etc/hostname", hostEndpoint) + // httpClient := &http.Client{} + // ctx := context.Background() + // req, _ := http.NewRequestWithContext(ctx, http.MethodGet, oidcConfigURL, nil) + // resp, err := httpClient.Do(req) + // if err != nil { + // log.Printf("headscale mock OIDC tests is not ready: %s\n", err) + + // return err + // } + // defer resp.Body.Close() + + // if resp.StatusCode != http.StatusOK { + // return errStatusCodeNotOK + // } + + // return nil + // }); err != nil { + // return err + // } + + return web, nil +}