initial hav2 test

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

integration/route_test.go

@@ -205,6 +205,12 @@ func TestHASubnetRouterFailover(t *testing.T) {
 	spec := ScenarioSpec{
 		NodesPerUser: 4,
 		Users:        []string{"user1"},
+		Networks: map[string][]string{
+			"usernet1": {"user1"},
+		},
+		ExtraService: map[string][]extraServiceFunc{
+			"usernet1": {Webservice},
+		},
 	}
 
 	scenario, err := NewScenario(spec)
@@ -1028,3 +1034,76 @@ func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subn
 	assert.Len(t, node.GetApprovedRoutes(), approved)
 	assert.Len(t, node.GetSubnetRoutes(), subnet)
 }
+
+func TestHASubnetRouterFailover2(t *testing.T) {
+	IntegrationSkip(t)
+	t.Parallel()
+
+	spec := ScenarioSpec{
+		NodesPerUser: 4,
+		Users:        []string{"user1"},
+		Networks: map[string][]string{
+			"usernet1": {"user1"},
+		},
+		ExtraService: map[string][]extraServiceFunc{
+			"usernet1": {Webservice},
+		},
+	}
+
+	scenario, err := NewScenario(spec)
+	require.NoErrorf(t, err, "failed to create scenario: %s", err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv([]tsic.Option{},
+		hsic.WithTestName("clienableroute"),
+		hsic.WithEmbeddedDERPServerOnly(),
+		hsic.WithTLS(),
+	)
+	assertNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	assertNoErrListClients(t, err)
+
+	err = scenario.WaitForTailscaleSync()
+	assertNoErrSync(t, err)
+
+	headscale, err := scenario.Headscale()
+	assertNoErrGetHeadscale(t, err)
+}
+
+// requirePeerSubnetRoutes asserts that the peer has the expected subnet routes.
+func requirePeerSubnetRoutes(t *testing.T, status *ipnstate.PeerStatus, expected []netip.Prefix) {
+	t.Helper()
+	if status.AllowedIPs.Len() <= 2 && len(expected) != 0 {
+		t.Fatalf("peer %s (%s) has no subnet routes, expected %v", status.HostName, status.ID, expected)
+		return
+	}
+
+	if len(expected) == 0 {
+		expected = []netip.Prefix{}
+	}
+
+	got := slicesx.Filter(nil, status.AllowedIPs.AsSlice(), func(p netip.Prefix) bool {
+		if tsaddr.IsExitRoute(p) {
+			return true
+		}
+		for _, ip := range status.TailscaleIPs {
+			if p.Contains(ip) {
+				return false
+			}
+		}
+
+		return true
+	})
+
+	if diff := cmp.Diff(expected, got, util.PrefixComparer, cmpopts.EquateEmpty()); diff != "" {
+		t.Fatalf("peer %s (%s) subnet routes, unexpected result (-want +got):\n%s", status.HostName, status.ID, diff)
+	}
+}
+
+func assertNodeRouteCount(t *testing.T, node *v1.Node, announced, approved, subnet int) {
+	t.Helper()
+	assert.Len(t, node.GetAvailableRoutes(), announced)
+	assert.Len(t, node.GetApprovedRoutes(), approved)
+	assert.Len(t, node.GetSubnetRoutes(), subnet)
+}

integration/scenario.go

@@ -100,9 +100,10 @@ type Scenario struct {
 
 	users map[string]*User
 
-	pool     *dockertest.Pool
+	pool          *dockertest.Pool
-	networks map[string]*dockertest.Network
+	networks      map[string]*dockertest.Network
-	mockOIDC scenarioOIDC
+	mockOIDC      scenarioOIDC
+	extraServices map[string][]*dockertest.Resource
 
 	mu sync.Mutex
 
@@ -120,7 +121,7 @@ type ScenarioSpec struct {
 	// NodesPerUser is how many nodes should be attached to each user.
 	NodesPerUser int
 
-	// Networks, if set, is the deparate Docker networks that should be
+	// Networks, if set, is the seperate Docker networks that should be
 	// created and a list of the users that should be placed in those networks.
 	// If not set, a single network will be created and all users+nodes will be
 	// added there.
@@ -128,6 +129,11 @@ type ScenarioSpec struct {
 	// connections between them might fall back to DERP.
 	Networks map[string][]string
 
+	// ExtraService, if set, is additional a map of network to additional
+	// container services that should be set up. These container services
+	// typically dont run Tailscale, e.g. web service to test subnet router.
+	ExtraService map[string][]extraServiceFunc
+
 	// OIDCUsers, if populated, will start a Mock OIDC server and populate
 	// the user login stack with the given users.
 	// If the NodesPerUser is set, it should align with this list to ensure
@@ -189,6 +195,16 @@ func NewScenario(spec ScenarioSpec) (*Scenario, error) {
 		}
 	}
 
+	for network, extras := range spec.ExtraService {
+		for _, extra := range extras {
+			svc, err := extra(s, network)
+			if err != nil {
+				return nil, err
+			}
+			s.extraServices[TestHashPrefix+"-"+network] = append(s.extraServices[TestHashPrefix+"-"+network], svc)
+		}
+	}
+
 	s.userToNetwork = userToNetwork
 
 	if spec.OIDCUsers != nil && len(spec.OIDCUsers) != 0 {
@@ -282,6 +298,13 @@ func (s *Scenario) ShutdownAssertNoPanics(t *testing.T) {
 		}
 	}
 
+	for _, svc := range s.extraServices {
+		err := svc.Close()
+		if err != nil {
+			log.Printf("failed to tear down service %q: %s", svc.Container.Name, err)
+		}
+	}
+
 	if s.mockOIDC.r != nil {
 		s.mockOIDC.r.Close()
 		if err := s.mockOIDC.r.Close(); err != nil {
@@ -1088,3 +1111,78 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
 
 	return nil
 }
+
+type extraServiceFunc func(*Scenario, string) (*dockertest.Resource, error)
+
+func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) {
+	// port, err := dockertestutil.RandomFreeHostPort()
+	// if err != nil {
+	// 	log.Fatalf("could not find an open port: %s", err)
+	// }
+	// portNotation := fmt.Sprintf("%d/tcp", port)
+
+	hash := util.MustGenerateRandomStringDNSSafe(hsicOIDCMockHashLength)
+
+	hostname := fmt.Sprintf("hs-webservice-%s", hash)
+
+	network, ok := s.networks[TestHashPrefix+"-"+networkName]
+	if !ok {
+		return nil, fmt.Errorf("network does not exist: %s", networkName)
+	}
+
+	webOpts := &dockertest.RunOptions{
+		Name: hostname,
+		Cmd:  []string{"/bin/sh", "-c", "python3 -m http.server --bind :: 80"},
+		// ExposedPorts: []string{portNotation},
+		// PortBindings: map[docker.Port][]docker.PortBinding{
+		// 	docker.Port(portNotation): {{HostPort: strconv.Itoa(port)}},
+		// },
+		Networks: []*dockertest.Network{network},
+		Env:      []string{},
+	}
+
+	webBOpts := &dockertest.BuildOptions{
+		Dockerfile: hsic.IntegrationTestDockerFileName,
+		ContextDir: dockerContextPath,
+	}
+
+	web, err := s.pool.BuildAndRunWithBuildOptions(
+		webBOpts,
+		webOpts,
+		dockertestutil.DockerRestartPolicy)
+	if err != nil {
+		return nil, err
+	}
+
+	// headscale needs to set up the provider with a specific
+	// IP addr to ensure we get the correct config from the well-known
+	// endpoint.
+	// ipAddr := web.GetIPInNetwork(network)
+
+	// log.Println("Waiting for headscale mock oidc to be ready for tests")
+	// hostEndpoint := net.JoinHostPort(ipAddr, strconv.Itoa(port))
+
+	// if err := s.pool.Retry(func() error {
+	// 	oidcConfigURL := fmt.Sprintf("http://%s/etc/hostname", hostEndpoint)
+	// 	httpClient := &http.Client{}
+	// 	ctx := context.Background()
+	// 	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, oidcConfigURL, nil)
+	// 	resp, err := httpClient.Do(req)
+	// 	if err != nil {
+	// 		log.Printf("headscale mock OIDC tests is not ready: %s\n", err)
+
+	// 		return err
+	// 	}
+	// 	defer resp.Body.Close()
+
+	// 	if resp.StatusCode != http.StatusOK {
+	// 		return errStatusCodeNotOK
+	// 	}
+
+	// 	return nil
+	// }); err != nil {
+	// 	return err
+	// }
+
+	return web, nil
+}