mirror of
https://github.com/juanfont/headscale.git
synced 2025-08-05 13:49:57 +02:00
delete all, rebuild
Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
This commit is contained in:
parent
bbe57f6cd4
commit
45566cc11d
@ -32,6 +32,7 @@ import (
|
||||
"github.com/juanfont/headscale/hscontrol/mapper"
|
||||
"github.com/juanfont/headscale/hscontrol/notifier"
|
||||
"github.com/juanfont/headscale/hscontrol/policy"
|
||||
"github.com/juanfont/headscale/hscontrol/routes"
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
zerolog "github.com/philip-bui/grpc-zerolog"
|
||||
@ -92,6 +93,7 @@ type Headscale struct {
|
||||
polManOnce sync.Once
|
||||
polMan policy.PolicyManager
|
||||
extraRecordMan *dns.ExtraRecordsMan
|
||||
primaryRoutes *routes.PrimaryRoutes
|
||||
|
||||
mapper *mapper.Mapper
|
||||
nodeNotifier *notifier.Notifier
|
||||
@ -134,6 +136,7 @@ func NewHeadscale(cfg *types.Config) (*Headscale, error) {
|
||||
registrationCache: registrationCache,
|
||||
pollNetMapStreamWG: sync.WaitGroup{},
|
||||
nodeNotifier: notifier.NewNotifier(cfg),
|
||||
primaryRoutes: routes.New(),
|
||||
}
|
||||
|
||||
app.db, err = db.NewHeadscaleDatabase(
|
||||
@ -495,6 +498,8 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router {
|
||||
|
||||
// TODO(kradalby): Do a variant of this, and polman which only updates the node that has changed.
|
||||
// Maybe we should attempt a new in memory state and not go via the DB?
|
||||
// Maybe this should be implemented as an event bus?
|
||||
// A bool is returned indicating if a full update was sent to all nodes
|
||||
func usersChangedHook(db *db.HSDatabase, polMan policy.PolicyManager, notif *notifier.Notifier) error {
|
||||
users, err := db.ListUsers()
|
||||
if err != nil {
|
||||
@ -516,6 +521,7 @@ func usersChangedHook(db *db.HSDatabase, polMan policy.PolicyManager, notif *not
|
||||
|
||||
// TODO(kradalby): Do a variant of this, and polman which only updates the node that has changed.
|
||||
// Maybe we should attempt a new in memory state and not go via the DB?
|
||||
// Maybe this should be implemented as an event bus?
|
||||
// A bool is returned indicating if a full update was sent to all nodes
|
||||
func nodesChangedHook(db *db.HSDatabase, polMan policy.PolicyManager, notif *notifier.Notifier) (bool, error) {
|
||||
nodes, err := db.ListNodes()
|
||||
|
@ -12,12 +12,10 @@ import (
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"github.com/rs/zerolog/log"
|
||||
"gorm.io/gorm"
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/types/key"
|
||||
"tailscale.com/types/ptr"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -267,9 +265,9 @@ func NodeSetExpiry(tx *gorm.DB,
|
||||
return tx.Model(&types.Node{}).Where("id = ?", nodeID).Update("expiry", expiry).Error
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) DeleteNode(node *types.Node, isLikelyConnected *xsync.MapOf[types.NodeID, bool]) ([]types.NodeID, error) {
|
||||
return Write(hsdb.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
|
||||
return DeleteNode(tx, node, isLikelyConnected)
|
||||
func (hsdb *HSDatabase) DeleteNode(node *types.Node) error {
|
||||
return hsdb.Write(func(tx *gorm.DB) error {
|
||||
return DeleteNode(tx, node)
|
||||
})
|
||||
}
|
||||
|
||||
@ -277,19 +275,13 @@ func (hsdb *HSDatabase) DeleteNode(node *types.Node, isLikelyConnected *xsync.Ma
|
||||
// Caller is responsible for notifying all of change.
|
||||
func DeleteNode(tx *gorm.DB,
|
||||
node *types.Node,
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
) ([]types.NodeID, error) {
|
||||
changed, err := deleteNodeRoutes(tx, node, isLikelyConnected)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
|
||||
) error {
|
||||
// Unscoped causes the node to be fully removed from the database.
|
||||
if err := tx.Unscoped().Delete(&types.Node{}, node.ID).Error; err != nil {
|
||||
return changed, err
|
||||
return err
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteEphemeralNode deletes a Node from the database, note that this method
|
||||
@ -495,141 +487,6 @@ func NodeSave(tx *gorm.DB, node *types.Node) error {
|
||||
return tx.Save(node).Error
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) GetAdvertisedRoutes(node *types.Node) ([]netip.Prefix, error) {
|
||||
return Read(hsdb.DB, func(rx *gorm.DB) ([]netip.Prefix, error) {
|
||||
return GetAdvertisedRoutes(rx, node)
|
||||
})
|
||||
}
|
||||
|
||||
// GetAdvertisedRoutes returns the routes that are be advertised by the given node.
|
||||
func GetAdvertisedRoutes(tx *gorm.DB, node *types.Node) ([]netip.Prefix, error) {
|
||||
routes := types.Routes{}
|
||||
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Where("node_id = ? AND advertised = ?", node.ID, true).Find(&routes).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, fmt.Errorf("getting advertised routes for node(%d): %w", node.ID, err)
|
||||
}
|
||||
|
||||
var prefixes []netip.Prefix
|
||||
for _, route := range routes {
|
||||
prefixes = append(prefixes, netip.Prefix(route.Prefix))
|
||||
}
|
||||
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) GetEnabledRoutes(node *types.Node) ([]netip.Prefix, error) {
|
||||
return Read(hsdb.DB, func(rx *gorm.DB) ([]netip.Prefix, error) {
|
||||
return GetEnabledRoutes(rx, node)
|
||||
})
|
||||
}
|
||||
|
||||
// GetEnabledRoutes returns the routes that are enabled for the node.
|
||||
func GetEnabledRoutes(tx *gorm.DB, node *types.Node) ([]netip.Prefix, error) {
|
||||
routes := types.Routes{}
|
||||
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Where("node_id = ? AND advertised = ? AND enabled = ?", node.ID, true, true).
|
||||
Find(&routes).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, fmt.Errorf("getting enabled routes for node(%d): %w", node.ID, err)
|
||||
}
|
||||
|
||||
var prefixes []netip.Prefix
|
||||
for _, route := range routes {
|
||||
prefixes = append(prefixes, netip.Prefix(route.Prefix))
|
||||
}
|
||||
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
func IsRoutesEnabled(tx *gorm.DB, node *types.Node, routeStr string) bool {
|
||||
route, err := netip.ParsePrefix(routeStr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
enabledRoutes, err := GetEnabledRoutes(tx, node)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, enabledRoute := range enabledRoutes {
|
||||
if route == enabledRoute {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) enableRoutes(
|
||||
node *types.Node,
|
||||
newRoutes ...netip.Prefix,
|
||||
) (*types.StateUpdate, error) {
|
||||
return Write(hsdb.DB, func(tx *gorm.DB) (*types.StateUpdate, error) {
|
||||
return enableRoutes(tx, node, newRoutes...)
|
||||
})
|
||||
}
|
||||
|
||||
// enableRoutes enables new routes based on a list of new routes.
|
||||
func enableRoutes(tx *gorm.DB,
|
||||
node *types.Node, newRoutes ...netip.Prefix,
|
||||
) (*types.StateUpdate, error) {
|
||||
advertisedRoutes, err := GetAdvertisedRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, newRoute := range newRoutes {
|
||||
if !slices.Contains(advertisedRoutes, newRoute) {
|
||||
return nil, fmt.Errorf(
|
||||
"route (%s) is not available on node %s: %w",
|
||||
node.Hostname,
|
||||
newRoute, ErrNodeRouteIsNotAvailable,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Separate loop so we don't leave things in a half-updated state
|
||||
for _, prefix := range newRoutes {
|
||||
route := types.Route{}
|
||||
err := tx.Preload("Node").
|
||||
Where("node_id = ? AND prefix = ?", node.ID, prefix.String()).
|
||||
First(&route).Error
|
||||
if err == nil {
|
||||
route.Enabled = true
|
||||
|
||||
// Mark already as primary if there is only this node offering this subnet
|
||||
// (and is not an exit route)
|
||||
if !route.IsExitRoute() {
|
||||
route.IsPrimary = isUniquePrefix(tx, route)
|
||||
}
|
||||
|
||||
err = tx.Save(&route).Error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to enable route: %w", err)
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to find route: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the node has the latest routes when notifying the other
|
||||
// nodes
|
||||
nRoutes, err := GetNodeRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read back routes: %w", err)
|
||||
}
|
||||
|
||||
node.Routes = nRoutes
|
||||
|
||||
return ptr.To(types.UpdatePeerChanged(node.ID)), nil
|
||||
}
|
||||
|
||||
func generateGivenName(suppliedName string, randomSuffix bool) (string, error) {
|
||||
suppliedName = util.ConvertWithFQDNRules(suppliedName)
|
||||
if len(suppliedName) > util.LabelHostnameLength {
|
||||
|
@ -15,7 +15,6 @@ import (
|
||||
"github.com/juanfont/headscale/hscontrol/policy"
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/check.v1"
|
||||
@ -102,7 +101,7 @@ func (s *Suite) TestHardDeleteNode(c *check.C) {
|
||||
trx := db.DB.Save(&node)
|
||||
c.Assert(trx.Error, check.IsNil)
|
||||
|
||||
_, err = db.DeleteNode(&node, xsync.NewMapOf[types.NodeID, bool]())
|
||||
err = db.DeleteNode(&node)
|
||||
c.Assert(err, check.IsNil)
|
||||
|
||||
_, err = db.getNode(types.UserID(user.ID), "testnode3")
|
||||
|
@ -1,676 +0,0 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sort"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/policy"
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"github.com/rs/zerolog/log"
|
||||
"gorm.io/gorm"
|
||||
"tailscale.com/net/tsaddr"
|
||||
"tailscale.com/types/ptr"
|
||||
"tailscale.com/util/set"
|
||||
)
|
||||
|
||||
var ErrRouteIsNotAvailable = errors.New("route is not available")
|
||||
|
||||
func GetRoutes(tx *gorm.DB) (types.Routes, error) {
|
||||
var routes types.Routes
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Preload("Node.User").
|
||||
Find(&routes).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func getAdvertisedAndEnabledRoutes(tx *gorm.DB) (types.Routes, error) {
|
||||
var routes types.Routes
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Preload("Node.User").
|
||||
Where("advertised = ? AND enabled = ?", true, true).
|
||||
Find(&routes).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func getRoutesByPrefix(tx *gorm.DB, pref netip.Prefix) (types.Routes, error) {
|
||||
var routes types.Routes
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Preload("Node.User").
|
||||
Where("prefix = ?", pref.String()).
|
||||
Find(&routes).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func GetNodeAdvertisedRoutes(tx *gorm.DB, node *types.Node) (types.Routes, error) {
|
||||
var routes types.Routes
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Preload("Node.User").
|
||||
Where("node_id = ? AND advertised = true", node.ID).
|
||||
Find(&routes).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) GetNodeRoutes(node *types.Node) (types.Routes, error) {
|
||||
return Read(hsdb.DB, func(rx *gorm.DB) (types.Routes, error) {
|
||||
return GetNodeRoutes(rx, node)
|
||||
})
|
||||
}
|
||||
|
||||
func GetNodeRoutes(tx *gorm.DB, node *types.Node) (types.Routes, error) {
|
||||
var routes types.Routes
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Preload("Node.User").
|
||||
Where("node_id = ?", node.ID).
|
||||
Find(&routes).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func GetRoute(tx *gorm.DB, id uint64) (*types.Route, error) {
|
||||
var route types.Route
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Preload("Node.User").
|
||||
First(&route, id).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &route, nil
|
||||
}
|
||||
|
||||
func EnableRoute(tx *gorm.DB, id uint64) (*types.StateUpdate, error) {
|
||||
route, err := GetRoute(tx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Tailscale requires both IPv4 and IPv6 exit routes to
|
||||
// be enabled at the same time, as per
|
||||
// https://github.com/juanfont/headscale/issues/804#issuecomment-1399314002
|
||||
if route.IsExitRoute() {
|
||||
return enableRoutes(
|
||||
tx,
|
||||
route.Node,
|
||||
tsaddr.AllIPv4(),
|
||||
tsaddr.AllIPv6(),
|
||||
)
|
||||
}
|
||||
|
||||
return enableRoutes(tx, route.Node, netip.Prefix(route.Prefix))
|
||||
}
|
||||
|
||||
func DisableRoute(tx *gorm.DB,
|
||||
id uint64,
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
) ([]types.NodeID, error) {
|
||||
route, err := GetRoute(tx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var routes types.Routes
|
||||
node := route.Node
|
||||
|
||||
// Tailscale requires both IPv4 and IPv6 exit routes to
|
||||
// be enabled at the same time, as per
|
||||
// https://github.com/juanfont/headscale/issues/804#issuecomment-1399314002
|
||||
var update []types.NodeID
|
||||
if !route.IsExitRoute() {
|
||||
route.Enabled = false
|
||||
err = tx.Save(route).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
update, err = failoverRouteTx(tx, isLikelyConnected, route)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
routes, err = GetNodeRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range routes {
|
||||
if routes[i].IsExitRoute() {
|
||||
routes[i].Enabled = false
|
||||
routes[i].IsPrimary = false
|
||||
|
||||
err = tx.Save(&routes[i]).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If update is empty, it means that one was not created
|
||||
// by failover (as a failover was not necessary), create
|
||||
// one and return to the caller.
|
||||
if update == nil {
|
||||
update = []types.NodeID{node.ID}
|
||||
}
|
||||
|
||||
return update, nil
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) DeleteRoute(
|
||||
id uint64,
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
) ([]types.NodeID, error) {
|
||||
return Write(hsdb.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
|
||||
return DeleteRoute(tx, id, isLikelyConnected)
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteRoute(
|
||||
tx *gorm.DB,
|
||||
id uint64,
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
) ([]types.NodeID, error) {
|
||||
route, err := GetRoute(tx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if route.Node == nil {
|
||||
// If the route is not assigned to a node, just delete it,
|
||||
// there are no updates to be sent as no nodes are
|
||||
// dependent on it
|
||||
if err := tx.Unscoped().Delete(&route).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var routes types.Routes
|
||||
node := route.Node
|
||||
|
||||
// Tailscale requires both IPv4 and IPv6 exit routes to
|
||||
// be enabled at the same time, as per
|
||||
// https://github.com/juanfont/headscale/issues/804#issuecomment-1399314002
|
||||
// This means that if we delete a route which is an exit route, delete both.
|
||||
var update []types.NodeID
|
||||
if route.IsExitRoute() {
|
||||
routes, err = GetNodeRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var routesToDelete types.Routes
|
||||
for _, r := range routes {
|
||||
if r.IsExitRoute() {
|
||||
routesToDelete = append(routesToDelete, r)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Unscoped().Delete(&routesToDelete).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
update, err = failoverRouteTx(tx, isLikelyConnected, route)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := tx.Unscoped().Delete(&route).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// If update is empty, it means that one was not created
|
||||
// by failover (as a failover was not necessary), create
|
||||
// one and return to the caller.
|
||||
if routes == nil {
|
||||
routes, err = GetNodeRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
node.Routes = routes
|
||||
|
||||
if update == nil {
|
||||
update = []types.NodeID{node.ID}
|
||||
}
|
||||
|
||||
return update, nil
|
||||
}
|
||||
|
||||
func deleteNodeRoutes(tx *gorm.DB, node *types.Node, isLikelyConnected *xsync.MapOf[types.NodeID, bool]) ([]types.NodeID, error) {
|
||||
routes, err := GetNodeRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting node routes: %w", err)
|
||||
}
|
||||
|
||||
var changed []types.NodeID
|
||||
for i := range routes {
|
||||
if err := tx.Unscoped().Delete(&routes[i]).Error; err != nil {
|
||||
return nil, fmt.Errorf("deleting route(%d): %w", &routes[i].ID, err)
|
||||
}
|
||||
|
||||
// TODO(kradalby): This is a bit too aggressive, we could probably
|
||||
// figure out which routes needs to be failed over rather than all.
|
||||
chn, err := failoverRouteTx(tx, isLikelyConnected, &routes[i])
|
||||
if err != nil {
|
||||
return changed, fmt.Errorf("failing over route after delete: %w", err)
|
||||
}
|
||||
|
||||
if chn != nil {
|
||||
changed = append(changed, chn...)
|
||||
}
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// isUniquePrefix returns if there is another node providing the same route already.
|
||||
func isUniquePrefix(tx *gorm.DB, route types.Route) bool {
|
||||
var count int64
|
||||
tx.Model(&types.Route{}).
|
||||
Where("prefix = ? AND node_id != ? AND advertised = ? AND enabled = ?",
|
||||
route.Prefix.String(),
|
||||
route.NodeID,
|
||||
true, true).Count(&count)
|
||||
|
||||
return count == 0
|
||||
}
|
||||
|
||||
func getPrimaryRoute(tx *gorm.DB, prefix netip.Prefix) (*types.Route, error) {
|
||||
var route types.Route
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Where("prefix = ? AND advertised = ? AND enabled = ? AND is_primary = ?", prefix.String(), true, true, true).
|
||||
First(&route).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
return &route, nil
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) GetNodePrimaryRoutes(node *types.Node) (types.Routes, error) {
|
||||
return Read(hsdb.DB, func(rx *gorm.DB) (types.Routes, error) {
|
||||
return GetNodePrimaryRoutes(rx, node)
|
||||
})
|
||||
}
|
||||
|
||||
// getNodePrimaryRoutes returns the routes that are enabled and marked as primary (for subnet failover)
|
||||
// Exit nodes are not considered for this, as they are never marked as Primary.
|
||||
func GetNodePrimaryRoutes(tx *gorm.DB, node *types.Node) (types.Routes, error) {
|
||||
var routes types.Routes
|
||||
err := tx.
|
||||
Preload("Node").
|
||||
Where("node_id = ? AND advertised = ? AND enabled = ? AND is_primary = ?", node.ID, true, true, true).
|
||||
Find(&routes).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) SaveNodeRoutes(node *types.Node) (bool, error) {
|
||||
return Write(hsdb.DB, func(tx *gorm.DB) (bool, error) {
|
||||
return SaveNodeRoutes(tx, node)
|
||||
})
|
||||
}
|
||||
|
||||
// SaveNodeRoutes takes a node and updates the database with
|
||||
// the new routes.
|
||||
// It returns a bool whether an update should be sent as the
|
||||
// saved route impacts nodes.
|
||||
func SaveNodeRoutes(tx *gorm.DB, node *types.Node) (bool, error) {
|
||||
sendUpdate := false
|
||||
|
||||
currentRoutes := types.Routes{}
|
||||
err := tx.Where("node_id = ?", node.ID).Find(¤tRoutes).Error
|
||||
if err != nil {
|
||||
return sendUpdate, err
|
||||
}
|
||||
|
||||
advertisedRoutes := map[netip.Prefix]bool{}
|
||||
for _, prefix := range node.Hostinfo.RoutableIPs {
|
||||
advertisedRoutes[prefix] = false
|
||||
}
|
||||
|
||||
log.Trace().
|
||||
Str("node", node.Hostname).
|
||||
Interface("advertisedRoutes", advertisedRoutes).
|
||||
Interface("currentRoutes", currentRoutes).
|
||||
Msg("updating routes")
|
||||
|
||||
for pos, route := range currentRoutes {
|
||||
if _, ok := advertisedRoutes[netip.Prefix(route.Prefix)]; ok {
|
||||
if !route.Advertised {
|
||||
currentRoutes[pos].Advertised = true
|
||||
err := tx.Save(¤tRoutes[pos]).Error
|
||||
if err != nil {
|
||||
return sendUpdate, err
|
||||
}
|
||||
|
||||
// If a route that is newly "saved" is already
|
||||
// enabled, set sendUpdate to true as it is now
|
||||
// available.
|
||||
if route.Enabled {
|
||||
sendUpdate = true
|
||||
}
|
||||
}
|
||||
advertisedRoutes[netip.Prefix(route.Prefix)] = true
|
||||
} else if route.Advertised {
|
||||
currentRoutes[pos].Advertised = false
|
||||
currentRoutes[pos].Enabled = false
|
||||
err := tx.Save(¤tRoutes[pos]).Error
|
||||
if err != nil {
|
||||
return sendUpdate, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for prefix, exists := range advertisedRoutes {
|
||||
if !exists {
|
||||
route := types.Route{
|
||||
NodeID: node.ID.Uint64(),
|
||||
Prefix: prefix,
|
||||
Advertised: true,
|
||||
Enabled: false,
|
||||
}
|
||||
err := tx.Create(&route).Error
|
||||
if err != nil {
|
||||
return sendUpdate, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sendUpdate, nil
|
||||
}
|
||||
|
||||
// FailoverNodeRoutesIfNecessary takes a node and checks if the node's route
|
||||
// need to be failed over to another host.
|
||||
// If needed, the failover will be attempted.
|
||||
func FailoverNodeRoutesIfNecessary(
|
||||
tx *gorm.DB,
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
node *types.Node,
|
||||
) (*types.StateUpdate, error) {
|
||||
nodeRoutes, err := GetNodeRoutes(tx, node)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
changedNodes := make(set.Set[types.NodeID])
|
||||
|
||||
nodeRouteLoop:
|
||||
for _, nodeRoute := range nodeRoutes {
|
||||
routes, err := getRoutesByPrefix(tx, netip.Prefix(nodeRoute.Prefix))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting routes by prefix: %w", err)
|
||||
}
|
||||
|
||||
for _, route := range routes {
|
||||
if route.IsPrimary {
|
||||
// if we have a primary route, and the node is connected
|
||||
// nothing needs to be done.
|
||||
if val, ok := isLikelyConnected.Load(route.Node.ID); ok && val {
|
||||
continue nodeRouteLoop
|
||||
}
|
||||
|
||||
// if not, we need to failover the route
|
||||
failover := failoverRoute(isLikelyConnected, &route, routes)
|
||||
if failover != nil {
|
||||
err := failover.save(tx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("saving failover routes: %w", err)
|
||||
}
|
||||
|
||||
changedNodes.Add(failover.old.Node.ID)
|
||||
changedNodes.Add(failover.new.Node.ID)
|
||||
|
||||
continue nodeRouteLoop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
chng := changedNodes.Slice()
|
||||
sort.SliceStable(chng, func(i, j int) bool {
|
||||
return chng[i] < chng[j]
|
||||
})
|
||||
|
||||
if len(changedNodes) != 0 {
|
||||
return ptr.To(types.UpdatePeerChanged(chng...)), nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// failoverRouteTx takes a route that is no longer available,
|
||||
// this can be either from:
|
||||
// - being disabled
|
||||
// - being deleted
|
||||
// - host going offline
|
||||
//
|
||||
// and tries to find a new route to take over its place.
|
||||
// If the given route was not primary, it returns early.
|
||||
func failoverRouteTx(
|
||||
tx *gorm.DB,
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
r *types.Route,
|
||||
) ([]types.NodeID, error) {
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// This route is not a primary route, and it is not
|
||||
// being served to nodes.
|
||||
if !r.IsPrimary {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// We do not have to failover exit nodes
|
||||
if r.IsExitRoute() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
routes, err := getRoutesByPrefix(tx, netip.Prefix(r.Prefix))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting routes by prefix: %w", err)
|
||||
}
|
||||
|
||||
fo := failoverRoute(isLikelyConnected, r, routes)
|
||||
if fo == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
err = fo.save(tx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("saving failover route: %w", err)
|
||||
}
|
||||
|
||||
log.Trace().
|
||||
Str("hostname", fo.new.Node.Hostname).
|
||||
Msgf("set primary to new route, was: id(%d), host(%s), now: id(%d), host(%s)", fo.old.ID, fo.old.Node.Hostname, fo.new.ID, fo.new.Node.Hostname)
|
||||
|
||||
// Return a list of the machinekeys of the changed nodes.
|
||||
return []types.NodeID{fo.old.Node.ID, fo.new.Node.ID}, nil
|
||||
}
|
||||
|
||||
type failover struct {
|
||||
old *types.Route
|
||||
new *types.Route
|
||||
}
|
||||
|
||||
func (f *failover) save(tx *gorm.DB) error {
|
||||
err := tx.Save(f.old).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("saving old primary: %w", err)
|
||||
}
|
||||
|
||||
err = tx.Save(f.new).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("saving new primary: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func failoverRoute(
|
||||
isLikelyConnected *xsync.MapOf[types.NodeID, bool],
|
||||
routeToReplace *types.Route,
|
||||
altRoutes types.Routes,
|
||||
) *failover {
|
||||
if routeToReplace == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// This route is not a primary route, and it is not
|
||||
// being served to nodes.
|
||||
if !routeToReplace.IsPrimary {
|
||||
return nil
|
||||
}
|
||||
|
||||
// We do not have to failover exit nodes
|
||||
if routeToReplace.IsExitRoute() {
|
||||
return nil
|
||||
}
|
||||
|
||||
var newPrimary *types.Route
|
||||
|
||||
// Find a new suitable route
|
||||
for idx, route := range altRoutes {
|
||||
if routeToReplace.ID == route.ID {
|
||||
continue
|
||||
}
|
||||
|
||||
if !route.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
if isLikelyConnected != nil {
|
||||
if val, ok := isLikelyConnected.Load(route.Node.ID); ok && val {
|
||||
newPrimary = &altRoutes[idx]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If a new route was not found/available,
|
||||
// return without an error.
|
||||
// We do not want to update the database as
|
||||
// the one currently marked as primary is the
|
||||
// best we got.
|
||||
if newPrimary == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
routeToReplace.IsPrimary = false
|
||||
newPrimary.IsPrimary = true
|
||||
|
||||
return &failover{
|
||||
old: routeToReplace,
|
||||
new: newPrimary,
|
||||
}
|
||||
}
|
||||
|
||||
func (hsdb *HSDatabase) EnableAutoApprovedRoutes(
|
||||
polMan policy.PolicyManager,
|
||||
node *types.Node,
|
||||
) error {
|
||||
return hsdb.Write(func(tx *gorm.DB) error {
|
||||
return EnableAutoApprovedRoutes(tx, polMan, node)
|
||||
})
|
||||
}
|
||||
|
||||
// EnableAutoApprovedRoutes enables any routes advertised by a node that match the ACL autoApprovers policy.
|
||||
func EnableAutoApprovedRoutes(
|
||||
tx *gorm.DB,
|
||||
polMan policy.PolicyManager,
|
||||
node *types.Node,
|
||||
) error {
|
||||
if node.IPv4 == nil && node.IPv6 == nil {
|
||||
return nil // This node has no IPAddresses, so can't possibly match any autoApprovers ACLs
|
||||
}
|
||||
|
||||
routes, err := GetNodeAdvertisedRoutes(tx, node)
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return fmt.Errorf("getting advertised routes for node(%s %d): %w", node.Hostname, node.ID, err)
|
||||
}
|
||||
|
||||
log.Trace().Interface("routes", routes).Msg("routes for autoapproving")
|
||||
|
||||
var approvedRoutes types.Routes
|
||||
|
||||
for _, advertisedRoute := range routes {
|
||||
if advertisedRoute.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
routeApprovers := polMan.ApproversForRoute(netip.Prefix(advertisedRoute.Prefix))
|
||||
|
||||
log.Trace().
|
||||
Str("node", node.Hostname).
|
||||
Uint("user.id", node.User.ID).
|
||||
Strs("routeApprovers", routeApprovers).
|
||||
Str("prefix", netip.Prefix(advertisedRoute.Prefix).String()).
|
||||
Msg("looking up route for autoapproving")
|
||||
|
||||
for _, approvedAlias := range routeApprovers {
|
||||
if approvedAlias == node.User.Username() {
|
||||
approvedRoutes = append(approvedRoutes, advertisedRoute)
|
||||
} else {
|
||||
// TODO(kradalby): figure out how to get this to depend on less stuff
|
||||
approvedIps, err := polMan.ExpandAlias(approvedAlias)
|
||||
if err != nil {
|
||||
return fmt.Errorf("expanding alias %q for autoApprovers: %w", approvedAlias, err)
|
||||
}
|
||||
|
||||
// approvedIPs should contain all of node's IPs if it matches the rule, so check for first
|
||||
if approvedIps.Contains(*node.IPv4) {
|
||||
approvedRoutes = append(approvedRoutes, advertisedRoute)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, approvedRoute := range approvedRoutes {
|
||||
_, err := EnableRoute(tx, uint64(approvedRoute.ID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("enabling approved route(%d): %w", approvedRoute.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
File diff suppressed because it is too large
Load Diff
@ -348,10 +348,7 @@ func (api headscaleV1APIServer) DeleteNode(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
changedNodes, err := api.h.db.DeleteNode(
|
||||
node,
|
||||
api.h.nodeNotifier.LikelyConnectedMap(),
|
||||
)
|
||||
err = api.h.db.DeleteNode(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -359,10 +356,6 @@ func (api headscaleV1APIServer) DeleteNode(
|
||||
ctx = types.NotifyCtx(ctx, "cli-deletenode", node.Hostname)
|
||||
api.h.nodeNotifier.NotifyAll(ctx, types.UpdatePeerRemoved(node.ID))
|
||||
|
||||
if changedNodes != nil {
|
||||
api.h.nodeNotifier.NotifyAll(ctx, types.UpdatePeerChanged(changedNodes...))
|
||||
}
|
||||
|
||||
return &v1.DeleteNodeResponse{}, nil
|
||||
}
|
||||
|
||||
@ -533,99 +526,110 @@ func (api headscaleV1APIServer) BackfillNodeIPs(
|
||||
return &v1.BackfillNodeIPsResponse{Changes: changes}, nil
|
||||
}
|
||||
|
||||
func (api headscaleV1APIServer) GetRoutes(
|
||||
ctx context.Context,
|
||||
request *v1.GetRoutesRequest,
|
||||
) (*v1.GetRoutesResponse, error) {
|
||||
routes, err := db.Read(api.h.db.DB, func(rx *gorm.DB) (types.Routes, error) {
|
||||
return db.GetRoutes(rx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// TODO(kradalby): Replace this with route management on a per node basis,
|
||||
// similar to how the tailscale client does it:
|
||||
//
|
||||
// List all (or specific) routes:
|
||||
// headscale nodes list-routes [-i <node_id>]
|
||||
//
|
||||
// Approval of routes will overwrite all, so you define the ones you want,
|
||||
// you can approve routes that are not yet defined:
|
||||
//
|
||||
// headscale nodes approve-routes -i <node_id> "comma,separated,routes"
|
||||
//
|
||||
// func (api headscaleV1APIServer) GetRoutes(
|
||||
// ctx context.Context,
|
||||
// request *v1.GetRoutesRequest,
|
||||
// ) (*v1.GetRoutesResponse, error) {
|
||||
// routes, err := db.Read(api.h.db.DB, func(rx *gorm.DB) (types.Routes, error) {
|
||||
// return db.GetRoutes(rx)
|
||||
// })
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
return &v1.GetRoutesResponse{
|
||||
Routes: types.Routes(routes).Proto(),
|
||||
}, nil
|
||||
}
|
||||
// return &v1.GetRoutesResponse{
|
||||
// Routes: types.Routes(routes).Proto(),
|
||||
// }, nil
|
||||
// }
|
||||
|
||||
func (api headscaleV1APIServer) EnableRoute(
|
||||
ctx context.Context,
|
||||
request *v1.EnableRouteRequest,
|
||||
) (*v1.EnableRouteResponse, error) {
|
||||
update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) (*types.StateUpdate, error) {
|
||||
return db.EnableRoute(tx, request.GetRouteId())
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// func (api headscaleV1APIServer) EnableRoute(
|
||||
// ctx context.Context,
|
||||
// request *v1.EnableRouteRequest,
|
||||
// ) (*v1.EnableRouteResponse, error) {
|
||||
// update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) (*types.StateUpdate, error) {
|
||||
// return db.EnableRoute(tx, request.GetRouteId())
|
||||
// })
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
if update != nil {
|
||||
ctx := types.NotifyCtx(ctx, "cli-enableroute", "unknown")
|
||||
api.h.nodeNotifier.NotifyAll(
|
||||
ctx, *update)
|
||||
}
|
||||
// if update != nil {
|
||||
// ctx := types.NotifyCtx(ctx, "cli-enableroute", "unknown")
|
||||
// api.h.nodeNotifier.NotifyAll(
|
||||
// ctx, *update)
|
||||
// }
|
||||
|
||||
return &v1.EnableRouteResponse{}, nil
|
||||
}
|
||||
// return &v1.EnableRouteResponse{}, nil
|
||||
// }
|
||||
|
||||
func (api headscaleV1APIServer) DisableRoute(
|
||||
ctx context.Context,
|
||||
request *v1.DisableRouteRequest,
|
||||
) (*v1.DisableRouteResponse, error) {
|
||||
update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
|
||||
return db.DisableRoute(tx, request.GetRouteId(), api.h.nodeNotifier.LikelyConnectedMap())
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// func (api headscaleV1APIServer) DisableRoute(
|
||||
// ctx context.Context,
|
||||
// request *v1.DisableRouteRequest,
|
||||
// ) (*v1.DisableRouteResponse, error) {
|
||||
// update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
|
||||
// return db.DisableRoute(tx, request.GetRouteId(), api.h.nodeNotifier.LikelyConnectedMap())
|
||||
// })
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
if update != nil {
|
||||
ctx := types.NotifyCtx(ctx, "cli-disableroute", "unknown")
|
||||
api.h.nodeNotifier.NotifyAll(ctx, types.UpdatePeerChanged(update...))
|
||||
}
|
||||
// if update != nil {
|
||||
// ctx := types.NotifyCtx(ctx, "cli-disableroute", "unknown")
|
||||
// api.h.nodeNotifier.NotifyAll(ctx, types.UpdatePeerChanged(update...))
|
||||
// }
|
||||
|
||||
return &v1.DisableRouteResponse{}, nil
|
||||
}
|
||||
// return &v1.DisableRouteResponse{}, nil
|
||||
// }
|
||||
|
||||
func (api headscaleV1APIServer) GetNodeRoutes(
|
||||
ctx context.Context,
|
||||
request *v1.GetNodeRoutesRequest,
|
||||
) (*v1.GetNodeRoutesResponse, error) {
|
||||
node, err := api.h.db.GetNodeByID(types.NodeID(request.GetNodeId()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// func (api headscaleV1APIServer) GetNodeRoutes(
|
||||
// ctx context.Context,
|
||||
// request *v1.GetNodeRoutesRequest,
|
||||
// ) (*v1.GetNodeRoutesResponse, error) {
|
||||
// node, err := api.h.db.GetNodeByID(types.NodeID(request.GetNodeId()))
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
routes, err := api.h.db.GetNodeRoutes(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// routes, err := api.h.db.GetNodeRoutes(node)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
return &v1.GetNodeRoutesResponse{
|
||||
Routes: types.Routes(routes).Proto(),
|
||||
}, nil
|
||||
}
|
||||
// return &v1.GetNodeRoutesResponse{
|
||||
// Routes: types.Routes(routes).Proto(),
|
||||
// }, nil
|
||||
// }
|
||||
|
||||
func (api headscaleV1APIServer) DeleteRoute(
|
||||
ctx context.Context,
|
||||
request *v1.DeleteRouteRequest,
|
||||
) (*v1.DeleteRouteResponse, error) {
|
||||
isConnected := api.h.nodeNotifier.LikelyConnectedMap()
|
||||
update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
|
||||
return db.DeleteRoute(tx, request.GetRouteId(), isConnected)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// func (api headscaleV1APIServer) DeleteRoute(
|
||||
// ctx context.Context,
|
||||
// request *v1.DeleteRouteRequest,
|
||||
// ) (*v1.DeleteRouteResponse, error) {
|
||||
// isConnected := api.h.nodeNotifier.LikelyConnectedMap()
|
||||
// update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
|
||||
// return db.DeleteRoute(tx, request.GetRouteId(), isConnected)
|
||||
// })
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
if update != nil {
|
||||
ctx := types.NotifyCtx(ctx, "cli-deleteroute", "unknown")
|
||||
api.h.nodeNotifier.NotifyAll(ctx, types.UpdatePeerChanged(update...))
|
||||
}
|
||||
// if update != nil {
|
||||
// ctx := types.NotifyCtx(ctx, "cli-deleteroute", "unknown")
|
||||
// api.h.nodeNotifier.NotifyAll(ctx, types.UpdatePeerChanged(update...))
|
||||
// }
|
||||
|
||||
return &v1.DeleteRouteResponse{}, nil
|
||||
}
|
||||
// return &v1.DeleteRouteResponse{}, nil
|
||||
// }
|
||||
|
||||
func (api headscaleV1APIServer) CreateApiKey(
|
||||
ctx context.Context,
|
||||
|
@ -49,17 +49,8 @@ func tailNode(
|
||||
[]netip.Prefix{},
|
||||
addrs...) // we append the node own IP, as it is required by the clients
|
||||
|
||||
primaryPrefixes := []netip.Prefix{}
|
||||
|
||||
for _, route := range node.Routes {
|
||||
if route.Enabled {
|
||||
if route.IsPrimary {
|
||||
allowedIPs = append(allowedIPs, netip.Prefix(route.Prefix))
|
||||
primaryPrefixes = append(primaryPrefixes, netip.Prefix(route.Prefix))
|
||||
} else if route.IsExitRoute() {
|
||||
allowedIPs = append(allowedIPs, netip.Prefix(route.Prefix))
|
||||
}
|
||||
}
|
||||
for _, route := range node.SubnetRoutes() {
|
||||
allowedIPs = append(allowedIPs, netip.Prefix(route))
|
||||
}
|
||||
|
||||
var derp int
|
||||
@ -114,8 +105,6 @@ func tailNode(
|
||||
|
||||
Tags: tags,
|
||||
|
||||
PrimaryRoutes: primaryPrefixes,
|
||||
|
||||
MachineAuthorized: !node.IsExpired(),
|
||||
Expired: node.IsExpired(),
|
||||
}
|
||||
|
145
hscontrol/routes/primary.go
Normal file
145
hscontrol/routes/primary.go
Normal file
@ -0,0 +1,145 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
xmaps "golang.org/x/exp/maps"
|
||||
"tailscale.com/util/deephash"
|
||||
"tailscale.com/util/set"
|
||||
)
|
||||
|
||||
type PrimaryRoutes struct {
|
||||
mu sync.Mutex
|
||||
|
||||
// routes is a map of prefixes that are adverties and approved and available
|
||||
// in the global headscale state.
|
||||
routes map[types.NodeID]set.Set[netip.Prefix]
|
||||
|
||||
// primaries is a map of prefixes to the node that is the primary for that prefix.
|
||||
primaries map[netip.Prefix]types.NodeID
|
||||
isPrimary map[types.NodeID]bool
|
||||
|
||||
primariesHash deephash.Sum
|
||||
}
|
||||
|
||||
func New() *PrimaryRoutes {
|
||||
return &PrimaryRoutes{
|
||||
routes: make(map[types.NodeID]set.Set[netip.Prefix]),
|
||||
primaries: make(map[netip.Prefix]types.NodeID),
|
||||
isPrimary: make(map[types.NodeID]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// updatePrimaryLocked recalculates the primary routes and updates the internal state.
|
||||
// It returns true if the primary routes have changed.
|
||||
// It is assumed that the caller holds the lock.
|
||||
// The algorthm is as follows:
|
||||
// 1. Reset the primaries map.
|
||||
// 2. Iterate over the routes and count the number of times a prefix is advertised.
|
||||
// 3. If a prefix is advertised by at least two nodes, it is a primary route.
|
||||
// 4. If the primary routes have changed, update the internal state and return true.
|
||||
// 5. Otherwise, return false.
|
||||
func (pr *PrimaryRoutes) updatePrimaryLocked() bool {
|
||||
// reset the primaries map, as we are going to recalculate it.
|
||||
newPrimaries := make(map[netip.Prefix]types.NodeID)
|
||||
pr.isPrimary = make(map[types.NodeID]bool)
|
||||
count := make(map[netip.Prefix]int)
|
||||
|
||||
// sort the node ids so we can iterate over them in a deterministic order.
|
||||
// this is important so the same node is chosen two times in a row
|
||||
// as the primary route.
|
||||
ids := types.NodeIDs(xmaps.Keys(pr.routes))
|
||||
sort.Sort(ids)
|
||||
for _, id := range ids {
|
||||
prefixes := pr.routes[id]
|
||||
for prefix := range prefixes {
|
||||
if _, ok := count[prefix]; !ok {
|
||||
count[prefix] = 1
|
||||
} else {
|
||||
count[prefix]++
|
||||
}
|
||||
if _, ok := newPrimaries[prefix]; !ok {
|
||||
newPrimaries[prefix] = id
|
||||
pr.isPrimary[id] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A primary is a route that is advertised by at least two nodes.
|
||||
for prefix, c := range count {
|
||||
if c < 2 {
|
||||
delete(newPrimaries, prefix)
|
||||
}
|
||||
}
|
||||
|
||||
primariesHash := deephash.Hash(&newPrimaries)
|
||||
if primariesHash == pr.primariesHash {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(newPrimaries) == 0 || len(newPrimaries) == len(pr.primaries) {
|
||||
return false
|
||||
}
|
||||
|
||||
pr.primaries = newPrimaries
|
||||
pr.primariesHash = primariesHash
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (pr *PrimaryRoutes) RegisterRoutes(node types.NodeID, prefix ...netip.Prefix) bool {
|
||||
pr.mu.Lock()
|
||||
defer pr.mu.Unlock()
|
||||
|
||||
if _, ok := pr.routes[node]; !ok {
|
||||
pr.routes[node] = make(set.Set[netip.Prefix], len(prefix))
|
||||
}
|
||||
|
||||
for _, p := range prefix {
|
||||
pr.routes[node].Add(p)
|
||||
}
|
||||
|
||||
return pr.updatePrimaryLocked()
|
||||
}
|
||||
|
||||
func (pr *PrimaryRoutes) DeregisterRoutes(node types.NodeID, prefix ...netip.Prefix) bool {
|
||||
pr.mu.Lock()
|
||||
defer pr.mu.Unlock()
|
||||
|
||||
if _, ok := pr.routes[node]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, p := range prefix {
|
||||
pr.routes[node].Delete(p)
|
||||
}
|
||||
|
||||
return pr.updatePrimaryLocked()
|
||||
}
|
||||
|
||||
func (pr *PrimaryRoutes) PrimaryRoutes(id types.NodeID) []netip.Prefix {
|
||||
if pr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
pr.mu.Lock()
|
||||
defer pr.mu.Unlock()
|
||||
|
||||
// Short circuit if the node is not a primary for any route.
|
||||
if _, ok := pr.isPrimary[id]; !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
var routes []netip.Prefix
|
||||
|
||||
for prefix, node := range pr.primaries {
|
||||
if node == id {
|
||||
routes = append(routes, prefix)
|
||||
}
|
||||
}
|
||||
|
||||
return routes
|
||||
}
|
190
hscontrol/routes/primary_test.go
Normal file
190
hscontrol/routes/primary_test.go
Normal file
@ -0,0 +1,190 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
)
|
||||
|
||||
// mp is a helper function that wraps netip.MustParsePrefix.
|
||||
func mp(prefix string) netip.Prefix {
|
||||
return netip.MustParsePrefix(prefix)
|
||||
}
|
||||
|
||||
func TestPrimaryRoutes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
operations func(pr *PrimaryRoutes) bool
|
||||
nodeID types.NodeID
|
||||
expectedRoutes []netip.Prefix
|
||||
expectedChange bool
|
||||
}{
|
||||
{
|
||||
name: "single-node-registers-single-route",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
return pr.RegisterRoutes(1, mp("192.168.1.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "multiple-nodes-register-different-routes",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("192.168.1.0/24"))
|
||||
return pr.RegisterRoutes(2, mp("192.168.2.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "multiple-nodes-register-overlapping-routes",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("192.168.1.0/24")) // false
|
||||
return pr.RegisterRoutes(2, mp("192.168.1.0/24")) // true
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
|
||||
expectedChange: true,
|
||||
},
|
||||
{
|
||||
name: "node-deregisters-a-route",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("192.168.1.0/24"))
|
||||
return pr.DeregisterRoutes(1, mp("192.168.1.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "node-deregisters-one-of-multiple-routes",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("192.168.1.0/24"), mp("192.168.2.0/24"))
|
||||
return pr.DeregisterRoutes(1, mp("192.168.1.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "node-registers-and-deregisters-routes-in-sequence",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("192.168.1.0/24"))
|
||||
pr.RegisterRoutes(2, mp("192.168.2.0/24"))
|
||||
pr.DeregisterRoutes(1, mp("192.168.1.0/24"))
|
||||
return pr.RegisterRoutes(1, mp("192.168.3.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "no-change-in-primary-routes",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
return pr.RegisterRoutes(1, mp("192.168.1.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "multiple-nodes-register-same-route",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("192.168.1.0/24")) // false
|
||||
pr.RegisterRoutes(2, mp("192.168.1.0/24")) // true
|
||||
return pr.RegisterRoutes(3, mp("192.168.1.0/24")) // false
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "multiple-nodes-register-same-route-and-exit",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
pr.RegisterRoutes(1, mp("0.0.0.0/0"), mp("192.168.1.0/24"))
|
||||
return pr.RegisterRoutes(2, mp("192.168.1.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: []netip.Prefix{mp("192.168.1.0/24")},
|
||||
expectedChange: true,
|
||||
},
|
||||
{
|
||||
name: "deregister-non-existent-route",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
return pr.DeregisterRoutes(1, mp("192.168.1.0/24"))
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "register-empty-prefix-list",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
return pr.RegisterRoutes(1)
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "deregister-empty-prefix-list",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
return pr.DeregisterRoutes(1)
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "concurrent-access",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
var change1, change2 bool
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
change1 = pr.RegisterRoutes(1, mp("192.168.1.0/24"))
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
change2 = pr.RegisterRoutes(2, mp("192.168.2.0/24"))
|
||||
}()
|
||||
wg.Wait()
|
||||
return change1 || change2
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
{
|
||||
name: "no-routes-registered",
|
||||
operations: func(pr *PrimaryRoutes) bool {
|
||||
// No operations
|
||||
return false
|
||||
},
|
||||
nodeID: 1,
|
||||
expectedRoutes: nil,
|
||||
expectedChange: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pr := New()
|
||||
change := tt.operations(pr)
|
||||
if change != tt.expectedChange {
|
||||
t.Errorf("change = %v, want %v", change, tt.expectedChange)
|
||||
}
|
||||
routes := pr.PrimaryRoutes(tt.nodeID)
|
||||
if diff := cmp.Diff(tt.expectedRoutes, routes, util.Comparers...); diff != "" {
|
||||
t.Errorf("PrimaryRoutes() mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -25,8 +26,11 @@ var (
|
||||
)
|
||||
|
||||
type NodeID uint64
|
||||
type NodeIDs []NodeID
|
||||
|
||||
// type NodeConnectedMap *xsync.MapOf[NodeID, bool]
|
||||
func (n NodeIDs) Len() int { return len(n) }
|
||||
func (n NodeIDs) Less(i, j int) bool { return n[i] < n[j] }
|
||||
func (n NodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
|
||||
|
||||
func (id NodeID) StableID() tailcfg.StableNodeID {
|
||||
return tailcfg.StableNodeID(strconv.FormatUint(uint64(id), util.Base10))
|
||||
@ -87,7 +91,15 @@ type Node struct {
|
||||
LastSeen *time.Time
|
||||
Expiry *time.Time
|
||||
|
||||
Routes []Route `gorm:"constraint:OnDelete:CASCADE;"`
|
||||
// DEPRECATED: Use the ApprovedRoutes field instead.
|
||||
// TODO(kradalby): remove when ApprovedRoutes is used all over the code.
|
||||
// Routes []Route `gorm:"constraint:OnDelete:CASCADE;"`
|
||||
|
||||
// ApprovedRoutes is a list of routes that the node is allowed to announce
|
||||
// as a subnet router. They are not necessarily the routes that the node
|
||||
// announces at the moment.
|
||||
// See [Node.Hostinfo]
|
||||
ApprovedRoutes []netip.Prefix `gorm:"serializer:json"`
|
||||
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
@ -185,17 +197,12 @@ func (node *Node) CanAccess(filter []tailcfg.FilterRule, node2 *Node) bool {
|
||||
|
||||
// TODO(kradalby): Regenerate this every time the filter change, instead of
|
||||
// every time we use it.
|
||||
// Part of #2416
|
||||
matchers := make([]matcher.Match, len(filter))
|
||||
for i, rule := range filter {
|
||||
matchers[i] = matcher.MatchFromFilterRule(rule)
|
||||
}
|
||||
|
||||
for _, route := range node2.Routes {
|
||||
if route.Enabled {
|
||||
allowedIPs = append(allowedIPs, netip.Prefix(route.Prefix).Addr())
|
||||
}
|
||||
}
|
||||
|
||||
for _, matcher := range matchers {
|
||||
if !matcher.SrcsContainsIPs(src) {
|
||||
continue
|
||||
@ -297,6 +304,29 @@ func (node *Node) GetFQDN(baseDomain string) (string, error) {
|
||||
return hostname, nil
|
||||
}
|
||||
|
||||
// AnnouncedRoutes returns the list of routes that the node announces.
|
||||
// It should be used instead of checking Hostinfo.RoutableIPs directly.
|
||||
func (node *Node) AnnouncedRoutes() []netip.Prefix {
|
||||
if node.Hostinfo == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return node.Hostinfo.RoutableIPs
|
||||
}
|
||||
|
||||
// SubnetRoutes returns the list of routes that the node announces and are approved.
|
||||
func (node *Node) SubnetRoutes() []netip.Prefix {
|
||||
var routes []netip.Prefix
|
||||
|
||||
for _, route := range node.AnnouncedRoutes() {
|
||||
if slices.Contains(node.ApprovedRoutes, route) {
|
||||
routes = append(routes, route)
|
||||
}
|
||||
}
|
||||
|
||||
return routes
|
||||
}
|
||||
|
||||
// func (node *Node) String() string {
|
||||
// return node.Hostname
|
||||
// }
|
||||
|
@ -1,102 +1,28 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
|
||||
v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
"gorm.io/gorm"
|
||||
"tailscale.com/net/tsaddr"
|
||||
)
|
||||
|
||||
// DEPRECATED: Approval of routes is denormalised onto the relevant node.
|
||||
// Struct is kept for GORM migrations only.
|
||||
type Route struct {
|
||||
gorm.Model
|
||||
|
||||
NodeID uint64 `gorm:"not null"`
|
||||
Node *Node
|
||||
|
||||
// TODO(kradalby): change this custom type to netip.Prefix
|
||||
Prefix netip.Prefix `gorm:"serializer:text"`
|
||||
|
||||
// Advertised is now only stored as part of [Node.Hostinfo].
|
||||
Advertised bool
|
||||
Enabled bool
|
||||
IsPrimary bool
|
||||
}
|
||||
|
||||
type Routes []Route
|
||||
|
||||
func (r *Route) String() string {
|
||||
return fmt.Sprintf("%s:%s", r.Node.Hostname, netip.Prefix(r.Prefix).String())
|
||||
}
|
||||
|
||||
func (r *Route) IsExitRoute() bool {
|
||||
return tsaddr.IsExitRoute(r.Prefix)
|
||||
}
|
||||
|
||||
func (r *Route) IsAnnouncable() bool {
|
||||
return r.Advertised && r.Enabled
|
||||
}
|
||||
|
||||
func (rs Routes) Prefixes() []netip.Prefix {
|
||||
prefixes := make([]netip.Prefix, len(rs))
|
||||
for i, r := range rs {
|
||||
prefixes[i] = netip.Prefix(r.Prefix)
|
||||
}
|
||||
|
||||
return prefixes
|
||||
}
|
||||
|
||||
// Primaries returns Primary routes from a list of routes.
|
||||
func (rs Routes) Primaries() Routes {
|
||||
res := make(Routes, 0)
|
||||
for _, route := range rs {
|
||||
if route.IsPrimary {
|
||||
res = append(res, route)
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (rs Routes) PrefixMap() map[netip.Prefix][]Route {
|
||||
res := map[netip.Prefix][]Route{}
|
||||
|
||||
for _, route := range rs {
|
||||
if _, ok := res[route.Prefix]; ok {
|
||||
res[route.Prefix] = append(res[route.Prefix], route)
|
||||
} else {
|
||||
res[route.Prefix] = []Route{route}
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (rs Routes) Proto() []*v1.Route {
|
||||
protoRoutes := []*v1.Route{}
|
||||
|
||||
for _, route := range rs {
|
||||
protoRoute := v1.Route{
|
||||
Id: uint64(route.ID),
|
||||
Prefix: route.Prefix.String(),
|
||||
Advertised: route.Advertised,
|
||||
Enabled: route.Enabled,
|
||||
IsPrimary: route.IsPrimary,
|
||||
CreatedAt: timestamppb.New(route.CreatedAt),
|
||||
UpdatedAt: timestamppb.New(route.UpdatedAt),
|
||||
}
|
||||
|
||||
if route.Node != nil {
|
||||
protoRoute.Node = route.Node.Proto()
|
||||
}
|
||||
|
||||
if route.DeletedAt.Valid {
|
||||
protoRoute.DeletedAt = timestamppb.New(route.DeletedAt.Time)
|
||||
}
|
||||
|
||||
protoRoutes = append(protoRoutes, &protoRoute)
|
||||
}
|
||||
|
||||
return protoRoutes
|
||||
|
||||
// Enabled is stored directly on the node as ApprovedRoutes.
|
||||
Enabled bool
|
||||
|
||||
// IsPrimary is only determined in memory as it is only relevant
|
||||
// when the server is up.
|
||||
IsPrimary bool
|
||||
}
|
||||
|
@ -1,89 +0,0 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
)
|
||||
|
||||
func TestPrefixMap(t *testing.T) {
|
||||
ipp := func(s string) netip.Prefix { return netip.MustParsePrefix(s) }
|
||||
|
||||
tests := []struct {
|
||||
rs Routes
|
||||
want map[netip.Prefix][]Route
|
||||
}{
|
||||
{
|
||||
rs: Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
},
|
||||
},
|
||||
want: map[netip.Prefix][]Route{
|
||||
ipp("10.0.0.0/24"): Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
rs: Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
},
|
||||
Route{
|
||||
Prefix: ipp("10.0.1.0/24"),
|
||||
},
|
||||
},
|
||||
want: map[netip.Prefix][]Route{
|
||||
ipp("10.0.0.0/24"): Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
},
|
||||
},
|
||||
ipp("10.0.1.0/24"): Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.1.0/24"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
rs: Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
Enabled: true,
|
||||
},
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
Enabled: false,
|
||||
},
|
||||
},
|
||||
want: map[netip.Prefix][]Route{
|
||||
ipp("10.0.0.0/24"): Routes{
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
Enabled: true,
|
||||
},
|
||||
Route{
|
||||
Prefix: ipp("10.0.0.0/24"),
|
||||
Enabled: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for idx, tt := range tests {
|
||||
t.Run(fmt.Sprintf("test-%d", idx), func(t *testing.T) {
|
||||
got := tt.rs.PrefixMap()
|
||||
if diff := cmp.Diff(tt.want, got, util.Comparers...); diff != "" {
|
||||
t.Errorf("PrefixMap() unexpected result (-want +got):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user