From 0475eb6ef7ffc70fa544c9f31f1e697d4235323e Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 21:58:28 +0100
Subject: [PATCH 01/14] Move DB call of pollmap to Machine inside a function

---
 api.go     |  1 +
 go.mod     |  4 ++--
 machine.go |  9 +++++++++
 poll.go    | 18 ++++++++++++------
 4 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/api.go b/api.go
index e2a56185..304f9a84 100644
--- a/api.go
+++ b/api.go
@@ -226,6 +226,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 			Msg("Cannot convert to node")
 		return nil, err
 	}
+
 	peers, err := h.getPeers(m)
 	if err != nil {
 		log.Error().
diff --git a/go.mod b/go.mod
index 51acff89..3a49c0fd 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/docker/docker v20.10.8+incompatible // indirect
 	github.com/efekarakus/termcolor v1.0.1
 	github.com/gin-gonic/gin v1.7.4
-	github.com/gofrs/uuid v4.0.0+incompatible // indirect
+	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/google/go-github v17.0.0+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
@@ -26,7 +26,7 @@ require (
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88
-	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e // indirect
+	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
diff --git a/machine.go b/machine.go
index 1d4939c1..7e266b67 100644
--- a/machine.go
+++ b/machine.go
@@ -240,6 +240,15 @@ func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
 	return &m, nil
 }
 
+// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
 // UpdateMachine takes a Machine struct pointer (typically already loaded from database
 // and updates it with the latest data from the database.
 func (h *Headscale) UpdateMachine(m *Machine) error {
diff --git a/poll.go b/poll.go
index 60bfa9ea..ca4f6764 100644
--- a/poll.go
+++ b/poll.go
@@ -51,13 +51,19 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		return
 	}
 
-	var m Machine
-	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey.HexString()); errors.Is(result.Error, gorm.ErrRecordNotFound) {
-		log.Warn().
+	m, err := h.GetMachineByMachineKey(mKey.HexString())
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			log.Warn().
+				Str("handler", "PollNetMap").
+				Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
+			c.String(http.StatusUnauthorized, "")
+			return
+		}
+		log.Error().
 			Str("handler", "PollNetMap").
-			Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
-		c.String(http.StatusUnauthorized, "")
-		return
+			Msgf("Failed to fetch machine from the database with Machine key: %s", mKey.HexString())
+		c.String(http.StatusInternalServerError, "")
 	}
 	log.Trace().
 		Str("handler", "PollNetMap").

From 0d4a006536a5d854f7c822f617a810354800ab1f Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 22:00:09 +0100
Subject: [PATCH 02/14] Consitently use Machine pointers

This commit rewrites a bunch of the code to always use *Machine instead
of a mix of both, and a mix of tailcfg.Node and Machine.

Now we use *Machine, and if tailcfg.Node is needed, it is converted just
before needed.
---
 api.go          |  4 +--
 machine.go      |  4 +--
 machine_test.go |  4 +--
 poll.go         | 25 ++++++++-------
 sharing_test.go | 82 ++++++++++++++++++++++++-------------------------
 5 files changed, 58 insertions(+), 61 deletions(-)

diff --git a/api.go b/api.go
index 304f9a84..72cb92f6 100644
--- a/api.go
+++ b/api.go
@@ -213,7 +213,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	c.Data(200, "application/json; charset=utf-8", respBody)
 }
 
-func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
@@ -286,7 +286,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 	return &data, nil
 }
 
-func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
 	resp := tailcfg.MapResponse{
 		KeepAlive: true,
 	}
diff --git a/machine.go b/machine.go
index 7e266b67..f49c5a89 100644
--- a/machine.go
+++ b/machine.go
@@ -296,7 +296,7 @@ func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
 }
 
 func (h *Headscale) notifyChangesToPeers(m *Machine) {
-	peers, err := h.getPeers(*m)
+	peers, err := h.getPeers(m)
 	if err != nil {
 		log.Error().
 			Str("func", "notifyChangesToPeers").
@@ -363,7 +363,7 @@ func (h *Headscale) closeUpdateChannel(m *Machine) {
 	h.clientsUpdateChannels.Delete(m.ID)
 }
 
-func (h *Headscale) sendRequestOnUpdateChannel(m *tailcfg.Node) error {
+func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
 	h.clientsUpdateChannelMutex.Lock()
 	defer h.clientsUpdateChannelMutex.Unlock()
 
diff --git a/machine_test.go b/machine_test.go
index d535be56..cf0d12e7 100644
--- a/machine_test.go
+++ b/machine_test.go
@@ -16,7 +16,7 @@ func (s *Suite) TestGetMachine(c *check.C) {
 	_, err = h.GetMachine("test", "testmachine")
 	c.Assert(err, check.NotNil)
 
-	m := Machine{
+	m := &Machine{
 		ID:             0,
 		MachineKey:     "foo",
 		NodeKey:        "bar",
@@ -27,7 +27,7 @@ func (s *Suite) TestGetMachine(c *check.C) {
 		RegisterMethod: "authKey",
 		AuthKeyID:      uint(pak.ID),
 	}
-	h.db.Save(&m)
+	h.db.Save(m)
 
 	m1, err := h.GetMachine("test", "testmachine")
 	c.Assert(err, check.IsNil)
diff --git a/poll.go b/poll.go
index ca4f6764..8032edee 100644
--- a/poll.go
+++ b/poll.go
@@ -140,7 +140,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("id", c.Param("id")).
 		Str("machine", m.Name).
 		Msg("Loading or creating update channel")
-	updateChan := h.getOrOpenUpdateChannel(&m)
+	updateChan := h.getOrOpenUpdateChannel(m)
 
 	pollDataChan := make(chan []byte)
 	// defer close(pollData)
@@ -159,7 +159,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
-		go h.notifyChangesToPeers(&m)
+		go h.notifyChangesToPeers(m)
 		return
 	} else if req.OmitPeers && req.Stream {
 		log.Warn().
@@ -184,7 +184,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Notifying peers")
-	go h.notifyChangesToPeers(&m)
+	go h.notifyChangesToPeers(m)
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
 	log.Trace().
@@ -199,7 +199,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 // to the connected clients.
 func (h *Headscale) PollNetMapStream(
 	c *gin.Context,
-	m Machine,
+	m *Machine,
 	req tailcfg.MapRequest,
 	mKey wgkey.Key,
 	pollDataChan chan []byte,
@@ -246,7 +246,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err = h.UpdateMachine(&m)
+			err = h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -292,7 +292,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err = h.UpdateMachine(&m)
+			err = h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -318,7 +318,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "update").
 				Msg("Received a request for update")
-			if h.isOutdated(&m) {
+			if h.isOutdated(m) {
 				log.Debug().
 					Str("handler", "PollNetMapStream").
 					Str("machine", m.Name).
@@ -356,7 +356,7 @@ func (h *Headscale) PollNetMapStream(
 					// TODO: Abstract away all the database calls, this can cause race conditions
 					// when an outdated machine object is kept alive, e.g. db is update from
 					// command line, but then overwritten.
-				err = h.UpdateMachine(&m)
+				err = h.UpdateMachine(m)
 				if err != nil {
 					log.Error().
 						Str("handler", "PollNetMapStream").
@@ -386,7 +386,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err := h.UpdateMachine(&m)
+			err := h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -401,7 +401,7 @@ func (h *Headscale) PollNetMapStream(
 
 			cancelKeepAlive <- struct{}{}
 
-			h.closeUpdateChannel(&m)
+			h.closeUpdateChannel(m)
 
 			close(pollDataChan)
 
@@ -417,7 +417,7 @@ func (h *Headscale) scheduledPollWorker(
 	keepAliveChan chan<- []byte,
 	mKey wgkey.Key,
 	req tailcfg.MapRequest,
-	m Machine,
+	m *Machine,
 ) {
 	keepAliveTicker := time.NewTicker(60 * time.Second)
 	updateCheckerTicker := time.NewTicker(30 * time.Second)
@@ -446,8 +446,7 @@ func (h *Headscale) scheduledPollWorker(
 		case <-updateCheckerTicker.C:
 			// Send an update request regardless of outdated or not, if data is sent
 			// to the node is determined in the updateChan consumer block
-			n, _ := m.toNode(true)
-			err := h.sendRequestOnUpdateChannel(n)
+			err := h.sendRequestOnUpdateChannel(m)
 			if err != nil {
 				log.Error().
 					Str("func", "keepAlive").
diff --git a/sharing_test.go b/sharing_test.go
index ec4951de..25de5846 100644
--- a/sharing_test.go
+++ b/sharing_test.go
@@ -2,7 +2,6 @@ package headscale
 
 import (
 	"gopkg.in/check.v1"
-	"tailscale.com/tailcfg"
 )
 
 func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
@@ -21,7 +20,7 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -33,12 +32,12 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -50,22 +49,22 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 1)
-	c.Assert((*p1sAfter)[0].ID, check.Equals, tailcfg.NodeID(m2.ID))
+	c.Assert(len(p1sAfter), check.Equals, 1)
+	c.Assert(p1sAfter[0].ID, check.Equals, m2.ID)
 }
 
 func (s *Suite) TestSameNamespace(c *check.C) {
@@ -84,7 +83,7 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -96,12 +95,12 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -113,16 +112,16 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m1, n1)
+	err = h.AddSharedMachineToNamespace(m1, n1)
 	c.Assert(err, check.Equals, errorSameNamespace)
 }
 
@@ -142,7 +141,7 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -154,12 +153,12 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -171,18 +170,18 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.Equals, errorMachineAlreadyShared)
 }
 
@@ -202,7 +201,7 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -214,12 +213,12 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -231,22 +230,21 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 1)
-	c.Assert(len((*p1sAfter)[0].AllowedIPs), check.Equals, 1)
+	c.Assert(len(p1sAfter), check.Equals, 1)
 }
 
 func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
@@ -274,7 +272,7 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -286,12 +284,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -303,12 +301,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
-	m3 := Machine{
+	m3 := &Machine{
 		ID:             2,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -320,12 +318,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.3",
 		AuthKeyID:      uint(pak3.ID),
 	}
-	h.db.Save(&m3)
+	h.db.Save(m3)
 
 	_, err = h.GetMachine(n3.Name, m3.Name)
 	c.Assert(err, check.IsNil)
 
-	m4 := Machine{
+	m4 := &Machine{
 		ID:             3,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -337,23 +335,23 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.4",
 		AuthKeyID:      uint(pak4.ID),
 	}
-	h.db.Save(&m4)
+	h.db.Save(m4)
 
 	_, err = h.GetMachine(n1.Name, m4.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 1) // nodes 1 and 4
+	c.Assert(len(p1s), check.Equals, 1) // nodes 1 and 4
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 2) // nodes 1, 2, 4
+	c.Assert(len(p1sAfter), check.Equals, 2) // nodes 1, 2, 4
 
 	pAlone, err := h.getPeers(m3)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*pAlone), check.Equals, 0) // node 3 is alone
+	c.Assert(len(pAlone), check.Equals, 0) // node 3 is alone
 }

From 3c3189caa612781fe271f11475f30461623b4f42 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 22:03:34 +0100
Subject: [PATCH 03/14] Move toNode, add type helpers, split peers and shared

This commit moves toNode to the bottom of the file, and adds a helper
function for lists of Machines to be converted.

It also adds string helpers for Machines and lists of machines.

Lastly it splits getPeers into getDirectPeers, which exist in the same
namespace, and getShared, which is nodes shared with the namespace.

getPeers is kept as a function putting together the two lists for
convenience.
---
 api.go     |  13 +-
 machine.go | 560 ++++++++++++++++++++++++++++++-----------------------
 2 files changed, 330 insertions(+), 243 deletions(-)

diff --git a/api.go b/api.go
index 72cb92f6..90d9be2b 100644
--- a/api.go
+++ b/api.go
@@ -242,11 +242,20 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 		DisplayName: m.Namespace.Name,
 	}
 
+	nodePeers, err := peers.toNodes(true)
+	if err != nil {
+		log.Error().
+			Str("func", "getMapResponse").
+			Err(err).
+			Msg("Failed to convert peers to Tailscale nodes")
+		return nil, err
+	}
+
 	resp := tailcfg.MapResponse{
 		KeepAlive: false,
 		Node:      node,
-		Peers:     *peers,
-		//TODO(kradalby): As per tailscale docs, if DNSConfig is nil,
+		Peers:     nodePeers,
+		// TODO(kradalby): As per tailscale docs, if DNSConfig is nil,
 		// it means its not updated, maybe we can have some logic
 		// to check and only pass updates when its updates.
 		// This is probably more relevant if we try to implement
diff --git a/machine.go b/machine.go
index f49c5a89..c87aba81 100644
--- a/machine.go
+++ b/machine.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -45,11 +46,329 @@ type Machine struct {
 	DeletedAt *time.Time
 }
 
+type (
+	Machines  []Machine
+	MachinesP []*Machine
+)
+
 // For the time being this method is rather naive
 func (m Machine) isAlreadyRegistered() bool {
 	return m.Registered
 }
 
+func (h *Headscale) getDirectPeers(m *Machine) (MachinesP, error) {
+	log.Trace().
+		Str("func", "getDirectPeers").
+		Str("machine", m.Name).
+		Msg("Finding peers")
+
+	machines := []Machine{}
+	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
+		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
+		log.Error().Err(err).Msg("Error accessing db")
+		return nil, err
+	}
+
+	peers := make(MachinesP, 0)
+	for _, peer := range machines {
+		peers = append(peers, &peer)
+	}
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getDirectPeers").
+		Str("machine", m.Name).
+		Msgf("Found peers: %s", peers.String())
+	return peers, nil
+}
+
+func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msg("Finding shared peers")
+
+	// We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for
+	sharedMachines := []SharedMachine{}
+	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
+		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
+		return nil, err
+	}
+
+	peers := make(MachinesP, 0)
+	for _, sharedMachine := range sharedMachines {
+		peers = append(peers, &sharedMachine.Machine)
+	}
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msgf("Found shared peers: %s", peers.String())
+	return peers, nil
+}
+
+func (h *Headscale) getPeers(m *Machine) (MachinesP, error) {
+	direct, err := h.getDirectPeers(m)
+	if err != nil {
+		log.Error().
+			Str("func", "getPeers").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return nil, err
+	}
+
+	shared, err := h.getShared(m)
+	if err != nil {
+		log.Error().
+			Str("func", "getDirectPeers").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return nil, err
+	}
+
+	return append(direct, shared...), nil
+}
+
+// GetMachine finds a Machine by name and namespace and returns the Machine struct
+func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) {
+	machines, err := h.ListMachinesInNamespace(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range *machines {
+		if m.Name == name {
+			return &m, nil
+		}
+	}
+	return nil, fmt.Errorf("machine not found")
+}
+
+// GetMachineByID finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
+// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
+// UpdateMachine takes a Machine struct pointer (typically already loaded from database
+// and updates it with the latest data from the database.
+func (h *Headscale) UpdateMachine(m *Machine) error {
+	if result := h.db.Find(m).First(&m); result.Error != nil {
+		return result.Error
+	}
+	return nil
+}
+
+// DeleteMachine softs deletes a Machine from the database
+func (h *Headscale) DeleteMachine(m *Machine) error {
+	m.Registered = false
+	namespaceID := m.NamespaceID
+	h.db.Save(&m) // we mark it as unregistered, just in case
+	if err := h.db.Delete(&m).Error; err != nil {
+		return err
+	}
+
+	return h.RequestMapUpdates(namespaceID)
+}
+
+// HardDeleteMachine hard deletes a Machine from the database
+func (h *Headscale) HardDeleteMachine(m *Machine) error {
+	namespaceID := m.NamespaceID
+	if err := h.db.Unscoped().Delete(&m).Error; err != nil {
+		return err
+	}
+	return h.RequestMapUpdates(namespaceID)
+}
+
+// GetHostInfo returns a Hostinfo struct for the machine
+func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
+	hostinfo := tailcfg.Hostinfo{}
+	if len(m.HostInfo) != 0 {
+		hi, err := m.HostInfo.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(hi, &hostinfo)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &hostinfo, nil
+}
+
+func (h *Headscale) notifyChangesToPeers(m *Machine) {
+	peers, err := h.getPeers(m)
+	if err != nil {
+		log.Error().
+			Str("func", "notifyChangesToPeers").
+			Str("machine", m.Name).
+			Msgf("Error getting peers: %s", err)
+		return
+	}
+	for _, peer := range peers {
+		log.Info().
+			Str("func", "notifyChangesToPeers").
+			Str("machine", m.Name).
+			Str("peer", peer.Name).
+			Str("address", peer.IPAddress).
+			Msgf("Notifying peer %s (%s)", peer.Name, peer.IPAddress)
+		err := h.sendRequestOnUpdateChannel(peer)
+		if err != nil {
+			log.Info().
+				Str("func", "notifyChangesToPeers").
+				Str("machine", m.Name).
+				Str("peer", peer.Name).
+				Msgf("Peer %s does not appear to be polling", peer.Name)
+		}
+		log.Trace().
+			Str("func", "notifyChangesToPeers").
+			Str("machine", m.Name).
+			Str("peer", peer.Name).
+			Str("address", peer.IPAddress).
+			Msgf("Notified peer %s (%s)", peer.Name, peer.IPAddress)
+	}
+}
+
+func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} {
+	var updateChan chan struct{}
+	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
+		if unwrapped, ok := storedChan.(chan struct{}); ok {
+			updateChan = unwrapped
+		} else {
+			log.Error().
+				Str("handler", "openUpdateChannel").
+				Str("machine", m.Name).
+				Msg("Failed to convert update channel to struct{}")
+		}
+	} else {
+		log.Debug().
+			Str("handler", "openUpdateChannel").
+			Str("machine", m.Name).
+			Msg("Update channel not found, creating")
+
+		updateChan = make(chan struct{})
+		h.clientsUpdateChannels.Store(m.ID, updateChan)
+	}
+	return updateChan
+}
+
+func (h *Headscale) closeUpdateChannel(m *Machine) {
+	h.clientsUpdateChannelMutex.Lock()
+	defer h.clientsUpdateChannelMutex.Unlock()
+
+	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
+		if unwrapped, ok := storedChan.(chan struct{}); ok {
+			close(unwrapped)
+		}
+	}
+	h.clientsUpdateChannels.Delete(m.ID)
+}
+
+func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
+	h.clientsUpdateChannelMutex.Lock()
+	defer h.clientsUpdateChannelMutex.Unlock()
+
+	pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID))
+	if ok {
+		log.Info().
+			Str("func", "requestUpdate").
+			Str("machine", m.Name).
+			Msgf("Notifying peer %s", m.Name)
+
+		if update, ok := pUp.(chan struct{}); ok {
+			log.Trace().
+				Str("func", "requestUpdate").
+				Str("machine", m.Name).
+				Msgf("Update channel is %#v", update)
+
+			update <- struct{}{}
+
+			log.Trace().
+				Str("func", "requestUpdate").
+				Str("machine", m.Name).
+				Msgf("Notified machine %s", m.Name)
+		}
+	} else {
+		log.Info().
+			Str("func", "requestUpdate").
+			Str("machine", m.Name).
+			Msgf("Machine %s does not appear to be polling", m.Name)
+		return errors.New("machine does not seem to be polling")
+	}
+	return nil
+}
+
+func (h *Headscale) isOutdated(m *Machine) bool {
+	err := h.UpdateMachine(m)
+	if err != nil {
+		return true
+	}
+
+	lastChange := h.getLastStateChange(m.Namespace.Name)
+	log.Trace().
+		Str("func", "keepAlive").
+		Str("machine", m.Name).
+		Time("last_successful_update", *m.LastSuccessfulUpdate).
+		Time("last_state_change", lastChange).
+		Msgf("Checking if %s is missing updates", m.Name)
+	return m.LastSuccessfulUpdate.Before(lastChange)
+}
+
+func (m Machine) String() string {
+	return m.Name
+}
+
+func (ms Machines) String() string {
+	temp := make([]string, len(ms))
+
+	for index, machine := range ms {
+		temp[index] = machine.Name
+	}
+
+	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
+}
+
+// TODO(kradalby): Remove when we have generics...
+func (ms MachinesP) String() string {
+	temp := make([]string, len(ms))
+
+	for index, machine := range ms {
+		temp[index] = machine.Name
+	}
+
+	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
+}
+
+func (ms MachinesP) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) {
+	nodes := make([]*tailcfg.Node, len(ms))
+
+	for index, machine := range ms {
+		node, err := machine.toNode(includeRoutes)
+		if err != nil {
+			return nil, err
+		}
+
+		nodes[index] = node
+	}
+
+	return nodes, nil
+}
+
 // toNode converts a Machine into a Tailscale Node. includeRoutes is false for shared nodes
 // as per the expected behaviour in the official SaaS
 func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
@@ -171,244 +490,3 @@ func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
 	}
 	return &n, nil
 }
-
-func (h *Headscale) getPeers(m Machine) (*[]*tailcfg.Node, error) {
-	log.Trace().
-		Str("func", "getPeers").
-		Str("machine", m.Name).
-		Msg("Finding peers")
-
-	machines := []Machine{}
-	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
-		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
-		log.Error().Err(err).Msg("Error accessing db")
-		return nil, err
-	}
-
-	// We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for
-	sharedMachines := []SharedMachine{}
-	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
-		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
-		return nil, err
-	}
-
-	peers := []*tailcfg.Node{}
-	for _, mn := range machines {
-		peer, err := mn.toNode(true)
-		if err != nil {
-			return nil, err
-		}
-		peers = append(peers, peer)
-	}
-	for _, sharedMachine := range sharedMachines {
-		peer, err := sharedMachine.Machine.toNode(false) // shared nodes do not expose their routes
-		if err != nil {
-			return nil, err
-		}
-		peers = append(peers, peer)
-	}
-	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
-
-	log.Trace().
-		Str("func", "getPeers").
-		Str("machine", m.Name).
-		Msgf("Found peers: %s", tailNodesToString(peers))
-	return &peers, nil
-}
-
-// GetMachine finds a Machine by name and namespace and returns the Machine struct
-func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) {
-	machines, err := h.ListMachinesInNamespace(namespace)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, m := range *machines {
-		if m.Name == name {
-			return &m, nil
-		}
-	}
-	return nil, fmt.Errorf("machine not found")
-}
-
-// GetMachineByID finds a Machine by ID and returns the Machine struct
-func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
-	m := Machine{}
-	if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil {
-		return nil, result.Error
-	}
-	return &m, nil
-}
-
-// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
-func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
-	m := Machine{}
-	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
-		return nil, result.Error
-	}
-	return &m, nil
-}
-
-// UpdateMachine takes a Machine struct pointer (typically already loaded from database
-// and updates it with the latest data from the database.
-func (h *Headscale) UpdateMachine(m *Machine) error {
-	if result := h.db.Find(m).First(&m); result.Error != nil {
-		return result.Error
-	}
-	return nil
-}
-
-// DeleteMachine softs deletes a Machine from the database
-func (h *Headscale) DeleteMachine(m *Machine) error {
-	m.Registered = false
-	namespaceID := m.NamespaceID
-	h.db.Save(&m) // we mark it as unregistered, just in case
-	if err := h.db.Delete(&m).Error; err != nil {
-		return err
-	}
-
-	return h.RequestMapUpdates(namespaceID)
-}
-
-// HardDeleteMachine hard deletes a Machine from the database
-func (h *Headscale) HardDeleteMachine(m *Machine) error {
-	namespaceID := m.NamespaceID
-	if err := h.db.Unscoped().Delete(&m).Error; err != nil {
-		return err
-	}
-	return h.RequestMapUpdates(namespaceID)
-}
-
-// GetHostInfo returns a Hostinfo struct for the machine
-func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
-	hostinfo := tailcfg.Hostinfo{}
-	if len(m.HostInfo) != 0 {
-		hi, err := m.HostInfo.MarshalJSON()
-		if err != nil {
-			return nil, err
-		}
-		err = json.Unmarshal(hi, &hostinfo)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return &hostinfo, nil
-}
-
-func (h *Headscale) notifyChangesToPeers(m *Machine) {
-	peers, err := h.getPeers(m)
-	if err != nil {
-		log.Error().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Msgf("Error getting peers: %s", err)
-		return
-	}
-	for _, p := range *peers {
-		log.Info().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", p.Name).
-			Str("address", p.Addresses[0].String()).
-			Msgf("Notifying peer %s (%s)", p.Name, p.Addresses[0])
-		err := h.sendRequestOnUpdateChannel(p)
-		if err != nil {
-			log.Info().
-				Str("func", "notifyChangesToPeers").
-				Str("machine", m.Name).
-				Str("peer", p.Name).
-				Msgf("Peer %s does not appear to be polling", p.Name)
-		}
-		log.Trace().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", p.Name).
-			Str("address", p.Addresses[0].String()).
-			Msgf("Notified peer %s (%s)", p.Name, p.Addresses[0])
-	}
-}
-
-func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} {
-	var updateChan chan struct{}
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			updateChan = unwrapped
-		} else {
-			log.Error().
-				Str("handler", "openUpdateChannel").
-				Str("machine", m.Name).
-				Msg("Failed to convert update channel to struct{}")
-		}
-	} else {
-		log.Debug().
-			Str("handler", "openUpdateChannel").
-			Str("machine", m.Name).
-			Msg("Update channel not found, creating")
-
-		updateChan = make(chan struct{})
-		h.clientsUpdateChannels.Store(m.ID, updateChan)
-	}
-	return updateChan
-}
-
-func (h *Headscale) closeUpdateChannel(m *Machine) {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			close(unwrapped)
-		}
-	}
-	h.clientsUpdateChannels.Delete(m.ID)
-}
-
-func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID))
-	if ok {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Notifying peer %s", m.Name)
-
-		if update, ok := pUp.(chan struct{}); ok {
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Update channel is %#v", update)
-
-			update <- struct{}{}
-
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Notified machine %s", m.Name)
-		}
-	} else {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Machine %s does not appear to be polling", m.Name)
-		return errors.New("machine does not seem to be polling")
-	}
-	return nil
-}
-
-func (h *Headscale) isOutdated(m *Machine) bool {
-	err := h.UpdateMachine(m)
-	if err != nil {
-		return true
-	}
-
-	lastChange := h.getLastStateChange(m.Namespace.Name)
-	log.Trace().
-		Str("func", "keepAlive").
-		Str("machine", m.Name).
-		Time("last_successful_update", *m.LastSuccessfulUpdate).
-		Time("last_state_change", lastChange).
-		Msgf("Checking if %s is missing updates", m.Name)
-	return m.LastSuccessfulUpdate.Before(lastChange)
-}

From 779301240972e6aa1291fcb014485467667b94a4 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:14:12 +0100
Subject: [PATCH 04/14] Add error if peer api is empty

---
 integration_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/integration_test.go b/integration_test.go
index b768fa59..dd792142 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -692,6 +693,9 @@ func getAPIURLs(tailscales map[string]dockertest.Resource) (map[netaddr.IP]strin
 			n := ft.Node
 			for _, a := range n.Addresses { // just add all the addresses
 				if _, ok := fts[a.IP()]; !ok {
+					if ft.PeerAPIURL == "" {
+						return nil, errors.New("api url is empty")
+					}
 					fts[a.IP()] = ft.PeerAPIURL
 				}
 			}

From 31b4f03f96329ef8d5c91dff339763d436b5be5e Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:14:28 +0100
Subject: [PATCH 05/14] Set integration logging to trace

---
 integration_test/etc/config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_test/etc/config.json b/integration_test/etc/config.json
index 8a6fd962..5454f2f7 100644
--- a/integration_test/etc/config.json
+++ b/integration_test/etc/config.json
@@ -7,5 +7,5 @@
   "db_type": "sqlite3",
   "db_path": "/tmp/integration_test_db.sqlite3",
   "acl_policy_path": "",
-  "log_level": "debug"
+  "log_level": "trace"
 }

From 2090a13dcd8ab18d25ec7ea99b50f233e512dfb6 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:15:20 +0100
Subject: [PATCH 06/14] Remove docker network, it wasnt used, comment out
 portmapping to host

---
 integration_test.go | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index dd792142..ecde3830 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -35,7 +35,6 @@ var (
 
 var (
 	pool      dockertest.Pool
-	network   dockertest.Network
 	headscale dockertest.Resource
 )
 
@@ -89,10 +88,6 @@ func TestIntegrationTestSuite(t *testing.T) {
 	if err := pool.Purge(&headscale); err != nil {
 		log.Printf("Could not purge resource: %s\n", err)
 	}
-
-	if err := network.Close(); err != nil {
-		log.Printf("Could not close network: %s\n", err)
-	}
 }
 
 func executeCommand(resource *dockertest.Resource, cmd []string, env []string) (string, error) {
@@ -184,9 +179,8 @@ func tailscaleContainer(namespace, identifier, version string) (string, *dockert
 	}
 	hostname := fmt.Sprintf("%s-tailscale-%s-%s", namespace, strings.Replace(version, ".", "-", -1), identifier)
 	tailscaleOptions := &dockertest.RunOptions{
-		Name:     hostname,
-		Networks: []*dockertest.Network{&network},
-		Cmd:      []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
+		Name: hostname,
+		Cmd:  []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
 	}
 
 	pts, err := pool.BuildAndRunWithBuildOptions(tailscaleBuildOptions, tailscaleOptions, dockerRestartPolicy)
@@ -210,12 +204,6 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		log.Fatalf("Could not connect to docker: %s", err)
 	}
 
-	if pnetwork, err := pool.CreateNetwork("headscale-test"); err == nil {
-		network = *pnetwork
-	} else {
-		log.Fatalf("Could not create network: %s", err)
-	}
-
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile",
 		ContextDir: ".",
@@ -232,11 +220,10 @@ func (s *IntegrationTestSuite) SetupSuite() {
 			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
 			fmt.Sprintf("%s/derp.yaml:/etc/headscale/derp.yaml", currentPath),
 		},
-		Networks: []*dockertest.Network{&network},
-		Cmd:      []string{"headscale", "serve"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"8080/tcp": {{HostPort: "8080"}},
-		},
+		Cmd: []string{"headscale", "serve"},
+		// PortBindings: map[docker.Port][]docker.PortBinding{
+		// 	"8080/tcp": {{HostPort: "8080"}},
+		// },
 	}
 
 	fmt.Println("Creating headscale container")

From 772541afaba27716e3b5bd2a861a993f347dbfbd Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:16:37 +0100
Subject: [PATCH 07/14] add comment about poor error handling when headscale
 isnt becoming available

---
 integration_test.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/integration_test.go b/integration_test.go
index ecde3830..7e75089d 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -258,7 +258,11 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		}
 		return nil
 	}); err != nil {
-		log.Fatalf("Could not connect to docker: %s", err)
+		// TODO(kradalby): If we cannot access headscale, or any other fatal error during
+		// test setup, we need to abort and tear down. However, testify does not seem to
+		// support that at the moment:
+		// https://github.com/stretchr/testify/issues/849
+		return // fmt.Errorf("Could not connect to headscale: %s", err)
 	}
 	fmt.Println("headscale container is ready")
 

From 931ef9482b4d7d66a9e42fe702fa367d90c971ae Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:17:05 +0100
Subject: [PATCH 08/14] Add checks to see if we can fetch the ip from map,
 remove possible null assignment

---
 integration_test.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index 7e75089d..30703d63 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -345,15 +345,16 @@ func (s *IntegrationTestSuite) TestGetIpAddresses() {
 
 		for hostname := range scales.tailscales {
 			s.T().Run(hostname, func(t *testing.T) {
-				ip := ips[hostname]
+				ip, ok := ips[hostname]
+
+				assert.True(t, ok)
+				assert.NotNil(t, ip)
 
 				fmt.Printf("IP for %s: %s\n", hostname, ip)
 
 				// c.Assert(ip.Valid(), check.IsTrue)
 				assert.True(t, ip.Is4())
 				assert.True(t, ipPrefix.Contains(ip))
-
-				ips[hostname] = ip
 			})
 		}
 	}

From c09428accad05a829c2aabfb4db8d2f63f0bdc1e Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:09:21 +0000
Subject: [PATCH 09/14] Revert "Remove docker network, it wasnt used, comment
 out portmapping to host"

This reverts commit 2090a13dcd8ab18d25ec7ea99b50f233e512dfb6.
---
 integration_test.go | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index 30703d63..58fa8582 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -35,6 +35,7 @@ var (
 
 var (
 	pool      dockertest.Pool
+	network   dockertest.Network
 	headscale dockertest.Resource
 )
 
@@ -88,6 +89,10 @@ func TestIntegrationTestSuite(t *testing.T) {
 	if err := pool.Purge(&headscale); err != nil {
 		log.Printf("Could not purge resource: %s\n", err)
 	}
+
+	if err := network.Close(); err != nil {
+		log.Printf("Could not close network: %s\n", err)
+	}
 }
 
 func executeCommand(resource *dockertest.Resource, cmd []string, env []string) (string, error) {
@@ -179,8 +184,9 @@ func tailscaleContainer(namespace, identifier, version string) (string, *dockert
 	}
 	hostname := fmt.Sprintf("%s-tailscale-%s-%s", namespace, strings.Replace(version, ".", "-", -1), identifier)
 	tailscaleOptions := &dockertest.RunOptions{
-		Name: hostname,
-		Cmd:  []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
+		Name:     hostname,
+		Networks: []*dockertest.Network{&network},
+		Cmd:      []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
 	}
 
 	pts, err := pool.BuildAndRunWithBuildOptions(tailscaleBuildOptions, tailscaleOptions, dockerRestartPolicy)
@@ -204,6 +210,12 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		log.Fatalf("Could not connect to docker: %s", err)
 	}
 
+	if pnetwork, err := pool.CreateNetwork("headscale-test"); err == nil {
+		network = *pnetwork
+	} else {
+		log.Fatalf("Could not create network: %s", err)
+	}
+
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile",
 		ContextDir: ".",
@@ -220,10 +232,11 @@ func (s *IntegrationTestSuite) SetupSuite() {
 			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
 			fmt.Sprintf("%s/derp.yaml:/etc/headscale/derp.yaml", currentPath),
 		},
-		Cmd: []string{"headscale", "serve"},
-		// PortBindings: map[docker.Port][]docker.PortBinding{
-		// 	"8080/tcp": {{HostPort: "8080"}},
-		// },
+		Networks: []*dockertest.Network{&network},
+		Cmd:      []string{"headscale", "serve"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"8080/tcp": {{HostPort: "8080"}},
+		},
 	}
 
 	fmt.Println("Creating headscale container")

From 07e32be5cecb91df8f7b5a7b41060c81140e29e3 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:39:28 +0000
Subject: [PATCH 10/14] Remove host port, we only need internal ports

---
 integration_test.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index 58fa8582..cb969412 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -234,9 +234,6 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		},
 		Networks: []*dockertest.Network{&network},
 		Cmd:      []string{"headscale", "serve"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"8080/tcp": {{HostPort: "8080"}},
-		},
 	}
 
 	fmt.Println("Creating headscale container")

From d3ef39a58f547365b9a4a55d2b78e687c9628c65 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:39:52 +0000
Subject: [PATCH 11/14] Correctly use the internal docker dns and port for
 headscale joining

---
 integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_test.go b/integration_test.go
index cb969412..a7805646 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -294,7 +294,7 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		)
 		assert.Nil(s.T(), err)
 
-		headscaleEndpoint := fmt.Sprintf("http://headscale:%s", headscale.GetPort("8080/tcp"))
+		headscaleEndpoint := "http://headscale:8080"
 
 		fmt.Printf("Joining tailscale containers to headscale at %s\n", headscaleEndpoint)
 		for hostname, tailscale := range scales.tailscales {

From 1d5b090579f21b94eee9364f4651fdf243e6abf6 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 16:28:07 +0000
Subject: [PATCH 12/14] Initial work on Prometheus metrics

This commit adds some Prometheus metrics to /metrics in headscale.

It will add the standard go metrics, some automatic gin metrics and some
initial headscale specific ones.

Some of them has been added to aid debugging #97 (loop bug)

In the future, we can use the metrics to get rid of the sleep in the
integration tests by checking that our expected number of nodes has been
registered:

```
headscale_machine_registrations_total
```
---
 api.go        | 14 +++++++++++++-
 app.go        |  7 +++++++
 go.mod        |  2 ++
 go.sum        | 26 ++++++++++++++++++++++++++
 machine.go    |  1 +
 metrics.go    | 41 +++++++++++++++++++++++++++++++++++++++++
 namespaces.go |  1 +
 poll.go       |  9 +++++++++
 8 files changed, 100 insertions(+), 1 deletion(-)
 create mode 100644 metrics.go

diff --git a/api.go b/api.go
index 90d9be2b..9cdf7100 100644
--- a/api.go
+++ b/api.go
@@ -64,6 +64,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			Str("handler", "Registration").
 			Err(err).
 			Msg("Cannot parse machine key")
+		machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc()
 		c.String(http.StatusInternalServerError, "Sad!")
 		return
 	}
@@ -74,6 +75,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			Str("handler", "Registration").
 			Err(err).
 			Msg("Cannot decode message")
+		machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc()
 		c.String(http.StatusInternalServerError, "Very sad!")
 		return
 	}
@@ -94,6 +96,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 				Str("handler", "Registration").
 				Err(err).
 				Msg("Could not create row")
+			machineRegistrations.WithLabelValues("unkown", "web", "error", m.Namespace.Name).Inc()
 			return
 		}
 	}
@@ -122,9 +125,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 					Str("handler", "Registration").
 					Err(err).
 					Msg("Cannot encode message")
+				machineRegistrations.WithLabelValues("update", "web", "error", m.Namespace.Name).Inc()
 				c.String(http.StatusInternalServerError, "")
 				return
 			}
+			machineRegistrations.WithLabelValues("update", "web", "success", m.Namespace.Name).Inc()
 			c.Data(200, "application/json; charset=utf-8", respBody)
 			return
 		}
@@ -141,9 +146,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 				Str("handler", "Registration").
 				Err(err).
 				Msg("Cannot encode message")
+			machineRegistrations.WithLabelValues("new", "web", "error", m.Namespace.Name).Inc()
 			c.String(http.StatusInternalServerError, "")
 			return
 		}
+		machineRegistrations.WithLabelValues("new", "web", "success", m.Namespace.Name).Inc()
 		c.Data(200, "application/json; charset=utf-8", respBody)
 		return
 	}
@@ -338,13 +345,15 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 				Err(err).
 				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "")
+			machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 			return
 		}
-		c.Data(200, "application/json; charset=utf-8", respBody)
+		c.Data(401, "application/json; charset=utf-8", respBody)
 		log.Error().
 			Str("func", "handleAuthKey").
 			Str("machine", m.Name).
 			Msg("Failed authentication via AuthKey")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		return
 	}
 
@@ -358,6 +367,7 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 			Str("func", "handleAuthKey").
 			Str("machine", m.Name).
 			Msg("Failed to find an available IP")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		return
 	}
 	log.Info().
@@ -383,9 +393,11 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 			Str("machine", m.Name).
 			Err(err).
 			Msg("Cannot encode message")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		c.String(http.StatusInternalServerError, "Extremely sad!")
 		return
 	}
+	machineRegistrations.WithLabelValues("new", "authkey", "success", m.Namespace.Name).Inc()
 	c.Data(200, "application/json; charset=utf-8", respBody)
 	log.Info().
 		Str("func", "handleAuthKey").
diff --git a/app.go b/app.go
index 8be137cf..107b78d6 100644
--- a/app.go
+++ b/app.go
@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/gin-gonic/gin"
+	"github.com/zsais/go-gin-prometheus"
 	"golang.org/x/crypto/acme/autocert"
 	"gorm.io/gorm"
 	"inet.af/netaddr"
@@ -140,6 +141,7 @@ func (h *Headscale) expireEphemeralNodesWorker() {
 				if err != nil {
 					log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database")
 				}
+				updateRequestsFromNode.WithLabelValues("ephemeral-node-update").Inc()
 				h.notifyChangesToPeers(&m)
 			}
 		}
@@ -163,6 +165,10 @@ func (h *Headscale) watchForKVUpdatesWorker() {
 // Serve launches a GIN server with the Headscale API
 func (h *Headscale) Serve() error {
 	r := gin.Default()
+
+	p := ginprometheus.NewPrometheus("gin")
+	p.Use(r)
+
 	r.GET("/health", func(c *gin.Context) { c.JSON(200, gin.H{"healthy": "ok"}) })
 	r.GET("/key", h.KeyHandler)
 	r.GET("/register", h.RegisterWebAPI)
@@ -233,6 +239,7 @@ func (h *Headscale) Serve() error {
 
 func (h *Headscale) setLastStateChangeToNow(namespace string) {
 	now := time.Now().UTC()
+	lastStateUpdate.WithLabelValues("", "headscale").Set(float64(now.Unix()))
 	h.lastStateChange.Store(namespace, now)
 }
 
diff --git a/go.mod b/go.mod
index 3a49c0fd..70428315 100644
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,7 @@ require (
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/ory/dockertest/v3 v3.7.0
+	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/pterm/pterm v0.12.30
 	github.com/rs/zerolog v1.25.0
 	github.com/spf13/cobra v1.2.1
@@ -28,6 +29,7 @@ require (
 	github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88
 	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/zsais/go-gin-prometheus v0.1.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
 	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
diff --git a/go.sum b/go.sum
index 1d97f18f..6dc92b6d 100644
--- a/go.sum
+++ b/go.sum
@@ -103,6 +103,7 @@ github.com/aws/aws-sdk-go v1.38.52/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
@@ -118,7 +119,9 @@ github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
 github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -518,6 +521,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB
 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
 github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
@@ -531,6 +535,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -538,6 +543,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -616,6 +622,7 @@ github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGw
 github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
 github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
 github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y=
@@ -669,6 +676,7 @@ github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8q
 github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
@@ -747,21 +755,32 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
 github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
@@ -929,6 +948,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4=
+github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4=
+github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
@@ -1143,6 +1164,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1158,6 +1180,8 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1190,6 +1214,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1450,6 +1475,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
diff --git a/machine.go b/machine.go
index 95aef851..cd2d124a 100644
--- a/machine.go
+++ b/machine.go
@@ -297,6 +297,7 @@ func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
 				Str("machine", m.Name).
 				Msgf("Update channel is %#v", update)
 
+			updateRequestsToNode.Inc()
 			update <- struct{}{}
 
 			log.Trace().
diff --git a/metrics.go b/metrics.go
new file mode 100644
index 00000000..d516fade
--- /dev/null
+++ b/metrics.go
@@ -0,0 +1,41 @@
+package headscale
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const prometheusNamespace = "headscale"
+
+var (
+	// This is a high cardinality metric (namespace x machines), we might want to make this
+	// configurable/opt-in in the future.
+	lastStateUpdate = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: prometheusNamespace,
+		Name:      "last_update_seconds",
+		Help:      "Time stamp in unix time when a machine or headscale was updated",
+	}, []string{"namespace", "machine"})
+
+	machineRegistrations = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "machine_registrations_total",
+		Help:      "The total amount of registered machine attempts",
+	}, []string{"action", "auth", "status", "namespace"})
+
+	updateRequestsFromNode = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_from_node_total",
+		Help:      "The number of updates requested by a node/update function",
+	}, []string{"state"})
+	updateRequestsToNode = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_to_node_total",
+		Help:      "The number of calls/messages issued on a specific nodes update channel",
+	})
+	//TODO(kradalby): This is very debugging, we might want to remove it.
+	updateRequestsReceivedOnChannel = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_received_on_channel_total",
+		Help:      "The number of update requests received on an update channel",
+	}, []string{"machine"})
+)
diff --git a/namespaces.go b/namespaces.go
index 8204f96d..75b6eab5 100644
--- a/namespaces.go
+++ b/namespaces.go
@@ -191,6 +191,7 @@ func (h *Headscale) checkForNamespacesPendingUpdates() {
 			continue
 		}
 		for _, m := range *machines {
+			updateRequestsFromNode.WithLabelValues("namespace-update").Inc()
 			h.notifyChangesToPeers(&m)
 		}
 	}
diff --git a/poll.go b/poll.go
index 7e547b4d..46e6cf8f 100644
--- a/poll.go
+++ b/poll.go
@@ -159,6 +159,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
+		updateRequestsFromNode.WithLabelValues("endpoint-update").Inc()
 		go h.notifyChangesToPeers(m)
 		return
 	} else if req.OmitPeers && req.Stream {
@@ -184,6 +185,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Notifying peers")
+	updateRequestsFromNode.WithLabelValues("full-update").Inc()
 	go h.notifyChangesToPeers(m)
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
@@ -258,7 +260,10 @@ func (h *Headscale) PollNetMapStream(
 			}
 			now := time.Now().UTC()
 			m.LastSeen = &now
+
+			lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix()))
 			m.LastSuccessfulUpdate = &now
+
 			h.db.Save(&m)
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -320,6 +325,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "update").
 				Msg("Received a request for update")
+			updateRequestsReceivedOnChannel.WithLabelValues(m.Name).Inc()
 			if h.isOutdated(m) {
 				log.Debug().
 					Str("handler", "PollNetMapStream").
@@ -369,7 +375,10 @@ func (h *Headscale) PollNetMapStream(
 						Msg("Cannot update machine from database")
 				}
 				now := time.Now().UTC()
+
+				lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix()))
 				m.LastSuccessfulUpdate = &now
+
 				h.db.Save(&m)
 			} else {
 				log.Trace().

From 2eb57e6288afb9925e81bf29d0f4acf45968ef32 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 17:39:01 +0000
Subject: [PATCH 13/14] Clean up pointer usage consistency.

This tries to make the same functions emit and consume the same type of
data all over the application.

If a function transform data, it should emit new data, not a pointer.
---
 api.go     |  9 +++++----
 machine.go | 41 ++++++++++++++++++++++-------------------
 poll.go    | 10 +++++-----
 3 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/api.go b/api.go
index 9cdf7100..17306b32 100644
--- a/api.go
+++ b/api.go
@@ -220,7 +220,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	c.Data(200, "application/json; charset=utf-8", respBody)
 }
 
-func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
+func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) {
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
@@ -277,6 +277,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
+		Interface("payload", resp).
 		Msgf("Generated map response: %s", tailMapResponseToString(resp))
 
 	var respBody []byte
@@ -299,10 +300,10 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 	data := make([]byte, 4)
 	binary.LittleEndian.PutUint32(data, uint32(len(respBody)))
 	data = append(data, respBody...)
-	return &data, nil
+	return data, nil
 }
 
-func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
+func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) {
 	resp := tailcfg.MapResponse{
 		KeepAlive: true,
 	}
@@ -325,7 +326,7 @@ func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapReque
 	data := make([]byte, 4)
 	binary.LittleEndian.PutUint32(data, uint32(len(respBody)))
 	data = append(data, respBody...)
-	return &data, nil
+	return data, nil
 }
 
 func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, req tailcfg.RegisterRequest, m Machine) {
diff --git a/machine.go b/machine.go
index cd2d124a..7a9df2e8 100644
--- a/machine.go
+++ b/machine.go
@@ -56,34 +56,29 @@ func (m Machine) isAlreadyRegistered() bool {
 	return m.Registered
 }
 
-func (h *Headscale) getDirectPeers(m *Machine) (MachinesP, error) {
+func (h *Headscale) getDirectPeers(m *Machine) (Machines, error) {
 	log.Trace().
 		Str("func", "getDirectPeers").
 		Str("machine", m.Name).
-		Msg("Finding peers")
+		Msg("Finding direct peers")
 
-	machines := []Machine{}
+	machines := Machines{}
 	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
 		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
 		log.Error().Err(err).Msg("Error accessing db")
 		return nil, err
 	}
 
-	peers := make(MachinesP, 0)
-	for _, peer := range machines {
-		peers = append(peers, &peer)
-	}
-
-	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+	sort.Slice(machines, func(i, j int) bool { return machines[i].ID < machines[j].ID })
 
 	log.Trace().
-		Str("func", "getDirectPeers").
+		Str("func", "getDirectmachines").
 		Str("machine", m.Name).
-		Msgf("Found peers: %s", peers.String())
-	return peers, nil
+		Msgf("Found direct machines: %s", machines.String())
+	return machines, nil
 }
 
-func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
+func (h *Headscale) getShared(m *Machine) (Machines, error) {
 	log.Trace().
 		Str("func", "getShared").
 		Str("machine", m.Name).
@@ -96,9 +91,9 @@ func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
 		return nil, err
 	}
 
-	peers := make(MachinesP, 0)
+	peers := make(Machines, 0)
 	for _, sharedMachine := range sharedMachines {
-		peers = append(peers, &sharedMachine.Machine)
+		peers = append(peers, sharedMachine.Machine)
 	}
 
 	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
@@ -110,7 +105,7 @@ func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
 	return peers, nil
 }
 
-func (h *Headscale) getPeers(m *Machine) (MachinesP, error) {
+func (h *Headscale) getPeers(m *Machine) (Machines, error) {
 	direct, err := h.getDirectPeers(m)
 	if err != nil {
 		log.Error().
@@ -129,7 +124,15 @@ func (h *Headscale) getPeers(m *Machine) (MachinesP, error) {
 		return nil, err
 	}
 
-	return append(direct, shared...), nil
+	peers := append(direct, shared...)
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msgf("Found total peers: %s", peers.String())
+
+	return peers, nil
 }
 
 // GetMachine finds a Machine by name and namespace and returns the Machine struct
@@ -227,7 +230,7 @@ func (h *Headscale) notifyChangesToPeers(m *Machine) {
 			Str("peer", peer.Name).
 			Str("address", peer.IPAddress).
 			Msgf("Notifying peer %s (%s)", peer.Name, peer.IPAddress)
-		err := h.sendRequestOnUpdateChannel(peer)
+		err := h.sendRequestOnUpdateChannel(&peer)
 		if err != nil {
 			log.Info().
 				Str("func", "notifyChangesToPeers").
@@ -357,7 +360,7 @@ func (ms MachinesP) String() string {
 	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
 }
 
-func (ms MachinesP) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) {
+func (ms Machines) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) {
 	nodes := make([]*tailcfg.Node, len(ms))
 
 	for index, machine := range ms {
diff --git a/poll.go b/poll.go
index 46e6cf8f..a33d341b 100644
--- a/poll.go
+++ b/poll.go
@@ -123,7 +123,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			Str("handler", "PollNetMap").
 			Str("machine", m.Name).
 			Msg("Client is starting up. Probably interested in a DERP map")
-		c.Data(200, "application/json; charset=utf-8", *data)
+		c.Data(200, "application/json; charset=utf-8", data)
 		return
 	}
 
@@ -155,7 +155,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			Str("handler", "PollNetMap").
 			Str("machine", m.Name).
 			Msg("Client sent endpoint update and is ok with a response without peer list")
-		c.Data(200, "application/json; charset=utf-8", *data)
+		c.Data(200, "application/json; charset=utf-8", data)
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
@@ -179,7 +179,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Sending initial map")
-	go func() { pollDataChan <- *data }()
+	go func() { pollDataChan <- data }()
 
 	log.Info().
 		Str("handler", "PollNetMap").
@@ -342,7 +342,7 @@ func (h *Headscale) PollNetMapStream(
 						Err(err).
 						Msg("Could not get the map update")
 				}
-				_, err = w.Write(*data)
+				_, err = w.Write(data)
 				if err != nil {
 					log.Error().
 						Str("handler", "PollNetMapStream").
@@ -473,7 +473,7 @@ func (h *Headscale) scheduledPollWorker(
 				Str("func", "keepAlive").
 				Str("machine", m.Name).
 				Msg("Sending keepalive")
-			keepAliveChan <- *data
+			keepAliveChan <- data
 
 		case <-updateCheckerTicker.C:
 			// Send an update request regardless of outdated or not, if data is sent

From f6a7564ec870a57c1b22c101578dd1edd9ba43b7 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 17:40:21 +0000
Subject: [PATCH 14/14] Add more test cases to prove that peers and shared
 peers work properly

---
 machine_test.go | 41 +++++++++++++++++++++++++++++++++++++++++
 sharing_test.go |  9 +++++++++
 2 files changed, 50 insertions(+)

diff --git a/machine_test.go b/machine_test.go
index cf0d12e7..dfe84d33 100644
--- a/machine_test.go
+++ b/machine_test.go
@@ -2,6 +2,7 @@ package headscale
 
 import (
 	"encoding/json"
+	"strconv"
 
 	"gopkg.in/check.v1"
 )
@@ -116,3 +117,43 @@ func (s *Suite) TestHardDeleteMachine(c *check.C) {
 	_, err = h.GetMachine(n.Name, "testmachine3")
 	c.Assert(err, check.NotNil)
 }
+
+func (s *Suite) TestGetDirectPeers(c *check.C) {
+	n, err := h.CreateNamespace("test")
+	c.Assert(err, check.IsNil)
+
+	pak, err := h.CreatePreAuthKey(n.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	_, err = h.GetMachineByID(0)
+	c.Assert(err, check.NotNil)
+
+	for i := 0; i <= 10; i++ {
+		m := Machine{
+			ID:             uint64(i),
+			MachineKey:     "foo" + strconv.Itoa(i),
+			NodeKey:        "bar" + strconv.Itoa(i),
+			DiscoKey:       "faa" + strconv.Itoa(i),
+			Name:           "testmachine" + strconv.Itoa(i),
+			NamespaceID:    n.ID,
+			Registered:     true,
+			RegisterMethod: "authKey",
+			AuthKeyID:      uint(pak.ID),
+		}
+		h.db.Save(&m)
+	}
+
+	m1, err := h.GetMachineByID(0)
+	c.Assert(err, check.IsNil)
+
+	_, err = m1.GetHostInfo()
+	c.Assert(err, check.IsNil)
+
+	peers, err := h.getDirectPeers(m1)
+	c.Assert(err, check.IsNil)
+
+	c.Assert(len(peers), check.Equals, 9)
+	c.Assert(peers[0].Name, check.Equals, "testmachine2")
+	c.Assert(peers[5].Name, check.Equals, "testmachine7")
+	c.Assert(peers[8].Name, check.Equals, "testmachine10")
+}
diff --git a/sharing_test.go b/sharing_test.go
index 25de5846..baa90d0c 100644
--- a/sharing_test.go
+++ b/sharing_test.go
@@ -245,6 +245,7 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(p1sAfter), check.Equals, 1)
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
 }
 
 func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
@@ -343,6 +344,7 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(p1s), check.Equals, 1) // nodes 1 and 4
+	c.Assert(p1s[0].Name, check.Equals, "test_get_shared_nodes_4")
 
 	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
@@ -350,6 +352,13 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(p1sAfter), check.Equals, 2) // nodes 1, 2, 4
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
+	c.Assert(p1sAfter[1].Name, check.Equals, "test_get_shared_nodes_4")
+
+	node1shared, err := h.getShared(m1)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(node1shared), check.Equals, 1) // nodes 1, 2, 4
+	c.Assert(node1shared[0].Name, check.Equals, "test_get_shared_nodes_2")
 
 	pAlone, err := h.getPeers(m3)
 	c.Assert(err, check.IsNil)