From 986725519fbe277ef6c9872b57cd9447d7a73bf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arthur=20Woimb=C3=A9e?= <arthur.woimbee@gmail.com>
Date: Fri, 1 Oct 2021 15:59:54 +0200
Subject: [PATCH 01/38] fix some typos in README

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 35f4c184..8dd9c8d6 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ Suggestions/PRs welcomed!
 
    ```shell
    touch db.sqlite
-   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8000:8000 headscale/headscale:x.x.x headscale namespaces create myfirstnamespace
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8080:8080 headscale/headscale:x.x.x headscale namespaces create myfirstnamespace
    ```
 
    or if your server is already running in docker:
@@ -100,7 +100,7 @@ Suggestions/PRs welcomed!
    the db.sqlite mount is only needed if you use sqlite
 
    ```shell
-   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8000:8000 headscale/headscale:x.x.x headscale serve
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8080:8080 headscale/headscale:x.x.x headscale serve
    ```
 
 6. If you used tailscale.com before in your nodes, make sure you clear the tailscald data folder
@@ -121,15 +121,15 @@ Suggestions/PRs welcomed!
 
 9. In the server, register your machine to a namespace with the CLI
    ```shell
-   headscale -n myfirstnamespace node register YOURMACHINEKEY
+   headscale -n myfirstnamespace nodes register YOURMACHINEKEY
    ```
    or docker:
    ```shell
-   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml headscale/headscale:x.x.x headscale -n myfirstnamespace node register YOURMACHINEKEY
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml headscale/headscale:x.x.x headscale -n myfirstnamespace nodes register YOURMACHINEKEY
    ```
    or if your server is already running in docker:
    ```shell
-   docker exec <container_name> headscale -n myfistnamespace node register YOURMACHINEKEY
+   docker exec <container_name> headscale -n myfirstnamespace nodes register YOURMACHINEKEY
    ```
 
 Alternatively, you can use Auth Keys to register your machines:

From 6ffea2225ddf5d47534b1b942a38b60044c042e3 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 15:28:19 +0100
Subject: [PATCH 02/38] Attempt to close failed streams

If we have a failed write toward any of our connections, attempt to
close the connection by returning "false" as in unsuccessful stream
---
 machine.go | 1 +
 poll.go    | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/machine.go b/machine.go
index 1d4939c1..2d4bb518 100644
--- a/machine.go
+++ b/machine.go
@@ -309,6 +309,7 @@ func (h *Headscale) notifyChangesToPeers(m *Machine) {
 				Str("machine", m.Name).
 				Str("peer", p.Name).
 				Msgf("Peer %s does not appear to be polling", p.Name)
+			return
 		}
 		log.Trace().
 			Str("func", "notifyChangesToPeers").
diff --git a/poll.go b/poll.go
index 60bfa9ea..c5da3a96 100644
--- a/poll.go
+++ b/poll.go
@@ -230,6 +230,7 @@ func (h *Headscale) PollNetMapStream(
 					Str("channel", "pollData").
 					Err(err).
 					Msg("Cannot write data")
+				return false
 			}
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -237,7 +238,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("channel", "pollData").
 				Int("bytes", len(data)).
 				Msg("Data from pollData channel written successfully")
-				// TODO: Abstract away all the database calls, this can cause race conditions
+				// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
 			err = h.UpdateMachine(&m)
@@ -276,6 +277,7 @@ func (h *Headscale) PollNetMapStream(
 					Str("channel", "keepAlive").
 					Err(err).
 					Msg("Cannot write keep alive message")
+				return false
 			}
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -283,7 +285,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("channel", "keepAlive").
 				Int("bytes", len(data)).
 				Msg("Keep alive sent successfully")
-				// TODO: Abstract away all the database calls, this can cause race conditions
+				// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
 			err = h.UpdateMachine(&m)
@@ -336,6 +338,7 @@ func (h *Headscale) PollNetMapStream(
 						Str("channel", "update").
 						Err(err).
 						Msg("Could not write the map response")
+					return false
 				}
 				log.Trace().
 					Str("handler", "PollNetMapStream").
@@ -347,7 +350,7 @@ func (h *Headscale) PollNetMapStream(
 					// we sometimes end in a state were the update
 					// is not picked up by a client and we use this
 					// to determine if we should "force" an update.
-					// TODO: Abstract away all the database calls, this can cause race conditions
+					// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
 					// when an outdated machine object is kept alive, e.g. db is update from
 					// command line, but then overwritten.
 				err = h.UpdateMachine(&m)

From ed728f57e06568eb4b618610fd353b4b9fe02a36 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 15:29:27 +0100
Subject: [PATCH 03/38] Remove WriteTimeout from HTTP

Golangs built in HTTP server does not allow different HTTP timeout for
different types of handlers, so we cannot have a write timeout as we
attempt to do long polling (my bad).

See linked article.

Also removed redundant server declaration
---
 app.go | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/app.go b/app.go
index dc398eb7..8be137cf 100644
--- a/app.go
+++ b/app.go
@@ -172,16 +172,18 @@ func (h *Headscale) Serve() error {
 	r.GET("/apple/:platform", h.ApplePlatformConfig)
 	var err error
 
-	timeout := 30 * time.Second
-
 	go h.watchForKVUpdates(5000)
 	go h.expireEphemeralNodes(5000)
 
 	s := &http.Server{
-		Addr:         h.cfg.Addr,
-		Handler:      r,
-		ReadTimeout:  timeout,
-		WriteTimeout: timeout,
+		Addr:        h.cfg.Addr,
+		Handler:     r,
+		ReadTimeout: 30 * time.Second,
+		// Go does not handle timeouts in HTTP very well, and there is
+		// no good way to handle streaming timeouts, therefore we need to
+		// keep this at unlimited and be careful to clean up connections
+		// https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/#aboutstreaming
+		WriteTimeout: 0,
 	}
 
 	if h.cfg.TLSLetsEncryptHostname != "" {
@@ -194,13 +196,9 @@ func (h *Headscale) Serve() error {
 			HostPolicy: autocert.HostWhitelist(h.cfg.TLSLetsEncryptHostname),
 			Cache:      autocert.DirCache(h.cfg.TLSLetsEncryptCacheDir),
 		}
-		s := &http.Server{
-			Addr:         h.cfg.Addr,
-			TLSConfig:    m.TLSConfig(),
-			Handler:      r,
-			ReadTimeout:  timeout,
-			WriteTimeout: timeout,
-		}
+
+		s.TLSConfig = m.TLSConfig()
+
 		if h.cfg.TLSLetsEncryptChallengeType == "TLS-ALPN-01" {
 			// Configuration via autocert with TLS-ALPN-01 (https://tools.ietf.org/html/rfc8737)
 			// The RFC requires that the validation is done on port 443; in other words, headscale
@@ -211,7 +209,6 @@ func (h *Headscale) Serve() error {
 			// port 80 for the certificate validation in addition to the headscale
 			// service, which can be configured to run on any other port.
 			go func() {
-
 				log.Fatal().
 					Err(http.ListenAndServe(h.cfg.TLSLetsEncryptListen, m.HTTPHandler(http.HandlerFunc(h.redirect)))).
 					Msg("failed to set up a HTTP server")

From cefe2d5bccda4ad0a6d36f70c3466bff79bcacfa Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 15:30:41 +0100
Subject: [PATCH 04/38] Improve and clarify log entry

---
 poll.go | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/poll.go b/poll.go
index c5da3a96..33b4b124 100644
--- a/poll.go
+++ b/poll.go
@@ -259,7 +259,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "pollData").
 				Int("bytes", len(data)).
-				Msg("Machine updated successfully after sending pollData")
+				Msg("Machine entry in database updated successfully after sending pollData")
 			return true
 
 		case data := <-keepAliveChan:
@@ -396,13 +396,33 @@ func (h *Headscale) PollNetMapStream(
 			m.LastSeen = &now
 			h.db.Save(&m)
 
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Canceling keepAlive channel")
 			cancelKeepAlive <- struct{}{}
 
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Closing update channel")
 			h.closeUpdateChannel(&m)
 
 			close(pollDataChan)
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Closing pollData channel")
 
 			close(keepAliveChan)
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Closing keepAliveChan channel")
 
 			return false
 		}

From 39abc4e97317b64597147200ed313216c86e24d5 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 15:38:53 +0100
Subject: [PATCH 05/38] Clarify error messages for nodes that are not connected

If a node does not have an update channel, it is probably not connected,
clarify the log messages and make sure we dont print that it was updated
successfully (continue, not return)
---
 machine.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/machine.go b/machine.go
index 2d4bb518..1cd2d95f 100644
--- a/machine.go
+++ b/machine.go
@@ -308,8 +308,8 @@ func (h *Headscale) notifyChangesToPeers(m *Machine) {
 				Str("func", "notifyChangesToPeers").
 				Str("machine", m.Name).
 				Str("peer", p.Name).
-				Msgf("Peer %s does not appear to be polling", p.Name)
-			return
+				Msgf("Peer %s does not have an open update client, skipping.", p.Name)
+			continue
 		}
 		log.Trace().
 			Str("func", "notifyChangesToPeers").
@@ -380,11 +380,12 @@ func (h *Headscale) sendRequestOnUpdateChannel(m *tailcfg.Node) error {
 				Msgf("Notified machine %s", m.Name)
 		}
 	} else {
+		err := errors.New("machine does not have an open update channel")
 		log.Info().
 			Str("func", "requestUpdate").
 			Str("machine", m.Name).
-			Msgf("Machine %s does not appear to be polling", m.Name)
-		return errors.New("machine does not seem to be polling")
+			Msgf("Machine %s does not have an open update channel", m.Name)
+		return err
 	}
 	return nil
 }

From 0435089ebabd5a1b14c6f87c097e65b956ca22dc Mon Sep 17 00:00:00 2001
From: Ward Vandewege <ward@jhvc.com>
Date: Sat, 2 Oct 2021 10:44:52 -0400
Subject: [PATCH 06/38] Fix a few typos in the tailscale command line
 arguments.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 8dd9c8d6..45c648ac 100644
--- a/README.md
+++ b/README.md
@@ -114,7 +114,7 @@ Suggestions/PRs welcomed!
 7. Add your first machine
 
    ```shell
-   tailscale up -login-server YOUR_HEADSCALE_URL
+   tailscale up --login-server YOUR_HEADSCALE_URL
    ```
 
 8. Navigate to the URL you will get with `tailscale up`, where you'll find your machine key.
@@ -154,7 +154,7 @@ Alternatively, you can use Auth Keys to register your machines:
 
 2. Use the authkey from your machine to register it
    ```shell
-   tailscale up -login-server YOUR_HEADSCALE_URL --authkey YOURAUTHKEY
+   tailscale up --login-server YOUR_HEADSCALE_URL --authkey YOURAUTHKEY
    ```
 
 If you create an authkey with the `--ephemeral` flag, that key will create ephemeral nodes. This implies that `--reusable` is true.

From 54daa0da23e44a5f9836485ae9e0e9582c7347a8 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 17:35:39 +0100
Subject: [PATCH 07/38] Fix spelling error

---
 poll.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/poll.go b/poll.go
index 33b4b124..73726610 100644
--- a/poll.go
+++ b/poll.go
@@ -400,7 +400,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("handler", "PollNetMapStream").
 				Str("machine", m.Name).
 				Str("channel", "Done").
-				Msg("Canceling keepAlive channel")
+				Msg("Cancelling keepAlive channel")
 			cancelKeepAlive <- struct{}{}
 
 			log.Trace().

From 78a0f3ca37e140f227cfd9c28331f32dff945fe4 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 18:39:09 +0100
Subject: [PATCH 08/38] Up ping timeout

---
 integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_test.go b/integration_test.go
index f62ca1da..925fc4ca 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -433,7 +433,7 @@ func (s *IntegrationTestSuite) TestPingAllPeers() {
 						command := []string{
 							"tailscale", "ping",
 							"--timeout=1s",
-							"--c=20",
+							"--c=100",
 							"--until-direct=true",
 							ip.String(),
 						}

From 0475eb6ef7ffc70fa544c9f31f1e697d4235323e Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 21:58:28 +0100
Subject: [PATCH 09/38] Move DB call of pollmap to Machine inside a function

---
 api.go     |  1 +
 go.mod     |  4 ++--
 machine.go |  9 +++++++++
 poll.go    | 18 ++++++++++++------
 4 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/api.go b/api.go
index e2a56185..304f9a84 100644
--- a/api.go
+++ b/api.go
@@ -226,6 +226,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 			Msg("Cannot convert to node")
 		return nil, err
 	}
+
 	peers, err := h.getPeers(m)
 	if err != nil {
 		log.Error().
diff --git a/go.mod b/go.mod
index 51acff89..3a49c0fd 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/docker/docker v20.10.8+incompatible // indirect
 	github.com/efekarakus/termcolor v1.0.1
 	github.com/gin-gonic/gin v1.7.4
-	github.com/gofrs/uuid v4.0.0+incompatible // indirect
+	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/google/go-github v17.0.0+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
@@ -26,7 +26,7 @@ require (
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88
-	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e // indirect
+	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
diff --git a/machine.go b/machine.go
index 1d4939c1..7e266b67 100644
--- a/machine.go
+++ b/machine.go
@@ -240,6 +240,15 @@ func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
 	return &m, nil
 }
 
+// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
 // UpdateMachine takes a Machine struct pointer (typically already loaded from database
 // and updates it with the latest data from the database.
 func (h *Headscale) UpdateMachine(m *Machine) error {
diff --git a/poll.go b/poll.go
index 60bfa9ea..ca4f6764 100644
--- a/poll.go
+++ b/poll.go
@@ -51,13 +51,19 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		return
 	}
 
-	var m Machine
-	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey.HexString()); errors.Is(result.Error, gorm.ErrRecordNotFound) {
-		log.Warn().
+	m, err := h.GetMachineByMachineKey(mKey.HexString())
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			log.Warn().
+				Str("handler", "PollNetMap").
+				Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
+			c.String(http.StatusUnauthorized, "")
+			return
+		}
+		log.Error().
 			Str("handler", "PollNetMap").
-			Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
-		c.String(http.StatusUnauthorized, "")
-		return
+			Msgf("Failed to fetch machine from the database with Machine key: %s", mKey.HexString())
+		c.String(http.StatusInternalServerError, "")
 	}
 	log.Trace().
 		Str("handler", "PollNetMap").

From 0d4a006536a5d854f7c822f617a810354800ab1f Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 22:00:09 +0100
Subject: [PATCH 10/38] Consitently use Machine pointers

This commit rewrites a bunch of the code to always use *Machine instead
of a mix of both, and a mix of tailcfg.Node and Machine.

Now we use *Machine, and if tailcfg.Node is needed, it is converted just
before needed.
---
 api.go          |  4 +--
 machine.go      |  4 +--
 machine_test.go |  4 +--
 poll.go         | 25 ++++++++-------
 sharing_test.go | 82 ++++++++++++++++++++++++-------------------------
 5 files changed, 58 insertions(+), 61 deletions(-)

diff --git a/api.go b/api.go
index 304f9a84..72cb92f6 100644
--- a/api.go
+++ b/api.go
@@ -213,7 +213,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	c.Data(200, "application/json; charset=utf-8", respBody)
 }
 
-func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
@@ -286,7 +286,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 	return &data, nil
 }
 
-func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
 	resp := tailcfg.MapResponse{
 		KeepAlive: true,
 	}
diff --git a/machine.go b/machine.go
index 7e266b67..f49c5a89 100644
--- a/machine.go
+++ b/machine.go
@@ -296,7 +296,7 @@ func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
 }
 
 func (h *Headscale) notifyChangesToPeers(m *Machine) {
-	peers, err := h.getPeers(*m)
+	peers, err := h.getPeers(m)
 	if err != nil {
 		log.Error().
 			Str("func", "notifyChangesToPeers").
@@ -363,7 +363,7 @@ func (h *Headscale) closeUpdateChannel(m *Machine) {
 	h.clientsUpdateChannels.Delete(m.ID)
 }
 
-func (h *Headscale) sendRequestOnUpdateChannel(m *tailcfg.Node) error {
+func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
 	h.clientsUpdateChannelMutex.Lock()
 	defer h.clientsUpdateChannelMutex.Unlock()
 
diff --git a/machine_test.go b/machine_test.go
index d535be56..cf0d12e7 100644
--- a/machine_test.go
+++ b/machine_test.go
@@ -16,7 +16,7 @@ func (s *Suite) TestGetMachine(c *check.C) {
 	_, err = h.GetMachine("test", "testmachine")
 	c.Assert(err, check.NotNil)
 
-	m := Machine{
+	m := &Machine{
 		ID:             0,
 		MachineKey:     "foo",
 		NodeKey:        "bar",
@@ -27,7 +27,7 @@ func (s *Suite) TestGetMachine(c *check.C) {
 		RegisterMethod: "authKey",
 		AuthKeyID:      uint(pak.ID),
 	}
-	h.db.Save(&m)
+	h.db.Save(m)
 
 	m1, err := h.GetMachine("test", "testmachine")
 	c.Assert(err, check.IsNil)
diff --git a/poll.go b/poll.go
index ca4f6764..8032edee 100644
--- a/poll.go
+++ b/poll.go
@@ -140,7 +140,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("id", c.Param("id")).
 		Str("machine", m.Name).
 		Msg("Loading or creating update channel")
-	updateChan := h.getOrOpenUpdateChannel(&m)
+	updateChan := h.getOrOpenUpdateChannel(m)
 
 	pollDataChan := make(chan []byte)
 	// defer close(pollData)
@@ -159,7 +159,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
-		go h.notifyChangesToPeers(&m)
+		go h.notifyChangesToPeers(m)
 		return
 	} else if req.OmitPeers && req.Stream {
 		log.Warn().
@@ -184,7 +184,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Notifying peers")
-	go h.notifyChangesToPeers(&m)
+	go h.notifyChangesToPeers(m)
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
 	log.Trace().
@@ -199,7 +199,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 // to the connected clients.
 func (h *Headscale) PollNetMapStream(
 	c *gin.Context,
-	m Machine,
+	m *Machine,
 	req tailcfg.MapRequest,
 	mKey wgkey.Key,
 	pollDataChan chan []byte,
@@ -246,7 +246,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err = h.UpdateMachine(&m)
+			err = h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -292,7 +292,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err = h.UpdateMachine(&m)
+			err = h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -318,7 +318,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "update").
 				Msg("Received a request for update")
-			if h.isOutdated(&m) {
+			if h.isOutdated(m) {
 				log.Debug().
 					Str("handler", "PollNetMapStream").
 					Str("machine", m.Name).
@@ -356,7 +356,7 @@ func (h *Headscale) PollNetMapStream(
 					// TODO: Abstract away all the database calls, this can cause race conditions
 					// when an outdated machine object is kept alive, e.g. db is update from
 					// command line, but then overwritten.
-				err = h.UpdateMachine(&m)
+				err = h.UpdateMachine(m)
 				if err != nil {
 					log.Error().
 						Str("handler", "PollNetMapStream").
@@ -386,7 +386,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err := h.UpdateMachine(&m)
+			err := h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -401,7 +401,7 @@ func (h *Headscale) PollNetMapStream(
 
 			cancelKeepAlive <- struct{}{}
 
-			h.closeUpdateChannel(&m)
+			h.closeUpdateChannel(m)
 
 			close(pollDataChan)
 
@@ -417,7 +417,7 @@ func (h *Headscale) scheduledPollWorker(
 	keepAliveChan chan<- []byte,
 	mKey wgkey.Key,
 	req tailcfg.MapRequest,
-	m Machine,
+	m *Machine,
 ) {
 	keepAliveTicker := time.NewTicker(60 * time.Second)
 	updateCheckerTicker := time.NewTicker(30 * time.Second)
@@ -446,8 +446,7 @@ func (h *Headscale) scheduledPollWorker(
 		case <-updateCheckerTicker.C:
 			// Send an update request regardless of outdated or not, if data is sent
 			// to the node is determined in the updateChan consumer block
-			n, _ := m.toNode(true)
-			err := h.sendRequestOnUpdateChannel(n)
+			err := h.sendRequestOnUpdateChannel(m)
 			if err != nil {
 				log.Error().
 					Str("func", "keepAlive").
diff --git a/sharing_test.go b/sharing_test.go
index ec4951de..25de5846 100644
--- a/sharing_test.go
+++ b/sharing_test.go
@@ -2,7 +2,6 @@ package headscale
 
 import (
 	"gopkg.in/check.v1"
-	"tailscale.com/tailcfg"
 )
 
 func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
@@ -21,7 +20,7 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -33,12 +32,12 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -50,22 +49,22 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 1)
-	c.Assert((*p1sAfter)[0].ID, check.Equals, tailcfg.NodeID(m2.ID))
+	c.Assert(len(p1sAfter), check.Equals, 1)
+	c.Assert(p1sAfter[0].ID, check.Equals, m2.ID)
 }
 
 func (s *Suite) TestSameNamespace(c *check.C) {
@@ -84,7 +83,7 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -96,12 +95,12 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -113,16 +112,16 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m1, n1)
+	err = h.AddSharedMachineToNamespace(m1, n1)
 	c.Assert(err, check.Equals, errorSameNamespace)
 }
 
@@ -142,7 +141,7 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -154,12 +153,12 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -171,18 +170,18 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.Equals, errorMachineAlreadyShared)
 }
 
@@ -202,7 +201,7 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -214,12 +213,12 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -231,22 +230,21 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 1)
-	c.Assert(len((*p1sAfter)[0].AllowedIPs), check.Equals, 1)
+	c.Assert(len(p1sAfter), check.Equals, 1)
 }
 
 func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
@@ -274,7 +272,7 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -286,12 +284,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -303,12 +301,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
-	m3 := Machine{
+	m3 := &Machine{
 		ID:             2,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -320,12 +318,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.3",
 		AuthKeyID:      uint(pak3.ID),
 	}
-	h.db.Save(&m3)
+	h.db.Save(m3)
 
 	_, err = h.GetMachine(n3.Name, m3.Name)
 	c.Assert(err, check.IsNil)
 
-	m4 := Machine{
+	m4 := &Machine{
 		ID:             3,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -337,23 +335,23 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.4",
 		AuthKeyID:      uint(pak4.ID),
 	}
-	h.db.Save(&m4)
+	h.db.Save(m4)
 
 	_, err = h.GetMachine(n1.Name, m4.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 1) // nodes 1 and 4
+	c.Assert(len(p1s), check.Equals, 1) // nodes 1 and 4
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 2) // nodes 1, 2, 4
+	c.Assert(len(p1sAfter), check.Equals, 2) // nodes 1, 2, 4
 
 	pAlone, err := h.getPeers(m3)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*pAlone), check.Equals, 0) // node 3 is alone
+	c.Assert(len(pAlone), check.Equals, 0) // node 3 is alone
 }

From 3c3189caa612781fe271f11475f30461623b4f42 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 22:03:34 +0100
Subject: [PATCH 11/38] Move toNode, add type helpers, split peers and shared

This commit moves toNode to the bottom of the file, and adds a helper
function for lists of Machines to be converted.

It also adds string helpers for Machines and lists of machines.

Lastly it splits getPeers into getDirectPeers, which exist in the same
namespace, and getShared, which is nodes shared with the namespace.

getPeers is kept as a function putting together the two lists for
convenience.
---
 api.go     |  13 +-
 machine.go | 560 ++++++++++++++++++++++++++++++-----------------------
 2 files changed, 330 insertions(+), 243 deletions(-)

diff --git a/api.go b/api.go
index 72cb92f6..90d9be2b 100644
--- a/api.go
+++ b/api.go
@@ -242,11 +242,20 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 		DisplayName: m.Namespace.Name,
 	}
 
+	nodePeers, err := peers.toNodes(true)
+	if err != nil {
+		log.Error().
+			Str("func", "getMapResponse").
+			Err(err).
+			Msg("Failed to convert peers to Tailscale nodes")
+		return nil, err
+	}
+
 	resp := tailcfg.MapResponse{
 		KeepAlive: false,
 		Node:      node,
-		Peers:     *peers,
-		//TODO(kradalby): As per tailscale docs, if DNSConfig is nil,
+		Peers:     nodePeers,
+		// TODO(kradalby): As per tailscale docs, if DNSConfig is nil,
 		// it means its not updated, maybe we can have some logic
 		// to check and only pass updates when its updates.
 		// This is probably more relevant if we try to implement
diff --git a/machine.go b/machine.go
index f49c5a89..c87aba81 100644
--- a/machine.go
+++ b/machine.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -45,11 +46,329 @@ type Machine struct {
 	DeletedAt *time.Time
 }
 
+type (
+	Machines  []Machine
+	MachinesP []*Machine
+)
+
 // For the time being this method is rather naive
 func (m Machine) isAlreadyRegistered() bool {
 	return m.Registered
 }
 
+func (h *Headscale) getDirectPeers(m *Machine) (MachinesP, error) {
+	log.Trace().
+		Str("func", "getDirectPeers").
+		Str("machine", m.Name).
+		Msg("Finding peers")
+
+	machines := []Machine{}
+	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
+		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
+		log.Error().Err(err).Msg("Error accessing db")
+		return nil, err
+	}
+
+	peers := make(MachinesP, 0)
+	for _, peer := range machines {
+		peers = append(peers, &peer)
+	}
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getDirectPeers").
+		Str("machine", m.Name).
+		Msgf("Found peers: %s", peers.String())
+	return peers, nil
+}
+
+func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msg("Finding shared peers")
+
+	// We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for
+	sharedMachines := []SharedMachine{}
+	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
+		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
+		return nil, err
+	}
+
+	peers := make(MachinesP, 0)
+	for _, sharedMachine := range sharedMachines {
+		peers = append(peers, &sharedMachine.Machine)
+	}
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msgf("Found shared peers: %s", peers.String())
+	return peers, nil
+}
+
+func (h *Headscale) getPeers(m *Machine) (MachinesP, error) {
+	direct, err := h.getDirectPeers(m)
+	if err != nil {
+		log.Error().
+			Str("func", "getPeers").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return nil, err
+	}
+
+	shared, err := h.getShared(m)
+	if err != nil {
+		log.Error().
+			Str("func", "getDirectPeers").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return nil, err
+	}
+
+	return append(direct, shared...), nil
+}
+
+// GetMachine finds a Machine by name and namespace and returns the Machine struct
+func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) {
+	machines, err := h.ListMachinesInNamespace(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range *machines {
+		if m.Name == name {
+			return &m, nil
+		}
+	}
+	return nil, fmt.Errorf("machine not found")
+}
+
+// GetMachineByID finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
+// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
+// UpdateMachine takes a Machine struct pointer (typically already loaded from database
+// and updates it with the latest data from the database.
+func (h *Headscale) UpdateMachine(m *Machine) error {
+	if result := h.db.Find(m).First(&m); result.Error != nil {
+		return result.Error
+	}
+	return nil
+}
+
+// DeleteMachine softs deletes a Machine from the database
+func (h *Headscale) DeleteMachine(m *Machine) error {
+	m.Registered = false
+	namespaceID := m.NamespaceID
+	h.db.Save(&m) // we mark it as unregistered, just in case
+	if err := h.db.Delete(&m).Error; err != nil {
+		return err
+	}
+
+	return h.RequestMapUpdates(namespaceID)
+}
+
+// HardDeleteMachine hard deletes a Machine from the database
+func (h *Headscale) HardDeleteMachine(m *Machine) error {
+	namespaceID := m.NamespaceID
+	if err := h.db.Unscoped().Delete(&m).Error; err != nil {
+		return err
+	}
+	return h.RequestMapUpdates(namespaceID)
+}
+
+// GetHostInfo returns a Hostinfo struct for the machine
+func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
+	hostinfo := tailcfg.Hostinfo{}
+	if len(m.HostInfo) != 0 {
+		hi, err := m.HostInfo.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(hi, &hostinfo)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &hostinfo, nil
+}
+
+func (h *Headscale) notifyChangesToPeers(m *Machine) {
+	peers, err := h.getPeers(m)
+	if err != nil {
+		log.Error().
+			Str("func", "notifyChangesToPeers").
+			Str("machine", m.Name).
+			Msgf("Error getting peers: %s", err)
+		return
+	}
+	for _, peer := range peers {
+		log.Info().
+			Str("func", "notifyChangesToPeers").
+			Str("machine", m.Name).
+			Str("peer", peer.Name).
+			Str("address", peer.IPAddress).
+			Msgf("Notifying peer %s (%s)", peer.Name, peer.IPAddress)
+		err := h.sendRequestOnUpdateChannel(peer)
+		if err != nil {
+			log.Info().
+				Str("func", "notifyChangesToPeers").
+				Str("machine", m.Name).
+				Str("peer", peer.Name).
+				Msgf("Peer %s does not appear to be polling", peer.Name)
+		}
+		log.Trace().
+			Str("func", "notifyChangesToPeers").
+			Str("machine", m.Name).
+			Str("peer", peer.Name).
+			Str("address", peer.IPAddress).
+			Msgf("Notified peer %s (%s)", peer.Name, peer.IPAddress)
+	}
+}
+
+func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} {
+	var updateChan chan struct{}
+	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
+		if unwrapped, ok := storedChan.(chan struct{}); ok {
+			updateChan = unwrapped
+		} else {
+			log.Error().
+				Str("handler", "openUpdateChannel").
+				Str("machine", m.Name).
+				Msg("Failed to convert update channel to struct{}")
+		}
+	} else {
+		log.Debug().
+			Str("handler", "openUpdateChannel").
+			Str("machine", m.Name).
+			Msg("Update channel not found, creating")
+
+		updateChan = make(chan struct{})
+		h.clientsUpdateChannels.Store(m.ID, updateChan)
+	}
+	return updateChan
+}
+
+func (h *Headscale) closeUpdateChannel(m *Machine) {
+	h.clientsUpdateChannelMutex.Lock()
+	defer h.clientsUpdateChannelMutex.Unlock()
+
+	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
+		if unwrapped, ok := storedChan.(chan struct{}); ok {
+			close(unwrapped)
+		}
+	}
+	h.clientsUpdateChannels.Delete(m.ID)
+}
+
+func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
+	h.clientsUpdateChannelMutex.Lock()
+	defer h.clientsUpdateChannelMutex.Unlock()
+
+	pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID))
+	if ok {
+		log.Info().
+			Str("func", "requestUpdate").
+			Str("machine", m.Name).
+			Msgf("Notifying peer %s", m.Name)
+
+		if update, ok := pUp.(chan struct{}); ok {
+			log.Trace().
+				Str("func", "requestUpdate").
+				Str("machine", m.Name).
+				Msgf("Update channel is %#v", update)
+
+			update <- struct{}{}
+
+			log.Trace().
+				Str("func", "requestUpdate").
+				Str("machine", m.Name).
+				Msgf("Notified machine %s", m.Name)
+		}
+	} else {
+		log.Info().
+			Str("func", "requestUpdate").
+			Str("machine", m.Name).
+			Msgf("Machine %s does not appear to be polling", m.Name)
+		return errors.New("machine does not seem to be polling")
+	}
+	return nil
+}
+
+func (h *Headscale) isOutdated(m *Machine) bool {
+	err := h.UpdateMachine(m)
+	if err != nil {
+		return true
+	}
+
+	lastChange := h.getLastStateChange(m.Namespace.Name)
+	log.Trace().
+		Str("func", "keepAlive").
+		Str("machine", m.Name).
+		Time("last_successful_update", *m.LastSuccessfulUpdate).
+		Time("last_state_change", lastChange).
+		Msgf("Checking if %s is missing updates", m.Name)
+	return m.LastSuccessfulUpdate.Before(lastChange)
+}
+
+func (m Machine) String() string {
+	return m.Name
+}
+
+func (ms Machines) String() string {
+	temp := make([]string, len(ms))
+
+	for index, machine := range ms {
+		temp[index] = machine.Name
+	}
+
+	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
+}
+
+// TODO(kradalby): Remove when we have generics...
+func (ms MachinesP) String() string {
+	temp := make([]string, len(ms))
+
+	for index, machine := range ms {
+		temp[index] = machine.Name
+	}
+
+	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
+}
+
+func (ms MachinesP) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) {
+	nodes := make([]*tailcfg.Node, len(ms))
+
+	for index, machine := range ms {
+		node, err := machine.toNode(includeRoutes)
+		if err != nil {
+			return nil, err
+		}
+
+		nodes[index] = node
+	}
+
+	return nodes, nil
+}
+
 // toNode converts a Machine into a Tailscale Node. includeRoutes is false for shared nodes
 // as per the expected behaviour in the official SaaS
 func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
@@ -171,244 +490,3 @@ func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
 	}
 	return &n, nil
 }
-
-func (h *Headscale) getPeers(m Machine) (*[]*tailcfg.Node, error) {
-	log.Trace().
-		Str("func", "getPeers").
-		Str("machine", m.Name).
-		Msg("Finding peers")
-
-	machines := []Machine{}
-	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
-		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
-		log.Error().Err(err).Msg("Error accessing db")
-		return nil, err
-	}
-
-	// We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for
-	sharedMachines := []SharedMachine{}
-	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
-		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
-		return nil, err
-	}
-
-	peers := []*tailcfg.Node{}
-	for _, mn := range machines {
-		peer, err := mn.toNode(true)
-		if err != nil {
-			return nil, err
-		}
-		peers = append(peers, peer)
-	}
-	for _, sharedMachine := range sharedMachines {
-		peer, err := sharedMachine.Machine.toNode(false) // shared nodes do not expose their routes
-		if err != nil {
-			return nil, err
-		}
-		peers = append(peers, peer)
-	}
-	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
-
-	log.Trace().
-		Str("func", "getPeers").
-		Str("machine", m.Name).
-		Msgf("Found peers: %s", tailNodesToString(peers))
-	return &peers, nil
-}
-
-// GetMachine finds a Machine by name and namespace and returns the Machine struct
-func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) {
-	machines, err := h.ListMachinesInNamespace(namespace)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, m := range *machines {
-		if m.Name == name {
-			return &m, nil
-		}
-	}
-	return nil, fmt.Errorf("machine not found")
-}
-
-// GetMachineByID finds a Machine by ID and returns the Machine struct
-func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
-	m := Machine{}
-	if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil {
-		return nil, result.Error
-	}
-	return &m, nil
-}
-
-// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
-func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
-	m := Machine{}
-	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
-		return nil, result.Error
-	}
-	return &m, nil
-}
-
-// UpdateMachine takes a Machine struct pointer (typically already loaded from database
-// and updates it with the latest data from the database.
-func (h *Headscale) UpdateMachine(m *Machine) error {
-	if result := h.db.Find(m).First(&m); result.Error != nil {
-		return result.Error
-	}
-	return nil
-}
-
-// DeleteMachine softs deletes a Machine from the database
-func (h *Headscale) DeleteMachine(m *Machine) error {
-	m.Registered = false
-	namespaceID := m.NamespaceID
-	h.db.Save(&m) // we mark it as unregistered, just in case
-	if err := h.db.Delete(&m).Error; err != nil {
-		return err
-	}
-
-	return h.RequestMapUpdates(namespaceID)
-}
-
-// HardDeleteMachine hard deletes a Machine from the database
-func (h *Headscale) HardDeleteMachine(m *Machine) error {
-	namespaceID := m.NamespaceID
-	if err := h.db.Unscoped().Delete(&m).Error; err != nil {
-		return err
-	}
-	return h.RequestMapUpdates(namespaceID)
-}
-
-// GetHostInfo returns a Hostinfo struct for the machine
-func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
-	hostinfo := tailcfg.Hostinfo{}
-	if len(m.HostInfo) != 0 {
-		hi, err := m.HostInfo.MarshalJSON()
-		if err != nil {
-			return nil, err
-		}
-		err = json.Unmarshal(hi, &hostinfo)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return &hostinfo, nil
-}
-
-func (h *Headscale) notifyChangesToPeers(m *Machine) {
-	peers, err := h.getPeers(m)
-	if err != nil {
-		log.Error().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Msgf("Error getting peers: %s", err)
-		return
-	}
-	for _, p := range *peers {
-		log.Info().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", p.Name).
-			Str("address", p.Addresses[0].String()).
-			Msgf("Notifying peer %s (%s)", p.Name, p.Addresses[0])
-		err := h.sendRequestOnUpdateChannel(p)
-		if err != nil {
-			log.Info().
-				Str("func", "notifyChangesToPeers").
-				Str("machine", m.Name).
-				Str("peer", p.Name).
-				Msgf("Peer %s does not appear to be polling", p.Name)
-		}
-		log.Trace().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", p.Name).
-			Str("address", p.Addresses[0].String()).
-			Msgf("Notified peer %s (%s)", p.Name, p.Addresses[0])
-	}
-}
-
-func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} {
-	var updateChan chan struct{}
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			updateChan = unwrapped
-		} else {
-			log.Error().
-				Str("handler", "openUpdateChannel").
-				Str("machine", m.Name).
-				Msg("Failed to convert update channel to struct{}")
-		}
-	} else {
-		log.Debug().
-			Str("handler", "openUpdateChannel").
-			Str("machine", m.Name).
-			Msg("Update channel not found, creating")
-
-		updateChan = make(chan struct{})
-		h.clientsUpdateChannels.Store(m.ID, updateChan)
-	}
-	return updateChan
-}
-
-func (h *Headscale) closeUpdateChannel(m *Machine) {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			close(unwrapped)
-		}
-	}
-	h.clientsUpdateChannels.Delete(m.ID)
-}
-
-func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID))
-	if ok {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Notifying peer %s", m.Name)
-
-		if update, ok := pUp.(chan struct{}); ok {
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Update channel is %#v", update)
-
-			update <- struct{}{}
-
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Notified machine %s", m.Name)
-		}
-	} else {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Machine %s does not appear to be polling", m.Name)
-		return errors.New("machine does not seem to be polling")
-	}
-	return nil
-}
-
-func (h *Headscale) isOutdated(m *Machine) bool {
-	err := h.UpdateMachine(m)
-	if err != nil {
-		return true
-	}
-
-	lastChange := h.getLastStateChange(m.Namespace.Name)
-	log.Trace().
-		Str("func", "keepAlive").
-		Str("machine", m.Name).
-		Time("last_successful_update", *m.LastSuccessfulUpdate).
-		Time("last_state_change", lastChange).
-		Msgf("Checking if %s is missing updates", m.Name)
-	return m.LastSuccessfulUpdate.Before(lastChange)
-}

From d637a9c3024160b81b25047b92d7c2757ba3c1bf Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Sat, 2 Oct 2021 22:56:48 +0100
Subject: [PATCH 12/38] Change ping count

---
 integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_test.go b/integration_test.go
index 925fc4ca..b768fa59 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -433,7 +433,7 @@ func (s *IntegrationTestSuite) TestPingAllPeers() {
 						command := []string{
 							"tailscale", "ping",
 							"--timeout=1s",
-							"--c=100",
+							"--c=10",
 							"--until-direct=true",
 							ip.String(),
 						}

From 1d81333685d06d15b93ef693970899ad784da7d6 Mon Sep 17 00:00:00 2001
From: Ward Vandewege <ward@jhvc.com>
Date: Sun, 3 Oct 2021 12:06:50 -0400
Subject: [PATCH 13/38] Make sure that goreleaser uses the appropriate version
 string when building the headscale executable.

---
 .goreleaser.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.goreleaser.yml b/.goreleaser.yml
index fd25563c..7b1ea605 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -19,7 +19,7 @@ builds:
     flags:
       - -mod=readonly
     ldflags:
-      - -s -w -X main.version={{.Version}}
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
   - id: linux-armhf
     main: ./cmd/headscale/headscale.go
     mod_timestamp: '{{ .CommitTimestamp }}'
@@ -39,7 +39,7 @@ builds:
     flags:
       - -mod=readonly
     ldflags:
-      - -s -w -X main.version={{.Version}}
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
 
 
   - id: linux-amd64
@@ -54,6 +54,8 @@ builds:
       - 7
     main: ./cmd/headscale/headscale.go
     mod_timestamp: '{{ .CommitTimestamp }}'
+    ldflags:
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
 
 archives:
   - id: golang-cross

From 8fa0fe65bacc000eac3a876e0dece01e42be55b4 Mon Sep 17 00:00:00 2001
From: Aaron Bieber <aaron@bolddaemon.com>
Date: Sun, 3 Oct 2021 12:26:38 -0600
Subject: [PATCH 14/38] Add the ability to specify registration ACME email and
 ACME URL.

---
 app.go                       | 8 ++++++++
 cmd/headscale/cli/utils.go   | 3 +++
 config.json.postgres.example | 2 ++
 config.json.sqlite.example   | 2 ++
 4 files changed, 15 insertions(+)

diff --git a/app.go b/app.go
index 8be137cf..1e6b7bcf 100644
--- a/app.go
+++ b/app.go
@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/gin-gonic/gin"
+	"golang.org/x/crypto/acme"
 	"golang.org/x/crypto/acme/autocert"
 	"gorm.io/gorm"
 	"inet.af/netaddr"
@@ -44,6 +45,9 @@ type Config struct {
 	TLSCertPath string
 	TLSKeyPath  string
 
+	ACMEURL   string
+	ACMEEmail string
+
 	DNSConfig *tailcfg.DNSConfig
 }
 
@@ -195,6 +199,10 @@ func (h *Headscale) Serve() error {
 			Prompt:     autocert.AcceptTOS,
 			HostPolicy: autocert.HostWhitelist(h.cfg.TLSLetsEncryptHostname),
 			Cache:      autocert.DirCache(h.cfg.TLSLetsEncryptCacheDir),
+			Client: &acme.Client{
+				DirectoryURL: h.cfg.ACMEURL,
+			},
+			Email: h.cfg.ACMEEmail,
 		}
 
 		s.TLSConfig = m.TLSConfig()
diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go
index 7ada6693..ac739a02 100644
--- a/cmd/headscale/cli/utils.go
+++ b/cmd/headscale/cli/utils.go
@@ -169,6 +169,9 @@ func getHeadscaleApp() (*headscale.Headscale, error) {
 		TLSCertPath: absPath(viper.GetString("tls_cert_path")),
 		TLSKeyPath:  absPath(viper.GetString("tls_key_path")),
 
+		ACMEEmail: absPath(viper.GetString("acme_email")),
+		ACMEURL:   absPath(viper.GetString("acme_url")),
+
 		DNSConfig: GetDNSConfig(),
 	}
 
diff --git a/config.json.postgres.example b/config.json.postgres.example
index aba72063..e9118204 100644
--- a/config.json.postgres.example
+++ b/config.json.postgres.example
@@ -10,6 +10,8 @@
     "db_name": "headscale",
     "db_user": "foo",
     "db_pass": "bar",
+    "acme_url": "https://acme-v02.api.letsencrypt.org/directory",
+    "acme_email": "",
     "tls_letsencrypt_hostname": "",
     "tls_letsencrypt_listen": ":http",
     "tls_letsencrypt_cache_dir": ".cache",
diff --git a/config.json.sqlite.example b/config.json.sqlite.example
index b22e5ace..5afa450f 100644
--- a/config.json.sqlite.example
+++ b/config.json.sqlite.example
@@ -6,6 +6,8 @@
     "ephemeral_node_inactivity_timeout": "30m",
     "db_type": "sqlite3",
     "db_path": "db.sqlite",
+    "acme_url": "https://acme-v02.api.letsencrypt.org/directory",
+    "acme_email": "",
     "tls_letsencrypt_hostname": "",
     "tls_letsencrypt_listen": ":http",
     "tls_letsencrypt_cache_dir": ".cache",

From 817cc1e567624f365f014bd31b985b509fa358b5 Mon Sep 17 00:00:00 2001
From: Aaron Bieber <aaron@bolddaemon.com>
Date: Sun, 3 Oct 2021 14:02:44 -0600
Subject: [PATCH 15/38] these are not files!

---
 cmd/headscale/cli/utils.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go
index ac739a02..ca09bf51 100644
--- a/cmd/headscale/cli/utils.go
+++ b/cmd/headscale/cli/utils.go
@@ -169,8 +169,8 @@ func getHeadscaleApp() (*headscale.Headscale, error) {
 		TLSCertPath: absPath(viper.GetString("tls_cert_path")),
 		TLSKeyPath:  absPath(viper.GetString("tls_key_path")),
 
-		ACMEEmail: absPath(viper.GetString("acme_email")),
-		ACMEURL:   absPath(viper.GetString("acme_url")),
+		ACMEEmail: viper.GetString("acme_email"),
+		ACMEURL:   viper.GetString("acme_url"),
 
 		DNSConfig: GetDNSConfig(),
 	}

From 566c2bc1fb1e73a84c85f147c8dc39994a8e8f9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Paul=20T=C3=B6tterman?= <ptman@users.noreply.github.com>
Date: Mon, 4 Oct 2021 14:58:36 +0300
Subject: [PATCH 16/38] Document client OS support in a table

---
 README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.md b/README.md
index 45c648ac..766e9465 100644
--- a/README.md
+++ b/README.md
@@ -30,6 +30,16 @@ Headscale implements this coordination server.
 - [x] Share nodes between ~~users~~ namespaces
 - [ ] MagicDNS / Smart DNS
 
+## Client OS support
+
+| OS | Supports headscale |
+| --- | --- |
+| Linux | Yes |
+| macOS | Yes |
+| Windows | Yes |
+| Android | [You need to compile the client yourself](https://github.com/juanfont/headscale/issues/58#issuecomment-885255270) |
+| iOS | Not yet |
+
 ## Roadmap 🤷
 
 Suggestions/PRs welcomed!

From 779301240972e6aa1291fcb014485467667b94a4 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:14:12 +0100
Subject: [PATCH 17/38] Add error if peer api is empty

---
 integration_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/integration_test.go b/integration_test.go
index b768fa59..dd792142 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -692,6 +693,9 @@ func getAPIURLs(tailscales map[string]dockertest.Resource) (map[netaddr.IP]strin
 			n := ft.Node
 			for _, a := range n.Addresses { // just add all the addresses
 				if _, ok := fts[a.IP()]; !ok {
+					if ft.PeerAPIURL == "" {
+						return nil, errors.New("api url is empty")
+					}
 					fts[a.IP()] = ft.PeerAPIURL
 				}
 			}

From 31b4f03f96329ef8d5c91dff339763d436b5be5e Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:14:28 +0100
Subject: [PATCH 18/38] Set integration logging to trace

---
 integration_test/etc/config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_test/etc/config.json b/integration_test/etc/config.json
index 8a6fd962..5454f2f7 100644
--- a/integration_test/etc/config.json
+++ b/integration_test/etc/config.json
@@ -7,5 +7,5 @@
   "db_type": "sqlite3",
   "db_path": "/tmp/integration_test_db.sqlite3",
   "acl_policy_path": "",
-  "log_level": "debug"
+  "log_level": "trace"
 }

From 2090a13dcd8ab18d25ec7ea99b50f233e512dfb6 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:15:20 +0100
Subject: [PATCH 19/38] Remove docker network, it wasnt used, comment out
 portmapping to host

---
 integration_test.go | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index dd792142..ecde3830 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -35,7 +35,6 @@ var (
 
 var (
 	pool      dockertest.Pool
-	network   dockertest.Network
 	headscale dockertest.Resource
 )
 
@@ -89,10 +88,6 @@ func TestIntegrationTestSuite(t *testing.T) {
 	if err := pool.Purge(&headscale); err != nil {
 		log.Printf("Could not purge resource: %s\n", err)
 	}
-
-	if err := network.Close(); err != nil {
-		log.Printf("Could not close network: %s\n", err)
-	}
 }
 
 func executeCommand(resource *dockertest.Resource, cmd []string, env []string) (string, error) {
@@ -184,9 +179,8 @@ func tailscaleContainer(namespace, identifier, version string) (string, *dockert
 	}
 	hostname := fmt.Sprintf("%s-tailscale-%s-%s", namespace, strings.Replace(version, ".", "-", -1), identifier)
 	tailscaleOptions := &dockertest.RunOptions{
-		Name:     hostname,
-		Networks: []*dockertest.Network{&network},
-		Cmd:      []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
+		Name: hostname,
+		Cmd:  []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
 	}
 
 	pts, err := pool.BuildAndRunWithBuildOptions(tailscaleBuildOptions, tailscaleOptions, dockerRestartPolicy)
@@ -210,12 +204,6 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		log.Fatalf("Could not connect to docker: %s", err)
 	}
 
-	if pnetwork, err := pool.CreateNetwork("headscale-test"); err == nil {
-		network = *pnetwork
-	} else {
-		log.Fatalf("Could not create network: %s", err)
-	}
-
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile",
 		ContextDir: ".",
@@ -232,11 +220,10 @@ func (s *IntegrationTestSuite) SetupSuite() {
 			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
 			fmt.Sprintf("%s/derp.yaml:/etc/headscale/derp.yaml", currentPath),
 		},
-		Networks: []*dockertest.Network{&network},
-		Cmd:      []string{"headscale", "serve"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"8080/tcp": {{HostPort: "8080"}},
-		},
+		Cmd: []string{"headscale", "serve"},
+		// PortBindings: map[docker.Port][]docker.PortBinding{
+		// 	"8080/tcp": {{HostPort: "8080"}},
+		// },
 	}
 
 	fmt.Println("Creating headscale container")

From 772541afaba27716e3b5bd2a861a993f347dbfbd Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:16:37 +0100
Subject: [PATCH 20/38] add comment about poor error handling when headscale
 isnt becoming available

---
 integration_test.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/integration_test.go b/integration_test.go
index ecde3830..7e75089d 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -258,7 +258,11 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		}
 		return nil
 	}); err != nil {
-		log.Fatalf("Could not connect to docker: %s", err)
+		// TODO(kradalby): If we cannot access headscale, or any other fatal error during
+		// test setup, we need to abort and tear down. However, testify does not seem to
+		// support that at the moment:
+		// https://github.com/stretchr/testify/issues/849
+		return // fmt.Errorf("Could not connect to headscale: %s", err)
 	}
 	fmt.Println("headscale container is ready")
 

From 931ef9482b4d7d66a9e42fe702fa367d90c971ae Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:17:05 +0100
Subject: [PATCH 21/38] Add checks to see if we can fetch the ip from map,
 remove possible null assignment

---
 integration_test.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index 7e75089d..30703d63 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -345,15 +345,16 @@ func (s *IntegrationTestSuite) TestGetIpAddresses() {
 
 		for hostname := range scales.tailscales {
 			s.T().Run(hostname, func(t *testing.T) {
-				ip := ips[hostname]
+				ip, ok := ips[hostname]
+
+				assert.True(t, ok)
+				assert.NotNil(t, ip)
 
 				fmt.Printf("IP for %s: %s\n", hostname, ip)
 
 				// c.Assert(ip.Valid(), check.IsTrue)
 				assert.True(t, ip.Is4())
 				assert.True(t, ipPrefix.Contains(ip))
-
-				ips[hostname] = ip
 			})
 		}
 	}

From c09428accad05a829c2aabfb4db8d2f63f0bdc1e Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:09:21 +0000
Subject: [PATCH 22/38] Revert "Remove docker network, it wasnt used, comment
 out portmapping to host"

This reverts commit 2090a13dcd8ab18d25ec7ea99b50f233e512dfb6.
---
 integration_test.go | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index 30703d63..58fa8582 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -35,6 +35,7 @@ var (
 
 var (
 	pool      dockertest.Pool
+	network   dockertest.Network
 	headscale dockertest.Resource
 )
 
@@ -88,6 +89,10 @@ func TestIntegrationTestSuite(t *testing.T) {
 	if err := pool.Purge(&headscale); err != nil {
 		log.Printf("Could not purge resource: %s\n", err)
 	}
+
+	if err := network.Close(); err != nil {
+		log.Printf("Could not close network: %s\n", err)
+	}
 }
 
 func executeCommand(resource *dockertest.Resource, cmd []string, env []string) (string, error) {
@@ -179,8 +184,9 @@ func tailscaleContainer(namespace, identifier, version string) (string, *dockert
 	}
 	hostname := fmt.Sprintf("%s-tailscale-%s-%s", namespace, strings.Replace(version, ".", "-", -1), identifier)
 	tailscaleOptions := &dockertest.RunOptions{
-		Name: hostname,
-		Cmd:  []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
+		Name:     hostname,
+		Networks: []*dockertest.Network{&network},
+		Cmd:      []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
 	}
 
 	pts, err := pool.BuildAndRunWithBuildOptions(tailscaleBuildOptions, tailscaleOptions, dockerRestartPolicy)
@@ -204,6 +210,12 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		log.Fatalf("Could not connect to docker: %s", err)
 	}
 
+	if pnetwork, err := pool.CreateNetwork("headscale-test"); err == nil {
+		network = *pnetwork
+	} else {
+		log.Fatalf("Could not create network: %s", err)
+	}
+
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile",
 		ContextDir: ".",
@@ -220,10 +232,11 @@ func (s *IntegrationTestSuite) SetupSuite() {
 			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
 			fmt.Sprintf("%s/derp.yaml:/etc/headscale/derp.yaml", currentPath),
 		},
-		Cmd: []string{"headscale", "serve"},
-		// PortBindings: map[docker.Port][]docker.PortBinding{
-		// 	"8080/tcp": {{HostPort: "8080"}},
-		// },
+		Networks: []*dockertest.Network{&network},
+		Cmd:      []string{"headscale", "serve"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"8080/tcp": {{HostPort: "8080"}},
+		},
 	}
 
 	fmt.Println("Creating headscale container")

From fcc6991d627c27ba56d8a88e4965c9e0118dda15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Paul=20T=C3=B6tterman?= <ptman@users.noreply.github.com>
Date: Mon, 4 Oct 2021 17:23:31 +0300
Subject: [PATCH 23/38] Update README.md

Co-authored-by: Kristoffer Dalby <kradalby@kradalby.no>
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 766e9465..5c3cc54d 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,7 @@ Headscale implements this coordination server.
 | OS | Supports headscale |
 | --- | --- |
 | Linux | Yes |
+| OpenBSD | Yes |
 | macOS | Yes |
 | Windows | Yes |
 | Android | [You need to compile the client yourself](https://github.com/juanfont/headscale/issues/58#issuecomment-885255270) |

From ed0b31d0727cccd1290dff3299f3bfee910d4ef1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Paul=20T=C3=B6tterman?= <ptman@users.noreply.github.com>
Date: Mon, 4 Oct 2021 17:23:38 +0300
Subject: [PATCH 24/38] Update README.md

Co-authored-by: Kristoffer Dalby <kradalby@kradalby.no>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5c3cc54d..5f691a6c 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Headscale implements this coordination server.
 | --- | --- |
 | Linux | Yes |
 | OpenBSD | Yes |
-| macOS | Yes |
+| macOS | Yes (see `/apple` on your headscale for more information) |
 | Windows | Yes |
 | Android | [You need to compile the client yourself](https://github.com/juanfont/headscale/issues/58#issuecomment-885255270) |
 | iOS | Not yet |

From 07e32be5cecb91df8f7b5a7b41060c81140e29e3 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:39:28 +0000
Subject: [PATCH 25/38] Remove host port, we only need internal ports

---
 integration_test.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration_test.go b/integration_test.go
index 58fa8582..cb969412 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -234,9 +234,6 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		},
 		Networks: []*dockertest.Network{&network},
 		Cmd:      []string{"headscale", "serve"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"8080/tcp": {{HostPort: "8080"}},
-		},
 	}
 
 	fmt.Println("Creating headscale container")

From d3ef39a58f547365b9a4a55d2b78e687c9628c65 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 14:39:52 +0000
Subject: [PATCH 26/38] Correctly use the internal docker dns and port for
 headscale joining

---
 integration_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_test.go b/integration_test.go
index cb969412..a7805646 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -294,7 +294,7 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		)
 		assert.Nil(s.T(), err)
 
-		headscaleEndpoint := fmt.Sprintf("http://headscale:%s", headscale.GetPort("8080/tcp"))
+		headscaleEndpoint := "http://headscale:8080"
 
 		fmt.Printf("Joining tailscale containers to headscale at %s\n", headscaleEndpoint)
 		for hostname, tailscale := range scales.tailscales {

From 1d5b090579f21b94eee9364f4651fdf243e6abf6 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 16:28:07 +0000
Subject: [PATCH 27/38] Initial work on Prometheus metrics

This commit adds some Prometheus metrics to /metrics in headscale.

It will add the standard go metrics, some automatic gin metrics and some
initial headscale specific ones.

Some of them has been added to aid debugging #97 (loop bug)

In the future, we can use the metrics to get rid of the sleep in the
integration tests by checking that our expected number of nodes has been
registered:

```
headscale_machine_registrations_total
```
---
 api.go        | 14 +++++++++++++-
 app.go        |  7 +++++++
 go.mod        |  2 ++
 go.sum        | 26 ++++++++++++++++++++++++++
 machine.go    |  1 +
 metrics.go    | 41 +++++++++++++++++++++++++++++++++++++++++
 namespaces.go |  1 +
 poll.go       |  9 +++++++++
 8 files changed, 100 insertions(+), 1 deletion(-)
 create mode 100644 metrics.go

diff --git a/api.go b/api.go
index 90d9be2b..9cdf7100 100644
--- a/api.go
+++ b/api.go
@@ -64,6 +64,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			Str("handler", "Registration").
 			Err(err).
 			Msg("Cannot parse machine key")
+		machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc()
 		c.String(http.StatusInternalServerError, "Sad!")
 		return
 	}
@@ -74,6 +75,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			Str("handler", "Registration").
 			Err(err).
 			Msg("Cannot decode message")
+		machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc()
 		c.String(http.StatusInternalServerError, "Very sad!")
 		return
 	}
@@ -94,6 +96,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 				Str("handler", "Registration").
 				Err(err).
 				Msg("Could not create row")
+			machineRegistrations.WithLabelValues("unkown", "web", "error", m.Namespace.Name).Inc()
 			return
 		}
 	}
@@ -122,9 +125,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 					Str("handler", "Registration").
 					Err(err).
 					Msg("Cannot encode message")
+				machineRegistrations.WithLabelValues("update", "web", "error", m.Namespace.Name).Inc()
 				c.String(http.StatusInternalServerError, "")
 				return
 			}
+			machineRegistrations.WithLabelValues("update", "web", "success", m.Namespace.Name).Inc()
 			c.Data(200, "application/json; charset=utf-8", respBody)
 			return
 		}
@@ -141,9 +146,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 				Str("handler", "Registration").
 				Err(err).
 				Msg("Cannot encode message")
+			machineRegistrations.WithLabelValues("new", "web", "error", m.Namespace.Name).Inc()
 			c.String(http.StatusInternalServerError, "")
 			return
 		}
+		machineRegistrations.WithLabelValues("new", "web", "success", m.Namespace.Name).Inc()
 		c.Data(200, "application/json; charset=utf-8", respBody)
 		return
 	}
@@ -338,13 +345,15 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 				Err(err).
 				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "")
+			machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 			return
 		}
-		c.Data(200, "application/json; charset=utf-8", respBody)
+		c.Data(401, "application/json; charset=utf-8", respBody)
 		log.Error().
 			Str("func", "handleAuthKey").
 			Str("machine", m.Name).
 			Msg("Failed authentication via AuthKey")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		return
 	}
 
@@ -358,6 +367,7 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 			Str("func", "handleAuthKey").
 			Str("machine", m.Name).
 			Msg("Failed to find an available IP")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		return
 	}
 	log.Info().
@@ -383,9 +393,11 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 			Str("machine", m.Name).
 			Err(err).
 			Msg("Cannot encode message")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		c.String(http.StatusInternalServerError, "Extremely sad!")
 		return
 	}
+	machineRegistrations.WithLabelValues("new", "authkey", "success", m.Namespace.Name).Inc()
 	c.Data(200, "application/json; charset=utf-8", respBody)
 	log.Info().
 		Str("func", "handleAuthKey").
diff --git a/app.go b/app.go
index 8be137cf..107b78d6 100644
--- a/app.go
+++ b/app.go
@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/gin-gonic/gin"
+	"github.com/zsais/go-gin-prometheus"
 	"golang.org/x/crypto/acme/autocert"
 	"gorm.io/gorm"
 	"inet.af/netaddr"
@@ -140,6 +141,7 @@ func (h *Headscale) expireEphemeralNodesWorker() {
 				if err != nil {
 					log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database")
 				}
+				updateRequestsFromNode.WithLabelValues("ephemeral-node-update").Inc()
 				h.notifyChangesToPeers(&m)
 			}
 		}
@@ -163,6 +165,10 @@ func (h *Headscale) watchForKVUpdatesWorker() {
 // Serve launches a GIN server with the Headscale API
 func (h *Headscale) Serve() error {
 	r := gin.Default()
+
+	p := ginprometheus.NewPrometheus("gin")
+	p.Use(r)
+
 	r.GET("/health", func(c *gin.Context) { c.JSON(200, gin.H{"healthy": "ok"}) })
 	r.GET("/key", h.KeyHandler)
 	r.GET("/register", h.RegisterWebAPI)
@@ -233,6 +239,7 @@ func (h *Headscale) Serve() error {
 
 func (h *Headscale) setLastStateChangeToNow(namespace string) {
 	now := time.Now().UTC()
+	lastStateUpdate.WithLabelValues("", "headscale").Set(float64(now.Unix()))
 	h.lastStateChange.Store(namespace, now)
 }
 
diff --git a/go.mod b/go.mod
index 3a49c0fd..70428315 100644
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,7 @@ require (
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/ory/dockertest/v3 v3.7.0
+	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/pterm/pterm v0.12.30
 	github.com/rs/zerolog v1.25.0
 	github.com/spf13/cobra v1.2.1
@@ -28,6 +29,7 @@ require (
 	github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88
 	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/zsais/go-gin-prometheus v0.1.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
 	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
diff --git a/go.sum b/go.sum
index 1d97f18f..6dc92b6d 100644
--- a/go.sum
+++ b/go.sum
@@ -103,6 +103,7 @@ github.com/aws/aws-sdk-go v1.38.52/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
@@ -118,7 +119,9 @@ github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
 github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -518,6 +521,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB
 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
 github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
@@ -531,6 +535,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -538,6 +543,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -616,6 +622,7 @@ github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGw
 github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
 github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
 github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y=
@@ -669,6 +676,7 @@ github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8q
 github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
@@ -747,21 +755,32 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
 github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
@@ -929,6 +948,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4=
+github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4=
+github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
@@ -1143,6 +1164,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1158,6 +1180,8 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1190,6 +1214,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1450,6 +1475,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
diff --git a/machine.go b/machine.go
index 95aef851..cd2d124a 100644
--- a/machine.go
+++ b/machine.go
@@ -297,6 +297,7 @@ func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
 				Str("machine", m.Name).
 				Msgf("Update channel is %#v", update)
 
+			updateRequestsToNode.Inc()
 			update <- struct{}{}
 
 			log.Trace().
diff --git a/metrics.go b/metrics.go
new file mode 100644
index 00000000..d516fade
--- /dev/null
+++ b/metrics.go
@@ -0,0 +1,41 @@
+package headscale
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const prometheusNamespace = "headscale"
+
+var (
+	// This is a high cardinality metric (namespace x machines), we might want to make this
+	// configurable/opt-in in the future.
+	lastStateUpdate = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: prometheusNamespace,
+		Name:      "last_update_seconds",
+		Help:      "Time stamp in unix time when a machine or headscale was updated",
+	}, []string{"namespace", "machine"})
+
+	machineRegistrations = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "machine_registrations_total",
+		Help:      "The total amount of registered machine attempts",
+	}, []string{"action", "auth", "status", "namespace"})
+
+	updateRequestsFromNode = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_from_node_total",
+		Help:      "The number of updates requested by a node/update function",
+	}, []string{"state"})
+	updateRequestsToNode = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_to_node_total",
+		Help:      "The number of calls/messages issued on a specific nodes update channel",
+	})
+	//TODO(kradalby): This is very debugging, we might want to remove it.
+	updateRequestsReceivedOnChannel = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_received_on_channel_total",
+		Help:      "The number of update requests received on an update channel",
+	}, []string{"machine"})
+)
diff --git a/namespaces.go b/namespaces.go
index 8204f96d..75b6eab5 100644
--- a/namespaces.go
+++ b/namespaces.go
@@ -191,6 +191,7 @@ func (h *Headscale) checkForNamespacesPendingUpdates() {
 			continue
 		}
 		for _, m := range *machines {
+			updateRequestsFromNode.WithLabelValues("namespace-update").Inc()
 			h.notifyChangesToPeers(&m)
 		}
 	}
diff --git a/poll.go b/poll.go
index 7e547b4d..46e6cf8f 100644
--- a/poll.go
+++ b/poll.go
@@ -159,6 +159,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
+		updateRequestsFromNode.WithLabelValues("endpoint-update").Inc()
 		go h.notifyChangesToPeers(m)
 		return
 	} else if req.OmitPeers && req.Stream {
@@ -184,6 +185,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Notifying peers")
+	updateRequestsFromNode.WithLabelValues("full-update").Inc()
 	go h.notifyChangesToPeers(m)
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
@@ -258,7 +260,10 @@ func (h *Headscale) PollNetMapStream(
 			}
 			now := time.Now().UTC()
 			m.LastSeen = &now
+
+			lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix()))
 			m.LastSuccessfulUpdate = &now
+
 			h.db.Save(&m)
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -320,6 +325,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "update").
 				Msg("Received a request for update")
+			updateRequestsReceivedOnChannel.WithLabelValues(m.Name).Inc()
 			if h.isOutdated(m) {
 				log.Debug().
 					Str("handler", "PollNetMapStream").
@@ -369,7 +375,10 @@ func (h *Headscale) PollNetMapStream(
 						Msg("Cannot update machine from database")
 				}
 				now := time.Now().UTC()
+
+				lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix()))
 				m.LastSuccessfulUpdate = &now
+
 				h.db.Save(&m)
 			} else {
 				log.Trace().

From 2eb57e6288afb9925e81bf29d0f4acf45968ef32 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 17:39:01 +0000
Subject: [PATCH 28/38] Clean up pointer usage consistency.

This tries to make the same functions emit and consume the same type of
data all over the application.

If a function transform data, it should emit new data, not a pointer.
---
 api.go     |  9 +++++----
 machine.go | 41 ++++++++++++++++++++++-------------------
 poll.go    | 10 +++++-----
 3 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/api.go b/api.go
index 9cdf7100..17306b32 100644
--- a/api.go
+++ b/api.go
@@ -220,7 +220,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	c.Data(200, "application/json; charset=utf-8", respBody)
 }
 
-func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
+func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) {
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
@@ -277,6 +277,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
+		Interface("payload", resp).
 		Msgf("Generated map response: %s", tailMapResponseToString(resp))
 
 	var respBody []byte
@@ -299,10 +300,10 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 	data := make([]byte, 4)
 	binary.LittleEndian.PutUint32(data, uint32(len(respBody)))
 	data = append(data, respBody...)
-	return &data, nil
+	return data, nil
 }
 
-func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) (*[]byte, error) {
+func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) {
 	resp := tailcfg.MapResponse{
 		KeepAlive: true,
 	}
@@ -325,7 +326,7 @@ func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapReque
 	data := make([]byte, 4)
 	binary.LittleEndian.PutUint32(data, uint32(len(respBody)))
 	data = append(data, respBody...)
-	return &data, nil
+	return data, nil
 }
 
 func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, req tailcfg.RegisterRequest, m Machine) {
diff --git a/machine.go b/machine.go
index cd2d124a..7a9df2e8 100644
--- a/machine.go
+++ b/machine.go
@@ -56,34 +56,29 @@ func (m Machine) isAlreadyRegistered() bool {
 	return m.Registered
 }
 
-func (h *Headscale) getDirectPeers(m *Machine) (MachinesP, error) {
+func (h *Headscale) getDirectPeers(m *Machine) (Machines, error) {
 	log.Trace().
 		Str("func", "getDirectPeers").
 		Str("machine", m.Name).
-		Msg("Finding peers")
+		Msg("Finding direct peers")
 
-	machines := []Machine{}
+	machines := Machines{}
 	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
 		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
 		log.Error().Err(err).Msg("Error accessing db")
 		return nil, err
 	}
 
-	peers := make(MachinesP, 0)
-	for _, peer := range machines {
-		peers = append(peers, &peer)
-	}
-
-	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+	sort.Slice(machines, func(i, j int) bool { return machines[i].ID < machines[j].ID })
 
 	log.Trace().
-		Str("func", "getDirectPeers").
+		Str("func", "getDirectmachines").
 		Str("machine", m.Name).
-		Msgf("Found peers: %s", peers.String())
-	return peers, nil
+		Msgf("Found direct machines: %s", machines.String())
+	return machines, nil
 }
 
-func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
+func (h *Headscale) getShared(m *Machine) (Machines, error) {
 	log.Trace().
 		Str("func", "getShared").
 		Str("machine", m.Name).
@@ -96,9 +91,9 @@ func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
 		return nil, err
 	}
 
-	peers := make(MachinesP, 0)
+	peers := make(Machines, 0)
 	for _, sharedMachine := range sharedMachines {
-		peers = append(peers, &sharedMachine.Machine)
+		peers = append(peers, sharedMachine.Machine)
 	}
 
 	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
@@ -110,7 +105,7 @@ func (h *Headscale) getShared(m *Machine) (MachinesP, error) {
 	return peers, nil
 }
 
-func (h *Headscale) getPeers(m *Machine) (MachinesP, error) {
+func (h *Headscale) getPeers(m *Machine) (Machines, error) {
 	direct, err := h.getDirectPeers(m)
 	if err != nil {
 		log.Error().
@@ -129,7 +124,15 @@ func (h *Headscale) getPeers(m *Machine) (MachinesP, error) {
 		return nil, err
 	}
 
-	return append(direct, shared...), nil
+	peers := append(direct, shared...)
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msgf("Found total peers: %s", peers.String())
+
+	return peers, nil
 }
 
 // GetMachine finds a Machine by name and namespace and returns the Machine struct
@@ -227,7 +230,7 @@ func (h *Headscale) notifyChangesToPeers(m *Machine) {
 			Str("peer", peer.Name).
 			Str("address", peer.IPAddress).
 			Msgf("Notifying peer %s (%s)", peer.Name, peer.IPAddress)
-		err := h.sendRequestOnUpdateChannel(peer)
+		err := h.sendRequestOnUpdateChannel(&peer)
 		if err != nil {
 			log.Info().
 				Str("func", "notifyChangesToPeers").
@@ -357,7 +360,7 @@ func (ms MachinesP) String() string {
 	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
 }
 
-func (ms MachinesP) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) {
+func (ms Machines) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) {
 	nodes := make([]*tailcfg.Node, len(ms))
 
 	for index, machine := range ms {
diff --git a/poll.go b/poll.go
index 46e6cf8f..a33d341b 100644
--- a/poll.go
+++ b/poll.go
@@ -123,7 +123,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			Str("handler", "PollNetMap").
 			Str("machine", m.Name).
 			Msg("Client is starting up. Probably interested in a DERP map")
-		c.Data(200, "application/json; charset=utf-8", *data)
+		c.Data(200, "application/json; charset=utf-8", data)
 		return
 	}
 
@@ -155,7 +155,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			Str("handler", "PollNetMap").
 			Str("machine", m.Name).
 			Msg("Client sent endpoint update and is ok with a response without peer list")
-		c.Data(200, "application/json; charset=utf-8", *data)
+		c.Data(200, "application/json; charset=utf-8", data)
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
@@ -179,7 +179,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Sending initial map")
-	go func() { pollDataChan <- *data }()
+	go func() { pollDataChan <- data }()
 
 	log.Info().
 		Str("handler", "PollNetMap").
@@ -342,7 +342,7 @@ func (h *Headscale) PollNetMapStream(
 						Err(err).
 						Msg("Could not get the map update")
 				}
-				_, err = w.Write(*data)
+				_, err = w.Write(data)
 				if err != nil {
 					log.Error().
 						Str("handler", "PollNetMapStream").
@@ -473,7 +473,7 @@ func (h *Headscale) scheduledPollWorker(
 				Str("func", "keepAlive").
 				Str("machine", m.Name).
 				Msg("Sending keepalive")
-			keepAliveChan <- *data
+			keepAliveChan <- data
 
 		case <-updateCheckerTicker.C:
 			// Send an update request regardless of outdated or not, if data is sent

From f6a7564ec870a57c1b22c101578dd1edd9ba43b7 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Mon, 4 Oct 2021 17:40:21 +0000
Subject: [PATCH 29/38] Add more test cases to prove that peers and shared
 peers work properly

---
 machine_test.go | 41 +++++++++++++++++++++++++++++++++++++++++
 sharing_test.go |  9 +++++++++
 2 files changed, 50 insertions(+)

diff --git a/machine_test.go b/machine_test.go
index cf0d12e7..dfe84d33 100644
--- a/machine_test.go
+++ b/machine_test.go
@@ -2,6 +2,7 @@ package headscale
 
 import (
 	"encoding/json"
+	"strconv"
 
 	"gopkg.in/check.v1"
 )
@@ -116,3 +117,43 @@ func (s *Suite) TestHardDeleteMachine(c *check.C) {
 	_, err = h.GetMachine(n.Name, "testmachine3")
 	c.Assert(err, check.NotNil)
 }
+
+func (s *Suite) TestGetDirectPeers(c *check.C) {
+	n, err := h.CreateNamespace("test")
+	c.Assert(err, check.IsNil)
+
+	pak, err := h.CreatePreAuthKey(n.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	_, err = h.GetMachineByID(0)
+	c.Assert(err, check.NotNil)
+
+	for i := 0; i <= 10; i++ {
+		m := Machine{
+			ID:             uint64(i),
+			MachineKey:     "foo" + strconv.Itoa(i),
+			NodeKey:        "bar" + strconv.Itoa(i),
+			DiscoKey:       "faa" + strconv.Itoa(i),
+			Name:           "testmachine" + strconv.Itoa(i),
+			NamespaceID:    n.ID,
+			Registered:     true,
+			RegisterMethod: "authKey",
+			AuthKeyID:      uint(pak.ID),
+		}
+		h.db.Save(&m)
+	}
+
+	m1, err := h.GetMachineByID(0)
+	c.Assert(err, check.IsNil)
+
+	_, err = m1.GetHostInfo()
+	c.Assert(err, check.IsNil)
+
+	peers, err := h.getDirectPeers(m1)
+	c.Assert(err, check.IsNil)
+
+	c.Assert(len(peers), check.Equals, 9)
+	c.Assert(peers[0].Name, check.Equals, "testmachine2")
+	c.Assert(peers[5].Name, check.Equals, "testmachine7")
+	c.Assert(peers[8].Name, check.Equals, "testmachine10")
+}
diff --git a/sharing_test.go b/sharing_test.go
index 25de5846..baa90d0c 100644
--- a/sharing_test.go
+++ b/sharing_test.go
@@ -245,6 +245,7 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(p1sAfter), check.Equals, 1)
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
 }
 
 func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
@@ -343,6 +344,7 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(p1s), check.Equals, 1) // nodes 1 and 4
+	c.Assert(p1s[0].Name, check.Equals, "test_get_shared_nodes_4")
 
 	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
@@ -350,6 +352,13 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(p1sAfter), check.Equals, 2) // nodes 1, 2, 4
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
+	c.Assert(p1sAfter[1].Name, check.Equals, "test_get_shared_nodes_4")
+
+	node1shared, err := h.getShared(m1)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(node1shared), check.Equals, 1) // nodes 1, 2, 4
+	c.Assert(node1shared[0].Name, check.Equals, "test_get_shared_nodes_2")
 
 	pAlone, err := h.getPeers(m3)
 	c.Assert(err, check.IsNil)

From 8abc7575cd7083060d19fb007dbf7117b2319738 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Tue, 5 Oct 2021 16:17:18 +0000
Subject: [PATCH 30/38] Tear out all the complicated update logic

There is some weird behaviour that seem to storm the update channel. And
our solution with a central map of update channels isnt particularly
elegant.

For now, replace all the complicated stuff with a simple channel that
checks roughly every 10s if the node is up to date. Only generate and
update if there has been changes.
---
 poll.go | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/poll.go b/poll.go
index a33d341b..df3112c4 100644
--- a/poll.go
+++ b/poll.go
@@ -140,10 +140,9 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("id", c.Param("id")).
 		Str("machine", m.Name).
 		Msg("Loading or creating update channel")
-	updateChan := h.getOrOpenUpdateChannel(m)
+	updateChan := make(chan struct{})
 
 	pollDataChan := make(chan []byte)
-	// defer close(pollData)
 
 	keepAliveChan := make(chan []byte)
 
@@ -160,7 +159,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
 		updateRequestsFromNode.WithLabelValues("endpoint-update").Inc()
-		go h.notifyChangesToPeers(m)
+		go func() { updateChan <- struct{}{} }()
 		return
 	} else if req.OmitPeers && req.Stream {
 		log.Warn().
@@ -186,7 +185,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("machine", m.Name).
 		Msg("Notifying peers")
 	updateRequestsFromNode.WithLabelValues("full-update").Inc()
-	go h.notifyChangesToPeers(m)
+	go func() { updateChan <- struct{}{} }()
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
 	log.Trace().
@@ -206,10 +205,10 @@ func (h *Headscale) PollNetMapStream(
 	mKey wgkey.Key,
 	pollDataChan chan []byte,
 	keepAliveChan chan []byte,
-	updateChan <-chan struct{},
+	updateChan chan struct{},
 	cancelKeepAlive chan struct{},
 ) {
-	go h.scheduledPollWorker(cancelKeepAlive, keepAliveChan, mKey, req, m)
+	go h.scheduledPollWorker(cancelKeepAlive, updateChan, keepAliveChan, mKey, req, m)
 
 	c.Stream(func(w io.Writer) bool {
 		log.Trace().
@@ -423,7 +422,8 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "Done").
 				Msg("Closing update channel")
-			h.closeUpdateChannel(m)
+			//h.closeUpdateChannel(m)
+			close(updateChan)
 
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -446,13 +446,14 @@ func (h *Headscale) PollNetMapStream(
 
 func (h *Headscale) scheduledPollWorker(
 	cancelChan <-chan struct{},
+	updateChan chan<- struct{},
 	keepAliveChan chan<- []byte,
 	mKey wgkey.Key,
 	req tailcfg.MapRequest,
 	m *Machine,
 ) {
 	keepAliveTicker := time.NewTicker(60 * time.Second)
-	updateCheckerTicker := time.NewTicker(30 * time.Second)
+	updateCheckerTicker := time.NewTicker(10 * time.Second)
 
 	for {
 		select {
@@ -476,16 +477,12 @@ func (h *Headscale) scheduledPollWorker(
 			keepAliveChan <- data
 
 		case <-updateCheckerTicker.C:
-			// Send an update request regardless of outdated or not, if data is sent
-			// to the node is determined in the updateChan consumer block
-			err := h.sendRequestOnUpdateChannel(m)
-			if err != nil {
-				log.Error().
-					Str("func", "keepAlive").
-					Str("machine", m.Name).
-					Err(err).
-					Msgf("Failed to send update request to %s", m.Name)
-			}
+			log.Debug().
+				Str("func", "scheduledPollWorker").
+				Str("machine", m.Name).
+				Msg("Sending update request")
+			updateRequestsFromNode.WithLabelValues("scheduled-update").Inc()
+			updateChan <- struct{}{}
 		}
 	}
 }

From a01a0d103924348fc2f76cd6d2cea8b20f6d71bd Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Tue, 5 Oct 2021 16:24:46 +0000
Subject: [PATCH 31/38] Remove unstable update channel, replace with state
 updates

---
 app.go        |   6 +--
 machine.go    | 106 --------------------------------------------------
 namespaces.go |  19 +++------
 3 files changed, 7 insertions(+), 124 deletions(-)

diff --git a/app.go b/app.go
index 27b9672d..a6e547f5 100644
--- a/app.go
+++ b/app.go
@@ -65,9 +65,6 @@ type Headscale struct {
 	aclPolicy *ACLPolicy
 	aclRules  *[]tailcfg.FilterRule
 
-	clientsUpdateChannels     sync.Map
-	clientsUpdateChannelMutex sync.Mutex
-
 	lastStateChange sync.Map
 }
 
@@ -145,10 +142,9 @@ func (h *Headscale) expireEphemeralNodesWorker() {
 				if err != nil {
 					log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database")
 				}
-				updateRequestsFromNode.WithLabelValues("ephemeral-node-update").Inc()
-				h.notifyChangesToPeers(&m)
 			}
 		}
+		h.setLastStateChangeToNow(ns.Name)
 	}
 }
 
diff --git a/machine.go b/machine.go
index 7a9df2e8..fcc8255c 100644
--- a/machine.go
+++ b/machine.go
@@ -2,7 +2,6 @@ package headscale
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"sort"
 	"strconv"
@@ -214,111 +213,6 @@ func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
 	return &hostinfo, nil
 }
 
-func (h *Headscale) notifyChangesToPeers(m *Machine) {
-	peers, err := h.getPeers(m)
-	if err != nil {
-		log.Error().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Msgf("Error getting peers: %s", err)
-		return
-	}
-	for _, peer := range peers {
-		log.Info().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", peer.Name).
-			Str("address", peer.IPAddress).
-			Msgf("Notifying peer %s (%s)", peer.Name, peer.IPAddress)
-		err := h.sendRequestOnUpdateChannel(&peer)
-		if err != nil {
-			log.Info().
-				Str("func", "notifyChangesToPeers").
-				Str("machine", m.Name).
-				Str("peer", peer.Name).
-				Msgf("Peer %s does not have an open update client, skipping.", peer.Name)
-			continue
-		}
-		log.Trace().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", peer.Name).
-			Str("address", peer.IPAddress).
-			Msgf("Notified peer %s (%s)", peer.Name, peer.IPAddress)
-	}
-}
-
-func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} {
-	var updateChan chan struct{}
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			updateChan = unwrapped
-		} else {
-			log.Error().
-				Str("handler", "openUpdateChannel").
-				Str("machine", m.Name).
-				Msg("Failed to convert update channel to struct{}")
-		}
-	} else {
-		log.Debug().
-			Str("handler", "openUpdateChannel").
-			Str("machine", m.Name).
-			Msg("Update channel not found, creating")
-
-		updateChan = make(chan struct{})
-		h.clientsUpdateChannels.Store(m.ID, updateChan)
-	}
-	return updateChan
-}
-
-func (h *Headscale) closeUpdateChannel(m *Machine) {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			close(unwrapped)
-		}
-	}
-	h.clientsUpdateChannels.Delete(m.ID)
-}
-
-func (h *Headscale) sendRequestOnUpdateChannel(m *Machine) error {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID))
-	if ok {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Notifying peer %s", m.Name)
-
-		if update, ok := pUp.(chan struct{}); ok {
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Update channel is %#v", update)
-
-			updateRequestsToNode.Inc()
-			update <- struct{}{}
-
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Notified machine %s", m.Name)
-		}
-	} else {
-		err := errors.New("machine does not have an open update channel")
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Machine %s does not have an open update channel", m.Name)
-		return err
-	}
-	return nil
-}
-
 func (h *Headscale) isOutdated(m *Machine) bool {
 	err := h.UpdateMachine(m)
 	if err != nil {
diff --git a/namespaces.go b/namespaces.go
index 75b6eab5..2bf62bb3 100644
--- a/namespaces.go
+++ b/namespaces.go
@@ -176,24 +176,17 @@ func (h *Headscale) checkForNamespacesPendingUpdates() {
 		return
 	}
 
-	names := []string{}
-	err = json.Unmarshal([]byte(v), &names)
+	namespaces := []string{}
+	err = json.Unmarshal([]byte(v), &namespaces)
 	if err != nil {
 		return
 	}
-	for _, name := range names {
+	for _, namespace := range namespaces {
 		log.Trace().
 			Str("func", "RequestMapUpdates").
-			Str("machine", name).
-			Msg("Sending updates to nodes in namespace")
-		machines, err := h.ListMachinesInNamespace(name)
-		if err != nil {
-			continue
-		}
-		for _, m := range *machines {
-			updateRequestsFromNode.WithLabelValues("namespace-update").Inc()
-			h.notifyChangesToPeers(&m)
-		}
+			Str("machine", namespace).
+			Msg("Sending updates to nodes in namespacespace")
+		h.setLastStateChangeToNow(namespace)
 	}
 	newV, err := h.getValue("namespaces_pending_updates")
 	if err != nil {

From 722084fbd36399b0dbc8d358d141b92b8f2e7fee Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Tue, 5 Oct 2021 16:51:42 +0000
Subject: [PATCH 32/38] Comment out aggressive logging

---
 api.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api.go b/api.go
index 17306b32..ce3b242e 100644
--- a/api.go
+++ b/api.go
@@ -277,7 +277,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Ma
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
-		Interface("payload", resp).
+		// Interface("payload", resp).
 		Msgf("Generated map response: %s", tailMapResponseToString(resp))
 
 	var respBody []byte

From c582c8d206fb3212f9aa71fbd4f1d74f7a76b1ec Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Tue, 5 Oct 2021 21:59:15 +0000
Subject: [PATCH 33/38] Update metrics for new code

---
 metrics.go | 10 +++++-----
 poll.go    | 24 +++++++++++++-----------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/metrics.go b/metrics.go
index d516fade..0d3dca34 100644
--- a/metrics.go
+++ b/metrics.go
@@ -26,16 +26,16 @@ var (
 		Namespace: prometheusNamespace,
 		Name:      "update_request_from_node_total",
 		Help:      "The number of updates requested by a node/update function",
-	}, []string{"state"})
-	updateRequestsToNode = promauto.NewCounter(prometheus.CounterOpts{
+	}, []string{"namespace", "machine", "state"})
+	updateRequestsSentToNode = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: prometheusNamespace,
-		Name:      "update_request_to_node_total",
+		Name:      "update_request_sent_to_node_total",
 		Help:      "The number of calls/messages issued on a specific nodes update channel",
-	})
+	}, []string{"namespace", "machine", "status"})
 	//TODO(kradalby): This is very debugging, we might want to remove it.
 	updateRequestsReceivedOnChannel = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: prometheusNamespace,
 		Name:      "update_request_received_on_channel_total",
 		Help:      "The number of update requests received on an update channel",
-	}, []string{"machine"})
+	}, []string{"namespace", "machine"})
 )
diff --git a/poll.go b/poll.go
index df3112c4..6a652805 100644
--- a/poll.go
+++ b/poll.go
@@ -158,7 +158,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
-		updateRequestsFromNode.WithLabelValues("endpoint-update").Inc()
+		updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "endpoint-update").Inc()
 		go func() { updateChan <- struct{}{} }()
 		return
 	} else if req.OmitPeers && req.Stream {
@@ -184,7 +184,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Notifying peers")
-	updateRequestsFromNode.WithLabelValues("full-update").Inc()
+	updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "full-update").Inc()
 	go func() { updateChan <- struct{}{} }()
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
@@ -324,7 +324,7 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "update").
 				Msg("Received a request for update")
-			updateRequestsReceivedOnChannel.WithLabelValues(m.Name).Inc()
+			updateRequestsReceivedOnChannel.WithLabelValues(m.Name, m.Namespace.Name).Inc()
 			if h.isOutdated(m) {
 				log.Debug().
 					Str("handler", "PollNetMapStream").
@@ -349,6 +349,7 @@ func (h *Headscale) PollNetMapStream(
 						Str("channel", "update").
 						Err(err).
 						Msg("Could not write the map response")
+					updateRequestsSentToNode.WithLabelValues(m.Name, m.Namespace.Name, "failed").Inc()
 					return false
 				}
 				log.Trace().
@@ -356,14 +357,15 @@ func (h *Headscale) PollNetMapStream(
 					Str("machine", m.Name).
 					Str("channel", "update").
 					Msg("Updated Map has been sent")
+				updateRequestsSentToNode.WithLabelValues(m.Name, m.Namespace.Name, "success").Inc()
 
-					// Keep track of the last successful update,
-					// we sometimes end in a state were the update
-					// is not picked up by a client and we use this
-					// to determine if we should "force" an update.
-					// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
-					// when an outdated machine object is kept alive, e.g. db is update from
-					// command line, but then overwritten.
+				// Keep track of the last successful update,
+				// we sometimes end in a state were the update
+				// is not picked up by a client and we use this
+				// to determine if we should "force" an update.
+				// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
+				// when an outdated machine object is kept alive, e.g. db is update from
+				// command line, but then overwritten.
 				err = h.UpdateMachine(m)
 				if err != nil {
 					log.Error().
@@ -481,7 +483,7 @@ func (h *Headscale) scheduledPollWorker(
 				Str("func", "scheduledPollWorker").
 				Str("machine", m.Name).
 				Msg("Sending update request")
-			updateRequestsFromNode.WithLabelValues("scheduled-update").Inc()
+			updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "scheduled-update").Inc()
 			updateChan <- struct{}{}
 		}
 	}

From ba391bc2eda93cedf749f2242b0caa42a036ae71 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Wed, 6 Oct 2021 19:32:15 +0000
Subject: [PATCH 34/38] Account for updates in shared namespaces

---
 app.go     |  2 +-
 machine.go | 23 +++++++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/app.go b/app.go
index a6e547f5..78412a24 100644
--- a/app.go
+++ b/app.go
@@ -250,8 +250,8 @@ func (h *Headscale) setLastStateChangeToNow(namespace string) {
 func (h *Headscale) getLastStateChange(namespace string) time.Time {
 	if wrapped, ok := h.lastStateChange.Load(namespace); ok {
 		lastChange, _ := wrapped.(time.Time)
-		return lastChange
 
+		return lastChange
 	}
 
 	now := time.Now().UTC()
diff --git a/machine.go b/machine.go
index fcc8255c..acc9e0e2 100644
--- a/machine.go
+++ b/machine.go
@@ -65,7 +65,7 @@ func (h *Headscale) getDirectPeers(m *Machine) (Machines, error) {
 	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
 		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
 		log.Error().Err(err).Msg("Error accessing db")
-		return nil, err
+		return Machines{}, err
 	}
 
 	sort.Slice(machines, func(i, j int) bool { return machines[i].ID < machines[j].ID })
@@ -87,7 +87,7 @@ func (h *Headscale) getShared(m *Machine) (Machines, error) {
 	sharedMachines := []SharedMachine{}
 	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
 		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
-		return nil, err
+		return Machines{}, err
 	}
 
 	peers := make(Machines, 0)
@@ -111,7 +111,7 @@ func (h *Headscale) getPeers(m *Machine) (Machines, error) {
 			Str("func", "getPeers").
 			Err(err).
 			Msg("Cannot fetch peers")
-		return nil, err
+		return Machines{}, err
 	}
 
 	shared, err := h.getShared(m)
@@ -120,7 +120,7 @@ func (h *Headscale) getPeers(m *Machine) (Machines, error) {
 			Str("func", "getDirectPeers").
 			Err(err).
 			Msg("Cannot fetch peers")
-		return nil, err
+		return Machines{}, err
 	}
 
 	peers := append(direct, shared...)
@@ -219,6 +219,21 @@ func (h *Headscale) isOutdated(m *Machine) bool {
 		return true
 	}
 
+	sharedMachines, _ := h.getShared(m)
+
+	// Check if any of our shared namespaces has updates that we have
+	// not propagated.
+	for _, sharedMachine := range sharedMachines {
+		lastChange := h.getLastStateChange(sharedMachine.Namespace.Name)
+		log.Trace().
+			Str("func", "keepAlive").
+			Str("machine", m.Name).
+			Time("last_successful_update", *m.LastSuccessfulUpdate).
+			Time("last_state_change", lastChange).
+			Msgf("Checking if %s is missing updates", m.Name)
+		return m.LastSuccessfulUpdate.Before(lastChange)
+	}
+
 	lastChange := h.getLastStateChange(m.Namespace.Name)
 	log.Trace().
 		Str("func", "keepAlive").

From 95f726fb31eed5d2430c8359a514796365a077d9 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Wed, 6 Oct 2021 19:56:14 +0000
Subject: [PATCH 35/38] Fix logic

---
 machine.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/machine.go b/machine.go
index acc9e0e2..ebe6f84b 100644
--- a/machine.go
+++ b/machine.go
@@ -231,7 +231,10 @@ func (h *Headscale) isOutdated(m *Machine) bool {
 			Time("last_successful_update", *m.LastSuccessfulUpdate).
 			Time("last_state_change", lastChange).
 			Msgf("Checking if %s is missing updates", m.Name)
-		return m.LastSuccessfulUpdate.Before(lastChange)
+		// Only return if we have a shared node with a newer update.
+		if m.LastSuccessfulUpdate.Before(lastChange) {
+			return true
+		}
 	}
 
 	lastChange := h.getLastStateChange(m.Namespace.Name)

From f0c54490ed9f8deec835cf59a59c16780a4374fe Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Wed, 6 Oct 2021 22:06:07 +0000
Subject: [PATCH 36/38] Allow multiple namespaces to be checked for state at
 the same time

---
 app.go     | 29 ++++++++++++++++++++++-------
 go.mod     |  1 +
 go.sum     |  2 ++
 machine.go | 23 +++++++++++------------
 4 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/app.go b/app.go
index 78412a24..acaa7f1b 100644
--- a/app.go
+++ b/app.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -247,14 +248,28 @@ func (h *Headscale) setLastStateChangeToNow(namespace string) {
 	h.lastStateChange.Store(namespace, now)
 }
 
-func (h *Headscale) getLastStateChange(namespace string) time.Time {
-	if wrapped, ok := h.lastStateChange.Load(namespace); ok {
-		lastChange, _ := wrapped.(time.Time)
+func (h *Headscale) getLastStateChange(namespaces ...string) time.Time {
+	times := []time.Time{}
+
+	for _, namespace := range namespaces {
+		if wrapped, ok := h.lastStateChange.Load(namespace); ok {
+			lastChange, _ := wrapped.(time.Time)
+
+			times = append(times, lastChange)
+		}
 
-		return lastChange
 	}
 
-	now := time.Now().UTC()
-	h.lastStateChange.Store(namespace, now)
-	return now
+	sort.Slice(times, func(i, j int) bool {
+		return times[i].After(times[j])
+	})
+
+	log.Trace().Msgf("Latest times %#v", times)
+
+	if len(times) == 0 {
+		return time.Now().UTC()
+
+	} else {
+		return times[0]
+	}
 }
diff --git a/go.mod b/go.mod
index 70428315..1fadd6b7 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/docker/cli v20.10.8+incompatible // indirect
 	github.com/docker/docker v20.10.8+incompatible // indirect
 	github.com/efekarakus/termcolor v1.0.1
+	github.com/fatih/set v0.2.1 // indirect
 	github.com/gin-gonic/gin v1.7.4
 	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/google/go-github v17.0.0+incompatible // indirect
diff --git a/go.sum b/go.sum
index 6dc92b6d..b429ca95 100644
--- a/go.sum
+++ b/go.sum
@@ -204,6 +204,8 @@ github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:Pjfxu
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
diff --git a/machine.go b/machine.go
index ebe6f84b..326c2fca 100644
--- a/machine.go
+++ b/machine.go
@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/set"
 	"github.com/rs/zerolog/log"
 
 	"gorm.io/datatypes"
@@ -221,23 +222,21 @@ func (h *Headscale) isOutdated(m *Machine) bool {
 
 	sharedMachines, _ := h.getShared(m)
 
+	namespaceSet := set.New(set.ThreadSafe)
+	namespaceSet.Add(m.Namespace.Name)
+
 	// Check if any of our shared namespaces has updates that we have
 	// not propagated.
 	for _, sharedMachine := range sharedMachines {
-		lastChange := h.getLastStateChange(sharedMachine.Namespace.Name)
-		log.Trace().
-			Str("func", "keepAlive").
-			Str("machine", m.Name).
-			Time("last_successful_update", *m.LastSuccessfulUpdate).
-			Time("last_state_change", lastChange).
-			Msgf("Checking if %s is missing updates", m.Name)
-		// Only return if we have a shared node with a newer update.
-		if m.LastSuccessfulUpdate.Before(lastChange) {
-			return true
-		}
+		namespaceSet.Add(sharedMachine.Namespace.Name)
 	}
 
-	lastChange := h.getLastStateChange(m.Namespace.Name)
+	namespaces := make([]string, namespaceSet.Size())
+	for index, namespace := range namespaceSet.List() {
+		namespaces[index] = namespace.(string)
+	}
+
+	lastChange := h.getLastStateChange(namespaces...)
 	log.Trace().
 		Str("func", "keepAlive").
 		Str("machine", m.Name).

From 20117c51a24eab616e8258359a87be6641d5da12 Mon Sep 17 00:00:00 2001
From: Zakhar Bessarab <zekker6@gmail.com>
Date: Thu, 7 Oct 2021 11:50:47 +0300
Subject: [PATCH 37/38] Add CI builds with artifacts for PRs and main

---
 .github/workflows/build.yml | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .github/workflows/build.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 00000000..90c48e92
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,36 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.16.3"
+
+      - name: Install dependencies
+        run: |
+          go version
+          go install golang.org/x/lint/golint@latest
+          sudo apt update
+          sudo apt install -y make
+
+      - name: Run lint
+        run: make build
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: headscale-linux
+          path: headscale
\ No newline at end of file

From 06f56411dd040d26269273c8377880d86321ba14 Mon Sep 17 00:00:00 2001
From: Kristoffer Dalby <kradalby@kradalby.no>
Date: Thu, 7 Oct 2021 15:45:45 +0100
Subject: [PATCH 38/38] Update machine.go

---
 machine.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/machine.go b/machine.go
index 326c2fca..21d774b4 100644
--- a/machine.go
+++ b/machine.go
@@ -217,6 +217,8 @@ func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
 func (h *Headscale) isOutdated(m *Machine) bool {
 	err := h.UpdateMachine(m)
 	if err != nil {
+		// It does not seem meaningful to propagate this error as the end result
+		// will have to be that the machine has to be considered outdated.
 		return true
 	}