Merge branch 'feature/relay-integration' into test/signal-dispatcher-with-relay

Pass the atomic store by address

client/cmd/status.go

@@ -31,9 +31,9 @@ type peerStateDetailOutput struct {
 	Status                 string           `json:"status" yaml:"status"`
 	LastStatusUpdate       time.Time        `json:"lastStatusUpdate" yaml:"lastStatusUpdate"`
 	ConnType               string           `json:"connectionType" yaml:"connectionType"`
-	Direct                 bool             `json:"direct" yaml:"direct"`
 	IceCandidateType       iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"`
 	IceCandidateEndpoint   iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"`
+	RelayAddress           string           `json:"relayAddress" yaml:"relayAddress"`
 	LastWireguardHandshake time.Time        `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"`
 	TransferReceived       int64            `json:"transferReceived" yaml:"transferReceived"`
 	TransferSent           int64            `json:"transferSent" yaml:"transferSent"`
@@ -335,16 +335,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput {
 
 func mapPeers(peers []*proto.PeerState) peersStateOutput {
 	var peersStateDetail []peerStateDetailOutput
-	localICE := ""
-	remoteICE := ""
-	localICEEndpoint := ""
-	remoteICEEndpoint := ""
-	connType := ""
 	peersConnected := 0
-	lastHandshake := time.Time{}
-	transferReceived := int64(0)
-	transferSent := int64(0)
 	for _, pbPeerState := range peers {
+		localICE := ""
+		remoteICE := ""
+		localICEEndpoint := ""
+		remoteICEEndpoint := ""
+		relayServerAddress := ""
+		connType := ""
+		lastHandshake := time.Time{}
+		transferReceived := int64(0)
+		transferSent := int64(0)
+
 		isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String()
 		if skipDetailByFilters(pbPeerState, isPeerConnected) {
 			continue
@@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			if pbPeerState.Relayed {
 				connType = "Relayed"
 			}
+			relayServerAddress = pbPeerState.GetRelayAddress()
 			lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local()
 			transferReceived = pbPeerState.GetBytesRx()
 			transferSent = pbPeerState.GetBytesTx()
@@ -372,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			Status:           pbPeerState.GetConnStatus(),
 			LastStatusUpdate: timeLocal,
 			ConnType:         connType,
-			Direct:           pbPeerState.GetDirect(),
 			IceCandidateType: iceCandidateType{
 				Local:  localICE,
 				Remote: remoteICE,
@@ -381,6 +383,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 				Local:  localICEEndpoint,
 				Remote: remoteICEEndpoint,
 			},
+			RelayAddress:           relayServerAddress,
 			FQDN:                   pbPeerState.GetFqdn(),
 			LastWireguardHandshake: lastHandshake,
 			TransferReceived:       transferReceived,
@@ -641,9 +644,9 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 				"  Status: %s\n"+
 				"  -- detail --\n"+
 				"  Connection type: %s\n"+
-				"  Direct: %t\n"+
 				"  ICE candidate (Local/Remote): %s/%s\n"+
 				"  ICE candidate endpoints (Local/Remote): %s/%s\n"+
+				"  Relay server address: %s\n"+
 				"  Last connection update: %s\n"+
 				"  Last WireGuard handshake: %s\n"+
 				"  Transfer status (received/sent) %s/%s\n"+
@@ -655,11 +658,11 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 			peerState.PubKey,
 			peerState.Status,
 			peerState.ConnType,
-			peerState.Direct,
 			localICE,
 			remoteICE,
 			localICEEndpoint,
 			remoteICEEndpoint,
+			peerState.RelayAddress,
 			timeAgo(peerState.LastStatusUpdate),
 			timeAgo(peerState.LastWireguardHandshake),
 			toIEC(peerState.TransferReceived),

client/cmd/status_test.go

@@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)),
 				Relayed:                    false,
-				Direct:                     true,
 				LocalIceCandidateType:      "",
 				RemoteIceCandidateType:     "",
 				LocalIceCandidateEndpoint:  "",
@@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)),
 				Relayed:                    true,
-				Direct:                     false,
 				LocalIceCandidateType:      "relay",
 				RemoteIceCandidateType:     "prflx",
 				LocalIceCandidateEndpoint:  "10.0.0.1:10001",
@@ -137,7 +135,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC),
 				ConnType:         "P2P",
-				Direct:           true,
 				IceCandidateType: iceCandidateType{
 					Local:  "",
 					Remote: "",
@@ -161,7 +158,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC),
 				ConnType:         "Relayed",
-				Direct:           false,
 				IceCandidateType: iceCandidateType{
 					Local:  "relay",
 					Remote: "prflx",
@@ -283,7 +279,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2001-01-01T01:01:01Z",
                 "connectionType": "P2P",
-                "direct": true,
                 "iceCandidateType": {
                   "local": "",
                   "remote": ""
@@ -292,6 +287,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "",
                   "remote": ""
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2001-01-01T01:01:02Z",
                 "transferReceived": 200,
                 "transferSent": 100,
@@ -308,7 +304,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2002-02-02T02:02:02Z",
                 "connectionType": "Relayed",
-                "direct": false,
                 "iceCandidateType": {
                   "local": "relay",
                   "remote": "prflx"
@@ -317,6 +312,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "10.0.0.1:10001",
                   "remote": "10.0.10.1:10002"
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2002-02-02T02:02:03Z",
                 "transferReceived": 2000,
                 "transferSent": 1000,
@@ -408,13 +404,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2001-01-01T01:01:01Z
           connectionType: P2P
-          direct: true
           iceCandidateType:
             local: ""
             remote: ""
           iceCandidateEndpoint:
             local: ""
             remote: ""
+          relayAddress: ""
           lastWireguardHandshake: 2001-01-01T01:01:02Z
           transferReceived: 200
           transferSent: 100
@@ -428,13 +424,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2002-02-02T02:02:02Z
           connectionType: Relayed
-          direct: false
           iceCandidateType:
             local: relay
             remote: prflx
           iceCandidateEndpoint:
             local: 10.0.0.1:10001
             remote: 10.0.10.1:10002
+          relayAddress: ""
           lastWireguardHandshake: 2002-02-02T02:02:03Z
           transferReceived: 2000
           transferSent: 1000
@@ -505,9 +501,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: P2P
-  Direct: true
   ICE candidate (Local/Remote): -/-
   ICE candidate endpoints (Local/Remote): -/-
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 200 B/100 B
@@ -521,9 +517,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: Relayed
-  Direct: false
   ICE candidate (Local/Remote): relay/prflx
   ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 2.0 KiB/1000 B

client/cmd/testutil_test.go

@@ -98,7 +98,11 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
+	rc := &mgmt.RelayConfig{
+		Address: "localhost:0",
+	}
+	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc)
 	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)

client/cmd/up.go

@@ -168,7 +168,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 	ctx, cancel = context.WithCancel(ctx)
 	SetupCloseHandler(ctx, cancel)
 
-	connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String()))
+	r := peer.NewRecorder(config.ManagementURL.String())
+	r.GetFullStatus()
+
+	connectClient := internal.NewConnectClient(ctx, config, r)
 	return connectClient.Run()
 }
 

client/internal/connect.go

@@ -26,6 +26,8 @@ import (
 	"github.com/netbirdio/netbird/iface"
 	mgm "github.com/netbirdio/netbird/management/client"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/relay/auth/hmac"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	signal "github.com/netbirdio/netbird/signal/client"
 	"github.com/netbirdio/netbird/util"
 	"github.com/netbirdio/netbird/version"
@@ -162,10 +164,8 @@ func (c *ConnectClient) run(
 	defer c.statusRecorder.ClientStop()
 	operation := func() error {
 		// if context cancelled we not start new backoff cycle
-		select {
-		case <-c.ctx.Done():
+		if c.isContextCancelled() {
 			return nil
-		default:
 		}
 
 		state.Set(StatusConnecting)
@@ -187,8 +187,7 @@ func (c *ConnectClient) run(
 
 		log.Debugf("connected to the Management service %s", c.config.ManagementURL.Host)
 		defer func() {
-			err = mgmClient.Close()
-			if err != nil {
+			if err = mgmClient.Close(); err != nil {
 				log.Warnf("failed to close the Management service client %v", err)
 			}
 		}()
@@ -211,7 +210,6 @@ func (c *ConnectClient) run(
 			KernelInterface: iface.WireGuardModuleIsLoaded(),
 			FQDN:            loginResp.GetPeerConfig().GetFqdn(),
 		}
-
 		c.statusRecorder.UpdateLocalPeerState(localPeerState)
 
 		signalURL := fmt.Sprintf("%s://%s",
@@ -244,6 +242,20 @@ func (c *ConnectClient) run(
 
 		c.statusRecorder.MarkSignalConnected()
 
+		relayURL, token := parseRelayInfo(loginResp)
+		relayManager := relayClient.NewManager(engineCtx, relayURL, myPrivateKey.PublicKey().String())
+		if relayURL != "" {
+			if token != nil {
+				relayManager.UpdateToken(token)
+			}
+			log.Infof("connecting to the Relay service %s", relayURL)
+			if err = relayManager.Serve(); err != nil {
+				log.Error(err)
+				return wrapErr(err)
+			}
+			c.statusRecorder.SetRelayMgr(relayManager)
+		}
+
 		peerConfig := loginResp.GetPeerConfig()
 
 		engineConfig, err := createEngineConfig(myPrivateKey, c.config, peerConfig)
@@ -255,11 +267,10 @@ func (c *ConnectClient) run(
 		checks := loginResp.GetChecks()
 
 		c.engineMutex.Lock()
-		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe, checks)
+		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe, checks)
 		c.engineMutex.Unlock()
 
-		err = c.engine.Start()
-		if err != nil {
+		if err := c.engine.Start(); err != nil {
 			log.Errorf("error while starting Netbird Connection Engine: %s", err)
 			return wrapErr(err)
 		}
@@ -299,6 +310,25 @@ func (c *ConnectClient) run(
 	return nil
 }
 
+func parseRelayInfo(resp *mgmProto.LoginResponse) (string, *hmac.Token) {
+	msg := resp.GetWiretrusteeConfig().GetRelay()
+	if msg == nil {
+		return "", nil
+	}
+
+	var url string
+	if msg.GetUrls() != nil && len(msg.GetUrls()) > 0 {
+		url = msg.GetUrls()[0]
+	}
+
+	token := &hmac.Token{
+		Payload:   msg.GetTokenPayload(),
+		Signature: msg.GetTokenSignature(),
+	}
+
+	return url, token
+}
+
 func (c *ConnectClient) Engine() *Engine {
 	var e *Engine
 	c.engineMutex.Lock()
@@ -307,6 +337,15 @@ func (c *ConnectClient) Engine() *Engine {
 	return e
 }
 
+func (c *ConnectClient) isContextCancelled() bool {
+	select {
+	case <-c.ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
+
 // createEngineConfig converts configuration received from Management Service to EngineConfig
 func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) {
 	nm := false

client/internal/engine.go

@@ -13,6 +13,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/pion/ice/v3"
@@ -24,6 +25,7 @@ import (
 	"github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl"
 	"github.com/netbirdio/netbird/client/internal/dns"
+
 	"github.com/netbirdio/netbird/client/internal/networkmonitor"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/relay"
@@ -39,6 +41,8 @@ import (
 	mgm "github.com/netbirdio/netbird/management/client"
 	"github.com/netbirdio/netbird/management/domain"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	signal "github.com/netbirdio/netbird/signal/client"
 	sProto "github.com/netbirdio/netbird/signal/proto"
@@ -101,7 +105,8 @@ type EngineConfig struct {
 // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers.
 type Engine struct {
 	// signal is a Signal Service client
-	signal signal.Client
+	signal   signal.Client
+	signaler *peer.Signaler
 	// mgmClient is a Management Service client
 	mgmClient mgm.Client
 	// peerConns is a map that holds all the peers that are known to this peer
@@ -122,7 +127,8 @@ type Engine struct {
 	// STUNs is a list of STUN servers used by ICE
 	STUNs []*stun.URI
 	// TURNs is a list of STUN servers used by ICE
-	TURNs []*stun.URI
+	TURNs    []*stun.URI
+	StunTurn atomic.Value
 
 	// clientRoutes is the most recent list of clientRoutes received from the Management Service
 	clientRoutes   route.HAMap
@@ -134,7 +140,7 @@ type Engine struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 
-	wgInterface    *iface.WGIface
+	wgInterface    iface.IWGIface
 	wgProxyFactory *wgproxy.Factory
 
 	udpMux *bind.UniversalUDPMuxDefault
@@ -160,10 +166,10 @@ type Engine struct {
 	relayProbe  *Probe
 	wgProbe     *Probe
 
-	wgConnWorker sync.WaitGroup
-
 	// checks are the client-applied posture checks that need to be evaluated on the client
 	checks []*mgmProto.Checks
+
+	relayManager *relayClient.Manager
 }
 
 // Peer is an instance of the Connection Peer
@@ -178,6 +184,7 @@ func NewEngine(
 	clientCancel context.CancelFunc,
 	signalClient signal.Client,
 	mgmClient mgm.Client,
+	relayManager *relayClient.Manager,
 	config *EngineConfig,
 	mobileDep MobileDependency,
 	statusRecorder *peer.Status,
@@ -188,6 +195,7 @@ func NewEngine(
 		clientCancel,
 		signalClient,
 		mgmClient,
+		relayManager,
 		config,
 		mobileDep,
 		statusRecorder,
@@ -205,6 +213,7 @@ func NewEngineWithProbes(
 	clientCancel context.CancelFunc,
 	signalClient signal.Client,
 	mgmClient mgm.Client,
+	relayManager *relayClient.Manager,
 	config *EngineConfig,
 	mobileDep MobileDependency,
 	statusRecorder *peer.Status,
@@ -214,12 +223,13 @@ func NewEngineWithProbes(
 	wgProbe *Probe,
 	checks []*mgmProto.Checks,
 ) *Engine {
-
 	return &Engine{
 		clientCtx:      clientCtx,
 		clientCancel:   clientCancel,
 		signal:         signalClient,
+		signaler:       peer.NewSignaler(signalClient, config.WgPrivateKey),
 		mgmClient:      mgmClient,
+		relayManager:   relayManager,
 		peerConns:      make(map[string]*peer.Conn),
 		syncMsgMux:     &sync.Mutex{},
 		config:         config,
@@ -265,24 +275,8 @@ func (e *Engine) Stop() error {
 	time.Sleep(500 * time.Millisecond)
 
 	e.close()
-	e.wgConnWorker.Wait()
-
-	maxWaitTime := 5 * time.Second
-	timeout := time.After(maxWaitTime)
-
-	for {
-		if !e.IsWGIfaceUp() {
-			log.Infof("stopped Netbird Engine")
-			return nil
-		}
-
-		select {
-		case <-timeout:
-			return fmt.Errorf("timeout when waiting for interface shutdown")
-		default:
-			time.Sleep(100 * time.Millisecond)
-		}
-	}
+	log.Infof("stopped Netbird Engine")
+	return nil
 }
 
 // Start creates a new WireGuard tunnel interface and listens to events from Signal and Management services
@@ -480,80 +474,42 @@ func (e *Engine) removePeer(peerKey string) error {
 	conn, exists := e.peerConns[peerKey]
 	if exists {
 		delete(e.peerConns, peerKey)
-		err := conn.Close()
-		if err != nil {
-			switch err.(type) {
-			case *peer.ConnectionAlreadyClosedError:
-				return nil
-			default:
-				return err
-			}
-		}
+		conn.Close()
 	}
 	return nil
 }
 
-func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client) error {
-	err := s.Send(&sProto.Message{
-		Key:       myKey.PublicKey().String(),
-		RemoteKey: remoteKey.String(),
-		Body: &sProto.Body{
-			Type:    sProto.Body_CANDIDATE,
-			Payload: candidate.Marshal(),
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func sendSignal(message *sProto.Message, s signal.Client) error {
-	return s.Send(message)
-}
-
-// SignalOfferAnswer signals either an offer or an answer to remote peer
-func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client,
-	isAnswer bool) error {
-	var t sProto.Body_Type
-	if isAnswer {
-		t = sProto.Body_ANSWER
-	} else {
-		t = sProto.Body_OFFER
-	}
-
-	msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{
-		UFrag: offerAnswer.IceCredentials.UFrag,
-		Pwd:   offerAnswer.IceCredentials.Pwd,
-	}, t, offerAnswer.RosenpassPubKey, offerAnswer.RosenpassAddr)
-	if err != nil {
-		return err
-	}
-
-	err = s.Send(msg)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (e *Engine) handleSync(update *mgmProto.SyncResponse) error {
 	e.syncMsgMux.Lock()
 	defer e.syncMsgMux.Unlock()
 
 	if update.GetWiretrusteeConfig() != nil {
-		err := e.updateTURNs(update.GetWiretrusteeConfig().GetTurns())
+		wCfg := update.GetWiretrusteeConfig()
+		err := e.updateTURNs(wCfg.GetTurns())
 		if err != nil {
 			return err
 		}
 
-		err = e.updateSTUNs(update.GetWiretrusteeConfig().GetStuns())
+		err = e.updateSTUNs(wCfg.GetStuns())
 		if err != nil {
 			return err
 		}
 
+		var stunTurn []*stun.URI
+		stunTurn = append(stunTurn, e.STUNs...)
+		stunTurn = append(stunTurn, e.TURNs...)
+		e.StunTurn.Store(stunTurn)
+
+		relayMsg := wCfg.GetRelay()
+		if relayMsg != nil {
+			c := &auth.Token{
+				Payload:   relayMsg.GetTokenPayload(),
+				Signature: relayMsg.GetTokenSignature(),
+			}
+			e.relayManager.UpdateToken(c)
+		}
+
+		// todo update relay address in the relay manager
 		// todo update signal
 	}
 
@@ -949,68 +905,13 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error {
 			log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err)
 		}
 
-		e.wgConnWorker.Add(1)
-		go e.connWorker(conn, peerKey)
+		conn.Open()
 	}
 	return nil
 }
 
-func (e *Engine) connWorker(conn *peer.Conn, peerKey string) {
-	defer e.wgConnWorker.Done()
-	for {
-
-		// randomize starting time a bit
-		minValue := 500
-		maxValue := 2000
-		duration := time.Duration(rand.Intn(maxValue-minValue)+minValue) * time.Millisecond
-		select {
-		case <-e.ctx.Done():
-			return
-		case <-time.After(duration):
-		}
-
-		// if peer has been removed -> give up
-		if !e.peerExists(peerKey) {
-			log.Debugf("peer %s doesn't exist anymore, won't retry connection", peerKey)
-			return
-		}
-
-		if !e.signal.Ready() {
-			log.Infof("signal client isn't ready, skipping connection attempt %s", peerKey)
-			continue
-		}
-
-		// we might have received new STUN and TURN servers meanwhile, so update them
-		e.syncMsgMux.Lock()
-		conn.UpdateStunTurn(append(e.STUNs, e.TURNs...))
-		e.syncMsgMux.Unlock()
-
-		err := conn.Open(e.ctx)
-		if err != nil {
-			log.Debugf("connection to peer %s failed: %v", peerKey, err)
-			var connectionClosedError *peer.ConnectionClosedError
-			switch {
-			case errors.As(err, &connectionClosedError):
-				// conn has been forced to close, so we exit the loop
-				return
-			default:
-			}
-		}
-	}
-}
-
-func (e *Engine) peerExists(peerKey string) bool {
-	e.syncMsgMux.Lock()
-	defer e.syncMsgMux.Unlock()
-	_, ok := e.peerConns[peerKey]
-	return ok
-}
-
 func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) {
 	log.Debugf("creating peer connection %s", pubKey)
-	var stunTurn []*stun.URI
-	stunTurn = append(stunTurn, e.STUNs...)
-	stunTurn = append(stunTurn, e.TURNs...)
 
 	wgConfig := peer.WgConfig{
 		RemoteKey:    pubKey,
@@ -1043,52 +944,29 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 	// randomize connection timeout
 	timeout := time.Duration(rand.Intn(PeerConnectionTimeoutMax-PeerConnectionTimeoutMin)+PeerConnectionTimeoutMin) * time.Millisecond
 	config := peer.ConnConfig{
-		Key:                  pubKey,
-		LocalKey:             e.config.WgPrivateKey.PublicKey().String(),
-		StunTurn:             stunTurn,
-		InterfaceBlackList:   e.config.IFaceBlackList,
-		DisableIPv6Discovery: e.config.DisableIPv6Discovery,
-		Timeout:              timeout,
-		UDPMux:               e.udpMux.UDPMuxDefault,
-		UDPMuxSrflx:          e.udpMux,
-		WgConfig:             wgConfig,
-		LocalWgPort:          e.config.WgPort,
-		NATExternalIPs:       e.parseNATExternalIPMappings(),
-		RosenpassPubKey:      e.getRosenpassPubKey(),
-		RosenpassAddr:        e.getRosenpassAddr(),
+		Key:             pubKey,
+		LocalKey:        e.config.WgPrivateKey.PublicKey().String(),
+		Timeout:         timeout,
+		WgConfig:        wgConfig,
+		LocalWgPort:     e.config.WgPort,
+		RosenpassPubKey: e.getRosenpassPubKey(),
+		RosenpassAddr:   e.getRosenpassAddr(),
+		ICEConfig: peer.ICEConfig{
+			StunTurn:             &e.StunTurn,
+			InterfaceBlackList:   e.config.IFaceBlackList,
+			DisableIPv6Discovery: e.config.DisableIPv6Discovery,
+			UDPMux:               e.udpMux.UDPMuxDefault,
+			UDPMuxSrflx:          e.udpMux,
+			NATExternalIPs:       e.parseNATExternalIPMappings(),
+		},
 	}
 
-	peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover)
+	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager)
 	if err != nil {
 		return nil, err
 	}
 
-	wgPubKey, err := wgtypes.ParseKey(pubKey)
-	if err != nil {
-		return nil, err
-	}
-
-	signalOffer := func(offerAnswer peer.OfferAnswer) error {
-		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false)
-	}
-
-	signalCandidate := func(candidate ice.Candidate) error {
-		return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal)
-	}
-
-	signalAnswer := func(offerAnswer peer.OfferAnswer) error {
-		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true)
-	}
-
-	peerConn.SetSignalCandidate(signalCandidate)
-	peerConn.SetSignalOffer(signalOffer)
-	peerConn.SetSignalAnswer(signalAnswer)
-	peerConn.SetSendSignalMessage(func(message *sProto.Message) error {
-		return sendSignal(message, e.signal)
-	})
-
 	if e.rpManager != nil {
-
 		peerConn.SetOnConnected(e.rpManager.OnConnected)
 		peerConn.SetOnDisconnected(e.rpManager.OnDisconnected)
 	}
@@ -1131,6 +1009,7 @@ func (e *Engine) receiveSignalEvents() {
 					Version:         msg.GetBody().GetNetBirdVersion(),
 					RosenpassPubKey: rosenpassPubKey,
 					RosenpassAddr:   rosenpassAddr,
+					RelaySrvAddress: msg.GetBody().GetRelayServerAddress(),
 				})
 			case sProto.Body_ANSWER:
 				remoteCred, err := signal.UnMarshalCredential(msg)
@@ -1153,6 +1032,7 @@ func (e *Engine) receiveSignalEvents() {
 					Version:         msg.GetBody().GetNetBirdVersion(),
 					RosenpassPubKey: rosenpassPubKey,
 					RosenpassAddr:   rosenpassAddr,
+					RelaySrvAddress: msg.GetBody().GetRelayServerAddress(),
 				})
 			case sProto.Body_CANDIDATE:
 				candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload)
@@ -1161,7 +1041,7 @@ func (e *Engine) receiveSignalEvents() {
 					return err
 				}
 
-				conn.OnRemoteCandidate(candidate, e.GetClientRoutes())
+				go conn.OnRemoteCandidate(candidate, e.GetClientRoutes())
 			case sProto.Body_MODE:
 			}
 
@@ -1457,7 +1337,7 @@ func (e *Engine) receiveProbeEvents() {
 
 			for _, peer := range e.peerConns {
 				key := peer.GetKey()
-				wgStats, err := peer.GetConf().WgConfig.WgInterface.GetStats(key)
+				wgStats, err := peer.WgConfig().WgInterface.GetStats(key)
 				if err != nil {
 					log.Debugf("failed to get wg stats for peer %s: %s", key, err)
 				}

client/internal/engine_test.go

@@ -36,6 +36,7 @@ import (
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	"github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/activity"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/management/server/telemetry"
 	"github.com/netbirdio/netbird/route"
 	signal "github.com/netbirdio/netbird/signal/client"
@@ -58,6 +59,12 @@ var (
 	}
 )
 
+func TestMain(m *testing.M) {
+	_ = util.InitLog("debug", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestEngine_SSH(t *testing.T) {
 	// todo resolve test execution on freebsd
 	if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" {
@@ -73,13 +80,23 @@ func TestEngine_SSH(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
-		WgIfaceName:      "utun101",
-		WgAddr:           "100.64.0.1/24",
-		WgPrivateKey:     key,
-		WgPort:           33100,
-		ServerSSHAllowed: true,
-	}, MobileDependency{}, peer.NewRecorder("https://mgm"), nil)
+	relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String())
+	engine := NewEngine(
+		ctx, cancel,
+		&signal.MockClient{},
+		&mgmt.MockClient{},
+		relayMgr,
+		&EngineConfig{
+			WgIfaceName:      "utun101",
+			WgAddr:           "100.64.0.1/24",
+			WgPrivateKey:     key,
+			WgPort:           33100,
+			ServerSSHAllowed: true,
+		},
+		MobileDependency{},
+		peer.NewRecorder("https://mgm"),
+		nil,
+	)
 
 	engine.dnsServer = &dns.MockServer{
 		UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil },
@@ -208,20 +225,28 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
-		WgIfaceName:  "utun102",
-		WgAddr:       "100.64.0.1/24",
-		WgPrivateKey: key,
-		WgPort:       33100,
-	}, MobileDependency{}, peer.NewRecorder("https://mgm"), nil)
-	newNet, err := stdnet.NewNet()
-	if err != nil {
-		t.Fatal(err)
-	}
-	engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil, nil)
-	if err != nil {
-		t.Fatal(err)
+	relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String())
+	engine := NewEngine(
+		ctx, cancel,
+		&signal.MockClient{},
+		&mgmt.MockClient{},
+		relayMgr,
+		&EngineConfig{
+			WgIfaceName:  "utun102",
+			WgAddr:       "100.64.0.1/24",
+			WgPrivateKey: key,
+			WgPort:       33100,
+		},
+		MobileDependency{},
+		peer.NewRecorder("https://mgm"),
+		nil)
+
+	wgIface := &iface.MockWGIface{
+		RemovePeerFunc: func(peerKey string) error {
+			return nil
+		},
 	}
+	engine.wgInterface = wgIface
 	engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, nil)
 	engine.dnsServer = &dns.MockServer{
 		UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil },
@@ -404,8 +429,8 @@ func TestEngine_Sync(t *testing.T) {
 		}
 		return nil
 	}
-
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, &EngineConfig{
+	relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String())
+	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, relayMgr, &EngineConfig{
 		WgIfaceName:  "utun103",
 		WgAddr:       "100.64.0.1/24",
 		WgPrivateKey: key,
@@ -564,7 +589,8 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
 			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
 			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
 
-			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+			relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String())
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{
 				WgIfaceName:  wgIfaceName,
 				WgAddr:       wgAddr,
 				WgPrivateKey: key,
@@ -734,7 +760,8 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
 			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
 			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
 
-			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+			relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String())
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{
 				WgIfaceName:  wgIfaceName,
 				WgAddr:       wgAddr,
 				WgPrivateKey: key,
@@ -1010,7 +1037,8 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin
 		WgPort:       wgPort,
 	}
 
-	e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil
+	relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String())
+	e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, relayMgr, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil
 	e.ctx = ctx
 	return e, err
 }
@@ -1078,7 +1106,10 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 	if err != nil {
 		return nil, "", err
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+	rc := &server.RelayConfig{
+		Address: "127.0.0.1:1234",
+	}
+	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc)
 	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		return nil, "", err

client/internal/peer/conn_test.go

@@ -2,25 +2,33 @@ package peer
 
 import (
 	"context"
+	"os"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/magiconair/properties/assert"
-	"github.com/pion/stun/v2"
 
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/client/internal/wgproxy"
 	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/util"
 )
 
 var connConf = ConnConfig{
-	Key:                "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
-	LocalKey:           "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
-	StunTurn:           []*stun.URI{},
-	InterfaceBlackList: nil,
-	Timeout:            time.Second,
-	LocalWgPort:        51820,
+	Key:         "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
+	LocalKey:    "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
+	Timeout:     time.Second,
+	LocalWgPort: 51820,
+	ICEConfig: ICEConfig{
+		InterfaceBlackList: nil,
+	},
+}
+
+func TestMain(m *testing.M) {
+	_ = util.InitLog("trace", "console")
+	code := m.Run()
+	os.Exit(code)
 }
 
 func TestNewConn_interfaceFilter(t *testing.T) {
@@ -40,7 +48,7 @@ func TestConn_GetKey(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -55,7 +63,7 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -63,7 +71,7 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
-		<-conn.remoteOffersCh
+		<-conn.handshaker.remoteOffersCh
 		wg.Done()
 	}()
 
@@ -92,7 +100,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -100,7 +108,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
-		<-conn.remoteAnswerCh
+		<-conn.handshaker.remoteAnswerCh
 		wg.Done()
 	}()
 
@@ -128,58 +136,33 @@ func TestConn_Status(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
 
 	tables := []struct {
-		name   string
-		status ConnStatus
-		want   ConnStatus
+		name        string
+		statusIce   ConnStatus
+		statusRelay ConnStatus
+		want        ConnStatus
 	}{
-		{"StatusConnected", StatusConnected, StatusConnected},
-		{"StatusDisconnected", StatusDisconnected, StatusDisconnected},
-		{"StatusConnecting", StatusConnecting, StatusConnecting},
+		{"StatusConnected", StatusConnected, StatusConnected, StatusConnected},
+		{"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected},
+		{"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting},
+		{"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting},
+		{"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected},
+		{"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting},
+		{"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected},
 	}
 
 	for _, table := range tables {
 		t.Run(table.name, func(t *testing.T) {
-			conn.status = table.status
+			conn.statusICE = table.statusIce
+			conn.statusRelay = table.statusRelay
 
 			got := conn.Status()
 			assert.Equal(t, got, table.want, "they should be equal")
 		})
 	}
 }
-
-func TestConn_Close(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
-	if err != nil {
-		return
-	}
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		<-conn.closeCh
-		wg.Done()
-	}()
-
-	go func() {
-		for {
-			err := conn.Close()
-			if err != nil {
-				continue
-			} else {
-				return
-			}
-		}
-	}()
-
-	wg.Wait()
-}

client/internal/peer/handshaker.go

@@ -0,0 +1,191 @@
+package peer
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/version"
+)
+
+var (
+	ErrSignalIsNotReady = errors.New("signal is not ready")
+)
+
+// IceCredentials ICE protocol credentials struct
+type IceCredentials struct {
+	UFrag string
+	Pwd   string
+}
+
+// OfferAnswer represents a session establishment offer or answer
+type OfferAnswer struct {
+	IceCredentials IceCredentials
+	// WgListenPort is a remote WireGuard listen port.
+	// This field is used when establishing a direct WireGuard connection without any proxy.
+	// We can set the remote peer's endpoint with this port.
+	WgListenPort int
+
+	// Version of NetBird Agent
+	Version string
+	// RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message
+	// This value is the local Rosenpass server public key when sending the message
+	RosenpassPubKey []byte
+	// RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message
+	// This value is the local Rosenpass server address when sending the message
+	RosenpassAddr string
+
+	// relay server address
+	RelaySrvAddress string
+}
+
+type Handshaker struct {
+	mu                  sync.Mutex
+	ctx                 context.Context
+	log                 *log.Entry
+	config              ConnConfig
+	signaler            *Signaler
+	ice                 *WorkerICE
+	relay               *WorkerRelay
+	onNewOfferListeners []func(*OfferAnswer)
+
+	// remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection
+	remoteOffersCh chan OfferAnswer
+	// remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection
+	remoteAnswerCh chan OfferAnswer
+}
+
+func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker {
+	return &Handshaker{
+		ctx:            ctx,
+		log:            log,
+		config:         config,
+		signaler:       signaler,
+		ice:            ice,
+		relay:          relay,
+		remoteOffersCh: make(chan OfferAnswer),
+		remoteAnswerCh: make(chan OfferAnswer),
+	}
+}
+
+func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAnswer)) {
+	h.onNewOfferListeners = append(h.onNewOfferListeners, offer)
+}
+
+func (h *Handshaker) Listen() {
+	for {
+		h.log.Debugf("wait for remote offer confirmation")
+		remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation()
+		if err != nil {
+			if _, ok := err.(*ConnectionClosedError); ok {
+				h.log.Tracef("stop handshaker")
+				return
+			}
+			h.log.Errorf("failed to received remote offer confirmation: %s", err)
+			continue
+		}
+
+		h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort)
+		for _, listener := range h.onNewOfferListeners {
+			go listener(remoteOfferAnswer)
+		}
+	}
+}
+
+func (h *Handshaker) SendOffer() error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.sendOffer()
+}
+
+// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
+// doesn't block, discards the message if connection wasn't ready
+func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool {
+	select {
+	case h.remoteOffersCh <- offer:
+		return true
+	default:
+		h.log.Debugf("OnRemoteOffer skipping message because is not ready")
+		// connection might not be ready yet to receive so we ignore the message
+		return false
+	}
+}
+
+// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
+// doesn't block, discards the message if connection wasn't ready
+func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool {
+	select {
+	case h.remoteAnswerCh <- answer:
+		return true
+	default:
+		// connection might not be ready yet to receive so we ignore the message
+		h.log.Debugf("OnRemoteAnswer skipping message because is not ready")
+		return false
+	}
+}
+
+func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) {
+	select {
+	case remoteOfferAnswer := <-h.remoteOffersCh:
+		// received confirmation from the remote peer -> ready to proceed
+		err := h.sendAnswer()
+		if err != nil {
+			return nil, err
+		}
+		return &remoteOfferAnswer, nil
+	case remoteOfferAnswer := <-h.remoteAnswerCh:
+		return &remoteOfferAnswer, nil
+	case <-h.ctx.Done():
+		// closed externally
+		return nil, NewConnectionClosedError(h.config.Key)
+	}
+}
+
+// sendOffer prepares local user credentials and signals them to the remote peer
+func (h *Handshaker) sendOffer() error {
+	if !h.signaler.Ready() {
+		return ErrSignalIsNotReady
+	}
+
+	iceUFrag, icePwd := h.ice.GetLocalUserCredentials()
+	offer := OfferAnswer{
+		IceCredentials:  IceCredentials{iceUFrag, icePwd},
+		WgListenPort:    h.config.LocalWgPort,
+		Version:         version.NetbirdVersion(),
+		RosenpassPubKey: h.config.RosenpassPubKey,
+		RosenpassAddr:   h.config.RosenpassAddr,
+	}
+
+	addr, err := h.relay.RelayInstanceAddress()
+	if err == nil {
+		offer.RelaySrvAddress = addr
+	}
+
+	return h.signaler.SignalOffer(offer, h.config.Key)
+}
+
+func (h *Handshaker) sendAnswer() error {
+	h.log.Debugf("sending answer")
+	uFrag, pwd := h.ice.GetLocalUserCredentials()
+
+	answer := OfferAnswer{
+		IceCredentials:  IceCredentials{uFrag, pwd},
+		WgListenPort:    h.config.LocalWgPort,
+		Version:         version.NetbirdVersion(),
+		RosenpassPubKey: h.config.RosenpassPubKey,
+		RosenpassAddr:   h.config.RosenpassAddr,
+	}
+	addr, err := h.relay.RelayInstanceAddress()
+	if err == nil {
+		answer.RelaySrvAddress = addr
+	}
+
+	err = h.signaler.SignalAnswer(answer, h.config.Key)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

client/internal/peer/signaler.go

@@ -0,0 +1,70 @@
+package peer
+
+import (
+	"github.com/pion/ice/v3"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	signal "github.com/netbirdio/netbird/signal/client"
+	sProto "github.com/netbirdio/netbird/signal/proto"
+)
+
+type Signaler struct {
+	signal       signal.Client
+	wgPrivateKey wgtypes.Key
+}
+
+func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler {
+	return &Signaler{
+		signal:       signal,
+		wgPrivateKey: wgPrivateKey,
+	}
+}
+
+func (s *Signaler) SignalOffer(offer OfferAnswer, remoteKey string) error {
+	return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER)
+}
+
+func (s *Signaler) SignalAnswer(offer OfferAnswer, remoteKey string) error {
+	return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER)
+}
+
+func (s *Signaler) SignalICECandidate(candidate ice.Candidate, remoteKey string) error {
+	return s.signal.Send(&sProto.Message{
+		Key:       s.wgPrivateKey.PublicKey().String(),
+		RemoteKey: remoteKey,
+		Body: &sProto.Body{
+			Type:    sProto.Body_CANDIDATE,
+			Payload: candidate.Marshal(),
+		},
+	})
+}
+
+func (s *Signaler) Ready() bool {
+	return s.signal.Ready()
+}
+
+// SignalOfferAnswer signals either an offer or an answer to remote peer
+func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error {
+	msg, err := signal.MarshalCredential(
+		s.wgPrivateKey,
+		offerAnswer.WgListenPort,
+		remoteKey,
+		&signal.Credential{
+			UFrag: offerAnswer.IceCredentials.UFrag,
+			Pwd:   offerAnswer.IceCredentials.Pwd,
+		},
+		bodyType,
+		offerAnswer.RosenpassPubKey,
+		offerAnswer.RosenpassAddr,
+		offerAnswer.RelaySrvAddress)
+	if err != nil {
+		return err
+	}
+
+	err = s.signal.Send(msg)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

client/internal/peer/status.go

@@ -13,6 +13,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/relay"
 	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/management/domain"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 )
 
 // State contains the latest state of a peer
@@ -24,11 +25,11 @@ type State struct {
 	ConnStatus                 ConnStatus
 	ConnStatusUpdate           time.Time
 	Relayed                    bool
-	Direct                     bool
 	LocalIceCandidateType      string
 	RemoteIceCandidateType     string
 	LocalIceCandidateEndpoint  string
 	RemoteIceCandidateEndpoint string
+	RelayServerAddress         string
 	LastWireguardHandshake     time.Time
 	BytesTx                    int64
 	BytesRx                    int64
@@ -142,6 +143,8 @@ type Status struct {
 	// Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events
 	// set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications()
 	peerListChangedForNotification bool
+
+	relayMgr *relayClient.Manager
 }
 
 // NewRecorder returns a new Status instance
@@ -156,6 +159,12 @@ func NewRecorder(mgmAddress string) *Status {
 	}
 }
 
+func (d *Status) SetRelayMgr(manager *relayClient.Manager) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	d.relayMgr = manager
+}
+
 // ReplaceOfflinePeers replaces
 func (d *Status) ReplaceOfflinePeers(replacement []State) {
 	d.mux.Lock()
@@ -231,17 +240,17 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		peerState.SetRoutes(receivedState.GetRoutes())
 	}
 
-	skipNotification := shouldSkipNotify(receivedState, peerState)
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
 
 	if receivedState.ConnStatus != peerState.ConnStatus {
 		peerState.ConnStatus = receivedState.ConnStatus
 		peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
-		peerState.Direct = receivedState.Direct
 		peerState.Relayed = receivedState.Relayed
 		peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
 		peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
 		peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
 		peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+		peerState.RelayServerAddress = receivedState.RelayServerAddress
 		peerState.RosenpassEnabled = receivedState.RosenpassEnabled
 	}
 
@@ -261,6 +270,146 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 	return nil
 }
 
+func (d *Status) UpdatePeerICEState(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	if receivedState.IP != "" {
+		peerState.IP = receivedState.IP
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.Relayed = receivedState.Relayed
+	peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
+	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
+	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
+	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+	peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerRelayedState(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.Relayed = receivedState.Relayed
+	peerState.RelayServerAddress = receivedState.RelayServerAddress
+	peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.Relayed = receivedState.Relayed
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.RelayServerAddress = ""
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.Relayed = receivedState.Relayed
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
+	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
+	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
+	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
 // UpdateWireGuardPeerState updates the WireGuard bits of the peer state
 func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) error {
 	d.mux.Lock()
@@ -280,13 +429,13 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats)
 	return nil
 }
 
-func shouldSkipNotify(received, curr State) bool {
+func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool {
 	switch {
-	case received.ConnStatus == StatusConnecting:
+	case receivedConnStatus == StatusConnecting:
 		return true
-	case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
+	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
 		return true
-	case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
+	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
 		return curr.IP != ""
 	default:
 		return false
@@ -503,7 +652,28 @@ func (d *Status) GetSignalState() SignalState {
 }
 
 func (d *Status) GetRelayStates() []relay.ProbeResult {
-	return d.relayStates
+	if d.relayMgr == nil {
+		return d.relayStates
+	}
+
+	// extend the list of stun, turn servers with relay address
+	relaysState := make([]relay.ProbeResult, len(d.relayStates), len(d.relayStates)+1)
+	copy(relaysState, d.relayStates)
+
+	relayState := relay.ProbeResult{}
+
+	// if the server connection is not established then we will use the general address
+	// in case of connection we will use the instance specific address
+	instanceAddr, err := d.relayMgr.RelayInstanceAddress()
+	if err != nil {
+		relayState.URI = d.relayMgr.ServerURL()
+		relayState.Err = err
+	} else {
+		relayState.URI = instanceAddr
+	}
+
+	relaysState = append(relaysState, relayState)
+	return relaysState
 }
 
 func (d *Status) GetDNSStates() []NSGroupState {
@@ -535,7 +705,6 @@ func (d *Status) GetFullStatus() FullStatus {
 	}
 
 	fullStatus.Peers = append(fullStatus.Peers, d.offlinePeers...)
-
 	return fullStatus
 }
 

client/internal/peer/status_test.go

@@ -2,8 +2,8 @@ package peer
 
 import (
 	"errors"
-	"testing"
 	"sync"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -43,7 +43,7 @@ func TestUpdatePeerState(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -64,7 +64,7 @@ func TestStatus_UpdatePeerFQDN(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -83,7 +83,7 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -108,7 +108,7 @@ func TestRemovePeer(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState

client/internal/peer/stdnet.go

@@ -6,6 +6,6 @@ import (
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 )
 
-func (conn *Conn) newStdNet() (*stdnet.Net, error) {
-	return stdnet.NewNet(conn.config.InterfaceBlackList)
+func (w *WorkerICE) newStdNet() (*stdnet.Net, error) {
+	return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList)
 }

client/internal/peer/stdnet_android.go

@@ -2,6 +2,6 @@ package peer
 
 import "github.com/netbirdio/netbird/client/internal/stdnet"
 
-func (conn *Conn) newStdNet() (*stdnet.Net, error) {
-	return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.config.InterfaceBlackList)
+func (w *WorkerICE) newStdNet() (*stdnet.Net, error) {
+	return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList)
 }

client/internal/peer/worker_ice.go

@@ -0,0 +1,457 @@
+package peer
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pion/ice/v3"
+	"github.com/pion/randutil"
+	"github.com/pion/stun/v2"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/stdnet"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/route"
+)
+
+const (
+	iceKeepAliveDefault           = 4 * time.Second
+	iceDisconnectedTimeoutDefault = 6 * time.Second
+	// iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package
+	iceRelayAcceptanceMinWaitDefault = 2 * time.Second
+
+	lenUFrag   = 16
+	lenPwd     = 32
+	runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+)
+
+var (
+	failedTimeout = 6 * time.Second
+)
+
+type ICEConfig struct {
+	// StunTurn is a list of STUN and TURN URLs
+	StunTurn *atomic.Value // []*stun.URI
+
+	// InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering
+	// (e.g. if eth0 is in the list, host candidate of this interface won't be used)
+	InterfaceBlackList   []string
+	DisableIPv6Discovery bool
+
+	UDPMux      ice.UDPMux
+	UDPMuxSrflx ice.UniversalUDPMux
+
+	NATExternalIPs []string
+}
+
+type ICEConnInfo struct {
+	RemoteConn                 net.Conn
+	RosenpassPubKey            []byte
+	RosenpassAddr              string
+	LocalIceCandidateType      string
+	RemoteIceCandidateType     string
+	RemoteIceCandidateEndpoint string
+	LocalIceCandidateEndpoint  string
+	Relayed                    bool
+	RelayedOnLocal             bool
+}
+
+type WorkerICECallbacks struct {
+	OnConnReady     func(ConnPriority, ICEConnInfo)
+	OnStatusChanged func(ConnStatus)
+}
+
+type WorkerICE struct {
+	ctx               context.Context
+	log               *log.Entry
+	config            ConnConfig
+	signaler          *Signaler
+	iFaceDiscover     stdnet.ExternalIFaceDiscover
+	statusRecorder    *Status
+	hasRelayOnLocally bool
+	conn              WorkerICECallbacks
+
+	selectedPriority ConnPriority
+
+	agent    *ice.Agent
+	muxAgent sync.Mutex
+
+	StunTurn []*stun.URI
+
+	sentExtraSrflx bool
+
+	localUfrag string
+	localPwd   string
+}
+
+func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) {
+	w := &WorkerICE{
+		ctx:               ctx,
+		log:               log,
+		config:            config,
+		signaler:          signaler,
+		iFaceDiscover:     ifaceDiscover,
+		statusRecorder:    statusRecorder,
+		hasRelayOnLocally: hasRelayOnLocally,
+		conn:              callBacks,
+	}
+
+	localUfrag, localPwd, err := generateICECredentials()
+	if err != nil {
+		return nil, err
+	}
+	w.localUfrag = localUfrag
+	w.localPwd = localPwd
+	return w, nil
+}
+
+func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
+	w.log.Debugf("OnNewOffer for ICE")
+	w.muxAgent.Lock()
+
+	if w.agent != nil {
+		w.log.Debugf("agent already exists, skipping the offer")
+		w.muxAgent.Unlock()
+		return
+	}
+
+	var preferredCandidateTypes []ice.CandidateType
+	if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" {
+		w.selectedPriority = connPriorityICEP2P
+		preferredCandidateTypes = candidateTypesP2P()
+	} else {
+		w.selectedPriority = connPriorityICETurn
+		preferredCandidateTypes = candidateTypes()
+	}
+
+	w.log.Debugf("recreate ICE agent")
+	agentCtx, agentCancel := context.WithCancel(w.ctx)
+	agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes)
+	if err != nil {
+		w.log.Errorf("failed to recreate ICE Agent: %s", err)
+		w.muxAgent.Unlock()
+		return
+	}
+	w.agent = agent
+	w.muxAgent.Unlock()
+
+	w.log.Debugf("gather candidates")
+	err = w.agent.GatherCandidates()
+	if err != nil {
+		w.log.Debugf("failed to gather candidates: %s", err)
+		return
+	}
+
+	// will block until connection succeeded
+	// but it won't release if ICE Agent went into Disconnected or Failed state,
+	// so we have to cancel it with the provided context once agent detected a broken connection
+	w.log.Debugf("turn agent dial")
+	remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer)
+	if err != nil {
+		w.log.Debugf("failed to dial the remote peer: %s", err)
+		return
+	}
+	w.log.Debugf("agent dial succeeded")
+
+	pair, err := w.agent.GetSelectedCandidatePair()
+	if err != nil {
+		return
+	}
+
+	if !isRelayCandidate(pair.Local) {
+		// dynamically set remote WireGuard port if other side specified a different one from the default one
+		remoteWgPort := iface.DefaultWgPort
+		if remoteOfferAnswer.WgListenPort != 0 {
+			remoteWgPort = remoteOfferAnswer.WgListenPort
+		}
+
+		// To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port
+		go w.punchRemoteWGPort(pair, remoteWgPort)
+	}
+
+	ci := ICEConnInfo{
+		RemoteConn:                 remoteConn,
+		RosenpassPubKey:            remoteOfferAnswer.RosenpassPubKey,
+		RosenpassAddr:              remoteOfferAnswer.RosenpassAddr,
+		LocalIceCandidateType:      pair.Local.Type().String(),
+		RemoteIceCandidateType:     pair.Remote.Type().String(),
+		LocalIceCandidateEndpoint:  fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()),
+		RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()),
+		Relayed:                    isRelayed(pair),
+		RelayedOnLocal:             isRelayCandidate(pair.Local),
+	}
+	w.log.Debugf("on ICE conn read to use ready")
+	go w.conn.OnConnReady(w.selectedPriority, ci)
+}
+
+// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
+func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String())
+	if w.agent == nil {
+		w.log.Warnf("ICE Agent is not initialized yet")
+		return
+	}
+
+	if candidateViaRoutes(candidate, haRoutes) {
+		return
+	}
+
+	err := w.agent.AddRemoteCandidate(candidate)
+	if err != nil {
+		w.log.Errorf("error while handling remote candidate")
+		return
+	}
+}
+
+func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	return w.localUfrag, w.localPwd
+}
+
+func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) {
+	transportNet, err := w.newStdNet()
+	if err != nil {
+		w.log.Errorf("failed to create pion's stdnet: %s", err)
+	}
+
+	iceKeepAlive := iceKeepAlive()
+	iceDisconnectedTimeout := iceDisconnectedTimeout()
+	iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
+
+	agentConfig := &ice.AgentConfig{
+		MulticastDNSMode:       ice.MulticastDNSModeDisabled,
+		NetworkTypes:           []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6},
+		Urls:                   w.config.ICEConfig.StunTurn.Load().([]*stun.URI),
+		CandidateTypes:         relaySupport,
+		InterfaceFilter:        stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList),
+		UDPMux:                 w.config.ICEConfig.UDPMux,
+		UDPMuxSrflx:            w.config.ICEConfig.UDPMuxSrflx,
+		NAT1To1IPs:             w.config.ICEConfig.NATExternalIPs,
+		Net:                    transportNet,
+		FailedTimeout:          &failedTimeout,
+		DisconnectedTimeout:    &iceDisconnectedTimeout,
+		KeepaliveInterval:      &iceKeepAlive,
+		RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait,
+		LocalUfrag:             w.localUfrag,
+		LocalPwd:               w.localPwd,
+	}
+
+	if w.config.ICEConfig.DisableIPv6Discovery {
+		agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4}
+	}
+
+	w.sentExtraSrflx = false
+	agent, err := ice.NewAgent(agentConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnCandidate(w.onICECandidate)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnConnectionStateChange(func(state ice.ConnectionState) {
+		w.log.Debugf("ICE ConnectionState has changed to %s", state.String())
+		if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected {
+			w.conn.OnStatusChanged(StatusDisconnected)
+
+			w.muxAgent.Lock()
+			agentCancel()
+			_ = agent.Close()
+			w.agent = nil
+
+			w.muxAgent.Unlock()
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) {
+		err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency())
+		if err != nil {
+			w.log.Debugf("failed to update latency for peer: %s", err)
+			return
+		}
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed setting binding response callback: %w", err)
+	}
+
+	return agent, nil
+}
+
+func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
+	// wait local endpoint configuration
+	time.Sleep(time.Second)
+	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort))
+	if err != nil {
+		w.log.Warnf("got an error while resolving the udp address, err: %s", err)
+		return
+	}
+
+	mux, ok := w.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault)
+	if !ok {
+		w.log.Warn("invalid udp mux conversion")
+		return
+	}
+	_, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr)
+	if err != nil {
+		w.log.Warnf("got an error while sending the punch packet, err: %s", err)
+	}
+}
+
+// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates
+// and then signals them to the remote peer
+func (w *WorkerICE) onICECandidate(candidate ice.Candidate) {
+	// nil means candidate gathering has been ended
+	if candidate == nil {
+		return
+	}
+
+	// TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored
+	w.log.Debugf("discovered local candidate %s", candidate.String())
+	go func() {
+		err := w.signaler.SignalICECandidate(candidate, w.config.Key)
+		if err != nil {
+			w.log.Errorf("failed signaling candidate to the remote peer %s %s", w.config.Key, err)
+		}
+	}()
+
+	if !w.shouldSendExtraSrflxCandidate(candidate) {
+		return
+	}
+
+	// sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port)
+	// this is useful when network has an existing port forwarding rule for the wireguard port and this peer
+	extraSrflx, err := extraSrflxCandidate(candidate)
+	if err != nil {
+		w.log.Errorf("failed creating extra server reflexive candidate %s", err)
+		return
+	}
+	w.sentExtraSrflx = true
+
+	go func() {
+		err = w.signaler.SignalICECandidate(extraSrflx, w.config.Key)
+		if err != nil {
+			w.log.Errorf("failed signaling the extra server reflexive candidate: %s", err)
+		}
+	}()
+}
+
+func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) {
+	w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
+		w.config.Key)
+}
+
+func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool {
+	if !w.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port {
+		return true
+	}
+	return false
+}
+
+func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
+	isControlling := w.config.LocalKey > w.config.Key
+	if isControlling {
+		return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+	} else {
+		return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+	}
+}
+
+func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) {
+	relatedAdd := candidate.RelatedAddress()
+	return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{
+		Network:   candidate.NetworkType().String(),
+		Address:   candidate.Address(),
+		Port:      relatedAdd.Port,
+		Component: candidate.Component(),
+		RelAddr:   relatedAdd.Address,
+		RelPort:   relatedAdd.Port,
+	})
+}
+
+func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool {
+	var routePrefixes []netip.Prefix
+	for _, routes := range clientRoutes {
+		if len(routes) > 0 && routes[0] != nil {
+			routePrefixes = append(routePrefixes, routes[0].Network)
+		}
+	}
+
+	addr, err := netip.ParseAddr(candidate.Address())
+	if err != nil {
+		log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err)
+		return false
+	}
+
+	for _, prefix := range routePrefixes {
+		// default route is
+		if prefix.Bits() == 0 {
+			continue
+		}
+
+		if prefix.Contains(addr) {
+			log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix)
+			return true
+		}
+	}
+	return false
+}
+
+func candidateTypes() []ice.CandidateType {
+	if hasICEForceRelayConn() {
+		return []ice.CandidateType{ice.CandidateTypeRelay}
+	}
+	// TODO: remove this once we have refactored userspace proxy into the bind package
+	if runtime.GOOS == "ios" {
+		return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+	}
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay}
+}
+
+func candidateTypesP2P() []ice.CandidateType {
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+}
+
+func isRelayCandidate(candidate ice.Candidate) bool {
+	return candidate.Type() == ice.CandidateTypeRelay
+}
+
+func isRelayed(pair *ice.CandidatePair) bool {
+	if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay {
+		return true
+	}
+	return false
+}
+
+func generateICECredentials() (string, string, error) {
+	ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+
+	pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+	return ufrag, pwd, nil
+
+}

client/internal/peer/worker_relay.go

@@ -0,0 +1,173 @@
+package peer
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync/atomic"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	relayClient "github.com/netbirdio/netbird/relay/client"
+)
+
+var (
+	wgHandshakePeriod   = 2 * time.Minute
+	wgHandshakeOvertime = 30000 * time.Millisecond
+)
+
+type RelayConnInfo struct {
+	relayedConn     net.Conn
+	rosenpassPubKey []byte
+	rosenpassAddr   string
+}
+
+type WorkerRelayCallbacks struct {
+	OnConnReady    func(RelayConnInfo)
+	OnDisconnected func()
+}
+
+type WorkerRelay struct {
+	parentCtx    context.Context
+	log          *log.Entry
+	config       ConnConfig
+	relayManager relayClient.ManagerService
+	conn         WorkerRelayCallbacks
+
+	ctxCancel                  context.CancelFunc
+	relaySupportedOnRemotePeer atomic.Bool
+}
+
+func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay {
+	r := &WorkerRelay{
+		parentCtx:    ctx,
+		log:          log,
+		config:       config,
+		relayManager: relayManager,
+		conn:         callbacks,
+	}
+	return r
+}
+
+func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
+	if !w.isRelaySupported(remoteOfferAnswer) {
+		w.log.Infof("Relay is not supported by remote peer")
+		w.relaySupportedOnRemotePeer.Store(false)
+		return
+	}
+	w.relaySupportedOnRemotePeer.Store(true)
+
+	// the relayManager will return with error in case if the connection has lost with relay server
+	currentRelayAddress, err := w.relayManager.RelayInstanceAddress()
+	if err != nil {
+		w.log.Errorf("failed to handle new offer: %s", err)
+		return
+	}
+
+	srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress)
+
+	relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key)
+	if err != nil {
+		// todo handle all type errors
+		if errors.Is(err, relayClient.ErrConnAlreadyExists) {
+			w.log.Infof("do not need to reopen relay connection")
+			return
+		}
+		w.log.Errorf("failed to open connection via Relay: %s", err)
+		return
+	}
+
+	ctx, ctxCancel := context.WithCancel(w.parentCtx)
+	w.ctxCancel = ctxCancel
+
+	err = w.relayManager.AddCloseListener(srv, w.disconnected)
+	if err != nil {
+		log.Errorf("failed to add close listener: %s", err)
+		_ = relayedConn.Close()
+		ctxCancel()
+		return
+	}
+
+	go w.wgStateCheck(ctx, relayedConn)
+
+	w.log.Debugf("peer conn opened via Relay: %s", srv)
+	go w.conn.OnConnReady(RelayConnInfo{
+		relayedConn:     relayedConn,
+		rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey,
+		rosenpassAddr:   remoteOfferAnswer.RosenpassAddr,
+	})
+}
+
+func (w *WorkerRelay) RelayInstanceAddress() (string, error) {
+	return w.relayManager.RelayInstanceAddress()
+}
+
+func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool {
+	return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally()
+}
+
+func (w *WorkerRelay) IsController() bool {
+	return w.config.LocalKey > w.config.Key
+}
+
+func (w *WorkerRelay) RelayIsSupportedLocally() bool {
+	return w.relayManager.HasRelayAddress()
+}
+
+// wgStateCheck help to check the state of the wireguard handshake and relay connection
+func (w *WorkerRelay) wgStateCheck(ctx context.Context, conn net.Conn) {
+	timer := time.NewTimer(wgHandshakeOvertime)
+	defer timer.Stop()
+	for {
+		select {
+		case <-timer.C:
+			lastHandshake, err := w.wgState()
+			if err != nil {
+				w.log.Errorf("failed to read wg stats: %v", err)
+				continue
+			}
+			w.log.Tracef("last handshake: %v", lastHandshake)
+
+			if time.Since(lastHandshake) > wgHandshakePeriod {
+				w.log.Infof("Wireguard handshake timed out, closing relay connection")
+				_ = conn.Close()
+				w.conn.OnDisconnected()
+				return
+			}
+			resetTime := time.Until(lastHandshake.Add(wgHandshakeOvertime + wgHandshakePeriod))
+			timer.Reset(resetTime)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool {
+	if !w.relayManager.HasRelayAddress() {
+		return false
+	}
+	return answer.RelaySrvAddress != ""
+}
+
+func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string {
+	if w.IsController() {
+		return myRelayAddress
+	}
+	return remoteRelayAddress
+}
+
+func (w *WorkerRelay) wgState() (time.Time, error) {
+	wgState, err := w.config.WgConfig.WgInterface.GetStats(w.config.Key)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return wgState.LastHandshake, nil
+}
+
+func (w *WorkerRelay) disconnected() {
+	if w.ctxCancel != nil {
+		w.ctxCancel()
+	}
+	w.conn.OnDisconnected()
+}

client/internal/relay/relay.go

@@ -17,7 +17,7 @@ import (
 
 // ProbeResult holds the info about the result of a relay probe request
 type ProbeResult struct {
-	URI  *stun.URI
+	URI  string
 	Err  error
 	Addr string
 }
@@ -176,7 +176,7 @@ func ProbeAll(
 		wg.Add(1)
 		go func(res *ProbeResult, stunURI *stun.URI) {
 			defer wg.Done()
-			res.URI = stunURI
+			res.URI = stunURI.String()
 			res.Addr, res.Err = fn(ctx, stunURI)
 		}(&results[i], uri)
 	}

client/internal/routemanager/client.go

@@ -22,7 +22,6 @@ import (
 type routerPeerStatus struct {
 	connected bool
 	relayed   bool
-	direct    bool
 	latency   time.Duration
 }
 
@@ -44,7 +43,7 @@ type clientNetwork struct {
 	ctx                 context.Context
 	cancel              context.CancelFunc
 	statusRecorder      *peer.Status
-	wgInterface         *iface.WGIface
+	wgInterface         iface.IWGIface
 	routes              map[route.ID]*route.Route
 	routeUpdate         chan routesUpdate
 	peerStateUpdate     chan struct{}
@@ -54,7 +53,7 @@ type clientNetwork struct {
 	updateSerial        uint64
 }
 
-func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface *iface.WGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork {
+func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface iface.IWGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork {
 	ctx, cancel := context.WithCancel(ctx)
 
 	client := &clientNetwork{
@@ -82,7 +81,6 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus {
 		routePeerStatuses[r.ID] = routerPeerStatus{
 			connected: peerStatus.ConnStatus == peer.StatusConnected,
 			relayed:   peerStatus.Relayed,
-			direct:    peerStatus.Direct,
 			latency:   peerStatus.Latency,
 		}
 	}
@@ -137,10 +135,6 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID]
 			tempScore++
 		}
 
-		if peerStatus.direct {
-			tempScore++
-		}
-
 		if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") {
 			chosen = r.ID
 			chosenScore = tempScore
@@ -384,7 +378,7 @@ func (c *clientNetwork) peersStateAndUpdateWatcher() {
 	}
 }
 
-func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface *iface.WGIface) RouteHandler {
+func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface iface.IWGIface) RouteHandler {
 	if rt.IsDynamic() {
 		dns := nbdns.NewServiceViaMemory(wgInterface)
 		return dynamic.NewRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouterInteval, statusRecorder, wgInterface, fmt.Sprintf("%s:%d", dns.RuntimeIP(), dns.RuntimePort()))

client/internal/routemanager/client_test.go

@@ -24,7 +24,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -43,7 +42,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   true,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -62,7 +60,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   true,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -81,7 +78,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: false,
 					relayed:   false,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -100,12 +96,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -129,41 +123,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 				"route2": {
 					connected: true,
 					relayed:   true,
-					direct:    true,
-				},
-			},
-			existingRoutes: map[route.ID]*route.Route{
-				"route1": {
-					ID:     "route1",
-					Metric: route.MaxMetric,
-					Peer:   "peer1",
-				},
-				"route2": {
-					ID:     "route2",
-					Metric: route.MaxMetric,
-					Peer:   "peer2",
-				},
-			},
-			currentRoute:    "",
-			expectedRouteID: "route1",
-		},
-		{
-			name: "multiple connected peers with one direct",
-			statuses: map[route.ID]routerPeerStatus{
-				"route1": {
-					connected: true,
-					relayed:   false,
-					direct:    true,
-				},
-				"route2": {
-					connected: true,
-					relayed:   false,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -241,13 +204,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   15 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},
@@ -272,13 +233,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   200 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},
@@ -303,13 +262,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   20 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},

client/internal/routemanager/dynamic/route.go

@@ -48,7 +48,7 @@ type Route struct {
 	currentPeerKey       string
 	cancel               context.CancelFunc
 	statusRecorder       *peer.Status
-	wgInterface          *iface.WGIface
+	wgInterface          iface.IWGIface
 	resolverAddr         string
 }
 
@@ -58,7 +58,7 @@ func NewRoute(
 	allowedIPsRefCounter *refcounter.AllowedIPsRefCounter,
 	interval time.Duration,
 	statusRecorder *peer.Status,
-	wgInterface *iface.WGIface,
+	wgInterface iface.IWGIface,
 	resolverAddr string,
 ) *Route {
 	return &Route{

client/internal/routemanager/manager.go

@@ -49,7 +49,7 @@ type DefaultManager struct {
 	serverRouter         serverRouter
 	sysOps               *systemops.SysOps
 	statusRecorder       *peer.Status
-	wgInterface          *iface.WGIface
+	wgInterface          iface.IWGIface
 	pubKey               string
 	notifier             *notifier.Notifier
 	routeRefCounter      *refcounter.RouteRefCounter
@@ -61,7 +61,7 @@ func NewManager(
 	ctx context.Context,
 	pubKey string,
 	dnsRouteInterval time.Duration,
-	wgInterface *iface.WGIface,
+	wgInterface iface.IWGIface,
 	statusRecorder *peer.Status,
 	initialRoutes []*route.Route,
 ) *DefaultManager {

client/internal/routemanager/server_android.go

@@ -11,6 +11,6 @@ import (
 	"github.com/netbirdio/netbird/iface"
 )
 
-func newServerRouter(context.Context, *iface.WGIface, firewall.Manager, *peer.Status) (serverRouter, error) {
+func newServerRouter(context.Context, iface.IWGIface, firewall.Manager, *peer.Status) (serverRouter, error) {
 	return nil, fmt.Errorf("server route not supported on this os")
 }

client/internal/routemanager/server_nonandroid.go

@@ -22,11 +22,11 @@ type defaultServerRouter struct {
 	ctx            context.Context
 	routes         map[route.ID]*route.Route
 	firewall       firewall.Manager
-	wgInterface    *iface.WGIface
+	wgInterface    iface.IWGIface
 	statusRecorder *peer.Status
 }
 
-func newServerRouter(ctx context.Context, wgInterface *iface.WGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) {
+func newServerRouter(ctx context.Context, wgInterface iface.IWGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) {
 	return &defaultServerRouter{
 		ctx:            ctx,
 		routes:         make(map[route.ID]*route.Route),

client/internal/routemanager/sysctl/sysctl_linux.go

@@ -23,7 +23,7 @@ const (
 )
 
 // Setup configures sysctl settings for RP filtering and source validation.
-func Setup(wgIface *iface.WGIface) (map[string]int, error) {
+func Setup(wgIface iface.IWGIface) (map[string]int, error) {
 	keys := map[string]int{}
 	var result *multierror.Error
 

client/internal/routemanager/systemops/systemops.go

@@ -19,7 +19,7 @@ type ExclusionCounter = refcounter.Counter[any, Nexthop]
 
 type SysOps struct {
 	refCounter  *ExclusionCounter
-	wgInterface *iface.WGIface
+	wgInterface iface.IWGIface
 	// prefixes is tracking all the current added prefixes im memory
 	// (this is used in iOS as all route updates require a full table update)
 	//nolint
@@ -30,7 +30,7 @@ type SysOps struct {
 	notifier *notifier.Notifier
 }
 
-func NewSysOps(wgInterface *iface.WGIface, notifier *notifier.Notifier) *SysOps {
+func NewSysOps(wgInterface iface.IWGIface, notifier *notifier.Notifier) *SysOps {
 	return &SysOps{
 		wgInterface: wgInterface,
 		notifier:    notifier,

client/internal/routemanager/systemops/systemops_generic.go

@@ -122,7 +122,7 @@ func (r *SysOps) addRouteForCurrentDefaultGateway(prefix netip.Prefix) error {
 
 // addRouteToNonVPNIntf adds a new route to the routing table for the given prefix and returns the next hop and interface.
 // If the next hop or interface is pointing to the VPN interface, it will return the initial values.
-func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf *iface.WGIface, initialNextHop Nexthop) (Nexthop, error) {
+func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf iface.IWGIface, initialNextHop Nexthop) (Nexthop, error) {
 	addr := prefix.Addr()
 	switch {
 	case addr.IsLoopback(),

client/internal/wgproxy/proxy_ebpf.go

@@ -181,7 +181,7 @@ func (p *WGEBPFProxy) proxyToRemote() {
 			conn, ok := p.turnConnStore[uint16(addr.Port)]
 			p.turnConnMutex.Unlock()
 			if !ok {
-				log.Infof("turn conn not found by port: %d", addr.Port)
+				log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port)
 				continue
 			}
 
@@ -206,7 +206,7 @@ func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) {
 }
 
 func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) {
-	log.Tracef("remove turn conn from store by port: %d", turnConnID)
+	log.Debugf("remove turn conn from store by port: %d", turnConnID)
 	p.turnConnMutex.Lock()
 	defer p.turnConnMutex.Unlock()
 	delete(p.turnConnStore, turnConnID)

client/internal/wgproxy/proxy_userspace.go

@@ -3,6 +3,7 @@ package wgproxy
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
 
 	log "github.com/sirupsen/logrus"
@@ -64,7 +65,6 @@ func (p *WGUserSpaceProxy) Free() error {
 // proxyToRemote proxies everything from Wireguard to the RemoteKey peer
 // blocks
 func (p *WGUserSpaceProxy) proxyToRemote() {
-
 	buf := make([]byte, 1500)
 	for {
 		select {
@@ -73,11 +73,17 @@ func (p *WGUserSpaceProxy) proxyToRemote() {
 		default:
 			n, err := p.localConn.Read(buf)
 			if err != nil {
+				log.Debugf("failed to read from wg interface conn: %s", err)
 				continue
 			}
 
 			_, err = p.remoteConn.Write(buf[:n])
 			if err != nil {
+				if err == io.EOF {
+					p.cancel()
+				} else {
+					log.Debugf("failed to write to remote conn: %s", err)
+				}
 				continue
 			}
 		}
@@ -96,11 +102,17 @@ func (p *WGUserSpaceProxy) proxyToLocal() {
 		default:
 			n, err := p.remoteConn.Read(buf)
 			if err != nil {
+				if err == io.EOF {
+					p.cancel()
+					return
+				}
+				log.Errorf("failed to read from remote conn: %s", err)
 				continue
 			}
 
 			_, err = p.localConn.Write(buf[:n])
 			if err != nil {
+				log.Debugf("failed to write to wg interface conn: %s", err)
 				continue
 			}
 		}

client/ios/NetBirdSDK/client.go

@@ -168,7 +168,6 @@ func (c *Client) GetStatusDetails() *StatusDetails {
 			BytesTx:                    p.BytesTx,
 			ConnStatus:                 p.ConnStatus.String(),
 			ConnStatusUpdate:           p.ConnStatusUpdate.Format("2006-01-02 15:04:05"),
-			Direct:                     p.Direct,
 			LastWireguardHandshake:     p.LastWireguardHandshake.String(),
 			Relayed:                    p.Relayed,
 			RosenpassEnabled:           p.RosenpassEnabled,

client/proto/daemon.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: daemon.proto
 
 package proto
@@ -899,7 +899,6 @@ type PeerState struct {
 	ConnStatus                 string                 `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"`
 	ConnStatusUpdate           *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"`
 	Relayed                    bool                   `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"`
-	Direct                     bool                   `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"`
 	LocalIceCandidateType      string                 `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"`
 	RemoteIceCandidateType     string                 `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"`
 	Fqdn                       string                 `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"`
@@ -911,6 +910,7 @@ type PeerState struct {
 	RosenpassEnabled           bool                   `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"`
 	Routes                     []string               `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"`
 	Latency                    *durationpb.Duration   `protobuf:"bytes,17,opt,name=latency,proto3" json:"latency,omitempty"`
+	RelayAddress               string                 `protobuf:"bytes,18,opt,name=relayAddress,proto3" json:"relayAddress,omitempty"`
 }
 
 func (x *PeerState) Reset() {
@@ -980,13 +980,6 @@ func (x *PeerState) GetRelayed() bool {
 	return false
 }
 
-func (x *PeerState) GetDirect() bool {
-	if x != nil {
-		return x.Direct
-	}
-	return false
-}
-
 func (x *PeerState) GetLocalIceCandidateType() string {
 	if x != nil {
 		return x.LocalIceCandidateType
@@ -1064,6 +1057,13 @@ func (x *PeerState) GetLatency() *durationpb.Duration {
 	return nil
 }
 
+func (x *PeerState) GetRelayAddress() string {
+	if x != nil {
+		return x.RelayAddress
+	}
+	return ""
+}
+
 // LocalPeerState contains the latest state of the local peer
 type LocalPeerState struct {
 	state         protoimpl.MessageState
@@ -2243,7 +2243,7 @@ var file_daemon_proto_rawDesc = []byte{
 	0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e,
 	0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c,
 	0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50,
-	0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xce, 0x05, 0x0a, 0x09, 0x50, 0x65,
+	0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xda, 0x05, 0x0a, 0x09, 0x50, 0x65,
 	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65,
 	0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12,
@@ -2255,209 +2255,210 @@ var file_daemon_proto_rawDesc = []byte{
 	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75,
 	0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79,
 	0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65,
-	0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28,
-	0x08, 0x52, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63,
-	0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79,
-	0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49,
+	0x64, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e,
+	0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64,
+	0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74,
+	0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70,
+	0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49,
 	0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12,
-	0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64,
-	0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64,
-	0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18,
-	0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c,
-	0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65,
-	0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19,
-	0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74,
-	0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d,
-	0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45,
-	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72,
-	0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74,
-	0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73,
-	0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68,
-	0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
-	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
-	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67,
-	0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a,
-	0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07,
-	0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73,
-	0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54,
-	0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e,
-	0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73,
-	0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a,
-	0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72,
-	0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79,
-	0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
-	0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c,
-	0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a,
-	0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a,
-	0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70,
-	0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49,
-	0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f,
-	0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12,
-	0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66,
-	0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73,
-	0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72,
-	0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12,
-	0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d,
-	0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f,
-	0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76,
-	0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28,
-	0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67,
-	0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f,
-	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63,
-	0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f,
-	0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57,
-	0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
-	0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
-	0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65,
-	0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79,
-	0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c,
-	0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69,
-	0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e,
-	0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65,
-	0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73,
-	0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12,
-	0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
-	0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72,
-	0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22,
-	0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41,
-	0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
-	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
-	0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65,
-	0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
-	0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
-	0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67,
-	0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61,
-	0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50,
-	0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50,
-	0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72,
-	0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
-	0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72,
-	0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28,
-	0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79,
-	0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a,
-	0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03,
-	0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72,
-	0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72,
-	0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74,
-	0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73,
-	0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
-	0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06,
-	0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74,
-	0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a,
-	0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52,
-	0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70,
-	0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e,
-	0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03,
-	0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75,
-	0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49,
-	0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74,
-	0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49,
-	0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73,
-	0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73,
-	0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69,
-	0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
-	0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73,
-	0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
-	0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50,
-	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64,
-	0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49,
-	0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
-	0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
-	0x02, 0x38, 0x01, 0x22, 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64,
-	0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f,
-	0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e,
-	0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12,
-	0x1e, 0x0a, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20,
-	0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22,
-	0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65,
-	0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
-	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22,
-	0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f,
-	0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a,
-	0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c,
-	0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a,
-	0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41,
-	0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08,
-	0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f,
-	0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a,
-	0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65,
-	0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f,
-	0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67,
-	0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d,
-	0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67,
-	0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74,
-	0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f,
-	0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
-	0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55,
-	0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
-	0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39,
-	0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
-	0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77,
-	0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
-	0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42,
-	0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61,
-	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47,
-	0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73,
-	0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f,
-	0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61,
-	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c,
-	0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d,
-	0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
-	0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65,
-	0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
-	0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53,
-	0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75,
-	0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65,
-	0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42,
-	0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
-	0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a,
-	0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65,
-	0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65,
+	0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66,
+	0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43,
+	0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65,
+	0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+	0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61,
+	0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
+	0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65,
+	0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
+	0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61,
+	0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c,
+	0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64,
+	0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78,
+	0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12,
+	0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03,
+	0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73,
+	0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20,
+	0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e,
+	0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18,
+	0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a,
+	0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
+	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+	0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e,
+	0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65,
+	0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41,
+	0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c,
+	0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62,
+	0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65,
+	0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72,
+	0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e,
+	0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66,
+	0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12,
+	0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62,
+	0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e,
+	0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72,
+	0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69,
+	0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70,
+	0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a,
+	0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72,
+	0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
+	0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65,
+	0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a,
+	0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12,
+	0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a,
+	0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72,
+	0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74,
+	0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
+	0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c,
+	0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f,
+	0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03,
+	0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65,
+	0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e,
+	0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a,
+	0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e,
+	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a,
+	0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e,
+	0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65,
+	0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64,
+	0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53,
+	0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65,
+	0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a,
+	0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e,
+	0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74,
+	0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73,
+	0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14,
+	0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53,
+	0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
+	0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75,
+	0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72,
+	0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61,
+	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74,
+	0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74,
+	0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75,
+	0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75,
+	0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a,
+	0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22,
+	0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73,
+	0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03,
+	0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a,
+	0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a,
+	0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
+	0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63,
+	0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63,
+	0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a,
+	0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74,
+	0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74,
+	0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a,
+	0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50,
+	0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22,
+	0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69,
+	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d,
+	0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73,
+	0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52,
+	0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x29, 0x0a, 0x13, 0x44,
+	0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67,
+	0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13,
+	0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c,
+	0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53,
+	0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76,
+	0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74,
+	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07,
+	0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e,
+	0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12,
+	0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41,
+	0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09,
+	0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41,
+	0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12,
+	0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c,
+	0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b,
+	0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b,
+	0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c,
+	0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61,
+	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69,
+	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55,
+	0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74,
+	0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74,
+	0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61,
+	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e,
+	0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65,
+	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
+	0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f,
+	0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45,
+	0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64,
+	0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
+	0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52,
+	0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53,
+	0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65,
+	0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f,
+	0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65,
+	0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63,
+	0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
+	0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65,
+	0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42,
+	0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64,
+	0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47,
+	0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65,
 	0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74,
-	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
-	0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65,
-	0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
+	0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c,
+	0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65,
+	0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67,
+	0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42,
+	0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
 }
 
 var (

client/proto/daemon.proto

@@ -168,7 +168,6 @@ message PeerState {
   string connStatus = 3;
   google.protobuf.Timestamp connStatusUpdate = 4;
   bool relayed = 5;
-  bool direct = 6;
   string localIceCandidateType = 7;
   string remoteIceCandidateType = 8;
   string fqdn = 9;
@@ -180,6 +179,7 @@ message PeerState {
   bool rosenpassEnabled = 15;
   repeated string routes = 16;
   google.protobuf.Duration latency = 17;
+  string relayAddress = 18;
 }
 
 // LocalPeerState contains the latest state of the local peer

client/server/debug.go

@@ -369,8 +369,8 @@ func seedFromStatus(a *anonymize.Anonymizer, status *peer.FullStatus) {
 	}
 
 	for _, relay := range status.Relays {
-		if relay.URI != nil {
-			a.AnonymizeURI(relay.URI.String())
+		if relay.URI != "" {
+			a.AnonymizeURI(relay.URI)
 		}
 	}
 }

client/server/server.go

@@ -745,11 +745,11 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus {
 			ConnStatus:                 peerState.ConnStatus.String(),
 			ConnStatusUpdate:           timestamppb.New(peerState.ConnStatusUpdate),
 			Relayed:                    peerState.Relayed,
-			Direct:                     peerState.Direct,
 			LocalIceCandidateType:      peerState.LocalIceCandidateType,
 			RemoteIceCandidateType:     peerState.RemoteIceCandidateType,
 			LocalIceCandidateEndpoint:  peerState.LocalIceCandidateEndpoint,
 			RemoteIceCandidateEndpoint: peerState.RemoteIceCandidateEndpoint,
+			RelayAddress:               peerState.RelayServerAddress,
 			Fqdn:                       peerState.FQDN,
 			LastWireguardHandshake:     timestamppb.New(peerState.LastWireguardHandshake),
 			BytesRx:                    peerState.BytesRx,
@@ -763,7 +763,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus {
 
 	for _, relayState := range fullStatus.Relays {
 		pbRelayState := &proto.RelayState{
-			URI:       relayState.URI.String(),
+			URI:       relayState.URI,
 			Available: relayState.Err == nil,
 		}
 		if err := relayState.Err; err != nil {

client/server/server_test.go

@@ -129,7 +129,10 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve
 	if err != nil {
 		return nil, "", err
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+	rc := &server.RelayConfig{
+		Address: "localhost:0",
+	}
+	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc)
 	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		return nil, "", err

encryption/cert.go

@@ -0,0 +1,19 @@
+package encryption
+
+import "crypto/tls"
+
+func LoadTLSConfig(certFile, keyFile string) (*tls.Config, error) {
+	serverCert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	config := &tls.Config{
+		Certificates: []tls.Certificate{serverCert},
+		ClientAuth:   tls.NoClientCert,
+		NextProtos: []string{
+			"h2", "http/1.1", // enable HTTP/2
+		},
+	}
+	return config, nil
+}

encryption/letsencrypt.go

@@ -9,7 +9,7 @@ import (
 )
 
 // CreateCertManager wraps common logic of generating Let's encrypt certificate.
-func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Manager, error) {
+func CreateCertManager(datadir string, letsencryptDomain ...string) (*autocert.Manager, error) {
 	certDir := filepath.Join(datadir, "letsencrypt")
 
 	if _, err := os.Stat(certDir); os.IsNotExist(err) {
@@ -24,7 +24,7 @@ func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Mana
 	certManager := &autocert.Manager{
 		Prompt:     autocert.AcceptTOS,
 		Cache:      autocert.DirCache(certDir),
-		HostPolicy: autocert.HostWhitelist(letsencryptDomain),
+		HostPolicy: autocert.HostWhitelist(letsencryptDomain...),
 	}
 
 	return certManager, nil

encryption/route53.go

@@ -0,0 +1,87 @@
+package encryption
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/caddyserver/certmagic"
+	"github.com/libdns/route53"
+	log "github.com/sirupsen/logrus"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/crypto/acme"
+)
+
+// Route53TLS by default, loads the AWS configuration from the environment.
+// env variables: AWS_REGION, AWS_PROFILE, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
+type Route53TLS struct {
+	DataDir string
+	Email   string
+	Domains []string
+	CA      string
+}
+
+func (r *Route53TLS) GetCertificate() (*tls.Config, error) {
+	if len(r.Domains) == 0 {
+		return nil, fmt.Errorf("no domains provided")
+	}
+
+	certmagic.Default.Logger = logger()
+	certmagic.Default.Storage = &certmagic.FileStorage{Path: r.DataDir}
+	certmagic.DefaultACME.Agreed = true
+	if r.Email != "" {
+		certmagic.DefaultACME.Email = r.Email
+	} else {
+		certmagic.DefaultACME.Email = emailFromDomain(r.Domains[0])
+	}
+
+	if r.CA == "" {
+		certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
+	} else {
+		certmagic.DefaultACME.CA = r.CA
+	}
+
+	certmagic.DefaultACME.DNS01Solver = &certmagic.DNS01Solver{
+		DNSManager: certmagic.DNSManager{
+			DNSProvider: &route53.Provider{},
+		},
+	}
+	cm := certmagic.NewDefault()
+	if err := cm.ManageSync(context.Background(), r.Domains); err != nil {
+		log.Errorf("failed to manage certificate: %v", err)
+		return nil, err
+	}
+
+	tlsConfig := &tls.Config{
+		GetCertificate: cm.GetCertificate,
+		NextProtos:     []string{"h2", "http/1.1", acme.ALPNProto},
+	}
+
+	return tlsConfig, nil
+}
+
+func emailFromDomain(domain string) string {
+	if domain == "" {
+		return ""
+	}
+
+	parts := strings.Split(domain, ".")
+	if len(parts) < 2 {
+		return ""
+	}
+	if parts[0] == "" {
+		return ""
+	}
+	return fmt.Sprintf("admin@%s.%s", parts[len(parts)-2], parts[len(parts)-1])
+}
+
+func logger() *zap.Logger {
+	return zap.New(zapcore.NewCore(
+		zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()),
+		os.Stderr,
+		zap.ErrorLevel,
+	))
+}

encryption/route53_test.go

@@ -0,0 +1,84 @@
+package encryption
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestRoute53TLSConfig(t *testing.T) {
+	t.SkipNow() // This test requires AWS credentials
+	exampleString := "Hello, world!"
+	rtls := &Route53TLS{
+		DataDir: t.TempDir(),
+		Email:   os.Getenv("LE_EMAIL_ROUTE53"),
+		Domains: []string{os.Getenv("DOMAIN")},
+	}
+	tlsConfig, err := rtls.GetCertificate()
+	if err != nil {
+		t.Errorf("Route53TLSConfig failed: %v", err)
+	}
+
+	server := &http.Server{
+		Addr: ":8443",
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write([]byte(exampleString))
+		}),
+		TLSConfig: tlsConfig,
+	}
+
+	go func() {
+		err := server.ListenAndServeTLS("", "")
+		if err != http.ErrServerClosed {
+			t.Errorf("Failed to start server: %v", err)
+		}
+	}()
+	defer func() {
+		if err := server.Shutdown(context.Background()); err != nil {
+			t.Errorf("Failed to shutdown server: %v", err)
+		}
+	}()
+
+	time.Sleep(1 * time.Second)
+	resp, err := http.Get("https://relay.godevltd.com:8443")
+	if err != nil {
+		t.Errorf("Failed to get response: %v", err)
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("Failed to read response body: %v", err)
+	}
+	if string(body) != exampleString {
+		t.Errorf("Unexpected response: %s", body)
+	}
+}
+
+func Test_emailFromDomain(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"example.com", "admin@example.com"},
+		{"x.example.com", "admin@example.com"},
+		{"x.x.example.com", "admin@example.com"},
+		{"*.example.com", "admin@example.com"},
+		{"example", ""},
+		{"", ""},
+		{".com", ""},
+	}
+	for _, tt := range tests {
+		t.Run("domain test", func(t *testing.T) {
+			if got := emailFromDomain(tt.input); got != tt.want {
+				t.Errorf("emailFromDomain() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.23.0
+	github.com/onsi/gomega v1.27.6
 	github.com/pion/ice/v3 v3.0.2
 	github.com/rs/cors v1.8.0
 	github.com/sirupsen/logrus v1.9.3
@@ -34,6 +34,7 @@ require (
 	fyne.io/systray v1.11.0
 	github.com/TheJumpCloud/jcapi-go v3.0.0+incompatible
 	github.com/c-robinson/iplib v1.0.3
+	github.com/caddyserver/certmagic v0.21.3
 	github.com/cilium/ebpf v0.15.0
 	github.com/coreos/go-iptables v0.7.0
 	github.com/creack/pty v1.1.18
@@ -50,18 +51,21 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-secure-stdlib/base62 v0.1.2
 	github.com/hashicorp/go-version v1.6.0
+	github.com/libdns/route53 v1.5.0
 	github.com/libp2p/go-netroute v0.2.1
 	github.com/magiconair/properties v1.8.7
 	github.com/mattn/go-sqlite3 v1.14.19
 	github.com/mdlayher/socket v0.4.1
-	github.com/miekg/dns v1.1.43
+	github.com/miekg/dns v1.1.59
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/nadoo/ipset v0.5.0
 	github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e
+	github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20240801161502-2171ea0babf1
 	github.com/okta/okta-sdk-golang/v2 v2.18.0
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pion/logging v0.2.2
+	github.com/pion/randutil v0.1.0
 	github.com/pion/stun/v2 v2.0.0
 	github.com/pion/transport/v3 v3.0.1
 	github.com/pion/turn/v3 v3.0.1
@@ -69,6 +73,7 @@ require (
 	github.com/rs/xid v1.3.0
 	github.com/shirou/gopsutil/v3 v3.24.4
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
+	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
 	github.com/stretchr/testify v1.9.0
 	github.com/testcontainers/testcontainers-go v0.31.0
 	github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0
@@ -80,6 +85,7 @@ require (
 	go.opentelemetry.io/otel/exporters/prometheus v0.48.0
 	go.opentelemetry.io/otel/metric v1.26.0
 	go.opentelemetry.io/otel/sdk/metric v1.26.0
+	go.uber.org/zap v1.27.0
 	goauthentik.io/api/v3 v3.2023051.3
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a
@@ -92,6 +98,7 @@ require (
 	gorm.io/driver/postgres v1.5.7
 	gorm.io/driver/sqlite v1.5.3
 	gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde
+	nhooyr.io/websocket v1.8.11
 )
 
 require (
@@ -105,8 +112,23 @@ require (
 	github.com/Microsoft/hcsshim v0.12.3 // indirect
 	github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect
+	github.com/caddyserver/zerossl v0.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/containerd v1.7.16 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -139,7 +161,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-uuid v1.0.2 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
@@ -148,13 +170,17 @@ require (
 	github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/libdns/libdns v0.2.2 // indirect
 	github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
 	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mholt/acmez/v2 v2.0.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
@@ -163,12 +189,12 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pegasus-kv/thrift v0.13.0 // indirect
 	github.com/pion/dtls/v2 v2.2.10 // indirect
 	github.com/pion/mdns v0.0.12 // indirect
-	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/transport/v2 v2.2.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -185,10 +211,12 @@ require (
 	github.com/tklauser/numcpus v0.8.0 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	github.com/yuin/goldmark v1.7.1 // indirect
+	github.com/zeebo/blake3 v0.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.26.0 // indirect
 	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/image v0.18.0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/text v0.16.0 // indirect

go.sum

@@ -79,6 +79,34 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
+github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
+github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
+github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 h1:MmLCRqP4U4Cw9gJ4bNrCG0mWqEtBlmAVleyelcHARMU=
+github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3/go.mod h1:AMPjK2YnRh0YgOID3PqhJA1BRNfXDfGOnSsKHtAe8yA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -87,6 +115,10 @@ github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwel
 github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU=
 github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
+github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0=
+github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI=
+github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA=
+github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4=
 github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -207,6 +239,8 @@ github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZs
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-text/render v0.1.0 h1:osrmVDZNHuP1RSu3pNG7Z77Sd2xSbcb/xWytAj9kyVs=
 github.com/go-text/render v0.1.0/go.mod h1:jqEuNMenrmj6QRnkdpeaP0oKGFLDNhDkVKwGjsWWYU4=
 github.com/go-text/typesetting v0.1.0 h1:vioSaLPYcHwPEPLT7gsjCGDCoYSbljxoHJzMnKwVvHw=
@@ -350,8 +384,9 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
@@ -382,6 +417,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -401,6 +440,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -413,6 +455,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
+github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
+github.com/libdns/route53 v1.5.0 h1:2SKdpPFl/qgWsXQvsLNJJAoX7rSxlk7zgoL4jnWdXVA=
+github.com/libdns/route53 v1.5.0/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
@@ -431,9 +477,11 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/
 github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
 github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=
 github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
+github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k=
+github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
-github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -477,6 +525,8 @@ github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-
 github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e/go.mod h1:nykwWZnxb+sJz2Z//CEq45CMRWSHllH8pODKRB8eY7Y=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0 h1:hirFRfx3grVA/9eEyjME5/z3nxdJlN9kfQpvWWPk32g=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
+github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20240801161502-2171ea0babf1 h1:YMPC80v0/bCWXjpCraSSpgh7/5lsNUZ1qoxVzzmRu5k=
+github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20240801161502-2171ea0babf1/go.mod h1:5/sjFmLb8O96B5737VCqhHyGRzNFIaN/Bu7ZodXc3qQ=
 github.com/netbirdio/wireguard-go v0.0.0-20240105182236-6c340dd55aed h1:t0UADZUJDaaZgfKrt8JUPrOLL9Mg/ryjP85RAH53qgs=
 github.com/netbirdio/wireguard-go v0.0.0-20240105182236-6c340dd55aed/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
 github.com/nicksnyder/go-i18n/v2 v2.4.0 h1:3IcvPOAvnCKwNm0TB0dLDTuawWEj+ax/RERNC+diLMM=
@@ -492,14 +542,14 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
-github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
-github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -590,6 +640,8 @@ github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrx
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
 github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
@@ -658,6 +710,12 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc=
 github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
+github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -693,8 +751,14 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4=
 goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -888,7 +952,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1185,6 +1248,8 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
+nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

iface/iface_moc.go

@@ -0,0 +1,103 @@
+package iface
+
+import (
+	"net"
+	"time"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/netbirdio/netbird/iface/bind"
+)
+
+type MockWGIface struct {
+	CreateFunc                 func() error
+	CreateOnAndroidFunc        func(routeRange []string, ip string, domains []string) error
+	IsUserspaceBindFunc        func() bool
+	NameFunc                   func() string
+	AddressFunc                func() WGAddress
+	ToInterfaceFunc            func() *net.Interface
+	UpFunc                     func() (*bind.UniversalUDPMuxDefault, error)
+	UpdateAddrFunc             func(newAddr string) error
+	UpdatePeerFunc             func(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
+	RemovePeerFunc             func(peerKey string) error
+	AddAllowedIPFunc           func(peerKey string, allowedIP string) error
+	RemoveAllowedIPFunc        func(peerKey string, allowedIP string) error
+	CloseFunc                  func() error
+	SetFilterFunc              func(filter PacketFilter) error
+	GetFilterFunc              func() PacketFilter
+	GetDeviceFunc              func() *DeviceWrapper
+	GetStatsFunc               func(peerKey string) (WGStats, error)
+	GetInterfaceGUIDStringFunc func() (string, error)
+}
+
+func (m *MockWGIface) GetInterfaceGUIDString() (string, error) {
+	return m.GetInterfaceGUIDStringFunc()
+}
+
+func (m *MockWGIface) Create() error {
+	return m.CreateFunc()
+}
+
+func (m *MockWGIface) CreateOnAndroid(routeRange []string, ip string, domains []string) error {
+	return m.CreateOnAndroidFunc(routeRange, ip, domains)
+}
+
+func (m *MockWGIface) IsUserspaceBind() bool {
+	return m.IsUserspaceBindFunc()
+}
+
+func (m *MockWGIface) Name() string {
+	return m.NameFunc()
+}
+
+func (m *MockWGIface) Address() WGAddress {
+	return m.AddressFunc()
+}
+
+func (m *MockWGIface) ToInterface() *net.Interface {
+	return m.ToInterfaceFunc()
+}
+
+func (m *MockWGIface) Up() (*bind.UniversalUDPMuxDefault, error) {
+	return m.UpFunc()
+}
+
+func (m *MockWGIface) UpdateAddr(newAddr string) error {
+	return m.UpdateAddrFunc(newAddr)
+}
+
+func (m *MockWGIface) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
+	return m.UpdatePeerFunc(peerKey, allowedIps, keepAlive, endpoint, preSharedKey)
+}
+
+func (m *MockWGIface) RemovePeer(peerKey string) error {
+	return m.RemovePeerFunc(peerKey)
+}
+
+func (m *MockWGIface) AddAllowedIP(peerKey string, allowedIP string) error {
+	return m.AddAllowedIPFunc(peerKey, allowedIP)
+}
+
+func (m *MockWGIface) RemoveAllowedIP(peerKey string, allowedIP string) error {
+	return m.RemoveAllowedIPFunc(peerKey, allowedIP)
+}
+
+func (m *MockWGIface) Close() error {
+	return m.CloseFunc()
+}
+
+func (m *MockWGIface) SetFilter(filter PacketFilter) error {
+	return m.SetFilterFunc(filter)
+}
+
+func (m *MockWGIface) GetFilter() PacketFilter {
+	return m.GetFilterFunc()
+}
+
+func (m *MockWGIface) GetDevice() *DeviceWrapper {
+	return m.GetDeviceFunc()
+}
+
+func (m *MockWGIface) GetStats(peerKey string) (WGStats, error) {
+	return m.GetStatsFunc(peerKey)
+}

iface/iwginterface.go

@@ -0,0 +1,32 @@
+//go:build !windows
+
+package iface
+
+import (
+	"net"
+	"time"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/netbirdio/netbird/iface/bind"
+)
+
+type IWGIface interface {
+	Create() error
+	CreateOnAndroid(routeRange []string, ip string, domains []string) error
+	IsUserspaceBind() bool
+	Name() string
+	Address() WGAddress
+	ToInterface() *net.Interface
+	Up() (*bind.UniversalUDPMuxDefault, error)
+	UpdateAddr(newAddr string) error
+	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
+	RemovePeer(peerKey string) error
+	AddAllowedIP(peerKey string, allowedIP string) error
+	RemoveAllowedIP(peerKey string, allowedIP string) error
+	Close() error
+	SetFilter(filter PacketFilter) error
+	GetFilter() PacketFilter
+	GetDevice() *DeviceWrapper
+	GetStats(peerKey string) (WGStats, error)
+}

iface/iwginterface_windows.go

@@ -0,0 +1,31 @@
+package iface
+
+import (
+	"net"
+	"time"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/netbirdio/netbird/iface/bind"
+)
+
+type IWGIface interface {
+	Create() error
+	CreateOnAndroid(routeRange []string, ip string, domains []string) error
+	IsUserspaceBind() bool
+	Name() string
+	Address() WGAddress
+	ToInterface() *net.Interface
+	Up() (*bind.UniversalUDPMuxDefault, error)
+	UpdateAddr(newAddr string) error
+	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
+	RemovePeer(peerKey string) error
+	AddAllowedIP(peerKey string, allowedIP string) error
+	RemoveAllowedIP(peerKey string, allowedIP string) error
+	Close() error
+	SetFilter(filter PacketFilter) error
+	GetFilter() PacketFilter
+	GetDevice() *DeviceWrapper
+	GetStats(peerKey string) (WGStats, error)
+	GetInterfaceGUIDString() (string, error)
+}

management/client/client_test.go

@@ -82,7 +82,10 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+	rc := &mgmt.RelayConfig{
+		Address: "localhost:0",
+	}
+	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc)
 	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)

management/cmd/management.go

@@ -195,7 +195,7 @@ var (
 				return fmt.Errorf("failed to build default manager: %v", err)
 			}
 
-			turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+			turnRelayTokenManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.RelayConfig)
 
 			trustedPeers := config.ReverseProxy.TrustedPeers
 			defaultTrustedPeers := []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0")}
@@ -271,7 +271,7 @@ var (
 			ephemeralManager.LoadInitialPeers(ctx)
 
 			gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
-			srv, err := server.NewServer(ctx, config, accountManager, peersUpdateManager, turnManager, appMetrics, ephemeralManager)
+			srv, err := server.NewServer(ctx, config, accountManager, peersUpdateManager, turnRelayTokenManager, appMetrics, ephemeralManager)
 			if err != nil {
 				return fmt.Errorf("failed creating gRPC API handler: %v", err)
 			}
@@ -538,6 +538,10 @@ func loadMgmtConfig(ctx context.Context, mgmtConfigPath string) (*server.Config,
 		}
 	}
 
+	if loadedConfig.RelayConfig != nil {
+		log.Infof("Relay address: %v", loadedConfig.RelayConfig.Address)
+	}
+
 	return loadedConfig, err
 }
 

management/cmd/management_test.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"testing"
+)
+
+const (
+	exampleConfig = `{	
+		"RelayConfig": {
+		   "Address": "rels://relay.stage.npeer.io"
+		},
+		"HttpConfig": {
+			"AuthAudience": "https://stageapp/",
+			"AuthIssuer": "https://something.eu.auth0.com/",
+			"OIDCConfigEndpoint": "https://something.eu.auth0.com/.well-known/openid-configuration"
+		}
+	}`
+)
+
+func Test_loadMgmtConfig(t *testing.T) {
+	tmpFile, err := createConfig()
+	if err != nil {
+		t.Fatalf("failed to create config: %s", err)
+	}
+
+	cfg, err := loadMgmtConfig(context.Background(), tmpFile)
+	if err != nil {
+		t.Fatalf("failed to load management config: %s", err)
+	}
+	if cfg.RelayConfig == nil {
+		t.Fatalf("config is nil")
+	}
+	if cfg.RelayConfig.Address == "" {
+		t.Fatalf("relay address is empty")
+	}
+}
+
+func createConfig() (string, error) {
+	tmpfile, err := os.CreateTemp("", "config.json")
+	if err != nil {
+		return "", err
+	}
+	_, err = tmpfile.Write([]byte(exampleConfig))
+	if err != nil {
+		return "", err
+	}
+
+	if err := tmpfile.Close(); err != nil {
+		return "", err
+	}
+	return tmpfile.Name(), nil
+}

management/proto/management.proto

@@ -177,6 +177,8 @@ message WiretrusteeConfig {
 
   // a Signal server config
   HostConfig signal = 3;
+
+  RelayConfig relay = 4;
 }
 
 // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management)
@@ -193,6 +195,13 @@ message HostConfig {
     DTLS = 4;
   }
 }
+
+message RelayConfig {
+  repeated string urls = 1;
+  string tokenPayload = 2;
+  string tokenSignature = 3;
+}
+
 // ProtectedHostConfig is similar to HostConfig but has additional user and password
 // Mostly used for TURN servers
 message ProtectedHostConfig {

management/server/config.go

@@ -32,9 +32,10 @@ const (
 
 // Config of the Management service
 type Config struct {
-	Stuns      []*Host
-	TURNConfig *TURNConfig
-	Signal     *Host
+	Stuns       []*Host
+	TURNConfig  *TURNConfig
+	RelayConfig *RelayConfig
+	Signal      *Host
 
 	Datadir                string
 	DataStoreEncryptionKey string
@@ -75,6 +76,10 @@ type TURNConfig struct {
 	Turns                []*Host
 }
 
+type RelayConfig struct {
+	Address string
+}
+
 // HttpServerConfig is a config of the HTTP Management service server
 type HttpServerConfig struct {
 	LetsEncryptDomain string

management/server/grpcserver.go

@@ -16,13 +16,12 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	nbContext "github.com/netbirdio/netbird/management/server/context"
-	"github.com/netbirdio/netbird/management/server/posture"
-
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/management/proto"
+	nbContext "github.com/netbirdio/netbird/management/server/context"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
 	nbpeer "github.com/netbirdio/netbird/management/server/peer"
+	"github.com/netbirdio/netbird/management/server/posture"
 	internalStatus "github.com/netbirdio/netbird/management/server/status"
 	"github.com/netbirdio/netbird/management/server/telemetry"
 )
@@ -32,17 +31,17 @@ type GRPCServer struct {
 	accountManager AccountManager
 	wgKey          wgtypes.Key
 	proto.UnimplementedManagementServiceServer
-	peersUpdateManager     *PeersUpdateManager
-	config                 *Config
-	turnCredentialsManager TURNCredentialsManager
-	jwtValidator           *jwtclaims.JWTValidator
-	jwtClaimsExtractor     *jwtclaims.ClaimsExtractor
-	appMetrics             telemetry.AppMetrics
-	ephemeralManager       *EphemeralManager
+	peersUpdateManager    *PeersUpdateManager
+	config                *Config
+	turnRelayTokenManager TURNRelayTokenManager
+	jwtValidator          *jwtclaims.JWTValidator
+	jwtClaimsExtractor    *jwtclaims.ClaimsExtractor
+	appMetrics            telemetry.AppMetrics
+	ephemeralManager      *EphemeralManager
 }
 
 // NewServer creates a new Management server
-func NewServer(ctx context.Context, config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnCredentialsManager TURNCredentialsManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) {
+func NewServer(ctx context.Context, config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnRelayTokenManager TURNRelayTokenManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) {
 	key, err := wgtypes.GeneratePrivateKey()
 	if err != nil {
 		return nil, err
@@ -88,14 +87,14 @@ func NewServer(ctx context.Context, config *Config, accountManager AccountManage
 	return &GRPCServer{
 		wgKey: key,
 		// peerKey -> event channel
-		peersUpdateManager:     peersUpdateManager,
-		accountManager:         accountManager,
-		config:                 config,
-		turnCredentialsManager: turnCredentialsManager,
-		jwtValidator:           jwtValidator,
-		jwtClaimsExtractor:     jwtClaimsExtractor,
-		appMetrics:             appMetrics,
-		ephemeralManager:       ephemeralManager,
+		peersUpdateManager:    peersUpdateManager,
+		accountManager:        accountManager,
+		config:                config,
+		turnRelayTokenManager: turnRelayTokenManager,
+		jwtValidator:          jwtValidator,
+		jwtClaimsExtractor:    jwtClaimsExtractor,
+		appMetrics:            appMetrics,
+		ephemeralManager:      ephemeralManager,
 	}, nil
 }
 
@@ -172,7 +171,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
 	s.ephemeralManager.OnPeerConnected(ctx, peer)
 
 	if s.config.TURNConfig.TimeBasedCredentials {
-		s.turnCredentialsManager.SetupRefresh(ctx, peer.ID)
+		s.turnRelayTokenManager.SetupRefresh(ctx, peer.ID)
 	}
 
 	if s.appMetrics != nil {
@@ -235,7 +234,7 @@ func (s *GRPCServer) sendUpdate(ctx context.Context, accountID string, peerKey w
 
 func (s *GRPCServer) cancelPeerRoutines(ctx context.Context, accountID string, peer *nbpeer.Peer) {
 	s.peersUpdateManager.CloseChannel(ctx, peer.ID)
-	s.turnCredentialsManager.CancelRefresh(peer.ID)
+	s.turnRelayTokenManager.CancelRefresh(peer.ID)
 	_ = s.accountManager.OnPeerDisconnected(ctx, accountID, peer.Key)
 	s.ephemeralManager.OnPeerDisconnected(ctx, peer)
 }
@@ -421,9 +420,14 @@ func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*p
 		s.ephemeralManager.OnPeerDisconnected(ctx, peer)
 	}
 
+	trt, err := s.turnRelayTokenManager.Generate()
+	if err != nil {
+		log.Errorf("failed generating TURN and Relay token: %v", err)
+	}
+
 	// if peer has reached this point then it has logged in
 	loginResp := &proto.LoginResponse{
-		WiretrusteeConfig: toWiretrusteeConfig(s.config, nil),
+		WiretrusteeConfig: toWiretrusteeConfig(s.config, nil, trt),
 		PeerConfig:        toPeerConfig(peer, netMap.Network, s.accountManager.GetDNSDomain()),
 		Checks:            toProtocolChecks(ctx, postureChecks),
 	}
@@ -481,7 +485,7 @@ func ToResponseProto(configProto Protocol) proto.HostConfig_Protocol {
 	}
 }
 
-func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *proto.WiretrusteeConfig {
+func toWiretrusteeConfig(config *Config, turnCredentials *TURNRelayToken, relayToken *TURNRelayToken) *proto.WiretrusteeConfig {
 	if config == nil {
 		return nil
 	}
@@ -497,8 +501,8 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot
 		var username string
 		var password string
 		if turnCredentials != nil {
-			username = turnCredentials.Username
-			password = turnCredentials.Password
+			username = turnCredentials.Payload
+			password = turnCredentials.Signature
 		} else {
 			username = turn.Username
 			password = turn.Password
@@ -513,6 +517,18 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot
 		})
 	}
 
+	var relayCfg *proto.RelayConfig
+	if config.RelayConfig != nil && config.RelayConfig.Address != "" {
+		relayCfg = &proto.RelayConfig{
+			Urls: []string{config.RelayConfig.Address},
+		}
+
+		if relayToken != nil {
+			relayCfg.TokenPayload = relayToken.Payload
+			relayCfg.TokenSignature = relayToken.Signature
+		}
+	}
+
 	return &proto.WiretrusteeConfig{
 		Stuns: stuns,
 		Turns: turns,
@@ -520,6 +536,7 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot
 			Uri:      config.Signal.URI,
 			Protocol: ToResponseProto(config.Signal.Proto),
 		},
+		Relay: relayCfg,
 	}
 }
 
@@ -533,9 +550,9 @@ func toPeerConfig(peer *nbpeer.Peer, network *Network, dnsName string) *proto.Pe
 	}
 }
 
-func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *TURNCredentials, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse {
+func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *TURNRelayToken, relayCredentials *TURNRelayToken, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse {
 	response := &proto.SyncResponse{
-		WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials),
+		WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials, relayCredentials),
 		PeerConfig:        toPeerConfig(peer, networkMap.Network, dnsName),
 		NetworkMap: &proto.NetworkMap{
 			Serial:    networkMap.Network.CurrentSerial(),
@@ -583,14 +600,15 @@ func (s *GRPCServer) IsHealthy(ctx context.Context, req *proto.Empty) (*proto.Em
 // sendInitialSync sends initial proto.SyncResponse to the peer requesting synchronization
 func (s *GRPCServer) sendInitialSync(ctx context.Context, peerKey wgtypes.Key, peer *nbpeer.Peer, networkMap *NetworkMap, postureChecks []*posture.Checks, srv proto.ManagementService_SyncServer) error {
 	// make secret time based TURN credentials optional
-	var turnCredentials *TURNCredentials
-	if s.config.TURNConfig.TimeBasedCredentials {
-		creds := s.turnCredentialsManager.GenerateCredentials()
-		turnCredentials = &creds
-	} else {
-		turnCredentials = nil
+	var turnCredentials *TURNRelayToken
+	trt, err := s.turnRelayTokenManager.Generate()
+	if err != nil {
+		log.Errorf("failed generating TURN and Relay token: %v", err)
 	}
-	plainResp := toSyncResponse(ctx, s.config, peer, turnCredentials, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil)
+	if s.config.TURNConfig.TimeBasedCredentials {
+		turnCredentials = trt
+	}
+	plainResp := toSyncResponse(ctx, s.config, peer, turnCredentials, trt, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil)
 
 	encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, plainResp)
 	if err != nil {

management/server/management_proto_test.go

@@ -429,7 +429,11 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, *DefaultAccoun
 	if err != nil {
 		return nil, nil, "", err
 	}
-	turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
+	rc := &RelayConfig{
+		Address: "localhost:0",
+	}
+	turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc)
 
 	ephemeralMgr := NewEphemeralManager(store, accountManager)
 	mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, ephemeralMgr)

management/server/management_test.go

@@ -552,7 +552,11 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) {
 	if err != nil {
 		log.Fatalf("failed creating a manager: %v", err)
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
+	rc := &server.RelayConfig{
+		Address: "localhost:0",
+	}
+	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc)
 	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	mgmtProto.RegisterManagementServiceServer(s, mgmtServer)

management/server/peer.go

@@ -964,7 +964,7 @@ func (am *DefaultAccountManager) updateAccountPeers(ctx context.Context, account
 
 			postureChecks := am.getPeerPostureChecks(account, p)
 			remotePeerNetworkMap := account.GetPeerNetworkMap(ctx, p.ID, customZone, approvedPeersMap, am.metrics.AccountManagerMetrics())
-			update := toSyncResponse(ctx, nil, p, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache)
+			update := toSyncResponse(ctx, nil, p, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache)
 			am.peersUpdateManager.SendUpdate(ctx, p.ID, &UpdateMessage{Update: update})
 		}(peer)
 	}

management/server/peer_test.go

@@ -848,9 +848,9 @@ func TestToSyncResponse(t *testing.T) {
 		DNSLabel:   "peer1",
 		SSHKey:     "peer1-ssh-key",
 	}
-	turnCredentials := &TURNCredentials{
-		Username: "turn-user",
-		Password: "turn-pass",
+	turnRelayToken := &TURNRelayToken{
+		Payload:   "turn-user",
+		Signature: "turn-pass",
 	}
 	networkMap := &NetworkMap{
 		Network:      &Network{Net: *ipnet, Serial: 1000},
@@ -916,7 +916,7 @@ func TestToSyncResponse(t *testing.T) {
 	}
 	dnsCache := &DNSConfigCache{}
 
-	response := toSyncResponse(context.Background(), config, peer, turnCredentials, networkMap, dnsName, checks, dnsCache)
+	response := toSyncResponse(context.Background(), config, peer, turnRelayToken, turnRelayToken, networkMap, dnsName, checks, dnsCache)
 
 	assert.NotNil(t, response)
 	// assert peer config

management/server/token_mgr.go

@@ -0,0 +1,132 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/management/proto"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+)
+
+// TURNRelayTokenManager used to manage TURN credentials
+type TURNRelayTokenManager interface {
+	Generate() (*TURNRelayToken, error)
+	SetupRefresh(ctx context.Context, peerKey string)
+	CancelRefresh(peerKey string)
+}
+
+// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server
+type TimeBasedAuthSecretsManager struct {
+	mux           sync.Mutex
+	turnCfg       *TURNConfig
+	relayAddr     string
+	hmacToken     *auth.TimedHMAC
+	updateManager *PeersUpdateManager
+	cancelMap     map[string]chan struct{}
+}
+
+type TURNRelayToken auth.Token
+
+func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayConfig *RelayConfig) *TimeBasedAuthSecretsManager {
+
+	var relayAddr string
+	if relayConfig != nil {
+		relayAddr = relayConfig.Address
+	}
+	return &TimeBasedAuthSecretsManager{
+		mux:           sync.Mutex{},
+		updateManager: updateManager,
+		turnCfg:       turnCfg,
+		relayAddr:     relayAddr,
+		hmacToken:     auth.NewTimedHMAC(turnCfg.Secret, turnCfg.CredentialsTTL.Duration),
+		cancelMap:     make(map[string]chan struct{}),
+	}
+}
+
+// Generate generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret
+func (m *TimeBasedAuthSecretsManager) Generate() (*TURNRelayToken, error) {
+	token, err := m.hmacToken.GenerateToken()
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate token: %s", err)
+	}
+
+	return (*TURNRelayToken)(token), nil
+}
+
+func (m *TimeBasedAuthSecretsManager) cancel(peerID string) {
+	if channel, ok := m.cancelMap[peerID]; ok {
+		close(channel)
+		delete(m.cancelMap, peerID)
+	}
+}
+
+// CancelRefresh cancels scheduled peer credentials refresh
+func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) {
+	m.mux.Lock()
+	defer m.mux.Unlock()
+	m.cancel(peerID)
+}
+
+// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer.
+// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone.
+func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) {
+	m.mux.Lock()
+	defer m.mux.Unlock()
+	m.cancel(peerID)
+	cancel := make(chan struct{}, 1)
+	m.cancelMap[peerID] = cancel
+	log.WithContext(ctx).Debugf("starting turn refresh for %s", peerID)
+
+	go func() {
+		// we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL)
+		ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-cancel:
+				log.WithContext(ctx).Debugf("stopping turn refresh for %s", peerID)
+				return
+			case <-ticker.C:
+				m.pushNewTokens(ctx, peerID)
+			}
+		}
+	}()
+}
+
+func (m *TimeBasedAuthSecretsManager) pushNewTokens(ctx context.Context, peerID string) {
+	token, err := m.hmacToken.GenerateToken()
+	if err != nil {
+		log.Errorf("failed to generate token for peer '%s': %s", peerID, err)
+		return
+	}
+
+	var turns []*proto.ProtectedHostConfig
+	for _, host := range m.turnCfg.Turns {
+		turns = append(turns, &proto.ProtectedHostConfig{
+			HostConfig: &proto.HostConfig{
+				Uri:      host.URI,
+				Protocol: ToResponseProto(host.Proto),
+			},
+			User:     token.Payload,
+			Password: token.Signature,
+		})
+	}
+
+	update := &proto.SyncResponse{
+		WiretrusteeConfig: &proto.WiretrusteeConfig{
+			Turns: turns,
+			Relay: &proto.RelayConfig{
+				Urls:           []string{m.relayAddr},
+				TokenPayload:   token.Payload,
+				TokenSignature: token.Signature,
+			},
+		},
+	}
+	log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID)
+	m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update})
+}

management/server/token_mgr_test.go

@@ -23,22 +23,25 @@ func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) {
 	secret := "some_secret"
 	peersManager := NewPeersUpdateManager(nil)
 
+	rc := &RelayConfig{
+		Address: "localhost:0",
+	}
 	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
 		CredentialsTTL: ttl,
 		Secret:         secret,
 		Turns:          []*Host{TurnTestHost},
-	})
+	}, rc)
 
-	credentials := tested.GenerateCredentials()
+	credentials, _ := tested.Generate()
 
-	if credentials.Username == "" {
+	if credentials.Payload == "" {
 		t.Errorf("expected generated TURN username not to be empty, got empty")
 	}
-	if credentials.Password == "" {
+	if credentials.Signature == "" {
 		t.Errorf("expected generated TURN password not to be empty, got empty")
 	}
 
-	validateMAC(t, credentials.Username, credentials.Password, []byte(secret))
+	validateMAC(t, credentials.Payload, credentials.Signature, []byte(secret))
 
 }
 
@@ -49,11 +52,14 @@ func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) {
 	peer := "some_peer"
 	updateChannel := peersManager.CreateChannel(context.Background(), peer)
 
+	rc := &RelayConfig{
+		Address: "localhost:0",
+	}
 	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
 		CredentialsTTL: ttl,
 		Secret:         secret,
 		Turns:          []*Host{TurnTestHost},
-	})
+	}, rc)
 
 	tested.SetupRefresh(context.Background(), peer)
 
@@ -97,11 +103,14 @@ func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) {
 	peersManager := NewPeersUpdateManager(nil)
 	peer := "some_peer"
 
+	rc := &RelayConfig{
+		Address: "localhost:0",
+	}
 	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
 		CredentialsTTL: ttl,
 		Secret:         secret,
 		Turns:          []*Host{TurnTestHost},
-	})
+	}, rc)
 
 	tested.SetupRefresh(context.Background(), peer)
 	if _, ok := tested.cancelMap[peer]; !ok {

management/server/turncredentials.go

@@ -1,126 +0,0 @@
-package server
-
-import (
-	"context"
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/base64"
-	"fmt"
-	"sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/netbirdio/netbird/management/proto"
-)
-
-// TURNCredentialsManager used to manage TURN credentials
-type TURNCredentialsManager interface {
-	GenerateCredentials() TURNCredentials
-	SetupRefresh(ctx context.Context, peerKey string)
-	CancelRefresh(peerKey string)
-}
-
-// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server
-type TimeBasedAuthSecretsManager struct {
-	mux           sync.Mutex
-	config        *TURNConfig
-	updateManager *PeersUpdateManager
-	cancelMap     map[string]chan struct{}
-}
-
-type TURNCredentials struct {
-	Username string
-	Password string
-}
-
-func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, config *TURNConfig) *TimeBasedAuthSecretsManager {
-	return &TimeBasedAuthSecretsManager{
-		mux:           sync.Mutex{},
-		config:        config,
-		updateManager: updateManager,
-		cancelMap:     make(map[string]chan struct{}),
-	}
-}
-
-// GenerateCredentials generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret
-func (m *TimeBasedAuthSecretsManager) GenerateCredentials() TURNCredentials {
-	mac := hmac.New(sha1.New, []byte(m.config.Secret))
-
-	timeAuth := time.Now().Add(m.config.CredentialsTTL.Duration).Unix()
-
-	username := fmt.Sprint(timeAuth)
-
-	_, err := mac.Write([]byte(username))
-	if err != nil {
-		log.Errorln("Generating turn password failed with error: ", err)
-	}
-
-	bytePassword := mac.Sum(nil)
-	password := base64.StdEncoding.EncodeToString(bytePassword)
-
-	return TURNCredentials{
-		Username: username,
-		Password: password,
-	}
-
-}
-
-func (m *TimeBasedAuthSecretsManager) cancel(peerID string) {
-	if channel, ok := m.cancelMap[peerID]; ok {
-		close(channel)
-		delete(m.cancelMap, peerID)
-	}
-}
-
-// CancelRefresh cancels scheduled peer credentials refresh
-func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) {
-	m.mux.Lock()
-	defer m.mux.Unlock()
-	m.cancel(peerID)
-}
-
-// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer.
-// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone.
-func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) {
-	m.mux.Lock()
-	defer m.mux.Unlock()
-	m.cancel(peerID)
-	cancel := make(chan struct{}, 1)
-	m.cancelMap[peerID] = cancel
-	log.WithContext(ctx).Debugf("starting turn refresh for %s", peerID)
-
-	go func() {
-		// we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL)
-		ticker := time.NewTicker(m.config.CredentialsTTL.Duration / 4 * 3)
-
-		for {
-			select {
-			case <-cancel:
-				log.WithContext(ctx).Debugf("stopping turn refresh for %s", peerID)
-				return
-			case <-ticker.C:
-				c := m.GenerateCredentials()
-				var turns []*proto.ProtectedHostConfig
-				for _, host := range m.config.Turns {
-					turns = append(turns, &proto.ProtectedHostConfig{
-						HostConfig: &proto.HostConfig{
-							Uri:      host.URI,
-							Protocol: ToResponseProto(host.Proto),
-						},
-						User:     c.Username,
-						Password: c.Password,
-					})
-				}
-
-				update := &proto.SyncResponse{
-					WiretrusteeConfig: &proto.WiretrusteeConfig{
-						Turns: turns,
-					},
-				}
-				log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID)
-				m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update})
-			}
-		}
-	}()
-}

relay/auth/allow_all.go

@@ -0,0 +1,10 @@
+package auth
+
+// AllowAllAuth is a Validator that allows all connections.
+// Used this for testing purposes only.
+type AllowAllAuth struct {
+}
+
+func (a *AllowAllAuth) Validate(any) error {
+	return nil
+}

relay/auth/doc.go

@@ -0,0 +1,26 @@
+/*
+Package auth manages the authentication process with the relay server.
+
+Key Components:
+
+Validator: The Validator interface defines the Validate method. Any type that provides this method can be used as a
+Validator.
+
+Methods:
+
+Validate(any): This method is defined in the Validator interface and is used to validate the authentication.
+
+Usage:
+
+To create a new AllowAllAuth validator, simply instantiate it:
+
+	validator := &auth.AllowAllAuth{}
+
+To validate the authentication, use the Validate method:
+
+	err := validator.Validate(any)
+
+This package provides a simple and effective way to manage authentication with the relay server, ensuring that the
+peers are authenticated properly.
+*/
+package auth

relay/auth/hmac/doc.go

@@ -0,0 +1,8 @@
+/*
+This package uses a similar HMAC method for authentication with the TURN server. The Management server provides the
+tokens for the peers. The peers manage these tokens in the token store. The token store is a simple thread safe store
+that keeps the tokens in memory. These tokens are used to authenticate the peers with the Relay server in the hello
+message.
+*/
+
+package hmac

relay/auth/hmac/store.go

@@ -0,0 +1,34 @@
+package hmac
+
+import (
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// TokenStore is a simple in-memory store for token
+// With this can update the token in thread safe way
+type TokenStore struct {
+	mu    sync.Mutex
+	token []byte
+}
+
+func (a *TokenStore) UpdateToken(token *Token) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if token == nil {
+		return
+	}
+
+	t, err := marshalToken(*token)
+	if err != nil {
+		log.Errorf("failed to marshal token: %s", err)
+	}
+	a.token = t
+}
+
+func (a *TokenStore) TokenBinary() []byte {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.token
+}

relay/auth/hmac/token.go

@@ -0,0 +1,105 @@
+package hmac
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/gob"
+	"fmt"
+	"strconv"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Token struct {
+	Payload   string
+	Signature string
+}
+
+func marshalToken(token Token) ([]byte, error) {
+	buffer := bytes.NewBuffer([]byte{})
+	encoder := gob.NewEncoder(buffer)
+	err := encoder.Encode(token)
+	if err != nil {
+		log.Errorf("failed to marshal token: %s", err)
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+func unmarshalToken(payload []byte) (Token, error) {
+	var creds Token
+	buffer := bytes.NewBuffer(payload)
+	decoder := gob.NewDecoder(buffer)
+	err := decoder.Decode(&creds)
+	return creds, err
+}
+
+// TimedHMAC generates token with TTL and using pre-shared secret known to TURN server
+type TimedHMAC struct {
+	secret     string
+	timeToLive time.Duration
+}
+
+// NewTimedHMAC creates a new TimedHMAC instance
+func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC {
+	return &TimedHMAC{
+		secret:     secret,
+		timeToLive: timeToLive,
+	}
+}
+
+// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC
+// hash of a timestamp with a preshared TURN secret
+func (m *TimedHMAC) GenerateToken() (*Token, error) {
+	timeAuth := time.Now().Add(m.timeToLive).Unix()
+	timeStamp := fmt.Sprint(timeAuth)
+
+	checksum, err := m.generate(timeStamp)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Token{
+		Payload:   timeStamp,
+		Signature: base64.StdEncoding.EncodeToString(checksum),
+	}, nil
+}
+
+// Validate checks if the token is valid
+func (m *TimedHMAC) Validate(token Token) error {
+	expectedMAC, err := m.generate(token.Payload)
+	if err != nil {
+		return err
+	}
+
+	expectedSignature := base64.StdEncoding.EncodeToString(expectedMAC)
+
+	if !hmac.Equal([]byte(expectedSignature), []byte(token.Signature)) {
+		return fmt.Errorf("signature mismatch")
+	}
+
+	timeAuthInt, err := strconv.ParseInt(token.Payload, 10, 64)
+	if err != nil {
+		return fmt.Errorf("invalid payload: %s", err)
+	}
+
+	if time.Now().Unix() > timeAuthInt {
+		return fmt.Errorf("expired token")
+	}
+
+	return nil
+}
+
+func (m *TimedHMAC) generate(payload string) ([]byte, error) {
+	mac := hmac.New(sha1.New, []byte(m.secret))
+	_, err := mac.Write([]byte(payload))
+	if err != nil {
+		log.Errorf("failed to generate token: %s", err)
+		return nil, err
+	}
+
+	return mac.Sum(nil), nil
+}

relay/auth/hmac/token_test.go

@@ -0,0 +1,103 @@
+package hmac
+
+import (
+	"encoding/base64"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestGenerateCredentials(t *testing.T) {
+	secret := "secret"
+	timeToLive := 1 * time.Hour
+	v := NewTimedHMAC(secret, timeToLive)
+
+	creds, err := v.GenerateToken()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if creds.Payload == "" {
+		t.Fatalf("expected non-empty payload")
+	}
+
+	_, err = strconv.ParseInt(creds.Payload, 10, 64)
+	if err != nil {
+		t.Fatalf("expected payload to be a valid unix timestamp, got %v", err)
+	}
+
+	_, err = base64.StdEncoding.DecodeString(creds.Signature)
+	if err != nil {
+		t.Fatalf("expected signature to be base64 encoded, got %v", err)
+	}
+}
+
+func TestValidateCredentials(t *testing.T) {
+	secret := "supersecret"
+	timeToLive := 1 * time.Hour
+	manager := NewTimedHMAC(secret, timeToLive)
+
+	// Test valid token
+	creds, err := manager.GenerateToken()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if err := manager.Validate(*creds); err != nil {
+		t.Fatalf("expected valid token: %s", err)
+	}
+}
+
+func TestInvalidSignature(t *testing.T) {
+	secret := "supersecret"
+	timeToLive := 1 * time.Hour
+	manager := NewTimedHMAC(secret, timeToLive)
+
+	creds, err := manager.GenerateToken()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	invalidCreds := &Token{
+		Payload:   creds.Payload,
+		Signature: "invalidsignature",
+	}
+
+	if err = manager.Validate(*invalidCreds); err == nil {
+		t.Fatalf("expected invalid token due to signature mismatch")
+	}
+}
+
+func TestExpired(t *testing.T) {
+	secret := "supersecret"
+	v := NewTimedHMAC(secret, -1*time.Hour)
+	expiredCreds, err := v.GenerateToken()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if err = v.Validate(*expiredCreds); err == nil {
+		t.Fatalf("expected invalid token due to expiration")
+	}
+}
+
+func TestInvalidPayload(t *testing.T) {
+	secret := "supersecret"
+	timeToLive := 1 * time.Hour
+	v := NewTimedHMAC(secret, timeToLive)
+
+	creds, err := v.GenerateToken()
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	// Test invalid payload
+	invalidPayloadCreds := &Token{
+		Payload:   "invalidtimestamp",
+		Signature: creds.Signature,
+	}
+
+	if err = v.Validate(*invalidPayloadCreds); err == nil {
+		t.Fatalf("expected invalid token due to invalid payload")
+	}
+}

relay/auth/hmac/validator.go

@@ -0,0 +1,27 @@
+package hmac
+
+import (
+	log "github.com/sirupsen/logrus"
+	"time"
+)
+
+type TimedHMACValidator struct {
+	*TimedHMAC
+}
+
+func NewTimedHMACValidator(secret string, duration time.Duration) *TimedHMACValidator {
+	ta := NewTimedHMAC(secret, duration)
+	return &TimedHMACValidator{
+		ta,
+	}
+}
+
+func (a *TimedHMACValidator) Validate(credentials any) error {
+	b := credentials.([]byte)
+	c, err := unmarshalToken(b)
+	if err != nil {
+		log.Errorf("failed to unmarshal token: %s", err)
+		return err
+	}
+	return a.TimedHMAC.Validate(c)
+}

relay/auth/validator.go

@@ -0,0 +1,6 @@
+package auth
+
+// Validator is an interface that defines the Validate method.
+type Validator interface {
+	Validate(any) error
+}

relay/client/addr.go

@@ -0,0 +1,13 @@
+package client
+
+type RelayAddr struct {
+	addr string
+}
+
+func (a RelayAddr) Network() string {
+	return "relay"
+}
+
+func (a RelayAddr) String() string {
+	return a.addr
+}

relay/client/client.go

@@ -0,0 +1,523 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/relay/client/dialer/ws"
+	"github.com/netbirdio/netbird/relay/healthcheck"
+	"github.com/netbirdio/netbird/relay/messages"
+)
+
+const (
+	bufferSize            = 8820
+	serverResponseTimeout = 8 * time.Second
+)
+
+var (
+	ErrConnAlreadyExists = fmt.Errorf("connection already exists")
+)
+
+type internalStopFlag struct {
+	sync.Mutex
+	stop bool
+}
+
+func newInternalStopFlag() *internalStopFlag {
+	return &internalStopFlag{}
+}
+
+func (isf *internalStopFlag) set() {
+	isf.Lock()
+	defer isf.Unlock()
+	isf.stop = true
+}
+
+func (isf *internalStopFlag) isSet() bool {
+	isf.Lock()
+	defer isf.Unlock()
+	return isf.stop
+}
+
+// Msg carry the payload from the server to the client. With this struct, the net.Conn can free the buffer.
+type Msg struct {
+	Payload []byte
+
+	bufPool *sync.Pool
+	bufPtr  *[]byte
+}
+
+func (m *Msg) Free() {
+	m.bufPool.Put(m.bufPtr)
+}
+
+type connContainer struct {
+	conn        *Conn
+	messages    chan Msg
+	msgChanLock sync.Mutex
+	closed      bool // flag to check if channel is closed
+}
+
+func newConnContainer(conn *Conn, messages chan Msg) *connContainer {
+	return &connContainer{
+		conn:     conn,
+		messages: messages,
+	}
+}
+
+func (cc *connContainer) writeMsg(msg Msg) {
+	cc.msgChanLock.Lock()
+	defer cc.msgChanLock.Unlock()
+	if cc.closed {
+		return
+	}
+	cc.messages <- msg
+}
+
+func (cc *connContainer) close() {
+	cc.msgChanLock.Lock()
+	defer cc.msgChanLock.Unlock()
+	if cc.closed {
+		return
+	}
+	close(cc.messages)
+	cc.closed = true
+}
+
+// Client is a client for the relay server. It is responsible for establishing a connection to the relay server and
+// managing connections to other peers. All exported functions are safe to call concurrently. After close the connection,
+// the client can be reused by calling Connect again. When the client is closed, all connections are closed too.
+// While the Connect is in progress, the OpenConn function will block until the connection is established with relay server.
+type Client struct {
+	log            *log.Entry
+	parentCtx      context.Context
+	connectionURL  string
+	authTokenStore *auth.TokenStore
+	hashedID       []byte
+
+	bufPool *sync.Pool
+
+	relayConn        net.Conn
+	conns            map[string]*connContainer
+	serviceIsRunning bool
+	mu               sync.Mutex // protect serviceIsRunning and conns
+	readLoopMutex    sync.Mutex
+	wgReadLoop       sync.WaitGroup
+	instanceURL      *RelayAddr
+	muInstanceURL    sync.Mutex
+
+	onDisconnectListener func()
+	listenerMutex        sync.Mutex
+}
+
+// NewClient creates a new client for the relay server. The client is not connected to the server until the Connect
+func NewClient(ctx context.Context, serverURL string, authTokenStore *auth.TokenStore, peerID string) *Client {
+	hashedID, hashedStringId := messages.HashID(peerID)
+	return &Client{
+		log:            log.WithField("client_id", hashedStringId),
+		parentCtx:      ctx,
+		connectionURL:  serverURL,
+		authTokenStore: authTokenStore,
+		hashedID:       hashedID,
+		bufPool: &sync.Pool{
+			New: func() any {
+				buf := make([]byte, bufferSize)
+				return &buf
+			},
+		},
+		conns: make(map[string]*connContainer),
+	}
+}
+
+// Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs.
+func (c *Client) Connect() error {
+	c.log.Infof("connecting to relay server: %s", c.connectionURL)
+	c.readLoopMutex.Lock()
+	defer c.readLoopMutex.Unlock()
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.serviceIsRunning {
+		return nil
+	}
+
+	err := c.connect()
+	if err != nil {
+		return err
+	}
+
+	c.serviceIsRunning = true
+
+	c.wgReadLoop.Add(1)
+	go c.readLoop(c.relayConn)
+
+	log.Infof("relay connection established with: %s", c.connectionURL)
+	return nil
+}
+
+// OpenConn create a new net.Conn for the destination peer ID. In case if the connection is in progress
+// to the relay server, the function will block until the connection is established or timed out. Otherwise,
+// it will return immediately.
+// todo: what should happen if call with the same peerID with multiple times?
+func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if !c.serviceIsRunning {
+		return nil, fmt.Errorf("relay connection is not established")
+	}
+
+	hashedID, hashedStringID := messages.HashID(dstPeerID)
+	_, ok := c.conns[hashedStringID]
+	if ok {
+		return nil, ErrConnAlreadyExists
+	}
+
+	log.Infof("open connection to peer: %s", hashedStringID)
+	msgChannel := make(chan Msg, 2)
+	conn := NewConn(c, hashedID, hashedStringID, msgChannel, c.instanceURL)
+
+	c.conns[hashedStringID] = newConnContainer(conn, msgChannel)
+	return conn, nil
+}
+
+// ServerInstanceURL returns the address of the relay server. It could change after the close and reopen the connection.
+func (c *Client) ServerInstanceURL() (string, error) {
+	c.muInstanceURL.Lock()
+	defer c.muInstanceURL.Unlock()
+	if c.instanceURL == nil {
+		return "", fmt.Errorf("relay connection is not established")
+	}
+	return c.instanceURL.String(), nil
+}
+
+// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed.
+func (c *Client) SetOnDisconnectListener(fn func()) {
+	c.listenerMutex.Lock()
+	defer c.listenerMutex.Unlock()
+	c.onDisconnectListener = fn
+}
+
+// HasConns returns true if there are connections.
+func (c *Client) HasConns() bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return len(c.conns) > 0
+}
+
+// Close closes the connection to the relay server and all connections to other peers.
+func (c *Client) Close() error {
+	return c.close(true)
+}
+
+func (c *Client) connect() error {
+	conn, err := ws.Dial(c.connectionURL)
+	if err != nil {
+		return err
+	}
+	c.relayConn = conn
+
+	err = c.handShake()
+	if err != nil {
+		cErr := conn.Close()
+		if cErr != nil {
+			log.Errorf("failed to close connection: %s", cErr)
+		}
+		c.relayConn = nil
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) handShake() error {
+	tb := c.authTokenStore.TokenBinary()
+
+	msg, err := messages.MarshalHelloMsg(c.hashedID, tb)
+	if err != nil {
+		log.Errorf("failed to marshal hello message: %s", err)
+		return err
+	}
+	_, err = c.relayConn.Write(msg)
+	if err != nil {
+		log.Errorf("failed to send hello message: %s", err)
+		return err
+	}
+	buf := make([]byte, messages.MaxHandshakeSize)
+	n, err := c.readWithTimeout(buf)
+	if err != nil {
+		log.Errorf("failed to read hello response: %s", err)
+		return err
+	}
+
+	msgType, err := messages.DetermineServerMsgType(buf[:n])
+	if err != nil {
+		log.Errorf("failed to determine message type: %s", err)
+		return err
+	}
+
+	if msgType != messages.MsgTypeHelloResponse {
+		log.Errorf("unexpected message type: %s", msgType)
+		return fmt.Errorf("unexpected message type")
+	}
+
+	ia, err := messages.UnmarshalHelloResponse(buf[:n])
+	if err != nil {
+		return err
+	}
+	c.muInstanceURL.Lock()
+	c.instanceURL = &RelayAddr{addr: ia}
+	c.muInstanceURL.Unlock()
+	return nil
+}
+
+func (c *Client) readLoop(relayConn net.Conn) {
+	internallyStoppedFlag := newInternalStopFlag()
+	hc := healthcheck.NewReceiver()
+	go c.listenForStopEvents(hc, relayConn, internallyStoppedFlag)
+
+	var (
+		errExit error
+		n       int
+	)
+	for {
+		bufPtr := c.bufPool.Get().(*[]byte)
+		buf := *bufPtr
+		n, errExit = relayConn.Read(buf)
+		if errExit != nil {
+			c.mu.Lock()
+			if c.serviceIsRunning && !internallyStoppedFlag.isSet() {
+				c.log.Debugf("failed to read message from relay server: %s", errExit)
+			}
+			c.mu.Unlock()
+			break
+		}
+
+		msgType, err := messages.DetermineServerMsgType(buf[:n])
+		if err != nil {
+			c.log.Errorf("failed to determine message type: %s", err)
+			continue
+		}
+
+		if !c.handleMsg(msgType, buf[:n], bufPtr, hc, internallyStoppedFlag) {
+			break
+		}
+	}
+
+	hc.Stop()
+
+	c.muInstanceURL.Lock()
+	c.instanceURL = nil
+	c.muInstanceURL.Unlock()
+
+	c.notifyDisconnected()
+	c.wgReadLoop.Done()
+	_ = c.close(false)
+}
+
+func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) {
+	switch msgType {
+	case messages.MsgTypeHealthCheck:
+		c.handleHealthCheck(hc, internallyStoppedFlag)
+		c.bufPool.Put(bufPtr)
+	case messages.MsgTypeTransport:
+		return c.handleTransportMsg(buf, bufPtr, internallyStoppedFlag)
+	case messages.MsgTypeClose:
+		log.Debugf("relay connection close by server")
+		c.bufPool.Put(bufPtr)
+		return false
+	}
+
+	return true
+}
+
+func (c *Client) handleHealthCheck(hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) {
+	msg := messages.MarshalHealthcheck()
+	_, wErr := c.relayConn.Write(msg)
+	if wErr != nil {
+		if c.serviceIsRunning && !internallyStoppedFlag.isSet() {
+			c.log.Errorf("failed to send heartbeat: %s", wErr)
+		}
+	}
+	hc.Heartbeat()
+}
+
+func (c *Client) handleTransportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool {
+	peerID, payload, err := messages.UnmarshalTransportMsg(buf)
+	if err != nil {
+		if c.serviceIsRunning && !internallyStoppedFlag.isSet() {
+			c.log.Errorf("failed to parse transport message: %v", err)
+		}
+
+		c.bufPool.Put(bufPtr)
+		return true
+	}
+	stringID := messages.HashIDToString(peerID)
+
+	c.mu.Lock()
+	if !c.serviceIsRunning {
+		c.mu.Unlock()
+		c.bufPool.Put(bufPtr)
+		return false
+	}
+	container, ok := c.conns[stringID]
+	c.mu.Unlock()
+	if !ok {
+		c.log.Errorf("peer not found: %s", stringID)
+		c.bufPool.Put(bufPtr)
+		return true
+	}
+	msg := Msg{
+		bufPool: c.bufPool,
+		bufPtr:  bufPtr,
+		Payload: payload,
+	}
+	container.writeMsg(msg)
+	return true
+}
+
+func (c *Client) writeTo(connReference *Conn, id string, dstID []byte, payload []byte) (int, error) {
+	c.mu.Lock()
+	conn, ok := c.conns[id]
+	c.mu.Unlock()
+	if !ok {
+		return 0, io.EOF
+	}
+
+	if conn.conn != connReference {
+		return 0, io.EOF
+	}
+
+	// todo: use buffer pool instead of create new transport msg.
+	msg, err := messages.MarshalTransportMsg(dstID, payload)
+	if err != nil {
+		log.Errorf("failed to marshal transport message: %s", err)
+		return 0, err
+	}
+
+	// the write always return with 0 length because the underling does not support the size feedback.
+	_, err = c.relayConn.Write(msg)
+	if err != nil {
+		log.Errorf("failed to write transport message: %s", err)
+	}
+	return len(payload), err
+}
+
+func (c *Client) listenForStopEvents(hc *healthcheck.Receiver, conn net.Conn, internalStopFlag *internalStopFlag) {
+	for {
+		select {
+		case _, ok := <-hc.OnTimeout:
+			if !ok {
+				return
+			}
+			c.log.Errorf("health check timeout")
+			internalStopFlag.set()
+			_ = conn.Close() // ignore the err because the readLoop will handle it
+			return
+		case <-c.parentCtx.Done():
+			err := c.close(true)
+			if err != nil {
+				log.Errorf("failed to teardown connection: %s", err)
+			}
+			return
+		}
+	}
+}
+
+func (c *Client) closeAllConns() {
+	for _, container := range c.conns {
+		container.close()
+	}
+	c.conns = make(map[string]*connContainer)
+}
+
+func (c *Client) closeConn(connReference *Conn, id string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	container, ok := c.conns[id]
+	if !ok {
+		return fmt.Errorf("connection already closed")
+	}
+
+	if container.conn != connReference {
+		return fmt.Errorf("conn reference mismatch")
+	}
+	container.close()
+	delete(c.conns, id)
+
+	return nil
+}
+
+func (c *Client) close(gracefullyExit bool) error {
+	c.readLoopMutex.Lock()
+	defer c.readLoopMutex.Unlock()
+
+	c.mu.Lock()
+	var err error
+	if !c.serviceIsRunning {
+		c.mu.Unlock()
+		return nil
+	}
+
+	c.serviceIsRunning = false
+	c.closeAllConns()
+	if gracefullyExit {
+		c.writeCloseMsg()
+		err = c.relayConn.Close()
+	}
+	c.mu.Unlock()
+
+	c.wgReadLoop.Wait()
+	c.log.Infof("relay connection closed with: %s", c.connectionURL)
+	return err
+}
+
+func (c *Client) notifyDisconnected() {
+	c.listenerMutex.Lock()
+	defer c.listenerMutex.Unlock()
+
+	if c.onDisconnectListener == nil {
+		return
+	}
+	go c.onDisconnectListener()
+}
+
+func (c *Client) writeCloseMsg() {
+	msg := messages.MarshalCloseMsg()
+	_, err := c.relayConn.Write(msg)
+	if err != nil {
+		c.log.Errorf("failed to send close message: %s", err)
+	}
+}
+
+func (c *Client) readWithTimeout(buf []byte) (int, error) {
+	ctx, cancel := context.WithTimeout(c.parentCtx, serverResponseTimeout)
+	defer cancel()
+
+	readDone := make(chan struct{})
+	var (
+		n   int
+		err error
+	)
+
+	go func() {
+		n, err = c.relayConn.Read(buf)
+		close(readDone)
+	}()
+
+	select {
+	case <-ctx.Done():
+		return 0, fmt.Errorf("read operation timed out")
+	case <-readDone:
+		return n, err
+	}
+}

relay/client/client_test.go

@@ -0,0 +1,631 @@
+package client
+
+import (
+	"context"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/relay/auth"
+	"github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/util"
+
+	"github.com/netbirdio/netbird/relay/server"
+)
+
+var (
+	av               = &auth.AllowAllAuth{}
+	hmacTokenStore   = &hmac.TokenStore{}
+	serverListenAddr = "127.0.0.1:1234"
+	serverURL        = "rel://127.0.0.1:1234"
+)
+
+func TestMain(m *testing.M) {
+	_ = util.InitLog("error", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestClient(t *testing.T) {
+	ctx := context.Background()
+
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		listenCfg := server.ListenerConfig{Address: serverListenAddr}
+		err := srv.Listen(listenCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for server to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+	t.Log("alice connecting to server")
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer clientAlice.Close()
+
+	t.Log("placeholder connecting to server")
+	clientPlaceHolder := NewClient(ctx, serverURL, hmacTokenStore, "clientPlaceHolder")
+	err = clientPlaceHolder.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer clientPlaceHolder.Close()
+
+	t.Log("Bob connecting to server")
+	clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob")
+	err = clientBob.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer clientBob.Close()
+
+	t.Log("Alice open connection to Bob")
+	connAliceToBob, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	t.Log("Bob open connection to Alice")
+	connBobToAlice, err := clientBob.OpenConn("alice")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	payload := "hello bob, I am alice"
+	_, err = connAliceToBob.Write([]byte(payload))
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+	log.Debugf("alice sent message to bob")
+
+	buf := make([]byte, 65535)
+	n, err := connBobToAlice.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+	log.Debugf("on new message from alice to bob")
+
+	if payload != string(buf[:n]) {
+		t.Fatalf("expected %s, got %s", payload, string(buf[:n]))
+	}
+}
+
+func TestRegistration(t *testing.T) {
+	ctx := context.Background()
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	// wait for server to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		_ = srv.Shutdown(ctx)
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close conn: %s", err)
+	}
+	err = srv.Shutdown(ctx)
+	if err != nil {
+		t.Errorf("failed to close server: %s", err)
+	}
+}
+
+func TestRegistrationTimeout(t *testing.T) {
+	ctx := context.Background()
+	fakeUDPListener, err := net.ListenUDP("udp", &net.UDPAddr{
+		Port: 1234,
+		IP:   net.ParseIP("0.0.0.0"),
+	})
+	if err != nil {
+		t.Fatalf("failed to bind UDP server: %s", err)
+	}
+	defer func(fakeUDPListener *net.UDPConn) {
+		_ = fakeUDPListener.Close()
+	}(fakeUDPListener)
+
+	fakeTCPListener, err := net.ListenTCP("tcp", &net.TCPAddr{
+		Port: 1234,
+		IP:   net.ParseIP("0.0.0.0"),
+	})
+	if err != nil {
+		t.Fatalf("failed to bind TCP server: %s", err)
+	}
+	defer func(fakeTCPListener *net.TCPListener) {
+		_ = fakeTCPListener.Close()
+	}(fakeTCPListener)
+
+	clientAlice := NewClient(ctx, "127.0.0.1:1234", hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err == nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+	log.Debugf("%s", err)
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close conn: %s", err)
+	}
+}
+
+func TestEcho(t *testing.T) {
+	ctx := context.Background()
+	idAlice := "alice"
+	idBob := "bob"
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice)
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer func() {
+		err := clientAlice.Close()
+		if err != nil {
+			t.Errorf("failed to close Alice client: %s", err)
+		}
+	}()
+
+	clientBob := NewClient(ctx, serverURL, hmacTokenStore, idBob)
+	err = clientBob.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer func() {
+		err := clientBob.Close()
+		if err != nil {
+			t.Errorf("failed to close Bob client: %s", err)
+		}
+	}()
+
+	connAliceToBob, err := clientAlice.OpenConn(idBob)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	connBobToAlice, err := clientBob.OpenConn(idAlice)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	payload := "hello bob, I am alice"
+	_, err = connAliceToBob.Write([]byte(payload))
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	buf := make([]byte, 65535)
+	n, err := connBobToAlice.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	_, err = connBobToAlice.Write(buf[:n])
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	n, err = connAliceToBob.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	if payload != string(buf[:n]) {
+		t.Fatalf("expected %s, got %s", payload, string(buf[:n]))
+	}
+}
+
+func TestBindToUnavailabePeer(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		log.Infof("closing server")
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+	_, err = clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	log.Infof("closing client")
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+}
+
+func TestBindReconnect(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		log.Infof("closing server")
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	_, err = clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob")
+	err = clientBob.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	chBob, err := clientBob.OpenConn("alice")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	log.Infof("closing client Alice")
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+
+	clientAlice = NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	chAlice, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	testString := "hello alice, I am bob"
+	_, err = chBob.Write([]byte(testString))
+	if err != nil {
+		t.Errorf("failed to write to channel: %s", err)
+	}
+
+	buf := make([]byte, 65535)
+	n, err := chAlice.Read(buf)
+	if err != nil {
+		t.Errorf("failed to read from channel: %s", err)
+	}
+
+	if testString != string(buf[:n]) {
+		t.Errorf("expected %s, got %s", testString, string(buf[:n]))
+	}
+
+	log.Infof("closing client")
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+}
+
+func TestCloseConn(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		log.Infof("closing server")
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	conn, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	log.Infof("closing connection")
+	err = conn.Close()
+	if err != nil {
+		t.Errorf("failed to close connection: %s", err)
+	}
+
+	_, err = conn.Read(make([]byte, 1))
+	if err == nil {
+		t.Errorf("unexpected reading from closed connection")
+	}
+
+	_, err = conn.Write([]byte("hello"))
+	if err == nil {
+		t.Errorf("unexpected writing from closed connection")
+	}
+}
+
+func TestCloseRelayConn(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			log.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+
+	conn, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	_ = clientAlice.relayConn.Close()
+
+	_, err = conn.Read(make([]byte, 1))
+	if err == nil {
+		t.Errorf("unexpected reading from closed connection")
+	}
+
+	_, err = clientAlice.OpenConn("bob")
+	if err == nil {
+		t.Errorf("unexpected opening connection to closed server")
+	}
+}
+
+func TestCloseByServer(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv1, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+
+	go func() {
+		err := srv1.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice)
+	err = relayClient.Connect()
+	if err != nil {
+		log.Fatalf("failed to connect to server: %s", err)
+	}
+
+	disconnected := make(chan struct{})
+	relayClient.SetOnDisconnectListener(func() {
+		log.Infof("client disconnected")
+		close(disconnected)
+	})
+
+	err = srv1.Shutdown(ctx)
+	if err != nil {
+		t.Fatalf("failed to close server: %s", err)
+	}
+
+	select {
+	case <-disconnected:
+	case <-time.After(3 * time.Second):
+		log.Fatalf("timeout waiting for client to disconnect")
+	}
+
+	_, err = relayClient.OpenConn("bob")
+	if err == nil {
+		t.Errorf("unexpected opening connection to closed server")
+	}
+}
+
+func TestCloseByClient(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice)
+	err = relayClient.Connect()
+	if err != nil {
+		log.Fatalf("failed to connect to server: %s", err)
+	}
+
+	err = relayClient.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+
+	_, err = relayClient.OpenConn("bob")
+	if err == nil {
+		t.Errorf("unexpected opening connection to closed server")
+	}
+
+	err = srv.Shutdown(ctx)
+	if err != nil {
+		t.Fatalf("failed to close server: %s", err)
+	}
+}
+
+func waitForServerToStart(errChan chan error) error {
+	select {
+	case err := <-errChan:
+		if err != nil {
+			return err
+		}
+	case <-time.After(300 * time.Millisecond):
+		return nil
+	}
+	return nil
+}

relay/client/conn.go

@@ -0,0 +1,76 @@
+package client
+
+import (
+	"io"
+	"net"
+	"time"
+)
+
+// Conn represent a connection to a relayed remote peer.
+type Conn struct {
+	client      *Client
+	dstID       []byte
+	dstStringID string
+	messageChan chan Msg
+	instanceURL *RelayAddr
+}
+
+// NewConn creates a new connection to a relayed remote peer.
+// client: the client instance, it used to send messages to the destination peer
+// dstID: the destination peer ID
+// dstStringID: the destination peer ID in string format
+// messageChan: the channel where the messages will be received
+// instanceURL: the relay instance URL, it used to get the proper server instance address for the remote peer
+func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg, instanceURL *RelayAddr) *Conn {
+	c := &Conn{
+		client:      client,
+		dstID:       dstID,
+		dstStringID: dstStringID,
+		messageChan: messageChan,
+		instanceURL: instanceURL,
+	}
+
+	return c
+}
+
+func (c *Conn) Write(p []byte) (n int, err error) {
+	return c.client.writeTo(c, c.dstStringID, c.dstID, p)
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	msg, ok := <-c.messageChan
+	if !ok {
+		return 0, io.EOF
+	}
+
+	n = copy(b, msg.Payload)
+	msg.Free()
+	return n, nil
+}
+
+func (c *Conn) Close() error {
+	return c.client.closeConn(c, c.dstStringID)
+}
+
+func (c *Conn) LocalAddr() net.Addr {
+	return c.client.relayConn.LocalAddr()
+}
+
+func (c *Conn) RemoteAddr() net.Addr {
+	return c.instanceURL
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	//TODO implement me
+	panic("SetDeadline is not implemented")
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	//TODO implement me
+	panic("SetReadDeadline is not implemented")
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	//TODO implement me
+	panic("SetReadDeadline is not implemented")
+}

relay/client/dialer/ws/addr.go

@@ -0,0 +1,13 @@
+package ws
+
+type WebsocketAddr struct {
+	addr string
+}
+
+func (a WebsocketAddr) Network() string {
+	return "websocket"
+}
+
+func (a WebsocketAddr) String() string {
+	return a.addr
+}

relay/client/dialer/ws/conn.go

@@ -0,0 +1,66 @@
+package ws
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"nhooyr.io/websocket"
+)
+
+type Conn struct {
+	ctx context.Context
+	*websocket.Conn
+	remoteAddr WebsocketAddr
+}
+
+func NewConn(wsConn *websocket.Conn, serverAddress string) net.Conn {
+	return &Conn{
+		ctx:        context.Background(),
+		Conn:       wsConn,
+		remoteAddr: WebsocketAddr{serverAddress},
+	}
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	t, ioReader, err := c.Conn.Reader(c.ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	if t != websocket.MessageBinary {
+		return 0, fmt.Errorf("unexpected message type")
+	}
+
+	return ioReader.Read(b)
+}
+
+func (c *Conn) Write(b []byte) (n int, err error) {
+	err = c.Conn.Write(c.ctx, websocket.MessageBinary, b)
+	return 0, err
+}
+
+func (c *Conn) RemoteAddr() net.Addr {
+	return c.remoteAddr
+}
+
+func (c *Conn) LocalAddr() net.Addr {
+	return WebsocketAddr{addr: "unknown"}
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	return fmt.Errorf("SetReadDeadline is not implemented")
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	return fmt.Errorf("SetWriteDeadline is not implemented")
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	return fmt.Errorf("SetDeadline is not implemented")
+}
+
+func (c *Conn) Close() error {
+	return c.Conn.CloseNow()
+}

relay/client/dialer/ws/ws.go

@@ -0,0 +1,59 @@
+package ws
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"nhooyr.io/websocket"
+
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func Dial(address string) (net.Conn, error) {
+	wsURL, err := prepareURL(address)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := &websocket.DialOptions{
+		HTTPClient: httpClientNbDialer(),
+	}
+
+	wsConn, resp, err := websocket.Dial(context.Background(), wsURL, opts)
+	if err != nil {
+		log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err)
+		return nil, err
+	}
+	if resp.Body != nil {
+		_ = resp.Body.Close()
+	}
+
+	conn := NewConn(wsConn, address)
+	return conn, nil
+}
+
+func prepareURL(address string) (string, error) {
+	if !strings.HasPrefix(address, "rel") {
+		return "", fmt.Errorf("unsupported scheme: %s", address)
+	}
+
+	return strings.Replace(address, "rel", "ws", 1), nil
+}
+
+func httpClientNbDialer() *http.Client {
+	customDialer := nbnet.NewDialer()
+
+	customTransport := &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return customDialer.DialContext(ctx, network, addr)
+		},
+	}
+
+	return &http.Client{
+		Transport: customTransport,
+	}
+}

relay/client/doc.go

@@ -0,0 +1,12 @@
+/*
+Package client contains the implementation of the Relay client.
+
+The Relay client is responsible for establishing a connection with the Relay server and sending and receiving messages,
+Keep persistent connection with the Relay server and handle the connection issues.
+It uses the WebSocket protocol for communication and optionally supports TLS (Transport Layer Security).
+
+If a peer wants to communicate with a peer on a different relay server, the manager will establish a new connection to
+the relay server. The connection with these relay servers will be closed if there is no active connection. The peers
+negotiate the common relay instance via signaling service.
+*/
+package client

relay/client/guard.go

@@ -0,0 +1,48 @@
+package client
+
+import (
+	"context"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	reconnectingTimeout = 5 * time.Second
+)
+
+// Guard manage the reconnection tries to the Relay server in case of disconnection event.
+type Guard struct {
+	ctx         context.Context
+	relayClient *Client
+}
+
+// NewGuard creates a new guard for the relay client.
+func NewGuard(context context.Context, relayClient *Client) *Guard {
+	g := &Guard{
+		ctx:         context,
+		relayClient: relayClient,
+	}
+	return g
+}
+
+// OnDisconnected is called when the relay client is disconnected from the relay server. It will trigger the reconnection
+// todo prevent multiple reconnection instances. In the current usage it should not happen, but it is better to prevent
+func (g *Guard) OnDisconnected() {
+	ticker := time.NewTicker(reconnectingTimeout)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			err := g.relayClient.Connect()
+			if err != nil {
+				log.Errorf("failed to reconnect to relay server: %s", err)
+				continue
+			}
+			return
+		case <-g.ctx.Done():
+			return
+		}
+	}
+}

relay/client/manager.go

@@ -0,0 +1,314 @@
+package client
+
+import (
+	"container/list"
+	"context"
+	"fmt"
+	"net"
+	"reflect"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	relayAuth "github.com/netbirdio/netbird/relay/auth/hmac"
+)
+
+var (
+	relayCleanupInterval = 60 * time.Second
+
+	errRelayClientNotConnected = fmt.Errorf("relay client not connected")
+)
+
+// RelayTrack hold the relay clients for the foreign relay servers.
+// With the mutex can ensure we can open new connection in case the relay connection has been established with
+// the relay server.
+type RelayTrack struct {
+	sync.RWMutex
+	relayClient *Client
+}
+
+func NewRelayTrack() *RelayTrack {
+	return &RelayTrack{}
+}
+
+type OnServerCloseListener func()
+
+// ManagerService is the interface for the relay manager.
+type ManagerService interface {
+	Serve() error
+	OpenConn(serverAddress, peerKey string) (net.Conn, error)
+	AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error
+	RelayInstanceAddress() (string, error)
+	ServerURL() string
+	HasRelayAddress() bool
+	UpdateToken(token *relayAuth.Token)
+}
+
+// Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL
+// and automatically reconnect to them in case disconnection.
+// The manager also manage temporary relay connection. If a client wants to communicate with a client on a
+// different relay server, the manager will establish a new connection to the relay server. The connection with these
+// relay servers will be closed if there is no active connection. Periodically the manager will check if there is any
+// unused relay connection and close it.
+type Manager struct {
+	ctx        context.Context
+	serverURL  string
+	peerID     string
+	tokenStore *relayAuth.TokenStore
+
+	relayClient    *Client
+	reconnectGuard *Guard
+
+	relayClients      map[string]*RelayTrack
+	relayClientsMutex sync.RWMutex
+
+	onDisconnectedListeners map[string]*list.List
+	listenerLock            sync.Mutex
+}
+
+// NewManager creates a new manager instance.
+// The serverURL address can be empty. In this case, the manager will not serve.
+func NewManager(ctx context.Context, serverURL string, peerID string) *Manager {
+	return &Manager{
+		ctx:                     ctx,
+		serverURL:               serverURL,
+		peerID:                  peerID,
+		tokenStore:              &relayAuth.TokenStore{},
+		relayClients:            make(map[string]*RelayTrack),
+		onDisconnectedListeners: make(map[string]*list.List),
+	}
+}
+
+// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop for
+// the unused relay connections. The manager will automatically reconnect to the relay server in case of disconnection.
+func (m *Manager) Serve() error {
+	if m.relayClient != nil {
+		return fmt.Errorf("manager already serving")
+	}
+
+	m.relayClient = NewClient(m.ctx, m.serverURL, m.tokenStore, m.peerID)
+	err := m.relayClient.Connect()
+	if err != nil {
+		log.Errorf("failed to connect to relay server: %s", err)
+		return err
+	}
+
+	m.reconnectGuard = NewGuard(m.ctx, m.relayClient)
+	m.relayClient.SetOnDisconnectListener(func() {
+		m.onServerDisconnected(m.serverURL)
+	})
+	m.startCleanupLoop()
+
+	return nil
+}
+
+// OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be
+// established via the relay server. If the peer is on a different relay server, the manager will establish a new
+// connection to the relay server. It returns back with a net.Conn what represent the remote peer connection.
+func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) {
+	if m.relayClient == nil {
+		return nil, errRelayClientNotConnected
+	}
+
+	foreign, err := m.isForeignServer(serverAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		netConn net.Conn
+	)
+	if !foreign {
+		log.Debugf("open peer connection via permanent server: %s", peerKey)
+		netConn, err = m.relayClient.OpenConn(peerKey)
+	} else {
+		log.Debugf("open peer connection via foreign server: %s", serverAddress)
+		netConn, err = m.openConnVia(serverAddress, peerKey)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return netConn, err
+}
+
+// AddCloseListener adds a listener to the given server instance address. The listener will be called if the connection
+// closed.
+func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error {
+	foreign, err := m.isForeignServer(serverAddress)
+	if err != nil {
+		return err
+	}
+
+	var listenerAddr string
+	if foreign {
+		listenerAddr = serverAddress
+	} else {
+		listenerAddr = m.serverURL
+	}
+	m.addListener(listenerAddr, onClosedListener)
+	return nil
+}
+
+// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is
+// lost. This address will be sent to the target peer to choose the common relay server for the communication.
+func (m *Manager) RelayInstanceAddress() (string, error) {
+	if m.relayClient == nil {
+		return "", errRelayClientNotConnected
+	}
+	return m.relayClient.ServerInstanceURL()
+}
+
+// ServerURL returns the address of the permanent relay server.
+func (m *Manager) ServerURL() string {
+	return m.serverURL
+}
+
+// HasRelayAddress returns true if the manager is serving. With this method can check if the peer can communicate with
+// Relay service.
+func (m *Manager) HasRelayAddress() bool {
+	return m.serverURL != ""
+}
+
+// UpdateToken updates the token in the token store.
+func (m *Manager) UpdateToken(token *relayAuth.Token) {
+	m.tokenStore.UpdateToken(token)
+}
+
+func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) {
+	// check if already has a connection to the desired relay server
+	m.relayClientsMutex.RLock()
+	rt, ok := m.relayClients[serverAddress]
+	if ok {
+		rt.RLock()
+		m.relayClientsMutex.RUnlock()
+		defer rt.RUnlock()
+		return rt.relayClient.OpenConn(peerKey)
+	}
+	m.relayClientsMutex.RUnlock()
+
+	// if not, establish a new connection but check it again (because changed the lock type) before starting the
+	// connection
+	m.relayClientsMutex.Lock()
+	rt, ok = m.relayClients[serverAddress]
+	if ok {
+		rt.RLock()
+		m.relayClientsMutex.Unlock()
+		defer rt.RUnlock()
+		return rt.relayClient.OpenConn(peerKey)
+	}
+
+	// create a new relay client and store it in the relayClients map
+	rt = NewRelayTrack()
+	rt.Lock()
+	m.relayClients[serverAddress] = rt
+	m.relayClientsMutex.Unlock()
+
+	relayClient := NewClient(m.ctx, serverAddress, m.tokenStore, m.peerID)
+	err := relayClient.Connect()
+	if err != nil {
+		rt.Unlock()
+		m.relayClientsMutex.Lock()
+		delete(m.relayClients, serverAddress)
+		m.relayClientsMutex.Unlock()
+		return nil, err
+	}
+	// if connection closed then delete the relay client from the list
+	relayClient.SetOnDisconnectListener(func() {
+		m.onServerDisconnected(serverAddress)
+	})
+	rt.relayClient = relayClient
+	rt.Unlock()
+
+	conn, err := relayClient.OpenConn(peerKey)
+	if err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+func (m *Manager) onServerDisconnected(serverAddress string) {
+	if serverAddress == m.serverURL {
+		go m.reconnectGuard.OnDisconnected()
+	}
+
+	m.notifyOnDisconnectListeners(serverAddress)
+}
+
+func (m *Manager) isForeignServer(address string) (bool, error) {
+	rAddr, err := m.relayClient.ServerInstanceURL()
+	if err != nil {
+		return false, fmt.Errorf("relay client not connected")
+	}
+	return rAddr != address, nil
+}
+
+func (m *Manager) startCleanupLoop() {
+	if m.ctx.Err() != nil {
+		return
+	}
+
+	ticker := time.NewTicker(relayCleanupInterval)
+	go func() {
+		defer ticker.Stop()
+		for {
+			select {
+			case <-m.ctx.Done():
+				return
+			case <-ticker.C:
+				m.cleanUpUnusedRelays()
+			}
+		}
+	}()
+}
+
+func (m *Manager) cleanUpUnusedRelays() {
+	m.relayClientsMutex.Lock()
+	defer m.relayClientsMutex.Unlock()
+
+	for addr, rt := range m.relayClients {
+		rt.Lock()
+		if rt.relayClient.HasConns() {
+			rt.Unlock()
+			continue
+		}
+		rt.relayClient.SetOnDisconnectListener(nil)
+		go func() {
+			_ = rt.relayClient.Close()
+		}()
+		log.Debugf("clean up unused relay server connection: %s", addr)
+		delete(m.relayClients, addr)
+		rt.Unlock()
+	}
+}
+
+func (m *Manager) addListener(serverAddress string, onClosedListener OnServerCloseListener) {
+	m.listenerLock.Lock()
+	defer m.listenerLock.Unlock()
+	l, ok := m.onDisconnectedListeners[serverAddress]
+	if !ok {
+		l = list.New()
+	}
+	for e := l.Front(); e != nil; e = e.Next() {
+		if reflect.ValueOf(e.Value).Pointer() == reflect.ValueOf(onClosedListener).Pointer() {
+			return
+		}
+	}
+	l.PushBack(onClosedListener)
+	m.onDisconnectedListeners[serverAddress] = l
+}
+
+func (m *Manager) notifyOnDisconnectListeners(serverAddress string) {
+	m.listenerLock.Lock()
+	defer m.listenerLock.Unlock()
+
+	l, ok := m.onDisconnectedListeners[serverAddress]
+	if !ok {
+		return
+	}
+	for e := l.Front(); e != nil; e = e.Next() {
+		go e.Value.(OnServerCloseListener)()
+	}
+	delete(m.onDisconnectedListeners, serverAddress)
+}

relay/client/manager_test.go

@@ -0,0 +1,432 @@
+package client
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/relay/server"
+)
+
+func TestEmptyURL(t *testing.T) {
+	mgr := NewManager(context.Background(), "", "alice")
+	err := mgr.Serve()
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+}
+
+func TestForeignConn(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	srvCfg2 := server.ListenerConfig{
+		Address: "localhost:2234",
+	}
+	srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan2 := make(chan error, 1)
+	go func() {
+		err := srv2.Listen(srvCfg2)
+		if err != nil {
+			errChan2 <- err
+		}
+	}()
+
+	defer func() {
+		err := srv2.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan2); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = clientAlice.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+
+	idBob := "bob"
+	log.Debugf("connect by bob")
+	clientBob := NewManager(mCtx, toURL(srvCfg2), idBob)
+	err = clientBob.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+	bobsSrvAddr, err := clientBob.RelayInstanceAddress()
+	if err != nil {
+		t.Fatalf("failed to get relay address: %s", err)
+	}
+	connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+	connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	payload := "hello bob, I am alice"
+	_, err = connAliceToBob.Write([]byte(payload))
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	buf := make([]byte, 65535)
+	n, err := connBobToAlice.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	_, err = connBobToAlice.Write(buf[:n])
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	n, err = connAliceToBob.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	if payload != string(buf[:n]) {
+		t.Fatalf("expected %s, got %s", payload, string(buf[:n]))
+	}
+}
+
+func TestForeginConnClose(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	srvCfg2 := server.ListenerConfig{
+		Address: "localhost:2234",
+	}
+	srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan2 := make(chan error, 1)
+	go func() {
+		err := srv2.Listen(srvCfg2)
+		if err != nil {
+			errChan2 <- err
+		}
+	}()
+
+	defer func() {
+		err := srv2.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan2); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	mgr := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = mgr.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+	conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	err = conn.Close()
+	if err != nil {
+		t.Fatalf("failed to close connection: %s", err)
+	}
+}
+
+func TestForeginAutoClose(t *testing.T) {
+	ctx := context.Background()
+	relayCleanupInterval = 1 * time.Second
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		t.Log("binding server 1.")
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		t.Logf("closing server 1.")
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+		t.Logf("server 1. closed")
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	srvCfg2 := server.ListenerConfig{
+		Address: "localhost:2234",
+	}
+	srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan2 := make(chan error, 1)
+	go func() {
+		t.Log("binding server 2.")
+		err := srv2.Listen(srvCfg2)
+		if err != nil {
+			errChan2 <- err
+		}
+	}()
+	defer func() {
+		t.Logf("closing server 2.")
+		err := srv2.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+		t.Logf("server 2 closed.")
+	}()
+
+	if err := waitForServerToStart(errChan2); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	t.Log("connect to server 1.")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	mgr := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = mgr.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+
+	t.Log("open connection to another peer")
+	conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	t.Log("close conn")
+	err = conn.Close()
+	if err != nil {
+		t.Fatalf("failed to close connection: %s", err)
+	}
+
+	t.Logf("waiting for relay cleanup: %s", relayCleanupInterval+1*time.Second)
+	time.Sleep(relayCleanupInterval + 1*time.Second)
+	if len(mgr.relayClients) != 0 {
+		t.Errorf("expected 0, got %d", len(mgr.relayClients))
+	}
+
+	t.Logf("closing manager")
+}
+
+func TestAutoReconnect(t *testing.T) {
+	ctx := context.Background()
+	reconnectingTimeout = 2 * time.Second
+
+	srvCfg := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv, err := server.NewServer(otel.Meter(""), srvCfg.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			log.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	clientAlice := NewManager(mCtx, toURL(srvCfg), "alice")
+	err = clientAlice.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+	ra, err := clientAlice.RelayInstanceAddress()
+	if err != nil {
+		t.Errorf("failed to get relay address: %s", err)
+	}
+	conn, err := clientAlice.OpenConn(ra, "bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	t.Log("closing client relay connection")
+	// todo figure out moc server
+	_ = clientAlice.relayClient.relayConn.Close()
+	t.Log("start test reading")
+	_, err = conn.Read(make([]byte, 1))
+	if err == nil {
+		t.Errorf("unexpected reading from closed connection")
+	}
+
+	log.Infof("waiting for reconnection")
+	time.Sleep(reconnectingTimeout + 1*time.Second)
+
+	log.Infof("reopent the connection")
+	_, err = clientAlice.OpenConn(ra, "bob")
+	if err != nil {
+		t.Errorf("failed to open channel: %s", err)
+	}
+}
+
+func TestNotifierDoubleAdd(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = clientAlice.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+
+	conn1, err := clientAlice.OpenConn(clientAlice.ServerURL(), "idBob")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	fnCloseListener := OnServerCloseListener(func() {
+		log.Infof("close listener")
+	})
+
+	err = clientAlice.AddCloseListener(clientAlice.ServerURL(), fnCloseListener)
+	if err != nil {
+		t.Fatalf("failed to add close listener: %s", err)
+	}
+
+	err = clientAlice.AddCloseListener(clientAlice.ServerURL(), fnCloseListener)
+	if err != nil {
+		t.Fatalf("failed to add close listener: %s", err)
+	}
+
+	err = conn1.Close()
+	if err != nil {
+		t.Errorf("failed to close connection: %s", err)
+	}
+
+}
+
+func toURL(address server.ListenerConfig) string {
+	return "rel://" + address.Address
+}

relay/cmd/env.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"os"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+// setFlagsFromEnvVars reads and updates flag values from environment variables with prefix NB_
+func setFlagsFromEnvVars(cmd *cobra.Command) {
+	flags := cmd.PersistentFlags()
+	flags.VisitAll(func(f *pflag.Flag) {
+		newEnvVar := flagNameToEnvVar(f.Name, "NB_")
+		value, present := os.LookupEnv(newEnvVar)
+		if !present {
+			return
+		}
+
+		err := flags.Set(f.Name, value)
+		if err != nil {
+			log.Infof("unable to configure flag %s using variable %s, err: %v", f.Name, newEnvVar, err)
+		}
+	})
+}
+
+// flagNameToEnvVar converts flag name to environment var name adding a prefix,
+// replacing dashes and making all uppercase (e.g. setup-keys is converted to NB_SETUP_KEYS according to the input prefix)
+func flagNameToEnvVar(cmdFlag string, prefix string) string {
+	parsed := strings.ReplaceAll(cmdFlag, "-", "_")
+	upper := strings.ToUpper(parsed)
+	return prefix + upper
+}

relay/cmd/main.go

@@ -0,0 +1,211 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/encryption"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/relay/server"
+	"github.com/netbirdio/netbird/signal/metrics"
+	"github.com/netbirdio/netbird/util"
+)
+
+const (
+	metricsPort = 9090
+)
+
+type Config struct {
+	ListenAddress string
+	// in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection
+	// it is a domain:port or ip:port
+	ExposedAddress     string
+	LetsencryptEmail   string
+	LetsencryptDataDir string
+	LetsencryptDomains []string
+	// in case of using Route 53 for DNS challenge the credentials should be provided in the environment variables or
+	// in the AWS credentials file
+	LetsencryptAWSRoute53 bool
+	TlsCertFile           string
+	TlsKeyFile            string
+	AuthSecret            string
+	LogLevel              string
+	LogFile               string
+}
+
+func (c Config) Validate() error {
+	if c.ExposedAddress == "" {
+		return fmt.Errorf("exposed address is required")
+	}
+	if c.AuthSecret == "" {
+		return fmt.Errorf("auth secret is required")
+	}
+	return nil
+}
+
+func (c Config) HasCertConfig() bool {
+	return c.TlsCertFile != "" && c.TlsKeyFile != ""
+}
+
+func (c Config) HasLetsEncrypt() bool {
+	return c.LetsencryptDataDir != "" && c.LetsencryptDomains != nil && len(c.LetsencryptDomains) > 0
+}
+
+var (
+	cobraConfig *Config
+	rootCmd     = &cobra.Command{
+		Use:           "relay",
+		Short:         "Relay service",
+		Long:          "Relay service for Netbird agents",
+		RunE:          execute,
+		SilenceUsage:  true,
+		SilenceErrors: true,
+	}
+)
+
+func init() {
+	_ = util.InitLog("trace", "console")
+	cobraConfig = &Config{}
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.ListenAddress, "listen-address", "l", ":443", "listen address")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.ExposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.LetsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.")
+	rootCmd.PersistentFlags().StringSliceVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS")
+	rootCmd.PersistentFlags().StringVar(&cobraConfig.LetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration")
+	rootCmd.PersistentFlags().BoolVar(&cobraConfig.LetsencryptAWSRoute53, "letsencrypt-aws-route53", false, "use AWS Route 53 for Let's Encrypt DNS challenge")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsCertFile, "tls-cert-file", "c", "", "")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsKeyFile, "tls-key-file", "k", "", "")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "auth secret")
+	rootCmd.PersistentFlags().StringVar(&cobraConfig.LogLevel, "log-level", "info", "log level")
+	rootCmd.PersistentFlags().StringVar(&cobraConfig.LogFile, "log-file", "console", "log file")
+
+	setFlagsFromEnvVars(rootCmd)
+}
+
+func waitForExitSignal() {
+	osSigs := make(chan os.Signal, 1)
+	signal.Notify(osSigs, syscall.SIGINT, syscall.SIGTERM)
+	<-osSigs
+}
+
+func execute(cmd *cobra.Command, args []string) error {
+	err := cobraConfig.Validate()
+	if err != nil {
+		return fmt.Errorf("invalid config: %s", err)
+	}
+
+	err = util.InitLog(cobraConfig.LogLevel, cobraConfig.LogFile)
+	if err != nil {
+		return fmt.Errorf("failed to initialize log: %s", err)
+	}
+
+	metricsServer, err := metrics.NewServer(metricsPort, "")
+	if err != nil {
+		return fmt.Errorf("setup metrics: %v", err)
+	}
+
+	go func() {
+		log.Infof("running metrics server: %s%s", metricsServer.Addr, metricsServer.Endpoint)
+		if err := metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+			log.Fatalf("Failed to start metrics server: %v", err)
+		}
+	}()
+
+	srvListenerCfg := server.ListenerConfig{
+		Address: cobraConfig.ListenAddress,
+	}
+
+	tlsConfig, tlsSupport, err := handleTLSConfig(cobraConfig)
+	if err != nil {
+		return fmt.Errorf("failed to setup TLS config: %s", err)
+	}
+	srvListenerCfg.TLSConfig = tlsConfig
+
+	authenticator := auth.NewTimedHMACValidator(cobraConfig.AuthSecret, 24*time.Hour)
+	srv, err := server.NewServer(metricsServer.Meter, cobraConfig.ExposedAddress, tlsSupport, authenticator)
+	if err != nil {
+		return fmt.Errorf("failed to create relay server: %v", err)
+	}
+	log.Infof("server will be available on: %s", srv.InstanceURL())
+	go func() {
+		if err := srv.Listen(srvListenerCfg); err != nil {
+			log.Fatalf("failed to bind server: %s", err)
+		}
+	}()
+
+	// it will block until exit signal
+	waitForExitSignal()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	var shutDownErrors error
+	if err := srv.Shutdown(ctx); err != nil {
+		shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close server: %s", err))
+	}
+
+	log.Infof("shutting down metrics server")
+	if err := metricsServer.Shutdown(ctx); err != nil {
+		shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err))
+	}
+	return shutDownErrors
+}
+
+func handleTLSConfig(cfg *Config) (*tls.Config, bool, error) {
+	if cfg.LetsencryptAWSRoute53 {
+		log.Debugf("using Let's Encrypt DNS resolver with Route 53 support")
+		r53 := encryption.Route53TLS{
+			DataDir: cfg.LetsencryptDataDir,
+			Email:   cfg.LetsencryptEmail,
+			Domains: cfg.LetsencryptDomains,
+		}
+		tlsCfg, err := r53.GetCertificate()
+		if err != nil {
+			return nil, false, fmt.Errorf("%s", err)
+		}
+		return tlsCfg, true, nil
+	}
+
+	if cfg.HasLetsEncrypt() {
+		log.Infof("setting up TLS with Let's Encrypt.")
+		tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...)
+		if err != nil {
+			return nil, false, fmt.Errorf("%s", err)
+		}
+		return tlsCfg, true, nil
+	}
+
+	if cfg.HasCertConfig() {
+		log.Debugf("using file based TLS config")
+		tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile)
+		if err != nil {
+			return nil, false, fmt.Errorf("%s", err)
+		}
+		return tlsCfg, true, nil
+	}
+	return nil, false, nil
+}
+
+func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) {
+	certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...)
+	if err != nil {
+		return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err)
+	}
+	return certManager.TLSConfig(), nil
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		log.Fatalf("%v", err)
+	}
+}

relay/doc.go

@@ -0,0 +1,14 @@
+//Package relay
+/*
+The `relay` package contains the implementation of the Relay server and client. The Relay server can be used to relay
+messages between peers on a single network channel. In this implementation the transport layer is the WebSocket
+protocol.
+
+Between the server and client communication has been design a custom protocol and message format. These messages are
+transported over the WebSocket connection. Optionally the server can use TLS to secure the communication.
+
+The service can support multiple Relay server instances. For this purpose the peers must know the server instance URL.
+This URL will be sent to the target peer to choose the common Relay server for the communication via Signal service.
+
+*/
+package relay

relay/healthcheck/doc.go

@@ -0,0 +1,17 @@
+/*
+The `healthcheck` package is responsible for managing the health checks between the client and the relay server. It
+ensures that the connection between the client and the server are alive and functioning properly.
+
+The `Sender` struct is responsible for sending health check signals to the receiver. The receiver listens for these
+signals and sends a new signal back to the sender to acknowledge that the signal has been received. If the sender does
+not receive an acknowledgment signal within a certain time frame, it will send a timeout signal via timeout channel
+and stop working.
+
+The `Receiver` struct is responsible for receiving the health check signals from the sender. If the receiver does not
+receive a signal within a certain time frame, it will send a timeout signal via the OnTimeout channel and stop working.
+
+In the Relay usage the signal is sent to the peer in message type Healthcheck. In case of timeout the connection is
+closed and the peer is removed from the relay.
+*/
+
+package healthcheck

relay/healthcheck/receiver.go

@@ -0,0 +1,83 @@
+package healthcheck
+
+import (
+	"context"
+	"time"
+)
+
+var (
+	heartbeatTimeout = healthCheckInterval + 3*time.Second
+)
+
+// Receiver is a healthcheck receiver
+// It will listen for heartbeat and check if the heartbeat is not received in a certain time
+// If the heartbeat is not received in a certain time, it will send a timeout signal and stop to work
+// It will also stop if the context is canceled
+// The heartbeat timeout is a bit longer than the sender's healthcheck interval
+type Receiver struct {
+	OnTimeout chan struct{}
+
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+	heartbeat chan struct{}
+	alive     bool
+}
+
+// NewReceiver creates a new healthcheck receiver and start the timer in the background
+func NewReceiver() *Receiver {
+	ctx, ctxCancel := context.WithCancel(context.Background())
+
+	r := &Receiver{
+		OnTimeout: make(chan struct{}, 1),
+		ctx:       ctx,
+		ctxCancel: ctxCancel,
+		heartbeat: make(chan struct{}, 1),
+	}
+
+	go r.waitForHealthcheck()
+	return r
+}
+
+// Heartbeat acknowledge the heartbeat has been received
+func (r *Receiver) Heartbeat() {
+	select {
+	case r.heartbeat <- struct{}{}:
+	default:
+	}
+}
+
+// Stop check the timeout and do not send new notifications
+func (r *Receiver) Stop() {
+	r.ctxCancel()
+}
+
+func (r *Receiver) waitForHealthcheck() {
+	ticker := time.NewTicker(heartbeatTimeout)
+	defer ticker.Stop()
+	defer r.ctxCancel()
+	defer close(r.OnTimeout)
+
+	for {
+		select {
+		case <-r.heartbeat:
+			r.alive = true
+		case <-ticker.C:
+			if r.alive {
+				r.alive = false
+				continue
+			}
+
+			r.notifyTimeout()
+			return
+		case <-r.ctx.Done():
+			return
+		}
+	}
+}
+
+func (r *Receiver) notifyTimeout() {
+	select {
+	case r.OnTimeout <- struct{}{}:
+	default:
+	}
+}

relay/healthcheck/receiver_test.go

@@ -0,0 +1,42 @@
+package healthcheck
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNewReceiver(t *testing.T) {
+	heartbeatTimeout = 5 * time.Second
+	r := NewReceiver()
+
+	select {
+	case <-r.OnTimeout:
+		t.Error("unexpected timeout")
+	case <-time.After(1 * time.Second):
+
+	}
+}
+
+func TestNewReceiverNotReceive(t *testing.T) {
+	heartbeatTimeout = 1 * time.Second
+	r := NewReceiver()
+
+	select {
+	case <-r.OnTimeout:
+	case <-time.After(2 * time.Second):
+		t.Error("timeout not received")
+	}
+}
+
+func TestNewReceiverAck(t *testing.T) {
+	heartbeatTimeout = 2 * time.Second
+	r := NewReceiver()
+
+	r.Heartbeat()
+
+	select {
+	case <-r.OnTimeout:
+		t.Error("unexpected timeout")
+	case <-time.After(3 * time.Second):
+	}
+}

relay/healthcheck/sender.go

@@ -0,0 +1,71 @@
+package healthcheck
+
+import (
+	"context"
+	"time"
+)
+
+var (
+	healthCheckInterval = 25 * time.Second
+	healthCheckTimeout  = 5 * time.Second
+)
+
+// Sender is a healthcheck sender
+// It will send healthcheck signal to the receiver
+// If the receiver does not receive the signal in a certain time, it will send a timeout signal and stop to work
+// It will also stop if the context is canceled
+type Sender struct {
+	// HealthCheck is a channel to send health check signal to the peer
+	HealthCheck chan struct{}
+	// Timeout is a channel to the health check signal is not received in a certain time
+	Timeout chan struct{}
+
+	ctx context.Context
+	ack chan struct{}
+}
+
+// NewSender creates a new healthcheck sender
+func NewSender(ctx context.Context) *Sender {
+	hc := &Sender{
+		HealthCheck: make(chan struct{}, 1),
+		Timeout:     make(chan struct{}, 1),
+		ctx:         ctx,
+		ack:         make(chan struct{}, 1),
+	}
+
+	go hc.healthCheck()
+	return hc
+}
+
+// OnHCResponse sends an acknowledgment signal to the sender
+func (hc *Sender) OnHCResponse() {
+	select {
+	case hc.ack <- struct{}{}:
+	default:
+	}
+}
+
+func (hc *Sender) healthCheck() {
+	ticker := time.NewTicker(healthCheckInterval)
+	defer ticker.Stop()
+
+	timeoutTimer := time.NewTimer(healthCheckInterval + healthCheckTimeout)
+	defer timeoutTimer.Stop()
+
+	defer close(hc.HealthCheck)
+	defer close(hc.Timeout)
+
+	for {
+		select {
+		case <-ticker.C:
+			hc.HealthCheck <- struct{}{}
+		case <-timeoutTimer.C:
+			hc.Timeout <- struct{}{}
+			return
+		case <-hc.ack:
+			timeoutTimer.Stop()
+		case <-hc.ctx.Done():
+			return
+		}
+	}
+}

relay/healthcheck/sender_test.go

@@ -0,0 +1,66 @@
+package healthcheck
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestMain(m *testing.M) {
+	// override the health check interval to speed up the test
+	healthCheckInterval = 1 * time.Second
+	healthCheckTimeout = 100 * time.Millisecond
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestNewHealthPeriod(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	hc := NewSender(ctx)
+
+	iterations := 0
+	for i := 0; i < 3; i++ {
+		select {
+		case <-hc.HealthCheck:
+			iterations++
+			hc.OnHCResponse()
+		case <-hc.Timeout:
+			t.Fatalf("health check is timed out")
+		case <-time.After(healthCheckInterval + 100*time.Millisecond):
+			t.Fatalf("health check not received")
+		}
+	}
+}
+
+func TestNewHealthFailed(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	hc := NewSender(ctx)
+
+	select {
+	case <-hc.Timeout:
+	case <-time.After(healthCheckInterval + healthCheckTimeout + 100*time.Millisecond):
+		t.Fatalf("health check is not timed out")
+	}
+}
+
+func TestNewHealthcheckStop(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	hc := NewSender(ctx)
+
+	time.Sleep(300 * time.Millisecond)
+	cancel()
+
+	select {
+	case <-hc.HealthCheck:
+		t.Fatalf("health check on received")
+	case <-hc.Timeout:
+		t.Fatalf("health check timedout")
+	case <-ctx.Done():
+		// expected
+	case <-time.After(1 * time.Second):
+		t.Fatalf("is not exited")
+	}
+}

relay/messages/doc.go

@@ -0,0 +1,5 @@
+/*
+Package messages provides the message types that are used to communicate between the relay and the client.
+This package is used to determine the type of message that is being sent and received between the relay and the client.
+*/
+package messages

relay/messages/id.go

@@ -0,0 +1,31 @@
+package messages
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+)
+
+const (
+	prefixLength = 4
+	IDSize       = prefixLength + sha256.Size
+)
+
+var (
+	prefix = []byte("sha-") // 4 bytes
+)
+
+// HashID generates a sha256 hash from the peerID and returns the hash and the human-readable string
+func HashID(peerID string) ([]byte, string) {
+	idHash := sha256.Sum256([]byte(peerID))
+	idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:])
+	var prefixedHash []byte
+	prefixedHash = append(prefixedHash, prefix...)
+	prefixedHash = append(prefixedHash, idHash[:]...)
+	return prefixedHash, idHashString
+}
+
+// HashIDToString converts a hash to a human-readable string
+func HashIDToString(idHash []byte) string {
+	return fmt.Sprintf("%s%s", idHash[:prefixLength], base64.StdEncoding.EncodeToString(idHash[prefixLength:]))
+}

relay/messages/id_test.go

@@ -0,0 +1,13 @@
+package messages
+
+import (
+	"testing"
+)
+
+func TestHashID(t *testing.T) {
+	hashedID, hashedStringId := HashID("alice")
+	enc := HashIDToString(hashedID)
+	if enc != hashedStringId {
+		t.Errorf("expected %s, got %s", hashedStringId, enc)
+	}
+}

relay/messages/message.go

@@ -0,0 +1,221 @@
+package messages
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	MsgTypeHello         MsgType = 0
+	MsgTypeHelloResponse MsgType = 1
+	MsgTypeTransport     MsgType = 2
+	MsgTypeClose         MsgType = 3
+	MsgTypeHealthCheck   MsgType = 4
+
+	sizeOfMsgType       = 1
+	sizeOfMagicBye      = 4
+	headerSizeTransport = sizeOfMsgType + IDSize                  // 1 byte for msg type, IDSize for peerID
+	headerSizeHello     = sizeOfMsgType + sizeOfMagicBye + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID
+
+	MaxHandshakeSize = 8192
+)
+
+var (
+	ErrInvalidMessageLength = fmt.Errorf("invalid message length")
+
+	magicHeader = []byte{0x21, 0x12, 0xA4, 0x42}
+
+	healthCheckMsg = []byte{byte(MsgTypeHealthCheck)}
+)
+
+type MsgType byte
+
+func (m MsgType) String() string {
+	switch m {
+	case MsgTypeHello:
+		return "hello"
+	case MsgTypeHelloResponse:
+		return "hello response"
+	case MsgTypeTransport:
+		return "transport"
+	case MsgTypeClose:
+		return "close"
+	case MsgTypeHealthCheck:
+		return "health check"
+	default:
+		return "unknown"
+	}
+}
+
+type HelloResponse struct {
+	InstanceAddress string
+}
+
+// DetermineClientMsgType determines the message type from the first byte of the message
+func DetermineClientMsgType(msg []byte) (MsgType, error) {
+	msgType := MsgType(msg[0])
+	switch msgType {
+	case MsgTypeHello:
+		return msgType, nil
+	case MsgTypeTransport:
+		return msgType, nil
+	case MsgTypeClose:
+		return msgType, nil
+	case MsgTypeHealthCheck:
+		return msgType, nil
+	default:
+		return 0, fmt.Errorf("invalid msg type, len: %d", len(msg))
+	}
+}
+
+// DetermineServerMsgType determines the message type from the first byte of the message
+func DetermineServerMsgType(msg []byte) (MsgType, error) {
+	msgType := MsgType(msg[0])
+	switch msgType {
+	case MsgTypeHelloResponse:
+		return msgType, nil
+	case MsgTypeTransport:
+		return msgType, nil
+	case MsgTypeClose:
+		return msgType, nil
+	case MsgTypeHealthCheck:
+		return msgType, nil
+	default:
+		return 0, fmt.Errorf("invalid msg type (len: %d)", len(msg))
+	}
+}
+
+// MarshalHelloMsg initial hello message
+// The Hello message is the first message sent by a client after establishing a connection with the Relay server. This
+// message is used to authenticate the client with the server. The authentication is done using an HMAC method.
+// The protocol does not limit to use HMAC, it can be any other method. If the authentication failed the server will
+// close the network connection without any response.
+func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) {
+	if len(peerID) != IDSize {
+		return nil, fmt.Errorf("invalid peerID length: %d", len(peerID))
+	}
+
+	// 5 = 1 byte for msg type, 4 byte for magic header
+	msg := make([]byte, 5, headerSizeHello+len(additions))
+	msg[0] = byte(MsgTypeHello)
+	copy(msg[1:5], magicHeader)
+	msg = append(msg, peerID...)
+	msg = append(msg, additions...)
+	return msg, nil
+}
+
+// UnmarshalHelloMsg extracts the peerID and the additional data from the hello message. The Additional data is used to
+// authenticate the client with the server.
+func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) {
+	if len(msg) < headerSizeHello {
+		return nil, nil, fmt.Errorf("invalid 'hello' message")
+	}
+	if !bytes.Equal(msg[1:5], magicHeader) {
+		return nil, nil, fmt.Errorf("invalid magic header")
+	}
+	return msg[5 : 5+IDSize], msg[headerSizeHello:], nil
+}
+
+// MarshalHelloResponse creates a response message to the hello message.
+// In case of success connection the server response with a Hello Response message. This message contains the server's
+// instance URL. This URL will be used by choose the common Relay server in case if the peers are in different Relay
+// servers.
+func MarshalHelloResponse(DomainAddress string) ([]byte, error) {
+	payload := HelloResponse{
+		InstanceAddress: DomainAddress,
+	}
+
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+
+	err := enc.Encode(payload)
+	if err != nil {
+		log.Errorf("failed to gob encode hello response: %s", err)
+		return nil, err
+	}
+
+	msg := make([]byte, 1, 1+buf.Len())
+	msg[0] = byte(MsgTypeHelloResponse)
+	msg = append(msg, buf.Bytes()...)
+	return msg, nil
+}
+
+// UnmarshalHelloResponse extracts the instance address from the hello response message
+func UnmarshalHelloResponse(msg []byte) (string, error) {
+	if len(msg) < 2 {
+		return "", fmt.Errorf("invalid 'hello response' message")
+	}
+	payload := HelloResponse{}
+	buf := bytes.NewBuffer(msg[1:])
+	dec := gob.NewDecoder(buf)
+
+	err := dec.Decode(&payload)
+	if err != nil {
+		log.Errorf("failed to gob decode hello response: %s", err)
+		return "", err
+	}
+	return payload.InstanceAddress, nil
+}
+
+// MarshalCloseMsg creates a close message.
+// The close message is used to close the connection gracefully between the client and the server. The server and the
+// client can send this message. After receiving this message, the server or client will close the connection.
+func MarshalCloseMsg() []byte {
+	msg := make([]byte, 1)
+	msg[0] = byte(MsgTypeClose)
+	return msg
+}
+
+// MarshalTransportMsg creates a transport message.
+// The transport message is used to exchange data between peers. The message contains the data to be exchanged and the
+// destination peer hashed ID.
+func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) {
+	if len(peerID) != IDSize {
+		return nil, fmt.Errorf("invalid peerID length: %d", len(peerID))
+	}
+
+	msg := make([]byte, headerSizeTransport, headerSizeTransport+len(payload))
+	msg[0] = byte(MsgTypeTransport)
+	copy(msg[1:], peerID)
+	msg = append(msg, payload...)
+	return msg, nil
+}
+
+// UnmarshalTransportMsg extracts the peerID and the payload from the transport message.
+func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) {
+	if len(buf) < headerSizeTransport {
+		return nil, nil, ErrInvalidMessageLength
+	}
+
+	return buf[1:headerSizeTransport], buf[headerSizeTransport:], nil
+}
+
+// UnmarshalTransportID extracts the peerID from the transport message.
+func UnmarshalTransportID(buf []byte) ([]byte, error) {
+	if len(buf) < headerSizeTransport {
+		log.Debugf("invalid message length: %d, expected: %d, %x", len(buf), headerSizeTransport, buf)
+		return nil, ErrInvalidMessageLength
+	}
+	return buf[1:headerSizeTransport], nil
+}
+
+// UpdateTransportMsg updates the peerID in the transport message.
+// With this function the server can reuse the given byte slice to update the peerID in the transport message. So do
+// need to allocate a new byte slice.
+func UpdateTransportMsg(msg []byte, peerID []byte) error {
+	if len(msg) < 1+len(peerID) {
+		return ErrInvalidMessageLength
+	}
+	copy(msg[1:], peerID)
+	return nil
+}
+
+// MarshalHealthcheck creates a health check message.
+// Health check message is sent by the server periodically. The client will respond with a health check response
+// message. If the client does not respond to the health check message, the server will close the connection.
+func MarshalHealthcheck() []byte {
+	return healthCheckMsg
+}

relay/messages/message_test.go

@@ -0,0 +1,43 @@
+package messages
+
+import (
+	"testing"
+)
+
+func TestMarshalHelloMsg(t *testing.T) {
+	peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+")
+	bHello, err := MarshalHelloMsg(peerID, nil)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	receivedPeerID, _, err := UnmarshalHelloMsg(bHello)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+	if string(receivedPeerID) != string(peerID) {
+		t.Errorf("expected %s, got %s", peerID, receivedPeerID)
+	}
+}
+
+func TestMarshalTransportMsg(t *testing.T) {
+	peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+")
+	payload := []byte("payload")
+	msg, err := MarshalTransportMsg(peerID, payload)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	id, respPayload, err := UnmarshalTransportMsg(msg)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	if string(id) != string(peerID) {
+		t.Errorf("expected %s, got %s", peerID, id)
+	}
+
+	if string(respPayload) != string(payload) {
+		t.Errorf("expected %s, got %s", payload, respPayload)
+	}
+}

relay/metrics/realy.go

@@ -0,0 +1,136 @@
+package metrics
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	idleTimeout = 30 * time.Second
+)
+
+type Metrics struct {
+	metric.Meter
+
+	TransferBytesSent metric.Int64Counter
+	TransferBytesRecv metric.Int64Counter
+
+	peers            metric.Int64UpDownCounter
+	peerActivityChan chan string
+	peerLastActive   map[string]time.Time
+	mutexActivity    sync.Mutex
+	ctx              context.Context
+}
+
+func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) {
+	bytesSent, err := meter.Int64Counter("relay_transfer_sent_bytes_total")
+	if err != nil {
+		return nil, err
+	}
+
+	bytesRecv, err := meter.Int64Counter("relay_transfer_received_bytes_total")
+	if err != nil {
+		return nil, err
+	}
+
+	peers, err := meter.Int64UpDownCounter("relay_peers")
+	if err != nil {
+		return nil, err
+	}
+
+	peersActive, err := meter.Int64ObservableGauge("relay_peers_active")
+	if err != nil {
+		return nil, err
+	}
+
+	peersIdle, err := meter.Int64ObservableGauge("relay_peers_idle")
+	if err != nil {
+		return nil, err
+	}
+
+	m := &Metrics{
+		Meter:             meter,
+		TransferBytesSent: bytesSent,
+		TransferBytesRecv: bytesRecv,
+		peers:             peers,
+
+		ctx:              ctx,
+		peerActivityChan: make(chan string, 10),
+		peerLastActive:   make(map[string]time.Time),
+	}
+
+	_, err = meter.RegisterCallback(
+		func(ctx context.Context, o metric.Observer) error {
+			active, idle := m.calculateActiveIdleConnections()
+			o.ObserveInt64(peersActive, active)
+			o.ObserveInt64(peersIdle, idle)
+			return nil
+		},
+		peersActive, peersIdle,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	go m.readPeerActivity()
+	return m, nil
+}
+
+// PeerConnected increments the number of connected peers and increments number of idle connections
+func (m *Metrics) PeerConnected(id string) {
+	m.peers.Add(m.ctx, 1)
+	m.mutexActivity.Lock()
+	defer m.mutexActivity.Unlock()
+
+	m.peerLastActive[id] = time.Time{}
+}
+
+// PeerDisconnected decrements the number of connected peers and decrements number of idle or active connections
+func (m *Metrics) PeerDisconnected(id string) {
+	m.peers.Add(m.ctx, -1)
+	m.mutexActivity.Lock()
+	defer m.mutexActivity.Unlock()
+
+	delete(m.peerLastActive, id)
+}
+
+// PeerActivity increases the active connections
+func (m *Metrics) PeerActivity(peerID string) {
+	select {
+	case m.peerActivityChan <- peerID:
+	default:
+		log.Errorf("peer activity channel is full, dropping activity metrics for peer %s", peerID)
+	}
+}
+
+func (m *Metrics) calculateActiveIdleConnections() (int64, int64) {
+	active, idle := int64(0), int64(0)
+	m.mutexActivity.Lock()
+	defer m.mutexActivity.Unlock()
+
+	for _, lastActive := range m.peerLastActive {
+		if time.Since(lastActive) > idleTimeout {
+			idle++
+		} else {
+			active++
+		}
+	}
+	return active, idle
+}
+
+func (m *Metrics) readPeerActivity() {
+	for {
+		select {
+		case peerID := <-m.peerActivityChan:
+			m.mutexActivity.Lock()
+			m.peerLastActive[peerID] = time.Now()
+			m.mutexActivity.Unlock()
+		case <-m.ctx.Done():
+			return
+		}
+	}
+}

relay/server/listener/listener.go

@@ -0,0 +1,11 @@
+package listener
+
+import (
+	"context"
+	"net"
+)
+
+type Listener interface {
+	Listen(func(conn net.Conn)) error
+	Shutdown(ctx context.Context) error
+}

relay/server/listener/ws/conn.go

@@ -0,0 +1,114 @@
+package ws
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"nhooyr.io/websocket"
+)
+
+const (
+	writeTimeout = 10 * time.Second
+)
+
+type Conn struct {
+	*websocket.Conn
+	lAddr *net.TCPAddr
+	rAddr *net.TCPAddr
+
+	closed   bool
+	closedMu sync.Mutex
+	ctx      context.Context
+}
+
+func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn {
+	return &Conn{
+		Conn:  wsConn,
+		lAddr: lAddr,
+		rAddr: rAddr,
+		ctx:   context.Background(),
+	}
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	t, r, err := c.Reader(c.ctx)
+	if err != nil {
+		return 0, c.ioErrHandling(err)
+	}
+
+	if t != websocket.MessageBinary {
+		log.Errorf("unexpected message type: %d", t)
+		return 0, fmt.Errorf("unexpected message type")
+	}
+
+	n, err = r.Read(b)
+	if err != nil {
+		return 0, c.ioErrHandling(err)
+	}
+	return n, err
+}
+
+// Write writes a binary message with the given payload.
+// It does not block until fill the internal buffer.
+// If the buffer filled up, wait until the buffer is drained or timeout.
+func (c *Conn) Write(b []byte) (int, error) {
+	ctx, ctxCancel := context.WithTimeout(c.ctx, writeTimeout)
+	defer ctxCancel()
+
+	err := c.Conn.Write(ctx, websocket.MessageBinary, b)
+	return len(b), err
+}
+
+func (c *Conn) LocalAddr() net.Addr {
+	return c.lAddr
+}
+
+func (c *Conn) RemoteAddr() net.Addr {
+	return c.rAddr
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	return fmt.Errorf("SetReadDeadline is not implemented")
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	return fmt.Errorf("SetWriteDeadline is not implemented")
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	return fmt.Errorf("SetDeadline is not implemented")
+}
+
+func (c *Conn) Close() error {
+	c.closedMu.Lock()
+	c.closed = true
+	c.closedMu.Unlock()
+	return c.Conn.CloseNow()
+}
+
+func (c *Conn) isClosed() bool {
+	c.closedMu.Lock()
+	defer c.closedMu.Unlock()
+	return c.closed
+}
+
+func (c *Conn) ioErrHandling(err error) error {
+	if c.isClosed() {
+		return io.EOF
+	}
+
+	var wErr *websocket.CloseError
+	if !errors.As(err, &wErr) {
+		return err
+	}
+	if wErr.Code == websocket.StatusNormalClosure {
+		return io.EOF
+	}
+	return err
+}

relay/server/listener/ws/listener.go

@@ -0,0 +1,83 @@
+package ws
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+
+	log "github.com/sirupsen/logrus"
+	"nhooyr.io/websocket"
+)
+
+type Listener struct {
+	// Address is the address to listen on.
+	Address string
+	// TLSConfig is the TLS configuration for the server.
+	TLSConfig *tls.Config
+
+	server   *http.Server
+	acceptFn func(conn net.Conn)
+}
+
+func (l *Listener) Listen(acceptFn func(conn net.Conn)) error {
+	l.acceptFn = acceptFn
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", l.onAccept)
+
+	l.server = &http.Server{
+		Addr:      l.Address,
+		Handler:   mux,
+		TLSConfig: l.TLSConfig,
+	}
+
+	log.Infof("WS server is listening on address: %s", l.Address)
+	var err error
+	if l.TLSConfig != nil {
+		err = l.server.ListenAndServeTLS("", "")
+	} else {
+		err = l.server.ListenAndServe()
+	}
+	if errors.Is(err, http.ErrServerClosed) {
+		return nil
+	}
+	return err
+}
+
+func (l *Listener) Shutdown(ctx context.Context) error {
+	if l.server == nil {
+		return nil
+	}
+
+	log.Infof("stop WS listener")
+	if err := l.server.Shutdown(ctx); err != nil {
+		return fmt.Errorf("server shutdown failed: %v", err)
+	}
+	log.Infof("WS listener stopped")
+	return nil
+}
+
+func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) {
+	wsConn, err := websocket.Accept(w, r, nil)
+	if err != nil {
+		log.Errorf("failed to accept ws connection from %s: %s", r.RemoteAddr, err)
+		return
+	}
+
+	rAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
+	if err != nil {
+		_ = wsConn.Close(websocket.StatusInternalError, "internal error")
+		return
+	}
+
+	lAddr, err := net.ResolveTCPAddr("tcp", l.server.Addr)
+	if err != nil {
+		_ = wsConn.Close(websocket.StatusInternalError, "internal error")
+		return
+	}
+
+	conn := NewConn(wsConn, lAddr, rAddr)
+	l.acceptFn(conn)
+}

relay/server/peer.go

@@ -0,0 +1,180 @@
+package server
+
+import (
+	"context"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/relay/healthcheck"
+	"github.com/netbirdio/netbird/relay/messages"
+	"github.com/netbirdio/netbird/relay/metrics"
+)
+
+const (
+	bufferSize = 8820
+)
+
+// Peer represents a peer connection
+type Peer struct {
+	metrics *metrics.Metrics
+	log     *log.Entry
+	idS     string
+	idB     []byte
+	conn    net.Conn
+	connMu  sync.RWMutex
+	store   *Store
+}
+
+// NewPeer creates a new Peer instance and prepare custom logging
+func NewPeer(metrics *metrics.Metrics, id []byte, conn net.Conn, store *Store) *Peer {
+	stringID := messages.HashIDToString(id)
+	return &Peer{
+		metrics: metrics,
+		log:     log.WithField("peer_id", stringID),
+		idS:     stringID,
+		idB:     id,
+		conn:    conn,
+		store:   store,
+	}
+}
+
+// Work reads data from the connection
+// It manages the protocol (healthcheck, transport, close). Read the message and determine the message type and handle
+// the message accordingly.
+func (p *Peer) Work() {
+	ctx, cancel := context.WithCancel(context.Background())
+	hc := healthcheck.NewSender(ctx)
+	go p.healthcheck(ctx, hc)
+	defer cancel()
+
+	buf := make([]byte, bufferSize)
+	for {
+		n, err := p.conn.Read(buf)
+		if err != nil {
+			if err != io.EOF {
+				p.log.Errorf("failed to read message: %s", err)
+			}
+			return
+		}
+
+		msg := buf[:n]
+
+		msgType, err := messages.DetermineClientMsgType(msg)
+		if err != nil {
+			p.log.Errorf("failed to determine message type: %s", err)
+			return
+		}
+		switch msgType {
+		case messages.MsgTypeHealthCheck:
+			hc.OnHCResponse()
+		case messages.MsgTypeTransport:
+			p.metrics.TransferBytesRecv.Add(ctx, int64(n))
+			p.metrics.PeerActivity(p.String())
+			p.handleTransportMsg(msg)
+		case messages.MsgTypeClose:
+			p.log.Infof("peer exited gracefully")
+			_ = p.conn.Close()
+			return
+		}
+	}
+}
+
+// Write writes data to the connection
+func (p *Peer) Write(b []byte) (int, error) {
+	p.connMu.RLock()
+	defer p.connMu.RUnlock()
+	return p.conn.Write(b)
+}
+
+// CloseGracefully closes the connection with the peer gracefully. Send a close message to the client and close the
+// connection.
+func (p *Peer) CloseGracefully(ctx context.Context) {
+	p.connMu.Lock()
+	_, err := p.writeWithTimeout(ctx, messages.MarshalCloseMsg())
+	if err != nil {
+		p.log.Errorf("failed to send close message to peer: %s", p.String())
+	}
+
+	err = p.conn.Close()
+	if err != nil {
+		p.log.Errorf("failed to close connection to peer: %s", err)
+	}
+
+	defer p.connMu.Unlock()
+}
+
+// String returns the peer ID
+func (p *Peer) String() string {
+	return p.idS
+}
+
+func (p *Peer) writeWithTimeout(ctx context.Context, buf []byte) (int, error) {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	writeDone := make(chan struct{})
+	var (
+		n   int
+		err error
+	)
+
+	go func() {
+		_, err = p.conn.Write(buf)
+		close(writeDone)
+	}()
+
+	select {
+	case <-ctx.Done():
+		return 0, ctx.Err()
+	case <-writeDone:
+		return n, err
+	}
+}
+
+func (p *Peer) healthcheck(ctx context.Context, hc *healthcheck.Sender) {
+	for {
+		select {
+		case <-hc.HealthCheck:
+			_, err := p.Write(messages.MarshalHealthcheck())
+			if err != nil {
+				p.log.Errorf("failed to send healthcheck message: %s", err)
+				return
+			}
+		case <-hc.Timeout:
+			p.log.Errorf("peer healthcheck timeout")
+			_ = p.conn.Close()
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (p *Peer) handleTransportMsg(msg []byte) {
+	peerID, err := messages.UnmarshalTransportID(msg)
+	if err != nil {
+		p.log.Errorf("failed to unmarshal transport message: %s", err)
+		return
+	}
+	stringPeerID := messages.HashIDToString(peerID)
+	dp, ok := p.store.Peer(stringPeerID)
+	if !ok {
+		p.log.Errorf("peer not found: %s", stringPeerID)
+		return
+	}
+	err = messages.UpdateTransportMsg(msg, p.idB)
+	if err != nil {
+		p.log.Errorf("failed to update transport message: %s", err)
+		return
+	}
+	n, err := dp.Write(msg)
+	if err != nil {
+		p.log.Errorf("failed to write transport message to: %s", dp.String())
+		return
+	}
+	p.metrics.TransferBytesSent.Add(context.Background(), int64(n))
+}

relay/server/relay.go

@@ -0,0 +1,163 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/netbirdio/netbird/relay/auth"
+	"github.com/netbirdio/netbird/relay/messages"
+	"github.com/netbirdio/netbird/relay/metrics"
+)
+
+// Relay represents the relay server
+type Relay struct {
+	metrics       *metrics.Metrics
+	metricsCancel context.CancelFunc
+	validator     auth.Validator
+
+	store       *Store
+	instanceURL string
+
+	closed  bool
+	closeMu sync.RWMutex
+}
+
+// NewRelay creates a new Relay instance
+//
+// Parameters:
+// meter: An instance of metric.Meter from the go.opentelemetry.io/otel/metric package. It is used to create and manage
+// metrics for the relay server.
+// exposedAddress: A string representing the address that the relay server is exposed on. The client will use this
+// address as the relay server's instance URL.
+// tlsSupport: A boolean indicating whether the relay server supports TLS (Transport Layer Security) or not. The
+// instance URL depends on this value.
+// validator: An instance of auth.Validator from the auth package. It is used to validate the authentication of the
+// peers.
+//
+// Returns:
+// A pointer to a Relay instance and an error. If the Relay instance is successfully created, the error is nil.
+// Otherwise, the error contains the details of what went wrong.
+func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, validator auth.Validator) (*Relay, error) {
+	ctx, metricsCancel := context.WithCancel(context.Background())
+	m, err := metrics.NewMetrics(ctx, meter)
+	if err != nil {
+		metricsCancel()
+		return nil, fmt.Errorf("creating app metrics: %v", err)
+	}
+
+	r := &Relay{
+		metrics:       m,
+		metricsCancel: metricsCancel,
+		validator:     validator,
+		store:         NewStore(),
+	}
+
+	if tlsSupport {
+		r.instanceURL = fmt.Sprintf("rels://%s", exposedAddress)
+	} else {
+		r.instanceURL = fmt.Sprintf("rel://%s", exposedAddress)
+	}
+	_, err = url.ParseRequestURI(r.instanceURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid exposed address: %v", err)
+	}
+
+	return r, nil
+}
+
+// Accept start to handle a new peer connection
+func (r *Relay) Accept(conn net.Conn) {
+	r.closeMu.RLock()
+	defer r.closeMu.RUnlock()
+	if r.closed {
+		return
+	}
+
+	peerID, err := r.handshake(conn)
+	if err != nil {
+		log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err)
+		cErr := conn.Close()
+		if cErr != nil {
+			log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr)
+		}
+		return
+	}
+
+	peer := NewPeer(r.metrics, peerID, conn, r.store)
+	peer.log.Infof("peer connected from: %s", conn.RemoteAddr())
+	r.store.AddPeer(peer)
+	r.metrics.PeerConnected(peer.String())
+	go func() {
+		peer.Work()
+		r.store.DeletePeer(peer)
+		peer.log.Debugf("relay connection closed")
+		r.metrics.PeerDisconnected(peer.String())
+	}()
+}
+
+// Shutdown closes the relay server
+// It closes the connection with all peers in gracefully and stops accepting new connections.
+func (r *Relay) Shutdown(ctx context.Context) {
+	log.Infof("close connection with all peers")
+	r.closeMu.Lock()
+	wg := sync.WaitGroup{}
+	peers := r.store.Peers()
+	for _, peer := range peers {
+		wg.Add(1)
+		go func(p *Peer) {
+			p.CloseGracefully(ctx)
+			wg.Done()
+		}(peer)
+	}
+	wg.Wait()
+	r.metricsCancel()
+	r.closeMu.Unlock()
+}
+
+// InstanceURL returns the instance URL of the relay server
+func (r *Relay) InstanceURL() string {
+	return r.instanceURL
+}
+
+func (r *Relay) handshake(conn net.Conn) ([]byte, error) {
+	buf := make([]byte, messages.MaxHandshakeSize)
+	n, err := conn.Read(buf)
+	if err != nil {
+		log.Debugf("failed to read message from: %s, %s", conn.RemoteAddr(), err)
+		return nil, err
+	}
+	msgType, err := messages.DetermineClientMsgType(buf[:n])
+	if err != nil {
+		return nil, err
+	}
+
+	if msgType != messages.MsgTypeHello {
+		tErr := fmt.Errorf("invalid message type")
+		log.Debugf("failed to handshake with: %s, %s", conn.RemoteAddr(), tErr)
+		return nil, tErr
+	}
+
+	peerID, authPayload, err := messages.UnmarshalHelloMsg(buf[:n])
+	if err != nil {
+		log.Debugf("failed to handshake with: %s, %s", conn.RemoteAddr(), err)
+		return nil, err
+	}
+
+	if err := r.validator.Validate(authPayload); err != nil {
+		log.Debugf("failed to authenticate connection with: %s, %s", conn.RemoteAddr(), err)
+		return nil, err
+	}
+
+	msg, _ := messages.MarshalHelloResponse(r.instanceURL)
+	_, err = conn.Write(msg)
+	if err != nil {
+		return nil, err
+	}
+	return peerID, nil
+}

relay/server/server.go

@@ -0,0 +1,76 @@
+package server
+
+import (
+	"context"
+	"crypto/tls"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/netbirdio/netbird/relay/auth"
+	"github.com/netbirdio/netbird/relay/server/listener"
+	"github.com/netbirdio/netbird/relay/server/listener/ws"
+)
+
+// ListenerConfig is the configuration for the listener.
+// Address: the address to bind the listener to. It could be an address behind a reverse proxy.
+// TLSConfig: the TLS configuration for the listener.
+type ListenerConfig struct {
+	Address   string
+	TLSConfig *tls.Config
+}
+
+// Server is the main entry point for the relay server.
+// It is the gate between the WebSocket listener and the Relay server logic.
+// In a new HTTP connection, the server will accept the connection and pass it to the Relay server via the Accept method.
+type Server struct {
+	relay      *Relay
+	wSListener listener.Listener
+}
+
+// NewServer creates a new relay server instance.
+// meter: the OpenTelemetry meter
+// exposedAddress: this address will be used as the instance URL. It should be a domain:port format.
+// tlsSupport: if true, the server will support TLS
+// authValidator: the auth validator to use for the server
+func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authValidator auth.Validator) (*Server, error) {
+	relay, err := NewRelay(meter, exposedAddress, tlsSupport, authValidator)
+	if err != nil {
+		return nil, err
+	}
+	return &Server{
+		relay: relay,
+	}, nil
+}
+
+// Listen starts the relay server.
+func (r *Server) Listen(cfg ListenerConfig) error {
+	r.wSListener = &ws.Listener{
+		Address:   cfg.Address,
+		TLSConfig: cfg.TLSConfig,
+	}
+
+	wslErr := r.wSListener.Listen(r.relay.Accept)
+	if wslErr != nil {
+		log.Errorf("failed to bind ws server: %s", wslErr)
+	}
+
+	return wslErr
+}
+
+// Shutdown stops the relay server. If there are active connections, they will be closed gracefully. In case of a context,
+// the connections will be forcefully closed.
+func (r *Server) Shutdown(ctx context.Context) (err error) {
+	// stop service new connections
+	if r.wSListener != nil {
+		err = r.wSListener.Shutdown(ctx)
+	}
+
+	r.relay.Shutdown(ctx)
+	return
+}
+
+// InstanceURL returns the instance URL of the relay server.
+func (r *Server) InstanceURL() string {
+	return r.relay.instanceURL
+}

relay/server/store.go

@@ -0,0 +1,64 @@
+package server
+
+import (
+	"sync"
+)
+
+// Store is a thread-safe store of peers
+// It is used to store the peers that are connected to the relay server
+type Store struct {
+	peers     map[string]*Peer // consider to use [32]byte as key. The Peer(id string) would be faster
+	peersLock sync.RWMutex
+}
+
+// NewStore creates a new Store instance
+func NewStore() *Store {
+	return &Store{
+		peers: make(map[string]*Peer),
+	}
+}
+
+// AddPeer adds a peer to the store
+// todo: consider to close peer conn if the peer already exists
+func (s *Store) AddPeer(peer *Peer) {
+	s.peersLock.Lock()
+	defer s.peersLock.Unlock()
+	s.peers[peer.String()] = peer
+}
+
+// DeletePeer deletes a peer from the store
+func (s *Store) DeletePeer(peer *Peer) {
+	s.peersLock.Lock()
+	defer s.peersLock.Unlock()
+
+	dp, ok := s.peers[peer.String()]
+	if !ok {
+		return
+	}
+	if dp != peer {
+		return
+	}
+
+	delete(s.peers, peer.String())
+}
+
+// Peer returns a peer by its ID
+func (s *Store) Peer(id string) (*Peer, bool) {
+	s.peersLock.RLock()
+	defer s.peersLock.RUnlock()
+
+	p, ok := s.peers[id]
+	return p, ok
+}
+
+// Peers returns all the peers in the store
+func (s *Store) Peers() []*Peer {
+	s.peersLock.RLock()
+	defer s.peersLock.RUnlock()
+
+	peers := make([]*Peer, 0, len(s.peers))
+	for _, p := range s.peers {
+		peers = append(peers, p)
+	}
+	return peers
+}

relay/server/store_test.go

@@ -0,0 +1,40 @@
+package server
+
+import (
+	"context"
+	"testing"
+
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/relay/metrics"
+)
+
+func TestStore_DeletePeer(t *testing.T) {
+	s := NewStore()
+
+	m, _ := metrics.NewMetrics(context.Background(), otel.Meter(""))
+
+	p := NewPeer(m, []byte("peer_one"), nil, nil)
+	s.AddPeer(p)
+	s.DeletePeer(p)
+	if _, ok := s.Peer(p.String()); ok {
+		t.Errorf("peer was not deleted")
+	}
+}
+
+func TestStore_DeleteDeprecatedPeer(t *testing.T) {
+	s := NewStore()
+
+	m, _ := metrics.NewMetrics(context.Background(), otel.Meter(""))
+
+	p1 := NewPeer(m, []byte("peer_id"), nil, nil)
+	p2 := NewPeer(m, []byte("peer_id"), nil, nil)
+
+	s.AddPeer(p1)
+	s.AddPeer(p2)
+	s.DeletePeer(p1)
+
+	if _, ok := s.Peer(p2.String()); !ok {
+		t.Errorf("second peer was deleted")
+	}
+}

relay/test/benchmark_test.go

@@ -0,0 +1,386 @@
+package test
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"net"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/logging"
+	"github.com/pion/turn/v3"
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/relay/auth"
+	"github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/relay/client"
+	"github.com/netbirdio/netbird/relay/server"
+	"github.com/netbirdio/netbird/util"
+)
+
+var (
+	av             = &auth.AllowAllAuth{}
+	hmacTokenStore = &hmac.TokenStore{}
+	pairs          = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
+	dataSize       = 1024 * 1024 * 10
+)
+
+func TestMain(m *testing.M) {
+	_ = util.InitLog("error", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestRelayDataTransfer(t *testing.T) {
+	t.SkipNow() // skip this test on CI because it is a benchmark test
+	testData, err := seedRandomData(dataSize)
+	if err != nil {
+		t.Fatalf("failed to seed random data: %s", err)
+	}
+
+	for _, peerPairs := range pairs {
+		t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) {
+			transfer(t, testData, peerPairs)
+		})
+	}
+}
+
+// TestTurnDataTransfer run turn server:
+// docker run --rm --name coturn -d --network=host coturn/coturn --user test:test
+func TestTurnDataTransfer(t *testing.T) {
+	t.SkipNow() // skip this test on CI because it is a benchmark test
+	testData, err := seedRandomData(dataSize)
+	if err != nil {
+		t.Fatalf("failed to seed random data: %s", err)
+	}
+
+	for _, peerPairs := range pairs {
+		t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) {
+			runTurnTest(t, testData, peerPairs)
+		})
+	}
+}
+
+func transfer(t *testing.T, testData []byte, peerPairs int) {
+	t.Helper()
+	ctx := context.Background()
+	port := 35000 + peerPairs
+	serverAddress := fmt.Sprintf("127.0.0.1:%d", port)
+	serverConnURL := fmt.Sprintf("rel://%s", serverAddress)
+
+	srv, err := server.NewServer(otel.Meter(""), serverConnURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		listenCfg := server.ListenerConfig{Address: serverAddress}
+		err := srv.Listen(listenCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for server to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientsSender := make([]*client.Client, peerPairs)
+	for i := 0; i < cap(clientsSender); i++ {
+		c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i))
+		err := c.Connect()
+		if err != nil {
+			t.Fatalf("failed to connect to server: %s", err)
+		}
+		clientsSender[i] = c
+	}
+
+	clientsReceiver := make([]*client.Client, peerPairs)
+	for i := 0; i < cap(clientsReceiver); i++ {
+		c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i))
+		err := c.Connect()
+		if err != nil {
+			t.Fatalf("failed to connect to server: %s", err)
+		}
+		clientsReceiver[i] = c
+	}
+
+	connsSender := make([]net.Conn, 0, peerPairs)
+	connsReceiver := make([]net.Conn, 0, peerPairs)
+	for i := 0; i < len(clientsSender); i++ {
+		conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i))
+		if err != nil {
+			t.Fatalf("failed to bind channel: %s", err)
+		}
+		connsSender = append(connsSender, conn)
+
+		conn, err = clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i))
+		if err != nil {
+			t.Fatalf("failed to bind channel: %s", err)
+		}
+		connsReceiver = append(connsReceiver, conn)
+	}
+
+	var transferDuration []time.Duration
+	wg := sync.WaitGroup{}
+	var writeErr error
+	var readErr error
+	for i := 0; i < len(connsSender); i++ {
+		wg.Add(2)
+		start := time.Now()
+		go func(i int) {
+			defer wg.Done()
+			pieceSize := 1024
+			testDataLen := len(testData)
+
+			for j := 0; j < testDataLen; j += pieceSize {
+				end := j + pieceSize
+				if end > testDataLen {
+					end = testDataLen
+				}
+				_, writeErr = connsSender[i].Write(testData[j:end])
+				if writeErr != nil {
+					return
+				}
+			}
+
+		}(i)
+
+		go func(i int, start time.Time) {
+			defer wg.Done()
+			buf := make([]byte, 8192)
+			rcv := 0
+			var n int
+			for receivedSize := 0; receivedSize < len(testData); {
+
+				n, readErr = connsReceiver[i].Read(buf)
+				if readErr != nil {
+					return
+				}
+
+				receivedSize += n
+				rcv += n
+			}
+			transferDuration = append(transferDuration, time.Since(start))
+		}(i, start)
+	}
+
+	wg.Wait()
+
+	if writeErr != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	if readErr != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	// calculate the megabytes per second from the average transferDuration against the dataSize
+	var totalDuration time.Duration
+	for _, d := range transferDuration {
+		totalDuration += d
+	}
+	avgDuration := totalDuration / time.Duration(len(transferDuration))
+	mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024
+	t.Logf("average transfer duration: %s", avgDuration)
+	t.Logf("average transfer speed: %.2f MB/s", mbps)
+
+	for i := 0; i < len(connsSender); i++ {
+		err := connsSender[i].Close()
+		if err != nil {
+			t.Errorf("failed to close connection: %s", err)
+		}
+
+		err = connsReceiver[i].Close()
+		if err != nil {
+			t.Errorf("failed to close connection: %s", err)
+		}
+	}
+}
+
+func runTurnTest(t *testing.T, testData []byte, maxPairs int) {
+	t.Helper()
+	var transferDuration []time.Duration
+	var wg sync.WaitGroup
+
+	for i := 0; i < maxPairs; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			d := runTurnDataTransfer(t, testData)
+			transferDuration = append(transferDuration, d)
+		}()
+
+	}
+	wg.Wait()
+
+	var totalDuration time.Duration
+	for _, d := range transferDuration {
+		totalDuration += d
+	}
+	avgDuration := totalDuration / time.Duration(len(transferDuration))
+	mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024
+	t.Logf("average transfer duration: %s", avgDuration)
+	t.Logf("average transfer speed: %.2f MB/s", mbps)
+}
+
+func runTurnDataTransfer(t *testing.T, testData []byte) time.Duration {
+	t.Helper()
+	testDataLen := len(testData)
+	relayAddress := "192.168.0.10:3478"
+	conn, err := net.Dial("tcp", relayAddress)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func(conn net.Conn) {
+		_ = conn.Close()
+	}(conn)
+
+	turnClient, err := getTurnClient(t, relayAddress, conn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer turnClient.Close()
+
+	relayConn, err := turnClient.Allocate()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func(relayConn net.PacketConn) {
+		_ = relayConn.Close()
+	}(relayConn)
+
+	receiverConn, err := net.Dial("udp", relayConn.LocalAddr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func(receiverConn net.Conn) {
+		_ = receiverConn.Close()
+	}(receiverConn)
+
+	var (
+		tb        int
+		start     time.Time
+		timerInit bool
+		readDone  = make(chan struct{})
+		ack       = make([]byte, 1)
+	)
+	go func() {
+		defer func() {
+			readDone <- struct{}{}
+		}()
+		buff := make([]byte, 8192)
+		for {
+			n, e := receiverConn.Read(buff)
+			if e != nil {
+				return
+			}
+			if !timerInit {
+				start = time.Now()
+				timerInit = true
+			}
+			tb += n
+			_, _ = receiverConn.Write(ack)
+
+			if tb >= testDataLen {
+				return
+			}
+		}
+	}()
+
+	pieceSize := 1024
+	ackBuff := make([]byte, 1)
+	pipelineSize := 10
+	for j := 0; j < testDataLen; j += pieceSize {
+		end := j + pieceSize
+		if end > testDataLen {
+			end = testDataLen
+		}
+		_, err := relayConn.WriteTo(testData[j:end], receiverConn.LocalAddr())
+		if err != nil {
+			t.Fatalf("failed to write to channel: %s", err)
+		}
+		if pipelineSize == 0 {
+			_, _, _ = relayConn.ReadFrom(ackBuff)
+		} else {
+			pipelineSize--
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	select {
+	case <-readDone:
+		if tb != testDataLen {
+			t.Fatalf("failed to read all data: %d/%d", tb, testDataLen)
+		}
+	case <-ctx.Done():
+		t.Fatal("timeout")
+	}
+	return time.Since(start)
+}
+
+func getTurnClient(t *testing.T, address string, conn net.Conn) (*turn.Client, error) {
+	t.Helper()
+	// Dial TURN Server
+	addrStr := fmt.Sprintf("%s:%d", address, 443)
+
+	fac := logging.NewDefaultLoggerFactory()
+	//fac.DefaultLogLevel = logging.LogLevelTrace
+
+	// Start a new TURN Client and wrap our net.Conn in a STUNConn
+	// This allows us to simulate datagram based communication over a net.Conn
+	cfg := &turn.ClientConfig{
+		TURNServerAddr: address,
+		Conn:           turn.NewSTUNConn(conn),
+		Username:       "test",
+		Password:       "test",
+		LoggerFactory:  fac,
+	}
+
+	client, err := turn.NewClient(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err)
+	}
+
+	// Start listening on the conn provided.
+	err = client.Listen()
+	if err != nil {
+		client.Close()
+		return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err)
+	}
+
+	return client, nil
+}
+
+func seedRandomData(size int) ([]byte, error) {
+	token := make([]byte, size)
+	_, err := rand.Read(token)
+	if err != nil {
+		return nil, err
+	}
+	return token, nil
+}
+
+func waitForServerToStart(errChan chan error) error {
+	select {
+	case err := <-errChan:
+		if err != nil {
+			return err
+		}
+	case <-time.After(300 * time.Millisecond):
+		return nil
+	}
+	return nil
+}