Fix log

Rethink the peer reconnection implementation

.github/workflows/golang-test-darwin.yml

@@ -42,4 +42,4 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...
+        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...

.github/workflows/golang-test-linux.yml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         arch: [ '386','amd64' ]
         store: [ 'sqlite', 'postgres']
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Install Go
         uses: actions/setup-go@v5
@@ -49,7 +49,7 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 6m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
 
   test_client_on_docker:
     runs-on: ubuntu-20.04
@@ -79,9 +79,6 @@ jobs:
       - name: check git status
         run: git --no-pager diff --exit-code
 
-      - name: Generate Iface Test bin
-        run: CGO_ENABLED=0 go test -c -o iface-testing.bin ./client/iface/
-
       - name: Generate Shared Sock Test bin
         run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock
 
@@ -98,7 +95,7 @@ jobs:
         run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal
 
       - name: Generate Peer Test bin
-        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/...
+        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/
 
       - run: chmod +x *testing.bin
 
@@ -106,7 +103,7 @@ jobs:
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/sharedsock --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/sharedsock-testing.bin -test.timeout 5m -test.parallel 1
 
       - name: Run Iface tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/iface --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/iface-testing.bin -test.timeout 5m -test.parallel 1
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/netbird -v /tmp/cache:/tmp/cache -v /tmp/modcache:/tmp/modcache -w /netbird -e GOCACHE=/tmp/cache -e GOMODCACHE=/tmp/modcache -e CGO_ENABLED=0 golang:1.23-alpine go test  -test.timeout 5m -test.parallel 1 ./client/iface/...
 
       - name: Run RouteManager tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/routemanager-testing.bin  -test.timeout 5m -test.parallel 1

.github/workflows/golangci-lint.yml

@@ -19,7 +19,7 @@ jobs:
       - name: codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          ignore_words_list: erro,clienta,hastable,iif
+          ignore_words_list: erro,clienta,hastable,iif,groupd
           skip: go.mod,go.sum
           only_warn: 1
   golangci:

.github/workflows/release.yml

@@ -9,7 +9,7 @@ on:
   pull_request:
 
 env:
-  SIGN_PIPE_VER: "v0.0.14"
+  SIGN_PIPE_VER: "v0.0.16"
   GORELEASER_VER: "v2.3.2"
   PRODUCT_NAME: "NetBird"
   COPYRIGHT: "Wiretrustee UG (haftungsbeschreankt)"
@@ -20,7 +20,7 @@ concurrency:
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       flags: ""
     steps:
@@ -223,4 +223,4 @@ jobs:
           repo: netbirdio/sign-pipelines
           ref: ${{ env.SIGN_PIPE_VER }}
           token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'
+          inputs: '{ "tag": "${{ github.ref }}", "skipRelease": false }'

.goreleaser.yaml

@@ -96,6 +96,9 @@ builds:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
     mod_timestamp: "{{ .CommitTimestamp }}"
 
+universal_binaries:
+  - id: netbird
+
 archives:
   - builds:
       - netbird

.goreleaser_ui_darwin.yaml

@@ -23,6 +23,9 @@ builds:
     tags:
       - load_wgnt_from_rsrc
 
+universal_binaries:
+  - id: netbird-ui-darwin
+
 archives:
   - builds:
       - netbird-ui-darwin

README.md

@@ -49,6 +49,8 @@
 
 ![netbird_2](https://github.com/netbirdio/netbird/assets/700848/46bc3b73-508d-4a0e-bb9a-f465d68646ab)
 
+### NetBird on Lawrence Systems (Video)
+[![Watch the video](https://img.youtube.com/vi/Kwrff6h0rEw/0.jpg)](https://www.youtube.com/watch?v=Kwrff6h0rEw)
 
 ### Key features
 
@@ -62,6 +64,7 @@
 |                                                                                                                              |                                                                                                          | <ul><li> - \[x] [Quantum-resistance with Rosenpass](https://netbird.io/knowledge-hub/the-first-quantum-resistant-mesh-vpn) </ul></li> |                                                                                                                                          | <ul><li> - \[x] OpenWRT </ul></li>                                                      |
 |                                                                                                                              |                                                                                                          | <ui><li> - \[x] [Periodic re-authentication](https://docs.netbird.io/how-to/enforce-periodic-user-authentication)</ul></li>           |                                                                                                                                          | <ul><li> - \[x] [Serverless](https://docs.netbird.io/how-to/netbird-on-faas) </ul></li> |
 |                                                                                                                              |                                                                                                          |                                                                                                                                       |                                                                                                                                          | <ul><li> - \[x] Docker </ul></li>                                                       |
+
 ### Quickstart with NetBird Cloud
 
 - Download and install NetBird at [https://app.netbird.io/install](https://app.netbird.io/install)

client/cmd/testutil_test.go

@@ -38,7 +38,7 @@ func startTestingServices(t *testing.T) string {
 	signalAddr := signalLis.Addr().String()
 	config.Signal.URI = signalAddr
 
-	_, mgmLis := startManagement(t, config, "../testdata/store.sqlite")
+	_, mgmLis := startManagement(t, config, "../testdata/store.sql")
 	mgmAddr := mgmLis.Addr().String()
 	return mgmAddr
 }
@@ -71,7 +71,7 @@ func startManagement(t *testing.T, config *mgmt.Config, testFile string) (*grpc.
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, cleanUp, err := mgmt.NewTestStoreFromSqlite(context.Background(), testFile, t.TempDir())
+	store, cleanUp, err := mgmt.NewTestStoreFromSQL(context.Background(), testFile, t.TempDir())
 	if err != nil {
 		t.Fatal(err)
 	}

client/firewall/create.go

@@ -3,7 +3,6 @@
 package firewall
 
 import (
-	"context"
 	"fmt"
 	"runtime"
 
@@ -11,10 +10,11 @@ import (
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 // NewFirewall creates a firewall manager instance
-func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, _ *statemanager.Manager) (firewall.Manager, error) {
 	if !iface.IsUserspaceBind() {
 		return nil, fmt.Errorf("not implemented for this OS: %s", runtime.GOOS)
 	}

client/firewall/create_linux.go

@@ -3,7 +3,7 @@
 package firewall
 
 import (
-	"context"
+	"errors"
 	"fmt"
 	"os"
 
@@ -15,6 +15,7 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	nbnftables "github.com/netbirdio/netbird/client/firewall/nftables"
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -32,54 +33,65 @@ const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK"
 // FWType is the type for the firewall type
 type FWType int
 
-func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
 	// on the linux system we try to user nftables or iptables
 	// in any case, because we need to allow netbird interface traffic
 	// so we use AllowNetbird traffic from these firewall managers
 	// for the userspace packet filtering firewall
-	var fm firewall.Manager
-	var errFw error
+	fm, err := createNativeFirewall(iface, stateManager)
 
+	if !iface.IsUserspaceBind() {
+		return fm, err
+	}
+
+	if err != nil {
+		log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err)
+	}
+	return createUserspaceFirewall(iface, fm)
+}
+
+func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+	fm, err := createFW(iface)
+	if err != nil {
+		return nil, fmt.Errorf("create firewall: %s", err)
+	}
+
+	if err = fm.Init(stateManager); err != nil {
+		return nil, fmt.Errorf("init firewall: %s", err)
+	}
+
+	return fm, nil
+}
+
+func createFW(iface IFaceMapper) (firewall.Manager, error) {
 	switch check() {
 	case IPTABLES:
 		log.Info("creating an iptables firewall manager")
-		fm, errFw = nbiptables.Create(context, iface)
-		if errFw != nil {
-			log.Errorf("failed to create iptables manager: %s", errFw)
-		}
+		return nbiptables.Create(iface)
 	case NFTABLES:
 		log.Info("creating an nftables firewall manager")
-		fm, errFw = nbnftables.Create(context, iface)
-		if errFw != nil {
-			log.Errorf("failed to create nftables manager: %s", errFw)
-		}
+		return nbnftables.Create(iface)
 	default:
-		errFw = fmt.Errorf("no firewall manager found")
 		log.Info("no firewall manager found, trying to use userspace packet filtering firewall")
+		return nil, errors.New("no firewall manager found")
+	}
+}
+
+func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager) (firewall.Manager, error) {
+	var errUsp error
+	if fm != nil {
+		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
+	} else {
+		fm, errUsp = uspfilter.Create(iface)
 	}
 
-	if iface.IsUserspaceBind() {
-		var errUsp error
-		if errFw == nil {
-			fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
-		} else {
-			fm, errUsp = uspfilter.Create(iface)
-		}
-		if errUsp != nil {
-			log.Debugf("failed to create userspace filtering firewall: %s", errUsp)
-			return nil, errUsp
-		}
-
-		if err := fm.AllowNetbird(); err != nil {
-			log.Errorf("failed to allow netbird interface traffic: %v", err)
-		}
-		return fm, nil
+	if errUsp != nil {
+		return nil, fmt.Errorf("create userspace firewall: %s", errUsp)
 	}
 
-	if errFw != nil {
-		return nil, errFw
+	if err := fm.AllowNetbird(); err != nil {
+		log.Errorf("failed to allow netbird interface traffic: %v", err)
 	}
-
 	return fm, nil
 }
 

client/firewall/iptables/acl_linux.go

@@ -11,6 +11,8 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
@@ -21,13 +23,23 @@ const (
 	chainNameOutputRules = "NETBIRD-ACL-OUTPUT"
 )
 
+type aclEntries map[string][][]string
+
+type entry struct {
+	spec     []string
+	position int
+}
+
 type aclManager struct {
 	iptablesClient     *iptables.IPTables
 	wgIface            iFaceMapper
 	routingFwChainName string
 
-	entries    map[string][][]string
-	ipsetStore *ipsetStore
+	entries         aclEntries
+	optionalEntries map[string][]entry
+	ipsetStore      *ipsetStore
+
+	stateManager *statemanager.Manager
 }
 
 func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routingFwChainName string) (*aclManager, error) {
@@ -36,27 +48,35 @@ func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routi
 		wgIface:            wgIface,
 		routingFwChainName: routingFwChainName,
 
-		entries:    make(map[string][][]string),
-		ipsetStore: newIpsetStore(),
+		entries:         make(map[string][][]string),
+		optionalEntries: make(map[string][]entry),
+		ipsetStore:      newIpsetStore(),
 	}
 
-	err := ipset.Init()
-	if err != nil {
-		return nil, fmt.Errorf("failed to init ipset: %w", err)
+	if err := ipset.Init(); err != nil {
+		return nil, fmt.Errorf("init ipset: %w", err)
 	}
 
+	return m, nil
+}
+
+func (m *aclManager) init(stateManager *statemanager.Manager) error {
+	m.stateManager = stateManager
+
 	m.seedInitialEntries()
+	m.seedInitialOptionalEntries()
 
-	err = m.cleanChains()
-	if err != nil {
-		return nil, err
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
 	}
 
-	err = m.createDefaultChains()
-	if err != nil {
-		return nil, err
+	if err := m.createDefaultChains(); err != nil {
+		return fmt.Errorf("create default chains: %w", err)
 	}
-	return m, nil
+
+	m.updateState()
+
+	return nil
 }
 
 func (m *aclManager) AddPeerFiltering(
@@ -137,6 +157,8 @@ func (m *aclManager) AddPeerFiltering(
 		chain:     chain,
 	}
 
+	m.updateState()
+
 	return []firewall.Rule{rule}, nil
 }
 
@@ -171,15 +193,23 @@ func (m *aclManager) DeletePeerRule(rule firewall.Rule) error {
 		}
 	}
 
-	err := m.iptablesClient.Delete(tableName, r.chain, r.specs...)
-	if err != nil {
-		log.Debugf("failed to delete rule, %s, %v: %s", r.chain, r.specs, err)
+	if err := m.iptablesClient.Delete(tableName, r.chain, r.specs...); err != nil {
+		return fmt.Errorf("failed to delete rule: %s, %v: %w", r.chain, r.specs, err)
 	}
-	return err
+
+	m.updateState()
+
+	return nil
 }
 
 func (m *aclManager) Reset() error {
-	return m.cleanChains()
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
+	}
+
+	m.updateState()
+
+	return nil
 }
 
 // todo write less destructive cleanup mechanism
@@ -232,6 +262,19 @@ func (m *aclManager) cleanChains() error {
 		}
 	}
 
+	ok, err = m.iptablesClient.ChainExists("mangle", "PREROUTING")
+	if err != nil {
+		return fmt.Errorf("list chains: %w", err)
+	}
+	if ok {
+		for _, rule := range m.entries["PREROUTING"] {
+			err := m.iptablesClient.DeleteIfExists("mangle", "PREROUTING", rule...)
+			if err != nil {
+				log.Errorf("failed to delete rule: %v, %s", rule, err)
+			}
+		}
+	}
+
 	for _, ipsetName := range m.ipsetStore.ipsetNames() {
 		if err := ipset.Flush(ipsetName); err != nil {
 			log.Errorf("flush ipset %q during reset: %v", ipsetName, err)
@@ -267,6 +310,17 @@ func (m *aclManager) createDefaultChains() error {
 		}
 	}
 
+	for chainName, entries := range m.optionalEntries {
+		for _, entry := range entries {
+			if err := m.iptablesClient.InsertUnique(tableName, chainName, entry.position, entry.spec...); err != nil {
+				log.Errorf("failed to insert optional entry %v: %v", entry.spec, err)
+				continue
+			}
+			m.entries[chainName] = append(m.entries[chainName], entry.spec)
+		}
+	}
+	clear(m.optionalEntries)
+
 	return nil
 }
 
@@ -295,10 +349,52 @@ func (m *aclManager) seedInitialEntries() {
 	m.appendToEntries("FORWARD", append([]string{"-o", m.wgIface.Name()}, established...))
 }
 
+func (m *aclManager) seedInitialOptionalEntries() {
+	m.optionalEntries["FORWARD"] = []entry{
+		{
+			spec:     []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmark), "-j", chainNameInputRules},
+			position: 2,
+		},
+	}
+
+	m.optionalEntries["PREROUTING"] = []entry{
+		{
+			spec:     []string{"-t", "mangle", "-i", m.wgIface.Name(), "-m", "addrtype", "--dst-type", "LOCAL", "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmark)},
+			position: 1,
+		},
+	}
+}
+
 func (m *aclManager) appendToEntries(chainName string, spec []string) {
 	m.entries[chainName] = append(m.entries[chainName], spec)
 }
 
+func (m *aclManager) updateState() {
+	if m.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := m.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.ACLEntries = m.entries
+	currentState.ACLIPsetStore = m.ipsetStore
+
+	if err := m.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
 // filterRuleSpecs returns the specs of a filtering rule
 func filterRuleSpecs(
 	ip net.IP, protocol string, sPort, dPort string, direction firewall.RuleDirection, action firewall.Action, ipsetName string,

client/firewall/iptables/manager_linux.go

@@ -8,10 +8,13 @@ import (
 	"sync"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 // Manager of iptables firewall
@@ -33,10 +36,10 @@ type iFaceMapper interface {
 }
 
 // Create iptables firewall manager
-func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	if err != nil {
-		return nil, fmt.Errorf("iptables is not installed in the system or not supported")
+		return nil, fmt.Errorf("init iptables: %w", err)
 	}
 
 	m := &Manager{
@@ -44,20 +47,49 @@ func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
 		ipv4Client: iptablesClient,
 	}
 
-	m.router, err = newRouter(context, iptablesClient, wgIface)
+	m.router, err = newRouter(iptablesClient, wgIface)
 	if err != nil {
-		log.Debugf("failed to initialize route related chains: %s", err)
-		return nil, err
+		return nil, fmt.Errorf("create router: %w", err)
 	}
+
 	m.aclMgr, err = newAclManager(iptablesClient, wgIface, chainRTFWD)
 	if err != nil {
-		log.Debugf("failed to initialize ACL manager: %s", err)
-		return nil, err
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	state := &ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}
+	stateManager.RegisterState(state)
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	if err := m.router.init(stateManager); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclMgr.init(stateManager); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	// persist early to ensure cleanup of chains
+	if err := stateManager.PersistState(context.Background()); err != nil {
+		log.Errorf("failed to persist state: %v", err)
+	}
+
+	return nil
+}
+
 // AddPeerFiltering adds a rule to the firewall
 //
 // Comment will be ignored because some system this feature is not supported
@@ -78,7 +110,7 @@ func (m *Manager) AddPeerFiltering(
 }
 
 func (m *Manager) AddRouteFiltering(
-	sources [] netip.Prefix,
+	sources []netip.Prefix,
 	destination netip.Prefix,
 	proto firewall.Protocol,
 	sPort *firewall.Port,
@@ -133,20 +165,27 @@ func (m *Manager) SetLegacyManagement(isLegacy bool) error {
 }
 
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	errAcl := m.aclMgr.Reset()
-	if errAcl != nil {
-		log.Errorf("failed to clean up ACL rules from firewall: %s", errAcl)
+	var merr *multierror.Error
+
+	if err := m.aclMgr.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset acl manager: %w", err))
 	}
-	errMgr := m.router.Reset()
-	if errMgr != nil {
-		log.Errorf("failed to clean up router rules from firewall: %s", errMgr)
-		return errMgr
+	if err := m.router.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset router: %w", err))
 	}
-	return errAcl
+
+	// attempt to delete state only if all other operations succeeded
+	if merr == nil {
+		if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("delete state: %w", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 // AllowNetbird allows netbird interface traffic

client/firewall/iptables/manager_linux_test.go

@@ -1,7 +1,6 @@
 package iptables
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"testing"
@@ -56,13 +55,14 @@ func TestIptablesManager(t *testing.T) {
 	require.NoError(t, err)
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), ifaceMock)
+	manager, err := Create(ifaceMock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 
 	time.Sleep(time.Second)
 
 	defer func() {
-		err := manager.Reset()
+		err := manager.Reset(nil)
 		require.NoError(t, err, "clear the manager state")
 
 		time.Sleep(time.Second)
@@ -122,7 +122,7 @@ func TestIptablesManager(t *testing.T) {
 		_, err = manager.AddPeerFiltering(ip, "udp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept Fake DNS traffic")
 		require.NoError(t, err, "failed to add rule")
 
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 
 		ok, err := ipv4Client.ChainExists("filter", chainNameInputRules)
@@ -154,13 +154,14 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	}
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), mock)
+	manager, err := Create(mock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 
 	time.Sleep(time.Second)
 
 	defer func() {
-		err := manager.Reset()
+		err := manager.Reset(nil)
 		require.NoError(t, err, "clear the manager state")
 
 		time.Sleep(time.Second)
@@ -219,7 +220,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	})
 
 	t.Run("reset check", func(t *testing.T) {
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 	})
 }
@@ -251,12 +252,13 @@ func TestIptablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(context.Background(), mock)
+			manager, err := Create(mock)
 			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second)
 
 			defer func() {
-				err := manager.Reset()
+				err := manager.Reset(nil)
 				require.NoError(t, err, "clear the manager state")
 
 				time.Sleep(time.Second)

client/firewall/iptables/router_linux.go

@@ -3,7 +3,6 @@
 package iptables
 
 import (
-	"context"
 	"fmt"
 	"net/netip"
 	"strconv"
@@ -18,6 +17,7 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl/id"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -48,28 +48,31 @@ type routeFilteringRuleParams struct {
 	SetName     string
 }
 
+type routeRules map[string][]string
+
+type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}]
+
 type router struct {
-	ctx              context.Context
-	stop             context.CancelFunc
 	iptablesClient   *iptables.IPTables
-	rules            map[string][]string
-	ipsetCounter     *refcounter.Counter[string, []netip.Prefix, struct{}]
+	rules            routeRules
+	ipsetCounter     *ipsetCounter
 	wgIface          iFaceMapper
 	legacyManagement bool
+
+	stateManager *statemanager.Manager
 }
 
-func newRouter(parentCtx context.Context, iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
-	ctx, cancel := context.WithCancel(parentCtx)
+func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
 	r := &router{
-		ctx:            ctx,
-		stop:           cancel,
 		iptablesClient: iptablesClient,
 		rules:          make(map[string][]string),
 		wgIface:        wgIface,
 	}
 
 	r.ipsetCounter = refcounter.New(
-		r.createIpSet,
+		func(name string, sources []netip.Prefix) (struct{}, error) {
+			return struct{}{}, r.createIpSet(name, sources)
+		},
 		func(name string, _ struct{}) error {
 			return r.deleteIpSet(name)
 		},
@@ -79,16 +82,23 @@ func newRouter(parentCtx context.Context, iptablesClient *iptables.IPTables, wgI
 		return nil, fmt.Errorf("init ipset: %w", err)
 	}
 
-	err := r.cleanUpDefaultForwardRules()
-	if err != nil {
-		log.Errorf("cleanup routing rules: %s", err)
-		return nil, err
+	return r, nil
+}
+
+func (r *router) init(stateManager *statemanager.Manager) error {
+	r.stateManager = stateManager
+
+	if err := r.cleanUpDefaultForwardRules(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
-	err = r.createContainers()
-	if err != nil {
-		log.Errorf("create containers for route: %s", err)
+
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
 	}
-	return r, err
+
+	r.updateState()
+
+	return nil
 }
 
 func (r *router) AddRouteFiltering(
@@ -129,6 +139,8 @@ func (r *router) AddRouteFiltering(
 
 	r.rules[string(ruleKey)] = rule
 
+	r.updateState()
+
 	return ruleKey, nil
 }
 
@@ -152,6 +164,8 @@ func (r *router) DeleteRouteRule(rule firewall.Rule) error {
 		log.Debugf("route rule %s not found", ruleKey)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
@@ -164,18 +178,18 @@ func (r *router) findSetNameInRule(rule []string) string {
 	return ""
 }
 
-func (r *router) createIpSet(setName string, sources []netip.Prefix) (struct{}, error) {
+func (r *router) createIpSet(setName string, sources []netip.Prefix) error {
 	if err := ipset.Create(setName, ipset.OptTimeout(0)); err != nil {
-		return struct{}{}, fmt.Errorf("create set %s: %w", setName, err)
+		return fmt.Errorf("create set %s: %w", setName, err)
 	}
 
 	for _, prefix := range sources {
 		if err := ipset.AddPrefix(setName, prefix); err != nil {
-			return struct{}{}, fmt.Errorf("add element to set %s: %w", setName, err)
+			return fmt.Errorf("add element to set %s: %w", setName, err)
 		}
 	}
 
-	return struct{}{}, nil
+	return nil
 }
 
 func (r *router) deleteIpSet(setName string) error {
@@ -206,6 +220,8 @@ func (r *router) AddNatRule(pair firewall.RouterPair) error {
 		return fmt.Errorf("add inverse nat rule: %w", err)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
@@ -223,6 +239,8 @@ func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
 		return fmt.Errorf("remove legacy routing rule: %w", err)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
@@ -278,8 +296,13 @@ func (r *router) RemoveAllLegacyRouteRules() error {
 		}
 		if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWD, rule...); err != nil {
 			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
 		}
 	}
+
+	r.updateState()
+
 	return nberrors.FormatErrorOrNil(merr)
 }
 
@@ -294,6 +317,8 @@ func (r *router) Reset() error {
 		merr = multierror.Append(merr, err)
 	}
 
+	r.updateState()
+
 	return nberrors.FormatErrorOrNil(merr)
 }
 
@@ -305,10 +330,7 @@ func (r *router) cleanUpDefaultForwardRules() error {
 
 	log.Debug("flushing routing related tables")
 	for _, chain := range []string{chainRTFWD, chainRTNAT} {
-		table := tableFilter
-		if chain == chainRTNAT {
-			table = tableNat
-		}
+		table := r.getTableForChain(chain)
 
 		ok, err := r.iptablesClient.ChainExists(table, chain)
 		if err != nil {
@@ -329,15 +351,19 @@ func (r *router) cleanUpDefaultForwardRules() error {
 func (r *router) createContainers() error {
 	for _, chain := range []string{chainRTFWD, chainRTNAT} {
 		if err := r.createAndSetupChain(chain); err != nil {
-			return fmt.Errorf("create chain %s: %v", chain, err)
+			return fmt.Errorf("create chain %s: %w", chain, err)
 		}
 	}
 
 	if err := r.insertEstablishedRule(chainRTFWD); err != nil {
-		return fmt.Errorf("insert established rule: %v", err)
+		return fmt.Errorf("insert established rule: %w", err)
 	}
 
-	return r.addJumpRules()
+	if err := r.addJumpRules(); err != nil {
+		return fmt.Errorf("add jump rules: %w", err)
+	}
+
+	return nil
 }
 
 func (r *router) createAndSetupChain(chain string) error {
@@ -430,12 +456,40 @@ func (r *router) removeNatRule(pair firewall.RouterPair) error {
 	return nil
 }
 
+func (r *router) updateState() {
+	if r.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := r.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.RouteRules = r.rules
+	currentState.RouteIPsetCounter = r.ipsetCounter
+
+	if err := r.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
 func genRuleSpec(jump string, source, destination netip.Prefix, intf string, inverse bool) []string {
 	intdir := "-i"
+	lointdir := "-o"
 	if inverse {
 		intdir = "-o"
+		lointdir = "-i"
 	}
-	return []string{intdir, intf, "-s", source.String(), "-d", destination.String(), "-j", jump}
+	return []string{intdir, intf, "!", lointdir, "lo", "-s", source.String(), "-d", destination.String(), "-j", jump}
 }
 
 func genRouteFilteringRuleSpec(params routeFilteringRuleParams) []string {

client/firewall/iptables/router_linux_test.go

@@ -3,7 +3,6 @@
 package iptables
 
 import (
-	"context"
 	"net/netip"
 	"os/exec"
 	"testing"
@@ -30,8 +29,9 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "failed to init iptables client")
 
-	manager, err := newRouter(context.TODO(), iptablesClient, ifaceMock)
+	manager, err := newRouter(iptablesClient, ifaceMock)
 	require.NoError(t, err, "should return a valid iptables manager")
+	require.NoError(t, manager.init(nil))
 
 	defer func() {
 		_ = manager.Reset()
@@ -74,8 +74,9 @@ func TestIptablesManager_AddNatRule(t *testing.T) {
 			iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 			require.NoError(t, err, "failed to init iptables client")
 
-			manager, err := newRouter(context.TODO(), iptablesClient, ifaceMock)
+			manager, err := newRouter(iptablesClient, ifaceMock)
 			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
 
 			defer func() {
 				err := manager.Reset()
@@ -132,8 +133,9 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) {
 		t.Run(testCase.Name, func(t *testing.T) {
 			iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 
-			manager, err := newRouter(context.TODO(), iptablesClient, ifaceMock)
+			manager, err := newRouter(iptablesClient, ifaceMock)
 			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
 			defer func() {
 				_ = manager.Reset()
 			}()
@@ -183,8 +185,9 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "Failed to create iptables client")
 
-	r, err := newRouter(context.Background(), iptablesClient, ifaceMock)
+	r, err := newRouter(iptablesClient, ifaceMock)
 	require.NoError(t, err, "Failed to create router manager")
+	require.NoError(t, r.init(nil))
 
 	defer func() {
 		err := r.Reset()

client/firewall/iptables/rulestore_linux.go

@@ -1,14 +1,16 @@
 package iptables
 
+import "encoding/json"
+
 type ipList struct {
 	ips map[string]struct{}
 }
 
-func newIpList(ip string) ipList {
+func newIpList(ip string) *ipList {
 	ips := make(map[string]struct{})
 	ips[ip] = struct{}{}
 
-	return ipList{
+	return &ipList{
 		ips: ips,
 	}
 }
@@ -17,27 +19,47 @@ func (s *ipList) addIP(ip string) {
 	s.ips[ip] = struct{}{}
 }
 
+// MarshalJSON implements json.Marshaler
+func (s *ipList) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{
+		IPs: s.ips,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipList) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ips = temp.IPs
+	return nil
+}
+
 type ipsetStore struct {
-	ipsets map[string]ipList // ipsetName -> ruleset
+	ipsets map[string]*ipList
 }
 
 func newIpsetStore() *ipsetStore {
 	return &ipsetStore{
-		ipsets: make(map[string]ipList),
+		ipsets: make(map[string]*ipList),
 	}
 }
 
-func (s *ipsetStore) ipset(ipsetName string) (ipList, bool) {
+func (s *ipsetStore) ipset(ipsetName string) (*ipList, bool) {
 	r, ok := s.ipsets[ipsetName]
 	return r, ok
 }
 
-func (s *ipsetStore) addIpList(ipsetName string, list ipList) {
+func (s *ipsetStore) addIpList(ipsetName string, list *ipList) {
 	s.ipsets[ipsetName] = list
 }
 
 func (s *ipsetStore) deleteIpset(ipsetName string) {
-	s.ipsets[ipsetName] = ipList{}
 	delete(s.ipsets, ipsetName)
 }
 
@@ -48,3 +70,24 @@ func (s *ipsetStore) ipsetNames() []string {
 	}
 	return names
 }
+
+// MarshalJSON implements json.Marshaler
+func (s *ipsetStore) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{
+		IPSets: s.ipsets,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipsetStore) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ipsets = temp.IPSets
+	return nil
+}

client/firewall/iptables/state_linux.go

@@ -0,0 +1,70 @@
+package iptables
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	sync.Mutex
+
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+
+	RouteRules        routeRules    `json:"route_rules,omitempty"`
+	RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"`
+
+	ACLEntries    aclEntries  `json:"acl_entries,omitempty"`
+	ACLIPsetStore *ipsetStore `json:"acl_ipset_store,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "iptables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	ipt, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create iptables manager: %w", err)
+	}
+
+	if s.RouteRules != nil {
+		ipt.router.rules = s.RouteRules
+	}
+	if s.RouteIPsetCounter != nil {
+		ipt.router.ipsetCounter.LoadData(s.RouteIPsetCounter)
+	}
+
+	if s.ACLEntries != nil {
+		ipt.aclMgr.entries = s.ACLEntries
+	}
+	if s.ACLIPsetStore != nil {
+		ipt.aclMgr.ipsetStore = s.ACLIPsetStore
+	}
+
+	if err := ipt.Reset(nil); err != nil {
+		return fmt.Errorf("reset iptables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/manager/firewall.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -52,6 +54,8 @@ const (
 // It declares methods which handle actions required by the
 // Netbird client for ACL and routing functionality
 type Manager interface {
+	Init(stateManager *statemanager.Manager) error
+
 	// AllowNetbird allows netbird interface traffic
 	AllowNetbird() error
 
@@ -91,7 +95,7 @@ type Manager interface {
 	SetLegacyManagement(legacy bool) error
 
 	// Reset firewall to the default state
-	Reset() error
+	Reset(stateManager *statemanager.Manager) error
 
 	// Flush the changes to firewall controller
 	Flush() error
@@ -132,7 +136,7 @@ func SetLegacyManagement(router LegacyManager, isLegacy bool) error {
 // GenerateSetName generates a unique name for an ipset based on the given sources.
 func GenerateSetName(sources []netip.Prefix) string {
 	// sort for consistent naming
-	sortPrefixes(sources)
+	SortPrefixes(sources)
 
 	var sourcesStr strings.Builder
 	for _, src := range sources {
@@ -170,9 +174,9 @@ func MergeIPRanges(prefixes []netip.Prefix) []netip.Prefix {
 	return merged
 }
 
-// sortPrefixes sorts the given slice of netip.Prefix in place.
+// SortPrefixes sorts the given slice of netip.Prefix in place.
 // It sorts first by IP address, then by prefix length (most specific to least specific).
-func sortPrefixes(prefixes []netip.Prefix) {
+func SortPrefixes(prefixes []netip.Prefix) {
 	sort.Slice(prefixes, func(i, j int) bool {
 		addrCmp := prefixes[i].Addr().Compare(prefixes[j].Addr())
 		if addrCmp != 0 {

client/firewall/nftables/acl_linux.go

@@ -11,12 +11,13 @@ import (
 	"time"
 
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/client/iface"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
@@ -29,6 +30,7 @@ const (
 	chainNameInputFilter   = "netbird-acl-input-filter"
 	chainNameOutputFilter  = "netbird-acl-output-filter"
 	chainNameForwardFilter = "netbird-acl-forward-filter"
+	chainNamePrerouting    = "netbird-rt-prerouting"
 
 	allowNetbirdInputRuleID = "allow Netbird incoming traffic"
 )
@@ -40,54 +42,44 @@ var (
 )
 
 type AclManager struct {
-	rConn               *nftables.Conn
-	sConn               *nftables.Conn
-	wgIface             iFaceMapper
-	routeingFwChainName string
+	rConn              *nftables.Conn
+	sConn              *nftables.Conn
+	wgIface            iFaceMapper
+	routingFwChainName string
 
 	workTable        *nftables.Table
 	chainInputRules  *nftables.Chain
 	chainOutputRules *nftables.Chain
-	chainFwFilter    *nftables.Chain
 
 	ipsetStore *ipsetStore
 	rules      map[string]*Rule
 }
 
-// iFaceMapper defines subset methods of interface required for manager
-type iFaceMapper interface {
-	Name() string
-	Address() iface.WGAddress
-	IsUserspaceBind() bool
-}
-
-func newAclManager(table *nftables.Table, wgIface iFaceMapper, routeingFwChainName string) (*AclManager, error) {
+func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) {
 	// sConn is used for creating sets and adding/removing elements from them
 	// it's differ then rConn (which does create new conn for each flush operation)
 	// and is permanent. Using same connection for both type of operations
 	// overloads netlink with high amount of rules ( > 10000)
 	sConn, err := nftables.New(nftables.AsLasting())
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create nf conn: %w", err)
 	}
 
-	m := &AclManager{
-		rConn:               &nftables.Conn{},
-		sConn:               sConn,
-		wgIface:             wgIface,
-		workTable:           table,
-		routeingFwChainName: routeingFwChainName,
+	return &AclManager{
+		rConn:              &nftables.Conn{},
+		sConn:              sConn,
+		wgIface:            wgIface,
+		workTable:          table,
+		routingFwChainName: routingFwChainName,
 
 		ipsetStore: newIpsetStore(),
 		rules:      make(map[string]*Rule),
-	}
+	}, nil
+}
 
-	err = m.createDefaultChains()
-	if err != nil {
-		return nil, err
-	}
-
-	return m, nil
+func (m *AclManager) init(workTable *nftables.Table) error {
+	m.workTable = workTable
+	return m.createDefaultChains()
 }
 
 // AddPeerFiltering rule to the firewall
@@ -462,9 +454,9 @@ func (m *AclManager) createDefaultChains() (err error) {
 	}
 
 	// netbird-acl-forward-filter
-	m.chainFwFilter = m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
-	m.addJumpRulesToRtForward() // to netbird-rt-fwd
-	m.addDropExpressions(m.chainFwFilter, expr.MetaKeyIIFNAME)
+	chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
+	m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd
+	m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME)
 
 	err = m.rConn.Flush()
 	if err != nil {
@@ -472,10 +464,96 @@ func (m *AclManager) createDefaultChains() (err error) {
 		return fmt.Errorf(flushError, err)
 	}
 
+	if err := m.allowRedirectedTraffic(chainFwFilter); err != nil {
+		log.Errorf("failed to allow redirected traffic: %s", err)
+	}
+
 	return nil
 }
 
-func (m *AclManager) addJumpRulesToRtForward() {
+// Makes redirected traffic originally destined for the host itself (now subject to the forward filter)
+// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the
+// netbird peer IP.
+func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error {
+	preroutingChain := m.rConn.AddChain(&nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    m.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	})
+
+	m.addPreroutingRule(preroutingChain)
+
+	m.addFwmarkToForward(chainFwFilter)
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (m *AclManager) addPreroutingRule(preroutingChain *nftables.Chain) {
+	m.rConn.AddRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: preroutingChain,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyIIFNAME,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     ifname(m.wgIface.Name()),
+			},
+			&expr.Fib{
+				Register:       1,
+				ResultADDRTYPE: true,
+				FlagDADDR:      true,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL),
+			},
+			&expr.Immediate{
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmark),
+			},
+			&expr.Meta{
+				Key:            expr.MetaKeyMARK,
+				Register:       1,
+				SourceRegister: true,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) {
+	m.rConn.InsertRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: chainFwFilter,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyMARK,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmark),
+			},
+			&expr.Verdict{
+				Kind:  expr.VerdictJump,
+				Chain: m.chainInputRules.Name,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) {
 	expressions := []expr.Any{
 		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
 		&expr.Cmp{
@@ -485,13 +563,13 @@ func (m *AclManager) addJumpRulesToRtForward() {
 		},
 		&expr.Verdict{
 			Kind:  expr.VerdictJump,
-			Chain: m.routeingFwChainName,
+			Chain: m.routingFwChainName,
 		},
 	}
 
 	_ = m.rConn.AddRule(&nftables.Rule{
 		Table: m.workTable,
-		Chain: m.chainFwFilter,
+		Chain: chainFwFilter,
 		Exprs: expressions,
 	})
 }
@@ -509,7 +587,7 @@ func (m *AclManager) createChain(name string) *nftables.Chain {
 	return chain
 }
 
-func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.ChainHook) *nftables.Chain {
+func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain {
 	polAccept := nftables.ChainPolicyAccept
 	chain := &nftables.Chain{
 		Name:     name,

client/firewall/nftables/manager_linux.go

@@ -14,6 +14,8 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -24,6 +26,13 @@ const (
 	chainNameInput  = "INPUT"
 )
 
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMapper interface {
+	Name() string
+	Address() iface.WGAddress
+	IsUserspaceBind() bool
+}
+
 // Manager of iptables firewall
 type Manager struct {
 	mutex   sync.Mutex
@@ -35,30 +44,68 @@ type Manager struct {
 }
 
 // Create nftables firewall manager
-func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
 	m := &Manager{
 		rConn:   &nftables.Conn{},
 		wgIface: wgIface,
 	}
 
-	workTable, err := m.createWorkTable()
-	if err != nil {
-		return nil, err
-	}
+	workTable := &nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4}
 
-	m.router, err = newRouter(context, workTable, wgIface)
+	var err error
+	m.router, err = newRouter(workTable, wgIface)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create router: %w", err)
 	}
 
 	m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
+// Init nftables firewall manager
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	workTable, err := m.createWorkTable()
+	if err != nil {
+		return fmt.Errorf("create work table: %w", err)
+	}
+
+	if err := m.router.init(workTable); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclManager.init(workTable); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	stateManager.RegisterState(&ShutdownState{})
+
+	// We only need to record minimal interface state for potential recreation.
+	// Unlike iptables, which requires tracking individual rules, nftables maintains
+	// a known state (our netbird table plus a few static rules). This allows for easy
+	// cleanup using Reset() without needing to store specific rules.
+	if err := stateManager.UpdateState(&ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	// persist early
+	if err := stateManager.PersistState(context.Background()); err != nil {
+		log.Errorf("failed to persist state: %v", err)
+	}
+
+	return nil
+}
+
 // AddPeerFiltering rule to the firewall
 //
 // If comment argument is empty firewall manager should set
@@ -183,68 +230,84 @@ func (m *Manager) AllowNetbird() error {
 
 // SetLegacyManagement sets the route manager to use legacy management
 func (m *Manager) SetLegacyManagement(isLegacy bool) error {
-	oldLegacy := m.router.legacyManagement
+	return firewall.SetLegacyManagement(m.router, isLegacy)
+}
 
-	if oldLegacy != isLegacy {
-		m.router.legacyManagement = isLegacy
-		log.Debugf("Set legacy management to %v", isLegacy)
+// Reset firewall to the default state
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if err := m.resetNetbirdInputRules(); err != nil {
+		return fmt.Errorf("reset netbird input rules: %v", err)
 	}
 
-	// client reconnected to a newer mgmt, we need to cleanup the legacy rules
-	if !isLegacy && oldLegacy {
-		if err := m.router.RemoveAllLegacyRouteRules(); err != nil {
-			return fmt.Errorf("remove legacy routing rules: %v", err)
-		}
+	if err := m.router.Reset(); err != nil {
+		return fmt.Errorf("reset router: %v", err)
+	}
 
-		log.Debugf("Legacy routing rules removed")
+	if err := m.cleanupNetbirdTables(); err != nil {
+		return fmt.Errorf("cleanup netbird tables: %v", err)
+	}
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+		return fmt.Errorf("delete state: %v", err)
 	}
 
 	return nil
 }
 
-// Reset firewall to the default state
-func (m *Manager) Reset() error {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-
+func (m *Manager) resetNetbirdInputRules() error {
 	chains, err := m.rConn.ListChains()
 	if err != nil {
-		return fmt.Errorf("list of chains: %w", err)
+		return fmt.Errorf("list chains: %w", err)
 	}
 
+	m.deleteNetbirdInputRules(chains)
+
+	return nil
+}
+
+func (m *Manager) deleteNetbirdInputRules(chains []*nftables.Chain) {
 	for _, c := range chains {
-		// delete Netbird allow input traffic rule if it exists
 		if c.Table.Name == "filter" && c.Name == "INPUT" {
 			rules, err := m.rConn.GetRules(c.Table, c)
 			if err != nil {
 				log.Errorf("get rules for chain %q: %v", c.Name, err)
 				continue
 			}
-			for _, r := range rules {
-				if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
-					if err := m.rConn.DelRule(r); err != nil {
-						log.Errorf("delete rule: %v", err)
-					}
-				}
+
+			m.deleteMatchingRules(rules)
+		}
+	}
+}
+
+func (m *Manager) deleteMatchingRules(rules []*nftables.Rule) {
+	for _, r := range rules {
+		if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
+			if err := m.rConn.DelRule(r); err != nil {
+				log.Errorf("delete rule: %v", err)
 			}
 		}
 	}
+}
 
-	if err := m.router.Reset(); err != nil {
-		return fmt.Errorf("reset forward rules: %v", err)
-	}
-
+func (m *Manager) cleanupNetbirdTables() error {
 	tables, err := m.rConn.ListTables()
 	if err != nil {
-		return fmt.Errorf("list of tables: %w", err)
+		return fmt.Errorf("list tables: %w", err)
 	}
+
 	for _, t := range tables {
 		if t.Name == tableNameNetbird {
 			m.rConn.DelTable(t)
 		}
 	}
-
-	return m.rConn.Flush()
+	return nil
 }
 
 // Flush rule/chain/set operations from the buffer
@@ -315,28 +378,33 @@ func insertReturnTrafficRule(conn *nftables.Conn, table *nftables.Table, chain *
 	rule := &nftables.Rule{
 		Table: table,
 		Chain: chain,
-		Exprs: []expr.Any{
-			&expr.Ct{
-				Key:      expr.CtKeySTATE,
-				Register: 1,
-			},
-			&expr.Bitwise{
-				SourceRegister: 1,
-				DestRegister:   1,
-				Len:            4,
-				Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
-				Xor:            binaryutil.NativeEndian.PutUint32(0),
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpNeq,
-				Register: 1,
-				Data:     []byte{0, 0, 0, 0},
-			},
-			&expr.Verdict{
-				Kind: expr.VerdictAccept,
-			},
-		},
+		Exprs: getEstablishedExprs(1),
 	}
 
 	conn.InsertRule(rule)
 }
+
+func getEstablishedExprs(register uint32) []expr.Any {
+	return []expr.Any{
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: register,
+		},
+		&expr.Bitwise{
+			SourceRegister: register,
+			DestRegister:   register,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: register,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+}

client/firewall/nftables/manager_linux_test.go

@@ -1,7 +1,6 @@
 package nftables
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"net/netip"
@@ -58,12 +57,13 @@ func (i *iFaceMock) IsUserspaceBind() bool { return false }
 func TestNftablesManager(t *testing.T) {
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), ifaceMock)
+	manager, err := Create(ifaceMock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 	time.Sleep(time.Second * 3)
 
 	defer func() {
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 		time.Sleep(time.Second)
 	}()
@@ -109,6 +109,7 @@ func TestNftablesManager(t *testing.T) {
 			Register: 1,
 			Data:     []byte{0, 0, 0, 0},
 		},
+		&expr.Counter{},
 		&expr.Verdict{
 			Kind: expr.VerdictAccept,
 		},
@@ -168,7 +169,7 @@ func TestNftablesManager(t *testing.T) {
 	// established rule remains
 	require.Len(t, rules, 1, "expected 1 rules after deletion")
 
-	err = manager.Reset()
+	err = manager.Reset(nil)
 	require.NoError(t, err, "failed to reset")
 }
 
@@ -191,12 +192,13 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(context.Background(), mock)
+			manager, err := Create(mock)
 			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second * 3)
 
 			defer func() {
-				if err := manager.Reset(); err != nil {
+				if err := manager.Reset(nil); err != nil {
 					t.Errorf("clear the manager state: %v", err)
 				}
 				time.Sleep(time.Second)

client/firewall/nftables/router_linux.go

@@ -2,7 +2,6 @@ package nftables
 
 import (
 	"bytes"
-	"context"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -10,6 +9,8 @@ import (
 	"net/netip"
 	"strings"
 
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/google/nftables"
 	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
@@ -24,7 +25,7 @@ import (
 
 const (
 	chainNameRoutingFw  = "netbird-rt-fwd"
-	chainNameRoutingNat = "netbird-rt-nat"
+	chainNameRoutingNat = "netbird-rt-postrouting"
 	chainNameForward    = "FORWARD"
 
 	userDataAcceptForwardRuleIif = "frwacceptiif"
@@ -38,8 +39,6 @@ var (
 )
 
 type router struct {
-	ctx         context.Context
-	stop        context.CancelFunc
 	conn        *nftables.Conn
 	workTable   *nftables.Table
 	filterTable *nftables.Table
@@ -52,12 +51,8 @@ type router struct {
 	legacyManagement bool
 }
 
-func newRouter(parentCtx context.Context, workTable *nftables.Table, wgIface iFaceMapper) (*router, error) {
-	ctx, cancel := context.WithCancel(parentCtx)
-
+func newRouter(workTable *nftables.Table, wgIface iFaceMapper) (*router, error) {
 	r := &router{
-		ctx:       ctx,
-		stop:      cancel,
 		conn:      &nftables.Conn{},
 		workTable: workTable,
 		chains:    make(map[string]*nftables.Chain),
@@ -76,20 +71,25 @@ func newRouter(parentCtx context.Context, workTable *nftables.Table, wgIface iFa
 		if errors.Is(err, errFilterTableNotFound) {
 			log.Warnf("table 'filter' not found for forward rules")
 		} else {
-			return nil, err
+			return nil, fmt.Errorf("load filter table: %w", err)
 		}
 	}
 
-	err = r.cleanUpDefaultForwardRules()
-	if err != nil {
+	return r, nil
+}
+
+func (r *router) init(workTable *nftables.Table) error {
+	r.workTable = workTable
+
+	if err := r.removeAcceptForwardRules(); err != nil {
 		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
 
-	err = r.createContainers()
-	if err != nil {
-		log.Errorf("failed to create containers for route: %s", err)
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
 	}
-	return r, err
+
+	return nil
 }
 
 // Reset cleans existing nftables default forward rules from the system
@@ -97,40 +97,7 @@ func (r *router) Reset() error {
 	// clear without deleting the ipsets, the nf table will be deleted by the caller
 	r.ipsetCounter.Clear()
 
-	return r.cleanUpDefaultForwardRules()
-}
-
-func (r *router) cleanUpDefaultForwardRules() error {
-	if r.filterTable == nil {
-		return nil
-	}
-
-	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
-	if err != nil {
-		return fmt.Errorf("list chains: %v", err)
-	}
-
-	for _, chain := range chains {
-		if chain.Table.Name != r.filterTable.Name || chain.Name != chainNameForward {
-			continue
-		}
-
-		rules, err := r.conn.GetRules(r.filterTable, chain)
-		if err != nil {
-			return fmt.Errorf("get rules: %v", err)
-		}
-
-		for _, rule := range rules {
-			if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
-				bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) {
-				if err := r.conn.DelRule(rule); err != nil {
-					return fmt.Errorf("delete rule: %v", err)
-				}
-			}
-		}
-	}
-
-	return r.conn.Flush()
+	return r.removeAcceptForwardRules()
 }
 
 func (r *router) loadFilterTable() (*nftables.Table, error) {
@@ -149,7 +116,6 @@ func (r *router) loadFilterTable() (*nftables.Table, error) {
 }
 
 func (r *router) createContainers() error {
-
 	r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{
 		Name:  chainNameRoutingFw,
 		Table: r.workTable,
@@ -157,25 +123,28 @@ func (r *router) createContainers() error {
 
 	insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw])
 
+	prio := *nftables.ChainPriorityNATSource - 1
+
 	r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
 		Name:     chainNameRoutingNat,
 		Table:    r.workTable,
 		Hooknum:  nftables.ChainHookPostrouting,
-		Priority: nftables.ChainPriorityNATSource - 1,
+		Priority: &prio,
 		Type:     nftables.ChainTypeNAT,
 	})
 
-	r.acceptForwardRules()
+	if err := r.acceptForwardRules(); err != nil {
+		log.Errorf("failed to add accept rules for the forward chain: %s", err)
+	}
 
-	err := r.refreshRulesMap()
-	if err != nil {
+	if err := r.refreshRulesMap(); err != nil {
 		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
 
-	err = r.conn.Flush()
-	if err != nil {
+	if err := r.conn.Flush(); err != nil {
 		return fmt.Errorf("nftables: unable to initialize table: %v", err)
 	}
+
 	return nil
 }
 
@@ -188,6 +157,7 @@ func (r *router) AddRouteFiltering(
 	dPort *firewall.Port,
 	action firewall.Action,
 ) (firewall.Rule, error) {
+
 	ruleKey := id.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action)
 	if _, ok := r.rules[string(ruleKey)]; ok {
 		return ruleKey, nil
@@ -248,9 +218,18 @@ func (r *router) AddRouteFiltering(
 		UserData: []byte(ruleKey),
 	}
 
-	r.rules[string(ruleKey)] = r.conn.AddRule(rule)
+	rule = r.conn.AddRule(rule)
 
-	return ruleKey, r.conn.Flush()
+	log.Tracef("Adding route rule %s", spew.Sdump(rule))
+	if err := r.conn.Flush(); err != nil {
+		return nil, fmt.Errorf(flushError, err)
+	}
+
+	r.rules[string(ruleKey)] = rule
+
+	log.Debugf("nftables: added route rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", sources, destination, proto, sPort, dPort, action)
+
+	return ruleKey, nil
 }
 
 func (r *router) getIpSetExprs(sources []netip.Prefix, exprs []expr.Any) ([]expr.Any, error) {
@@ -288,6 +267,10 @@ func (r *router) DeleteRouteRule(rule firewall.Rule) error {
 		return nil
 	}
 
+	if nftRule.Handle == 0 {
+		return fmt.Errorf("route rule %s has no handle", ruleKey)
+	}
+
 	setName := r.findSetNameInRule(nftRule)
 
 	if err := r.deleteNftRule(nftRule, ruleKey); err != nil {
@@ -440,11 +423,15 @@ func (r *router) addNatRule(pair firewall.RouterPair) error {
 	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
 
 	dir := expr.MetaKeyIIFNAME
+	notDir := expr.MetaKeyOIFNAME
 	if pair.Inverse {
 		dir = expr.MetaKeyOIFNAME
+		notDir = expr.MetaKeyIIFNAME
 	}
 
+	lo := ifname("lo")
 	intf := ifname(r.wgIface.Name())
+
 	exprs := []expr.Any{
 		&expr.Meta{
 			Key:      dir,
@@ -455,6 +442,17 @@ func (r *router) addNatRule(pair firewall.RouterPair) error {
 			Register: 1,
 			Data:     intf,
 		},
+
+		// We need to exclude the loopback interface as this changes the ebpf proxy port
+		&expr.Meta{
+			Key:      notDir,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     lo,
+		},
 	}
 
 	exprs = append(exprs, sourceExp...)
@@ -553,7 +551,10 @@ func (r *router) RemoveAllLegacyRouteRules() error {
 		}
 		if err := r.conn.DelRule(rule); err != nil {
 			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
 		}
+
 	}
 	return nberrors.FormatErrorOrNil(merr)
 }
@@ -562,19 +563,60 @@ func (r *router) RemoveAllLegacyRouteRules() error {
 // that our traffic is not dropped by existing rules there.
 // The existing FORWARD rules/policies decide outbound traffic towards our interface.
 // In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule.
-func (r *router) acceptForwardRules() {
+func (r *router) acceptForwardRules() error {
 	if r.filterTable == nil {
 		log.Debugf("table 'filter' not found for forward rules, skipping accept rules")
-		return
+		return nil
 	}
 
+	fw := "iptables"
+
+	defer func() {
+		log.Debugf("Used %s to add accept forward rules", fw)
+	}()
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		// filter table exists but iptables is not
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+
+		fw = "nftables"
+		return r.acceptForwardRulesNftables()
+	}
+
+	return r.acceptForwardRulesIptables(ipt)
+}
+
+func (r *router) acceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("add iptables rule: %v", err))
+		} else {
+			log.Debugf("added iptables rule: %v", rule)
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *router) getAcceptForwardRules() [][]string {
+	intf := r.wgIface.Name()
+	return [][]string{
+		{"-i", intf, "-j", "ACCEPT"},
+		{"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
+	}
+}
+
+func (r *router) acceptForwardRulesNftables() error {
 	intf := ifname(r.wgIface.Name())
 
 	// Rule for incoming interface (iif) with counter
 	iifRule := &nftables.Rule{
 		Table: r.filterTable,
 		Chain: &nftables.Chain{
-			Name:     "FORWARD",
+			Name:     chainNameForward,
 			Table:    r.filterTable,
 			Type:     nftables.ChainTypeFilter,
 			Hooknum:  nftables.ChainHookForward,
@@ -594,6 +636,15 @@ func (r *router) acceptForwardRules() {
 	}
 	r.conn.InsertRule(iifRule)
 
+	oifExprs := []expr.Any{
+		&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     intf,
+		},
+	}
+
 	// Rule for outgoing interface (oif) with counter
 	oifRule := &nftables.Rule{
 		Table: r.filterTable,
@@ -604,36 +655,72 @@ func (r *router) acceptForwardRules() {
 			Hooknum:  nftables.ChainHookForward,
 			Priority: nftables.ChainPriorityFilter,
 		},
-		Exprs: []expr.Any{
-			&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     intf,
-			},
-			&expr.Ct{
-				Key:      expr.CtKeySTATE,
-				Register: 2,
-			},
-			&expr.Bitwise{
-				SourceRegister: 2,
-				DestRegister:   2,
-				Len:            4,
-				Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
-				Xor:            binaryutil.NativeEndian.PutUint32(0),
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpNeq,
-				Register: 2,
-				Data:     []byte{0, 0, 0, 0},
-			},
-			&expr.Counter{},
-			&expr.Verdict{Kind: expr.VerdictAccept},
-		},
+		Exprs:    append(oifExprs, getEstablishedExprs(2)...),
 		UserData: []byte(userDataAcceptForwardRuleOif),
 	}
 
 	r.conn.InsertRule(oifRule)
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRules() error {
+	if r.filterTable == nil {
+		return nil
+	}
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+		return r.removeAcceptForwardRulesNftables()
+	}
+
+	return r.removeAcceptForwardRulesIptables(ipt)
+}
+
+func (r *router) removeAcceptForwardRulesNftables() error {
+	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return fmt.Errorf("list chains: %v", err)
+	}
+
+	for _, chain := range chains {
+		if chain.Table.Name != r.filterTable.Name || chain.Name != chainNameForward {
+			continue
+		}
+
+		rules, err := r.conn.GetRules(r.filterTable, chain)
+		if err != nil {
+			return fmt.Errorf("get rules: %v", err)
+		}
+
+		for _, rule := range rules {
+			if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
+				bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) {
+				if err := r.conn.DelRule(rule); err != nil {
+					return fmt.Errorf("delete rule: %v", err)
+				}
+			}
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("remove iptables rule: %v", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 // RemoveNatRule removes a nftables rule pair from nat chains
@@ -658,7 +745,7 @@ func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
 		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
 	}
 
-	log.Debugf("nftables: removed rules for %s", pair.Destination)
+	log.Debugf("nftables: removed nat rules for %s", pair.Destination)
 	return nil
 }
 

client/firewall/nftables/router_linux_test.go

@@ -3,7 +3,6 @@
 package nftables
 
 import (
-	"context"
 	"encoding/binary"
 	"net/netip"
 	"os/exec"
@@ -40,8 +39,9 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
 
 	for _, testCase := range test.InsertRuleTestCases {
 		t.Run(testCase.Name, func(t *testing.T) {
-			manager, err := newRouter(context.TODO(), table, ifaceMock)
+			manager, err := newRouter(table, ifaceMock)
 			require.NoError(t, err, "failed to create router")
+			require.NoError(t, manager.init(table))
 
 			nftablesTestingClient := &nftables.Conn{}
 
@@ -69,6 +69,12 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
 						Register: 1,
 						Data:     ifname(ifaceMock.Name()),
 					},
+					&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
+					&expr.Cmp{
+						Op:       expr.CmpOpNeq,
+						Register: 1,
+						Data:     ifname("lo"),
+					},
 				)
 
 				natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
@@ -97,6 +103,12 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
 						Register: 1,
 						Data:     ifname(ifaceMock.Name()),
 					},
+					&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+					&expr.Cmp{
+						Op:       expr.CmpOpNeq,
+						Register: 1,
+						Data:     ifname("lo"),
+					},
 				)
 
 				inNatRuleKey := firewall.GenKey(firewall.NatFormat, firewall.GetInversePair(testCase.InputPair))
@@ -130,8 +142,9 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) {
 
 	for _, testCase := range test.RemoveRuleTestCases {
 		t.Run(testCase.Name, func(t *testing.T) {
-			manager, err := newRouter(context.TODO(), table, ifaceMock)
+			manager, err := newRouter(table, ifaceMock)
 			require.NoError(t, err, "failed to create router")
+			require.NoError(t, manager.init(table))
 
 			nftablesTestingClient := &nftables.Conn{}
 
@@ -198,8 +211,9 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 
 	defer deleteWorkTable()
 
-	r, err := newRouter(context.Background(), workTable, ifaceMock)
+	r, err := newRouter(workTable, ifaceMock)
 	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
 
 	defer func(r *router) {
 		require.NoError(t, r.Reset(), "Failed to reset rules")
@@ -314,6 +328,10 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 			ruleKey, err := r.AddRouteFiltering(tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.action)
 			require.NoError(t, err, "AddRouteFiltering failed")
 
+			t.Cleanup(func() {
+				require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule")
+			})
+
 			// Check if the rule is in the internal map
 			rule, ok := r.rules[ruleKey.GetRuleID()]
 			assert.True(t, ok, "Rule not found in internal map")
@@ -346,10 +364,6 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 
 			// Verify actual nftables rule content
 			verifyRule(t, nftRule, tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.direction, tt.action, tt.expectSet)
-
-			// Clean up
-			err = r.DeleteRouteRule(ruleKey)
-			require.NoError(t, err, "Failed to delete rule")
 		})
 	}
 }
@@ -364,8 +378,9 @@ func TestNftablesCreateIpSet(t *testing.T) {
 
 	defer deleteWorkTable()
 
-	r, err := newRouter(context.Background(), workTable, ifaceMock)
+	r, err := newRouter(workTable, ifaceMock)
 	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
 
 	defer func() {
 		require.NoError(t, r.Reset(), "Failed to reset router")

client/firewall/nftables/state.go

@@ -0,0 +1 @@
+package nftables

client/firewall/nftables/state_linux.go

@@ -0,0 +1,47 @@
+package nftables
+
+import (
+	"fmt"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "nftables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	nft, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create nftables manager: %w", err)
+	}
+
+	if err := nft.Reset(nil); err != nil {
+		return fmt.Errorf("reset nftables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/uspfilter/allow_netbird.go

@@ -2,8 +2,10 @@
 
 package uspfilter
 
+import "github.com/netbirdio/netbird/client/internal/statemanager"
+
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
@@ -11,7 +13,7 @@ func (m *Manager) Reset() error {
 	m.incomingRules = make(map[string]RuleSet)
 
 	if m.nativeFirewall != nil {
-		return m.nativeFirewall.Reset()
+		return m.nativeFirewall.Reset(stateManager)
 	}
 	return nil
 }

client/firewall/uspfilter/allow_netbird_windows.go

@@ -6,6 +6,8 @@ import (
 	"syscall"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 type action string
@@ -17,7 +19,7 @@ const (
 )
 
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(*statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 

client/firewall/uspfilter/uspfilter.go

@@ -14,6 +14,7 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const layerTypeAll = 0
@@ -97,6 +98,10 @@ func create(iface IFaceMapper) (*Manager, error) {
 	return m, nil
 }
 
+func (m *Manager) Init(*statemanager.Manager) error {
+	return nil
+}
+
 func (m *Manager) IsServerRouteSupported() bool {
 	if m.nativeFirewall == nil {
 		return false
@@ -190,7 +195,7 @@ func (m *Manager) AddPeerFiltering(
 	return []firewall.Rule{&r}, nil
 }
 
-func (m *Manager) AddRouteFiltering(sources [] netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action ) (firewall.Rule, error) {
+func (m *Manager) AddRouteFiltering(sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action) (firewall.Rule, error) {
 	if m.nativeFirewall == nil {
 		return nil, errRouteNotSupported
 	}
@@ -232,8 +237,11 @@ func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
 }
 
 // SetLegacyManagement doesn't need to be implemented for this manager
-func (m *Manager) SetLegacyManagement(_ bool) error {
-	return nil
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	if m.nativeFirewall == nil {
+		return errRouteNotSupported
+	}
+	return m.nativeFirewall.SetLegacyManagement(isLegacy)
 }
 
 // Flush doesn't need to be implemented for this manager

client/firewall/uspfilter/uspfilter_test.go

@@ -259,7 +259,7 @@ func TestManagerReset(t *testing.T) {
 		return
 	}
 
-	err = m.Reset()
+	err = m.Reset(nil)
 	if err != nil {
 		t.Errorf("failed to reset Manager: %v", err)
 		return
@@ -330,7 +330,7 @@ func TestNotMatchByIP(t *testing.T) {
 		return
 	}
 
-	if err = m.Reset(); err != nil {
+	if err = m.Reset(nil); err != nil {
 		t.Errorf("failed to reset Manager: %v", err)
 		return
 	}
@@ -396,7 +396,7 @@ func TestUSPFilterCreatePerformance(t *testing.T) {
 			time.Sleep(time.Second)
 
 			defer func() {
-				if err := manager.Reset(); err != nil {
+				if err := manager.Reset(nil); err != nil {
 					t.Errorf("clear the manager state: %v", err)
 				}
 				time.Sleep(time.Second)

client/iface/bind/bind.go

@@ -1,142 +0,0 @@
-package bind
-
-import (
-	"fmt"
-	"net"
-	"runtime"
-	"sync"
-
-	"github.com/pion/stun/v2"
-	"github.com/pion/transport/v3"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/net/ipv4"
-	wgConn "golang.zx2c4.com/wireguard/conn"
-)
-
-type receiverCreator struct {
-	iceBind *ICEBind
-}
-
-func (rc receiverCreator) CreateIPv4ReceiverFn(msgPool *sync.Pool, pc *ipv4.PacketConn, conn *net.UDPConn) wgConn.ReceiveFunc {
-	return rc.iceBind.createIPv4ReceiverFn(msgPool, pc, conn)
-}
-
-type ICEBind struct {
-	*wgConn.StdNetBind
-
-	muUDPMux sync.Mutex
-
-	transportNet transport.Net
-	udpMux       *UniversalUDPMuxDefault
-
-	filterFn FilterFn
-}
-
-func NewICEBind(transportNet transport.Net, filterFn FilterFn) *ICEBind {
-	ib := &ICEBind{
-		transportNet: transportNet,
-		filterFn:     filterFn,
-	}
-
-	rc := receiverCreator{
-		ib,
-	}
-	ib.StdNetBind = wgConn.NewStdNetBindWithReceiverCreator(rc)
-	return ib
-}
-
-// GetICEMux returns the ICE UDPMux that was created and used by ICEBind
-func (s *ICEBind) GetICEMux() (*UniversalUDPMuxDefault, error) {
-	s.muUDPMux.Lock()
-	defer s.muUDPMux.Unlock()
-	if s.udpMux == nil {
-		return nil, fmt.Errorf("ICEBind has not been initialized yet")
-	}
-
-	return s.udpMux, nil
-}
-
-func (s *ICEBind) createIPv4ReceiverFn(ipv4MsgsPool *sync.Pool, pc *ipv4.PacketConn, conn *net.UDPConn) wgConn.ReceiveFunc {
-	s.muUDPMux.Lock()
-	defer s.muUDPMux.Unlock()
-
-	s.udpMux = NewUniversalUDPMuxDefault(
-		UniversalUDPMuxParams{
-			UDPConn:  conn,
-			Net:      s.transportNet,
-			FilterFn: s.filterFn,
-		},
-	)
-	return func(bufs [][]byte, sizes []int, eps []wgConn.Endpoint) (n int, err error) {
-		msgs := ipv4MsgsPool.Get().(*[]ipv4.Message)
-		defer ipv4MsgsPool.Put(msgs)
-		for i := range bufs {
-			(*msgs)[i].Buffers[0] = bufs[i]
-		}
-		var numMsgs int
-		if runtime.GOOS == "linux" {
-			numMsgs, err = pc.ReadBatch(*msgs, 0)
-			if err != nil {
-				return 0, err
-			}
-		} else {
-			msg := &(*msgs)[0]
-			msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
-			if err != nil {
-				return 0, err
-			}
-			numMsgs = 1
-		}
-		for i := 0; i < numMsgs; i++ {
-			msg := &(*msgs)[i]
-
-			// todo: handle err
-			ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr)
-			if ok {
-				sizes[i] = 0
-			} else {
-				sizes[i] = msg.N
-			}
-
-			addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
-			ep := &wgConn.StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
-			wgConn.GetSrcFromControl(msg.OOB[:msg.NN], ep)
-			eps[i] = ep
-		}
-		return numMsgs, nil
-	}
-}
-
-func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) {
-	for i := range buffers {
-		if !stun.IsMessage(buffers[i]) {
-			continue
-		}
-
-		msg, err := s.parseSTUNMessage(buffers[i][:n])
-		if err != nil {
-			buffers[i] = []byte{}
-			return true, err
-		}
-
-		muxErr := s.udpMux.HandleSTUNMessage(msg, addr)
-		if muxErr != nil {
-			log.Warnf("failed to handle STUN packet")
-		}
-
-		buffers[i] = []byte{}
-		return true, nil
-	}
-	return false, nil
-}
-
-func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) {
-	msg := &stun.Message{
-		Raw: raw,
-	}
-	if err := msg.Decode(); err != nil {
-		return nil, err
-	}
-
-	return msg, nil
-}

client/iface/bind/endpoint.go

@@ -0,0 +1,5 @@
+package bind
+
+import wgConn "golang.zx2c4.com/wireguard/conn"
+
+type Endpoint = wgConn.StdNetEndpoint

client/iface/bind/ice_bind.go

@@ -0,0 +1,275 @@
+package bind
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/pion/stun/v2"
+	"github.com/pion/transport/v3"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/ipv4"
+	wgConn "golang.zx2c4.com/wireguard/conn"
+)
+
+type RecvMessage struct {
+	Endpoint *Endpoint
+	Buffer   []byte
+}
+
+type receiverCreator struct {
+	iceBind *ICEBind
+}
+
+func (rc receiverCreator) CreateIPv4ReceiverFn(msgPool *sync.Pool, pc *ipv4.PacketConn, conn *net.UDPConn) wgConn.ReceiveFunc {
+	return rc.iceBind.createIPv4ReceiverFn(msgPool, pc, conn)
+}
+
+// ICEBind is a bind implementation with two main features:
+// 1. filter out STUN messages and handle them
+// 2. forward the received packets to the WireGuard interface from the relayed connection
+//
+// ICEBind.endpoints var is a map that stores the connection for each relayed peer. Fake address is just an IP address
+// without port, in the format of 127.1.x.x where x.x is the last two octets of the peer address. We try to avoid to
+// use the port because in the Send function the wgConn.Endpoint the port info is not exported.
+type ICEBind struct {
+	*wgConn.StdNetBind
+	RecvChan chan RecvMessage
+
+	transportNet transport.Net
+	filterFn     FilterFn
+	endpoints    map[netip.Addr]net.Conn
+	endpointsMu  sync.Mutex
+	// every time when Close() is called (i.e. BindUpdate()) we need to close exit from the receiveRelayed and create a
+	// new closed channel. With the closedChanMu we can safely close the channel and create a new one
+	closedChan   chan struct{}
+	closedChanMu sync.RWMutex // protect the closeChan recreation from reading from it.
+	closed       bool
+
+	muUDPMux sync.Mutex
+	udpMux   *UniversalUDPMuxDefault
+}
+
+func NewICEBind(transportNet transport.Net, filterFn FilterFn) *ICEBind {
+	b, _ := wgConn.NewStdNetBind().(*wgConn.StdNetBind)
+	ib := &ICEBind{
+		StdNetBind:   b,
+		RecvChan:     make(chan RecvMessage, 1),
+		transportNet: transportNet,
+		filterFn:     filterFn,
+		endpoints:    make(map[netip.Addr]net.Conn),
+		closedChan:   make(chan struct{}),
+		closed:       true,
+	}
+
+	rc := receiverCreator{
+		ib,
+	}
+	ib.StdNetBind = wgConn.NewStdNetBindWithReceiverCreator(rc)
+	return ib
+}
+
+func (s *ICEBind) Open(uport uint16) ([]wgConn.ReceiveFunc, uint16, error) {
+	s.closed = false
+	s.closedChanMu.Lock()
+	s.closedChan = make(chan struct{})
+	s.closedChanMu.Unlock()
+	fns, port, err := s.StdNetBind.Open(uport)
+	if err != nil {
+		return nil, 0, err
+	}
+	fns = append(fns, s.receiveRelayed)
+	return fns, port, nil
+}
+
+func (s *ICEBind) Close() error {
+	if s.closed {
+		return nil
+	}
+	s.closed = true
+
+	close(s.closedChan)
+
+	return s.StdNetBind.Close()
+}
+
+// GetICEMux returns the ICE UDPMux that was created and used by ICEBind
+func (s *ICEBind) GetICEMux() (*UniversalUDPMuxDefault, error) {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+	if s.udpMux == nil {
+		return nil, fmt.Errorf("ICEBind has not been initialized yet")
+	}
+
+	return s.udpMux, nil
+}
+
+func (b *ICEBind) SetEndpoint(peerAddress *net.UDPAddr, conn net.Conn) (*net.UDPAddr, error) {
+	fakeUDPAddr, err := fakeAddress(peerAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// force IPv4
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		return nil, fmt.Errorf("failed to convert IP to netip.Addr")
+	}
+
+	b.endpointsMu.Lock()
+	b.endpoints[fakeAddr] = conn
+	b.endpointsMu.Unlock()
+
+	return fakeUDPAddr, nil
+}
+
+func (b *ICEBind) RemoveEndpoint(fakeUDPAddr *net.UDPAddr) {
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		log.Warnf("failed to convert IP to netip.Addr")
+		return
+	}
+
+	b.endpointsMu.Lock()
+	defer b.endpointsMu.Unlock()
+	delete(b.endpoints, fakeAddr)
+}
+
+func (b *ICEBind) Send(bufs [][]byte, ep wgConn.Endpoint) error {
+	b.endpointsMu.Lock()
+	conn, ok := b.endpoints[ep.DstIP()]
+	b.endpointsMu.Unlock()
+	if !ok {
+		return b.StdNetBind.Send(bufs, ep)
+	}
+
+	for _, buf := range bufs {
+		if _, err := conn.Write(buf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *ICEBind) createIPv4ReceiverFn(ipv4MsgsPool *sync.Pool, pc *ipv4.PacketConn, conn *net.UDPConn) wgConn.ReceiveFunc {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+
+	s.udpMux = NewUniversalUDPMuxDefault(
+		UniversalUDPMuxParams{
+			UDPConn:  conn,
+			Net:      s.transportNet,
+			FilterFn: s.filterFn,
+		},
+	)
+	return func(bufs [][]byte, sizes []int, eps []wgConn.Endpoint) (n int, err error) {
+		msgs := ipv4MsgsPool.Get().(*[]ipv4.Message)
+		defer ipv4MsgsPool.Put(msgs)
+		for i := range bufs {
+			(*msgs)[i].Buffers[0] = bufs[i]
+		}
+		var numMsgs int
+		if runtime.GOOS == "linux" {
+			numMsgs, err = pc.ReadBatch(*msgs, 0)
+			if err != nil {
+				return 0, err
+			}
+		} else {
+			msg := &(*msgs)[0]
+			msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
+			if err != nil {
+				return 0, err
+			}
+			numMsgs = 1
+		}
+		for i := 0; i < numMsgs; i++ {
+			msg := &(*msgs)[i]
+
+			// todo: handle err
+			ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr)
+			if ok {
+				sizes[i] = 0
+			} else {
+				sizes[i] = msg.N
+			}
+
+			addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
+			ep := &wgConn.StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
+			wgConn.GetSrcFromControl(msg.OOB[:msg.NN], ep)
+			eps[i] = ep
+		}
+		return numMsgs, nil
+	}
+}
+
+func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) {
+	for i := range buffers {
+		if !stun.IsMessage(buffers[i]) {
+			continue
+		}
+
+		msg, err := s.parseSTUNMessage(buffers[i][:n])
+		if err != nil {
+			buffers[i] = []byte{}
+			return true, err
+		}
+
+		muxErr := s.udpMux.HandleSTUNMessage(msg, addr)
+		if muxErr != nil {
+			log.Warnf("failed to handle STUN packet")
+		}
+
+		buffers[i] = []byte{}
+		return true, nil
+	}
+	return false, nil
+}
+
+func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) {
+	msg := &stun.Message{
+		Raw: raw,
+	}
+	if err := msg.Decode(); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}
+
+// receiveRelayed is a receive function that is used to receive packets from the relayed connection and forward to the
+// WireGuard. Critical part is do not block if the Closed() has been called.
+func (c *ICEBind) receiveRelayed(buffs [][]byte, sizes []int, eps []wgConn.Endpoint) (int, error) {
+	c.closedChanMu.RLock()
+	defer c.closedChanMu.RUnlock()
+
+	select {
+	case <-c.closedChan:
+		return 0, net.ErrClosed
+	case msg, ok := <-c.RecvChan:
+		if !ok {
+			return 0, net.ErrClosed
+		}
+		copy(buffs[0], msg.Buffer)
+		sizes[0] = len(msg.Buffer)
+		eps[0] = wgConn.Endpoint(msg.Endpoint)
+		return 1, nil
+	}
+}
+
+// fakeAddress returns a fake address that is used to as an identifier for the peer.
+// The fake address is in the format of 127.1.x.x where x.x is the last two octets of the peer address.
+func fakeAddress(peerAddress *net.UDPAddr) (*net.UDPAddr, error) {
+	octets := strings.Split(peerAddress.IP.String(), ".")
+	if len(octets) != 4 {
+		return nil, fmt.Errorf("invalid IP format")
+	}
+
+	newAddr := &net.UDPAddr{
+		IP:   net.ParseIP(fmt.Sprintf("127.1.%s.%s", octets[2], octets[3])),
+		Port: peerAddress.Port,
+	}
+	return newAddr, nil
+}

client/iface/device/device_android.go

@@ -5,7 +5,6 @@ package device
 import (
 	"strings"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/device"
@@ -31,13 +30,13 @@ type WGTunDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewTunDevice(address WGAddress, port int, key string, mtu int, transportNet transport.Net, tunAdapter TunAdapter, filterFn bind.FilterFn) *WGTunDevice {
+func NewTunDevice(address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind, tunAdapter TunAdapter) *WGTunDevice {
 	return &WGTunDevice{
 		address:    address,
 		port:       port,
 		key:        key,
 		mtu:        mtu,
-		iceBind:    bind.NewICEBind(transportNet, filterFn),
+		iceBind:    iceBind,
 		tunAdapter: tunAdapter,
 	}
 }

client/iface/device/device_darwin.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os/exec"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
@@ -29,14 +28,14 @@ type TunDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) *TunDevice {
+func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *TunDevice {
 	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 

client/iface/device/device_ios.go

@@ -6,7 +6,6 @@ package device
 import (
 	"os"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/device"
@@ -30,13 +29,13 @@ type TunDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewTunDevice(name string, address WGAddress, port int, key string, transportNet transport.Net, tunFd int, filterFn bind.FilterFn) *TunDevice {
+func NewTunDevice(name string, address WGAddress, port int, key string, iceBind *bind.ICEBind, tunFd int) *TunDevice {
 	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 		tunFd:   tunFd,
 	}
 }

client/iface/device/device_netstack.go

@@ -6,7 +6,6 @@ package device
 import (
 	"fmt"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 
@@ -31,7 +30,7 @@ type TunNetstackDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, mtu int, transportNet transport.Net, listenAddress string, filterFn bind.FilterFn) *TunNetstackDevice {
+func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, mtu int, iceBind *bind.ICEBind, listenAddress string) *TunNetstackDevice {
 	return &TunNetstackDevice{
 		name:          name,
 		address:       address,
@@ -39,7 +38,7 @@ func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, m
 		key:           key,
 		mtu:           mtu,
 		listenAddress: listenAddress,
-		iceBind:       bind.NewICEBind(transportNet, filterFn),
+		iceBind:       iceBind,
 	}
 }
 

client/iface/device/device_usp_unix.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
@@ -30,7 +29,7 @@ type USPDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) *USPDevice {
+func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *USPDevice {
 	log.Infof("using userspace bind mode")
 
 	checkUser()
@@ -41,7 +40,8 @@ func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn)}
+		iceBind: iceBind,
+	}
 }
 
 func (t *USPDevice) Create() (WGConfigurer, error) {

client/iface/device/device_windows.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/netip"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/device"
@@ -32,14 +31,14 @@ type TunDevice struct {
 	configurer      WGConfigurer
 }
 
-func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) *TunDevice {
+func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *TunDevice {
 	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 

client/iface/iface.go

@@ -6,12 +6,16 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
+	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
+	"github.com/netbirdio/netbird/client/errors"
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 const (
@@ -22,14 +26,35 @@ const (
 
 type WGAddress = device.WGAddress
 
+type wgProxyFactory interface {
+	GetProxy() wgproxy.Proxy
+	Free() error
+}
+
+type WGIFaceOpts struct {
+	IFaceName    string
+	Address      string
+	WGPort       int
+	WGPrivKey    string
+	MTU          int
+	MobileArgs   *device.MobileIFaceArguments
+	TransportNet transport.Net
+	FilterFn     bind.FilterFn
+}
+
 // WGIface represents an interface instance
 type WGIface struct {
 	tun           WGTunDevice
 	userspaceBind bool
 	mu            sync.Mutex
 
-	configurer device.WGConfigurer
-	filter     device.PacketFilter
+	configurer     device.WGConfigurer
+	filter         device.PacketFilter
+	wgProxyFactory wgProxyFactory
+}
+
+func (w *WGIface) GetProxy() wgproxy.Proxy {
+	return w.wgProxyFactory.GetProxy()
 }
 
 // IsUserspaceBind indicates whether this interfaces is userspace with bind.ICEBind
@@ -124,22 +149,26 @@ func (w *WGIface) Close() error {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 
-	err := w.tun.Close()
-	if err != nil {
-		return fmt.Errorf("failed to close wireguard interface %s: %w", w.Name(), err)
+	var result *multierror.Error
+
+	if err := w.wgProxyFactory.Free(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to free WireGuard proxy: %w", err))
 	}
 
-	err = w.waitUntilRemoved()
-	if err != nil {
+	if err := w.tun.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close wireguard interface %s: %w", w.Name(), err))
+	}
+
+	if err := w.waitUntilRemoved(); err != nil {
 		log.Warnf("failed to remove WireGuard interface %s: %v", w.Name(), err)
-		err = w.Destroy()
-		if err != nil {
-			return fmt.Errorf("failed to remove WireGuard interface %s: %w", w.Name(), err)
+		if err := w.Destroy(); err != nil {
+			result = multierror.Append(result, fmt.Errorf("failed to remove WireGuard interface %s: %w", w.Name(), err))
+			return errors.FormatErrorOrNil(result)
 		}
 		log.Infof("interface %s successfully removed", w.Name())
 	}
 
-	return nil
+	return errors.FormatErrorOrNil(result)
 }
 
 // SetFilter sets packet filters for the userspace implementation

client/iface/iface_android.go

@@ -1,43 +0,0 @@
-package iface
-
-import (
-	"fmt"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{
-		tun:           device.NewTunDevice(wgAddress, wgPort, wgPrivKey, mtu, transportNet, args.TunAdapter, filterFn),
-		userspaceBind: true,
-	}
-	return wgIFace, nil
-}
-
-// CreateOnAndroid creates a new Wireguard interface, sets a given IP and brings it up.
-// Will reuse an existing one.
-func (w *WGIface) CreateOnAndroid(routes []string, dns string, searchDomains []string) error {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-
-	cfgr, err := w.tun.Create(routes, dns, searchDomains)
-	if err != nil {
-		return err
-	}
-	w.configurer = cfgr
-	return nil
-}
-
-// Create this function make sense on mobile only
-func (w *WGIface) Create() error {
-	return fmt.Errorf("this function has not implemented on this platform")
-}

client/iface/iface_create.go

@@ -2,6 +2,8 @@
 
 package iface
 
+import "fmt"
+
 // Create creates a new Wireguard interface, sets a given IP and brings it up.
 // Will reuse an existing one.
 // this function is different on Android
@@ -17,3 +19,8 @@ func (w *WGIface) Create() error {
 	w.configurer = cfgr
 	return nil
 }
+
+// CreateOnAndroid this function make sense on mobile only
+func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
+	return fmt.Errorf("this function has not implemented on non mobile")
+}

client/iface/iface_create_android.go

@@ -0,0 +1,24 @@
+package iface
+
+import (
+	"fmt"
+)
+
+// CreateOnAndroid creates a new Wireguard interface, sets a given IP and brings it up.
+// Will reuse an existing one.
+func (w *WGIface) CreateOnAndroid(routes []string, dns string, searchDomains []string) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	cfgr, err := w.tun.Create(routes, dns, searchDomains)
+	if err != nil {
+		return err
+	}
+	w.configurer = cfgr
+	return nil
+}
+
+// Create this function make sense on mobile only
+func (w *WGIface) Create() error {
+	return fmt.Errorf("this function has not implemented on this platform")
+}

client/iface/iface_create_darwin.go

@@ -7,39 +7,8 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-	"github.com/netbirdio/netbird/client/iface/netstack"
 )
 
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, _ *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{
-		userspaceBind: true,
-	}
-
-	if netstack.IsEnabled() {
-		wgIFace.tun = device.NewNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr(), filterFn)
-		return wgIFace, nil
-	}
-
-	wgIFace.tun = device.NewTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, filterFn)
-
-	return wgIFace, nil
-}
-
-// CreateOnAndroid this function make sense on mobile only
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("this function has not implemented on this platform")
-}
-
 // Create creates a new Wireguard interface, sets a given IP and brings it up.
 // Will reuse an existing one.
 // this function is different on Android
@@ -65,3 +34,8 @@ func (w *WGIface) Create() error {
 
 	return backoff.Retry(operation, backOff)
 }
+
+// CreateOnAndroid this function make sense on mobile only
+func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
+	return fmt.Errorf("this function has not implemented on this platform")
+}

client/iface/iface_guid_windows.go

@@ -0,0 +1,10 @@
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+// GetInterfaceGUIDString returns an interface GUID. This is useful on Windows only
+func (w *WGIface) GetInterfaceGUIDString() (string, error) {
+	return w.tun.(*device.TunDevice).GetInterfaceGUIDString()
+}

client/iface/iface_ios.go

@@ -1,31 +0,0 @@
-//go:build ios
-
-package iface
-
-import (
-	"fmt"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-	wgIFace := &WGIface{
-		tun:           device.NewTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, transportNet, args.TunFd, filterFn),
-		userspaceBind: true,
-	}
-	return wgIFace, nil
-}
-
-// CreateOnAndroid creates a new Wireguard interface, sets a given IP and brings it up.
-// Will reuse an existing one.
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("this function has not implemented on this platform")
-}

client/iface/iface_moc.go

@@ -9,6 +9,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type MockWGIface struct {
@@ -30,6 +31,7 @@ type MockWGIface struct {
 	GetDeviceFunc              func() *device.FilteredDevice
 	GetStatsFunc               func(peerKey string) (configurer.WGStats, error)
 	GetInterfaceGUIDStringFunc func() (string, error)
+	GetProxyFunc               func() wgproxy.Proxy
 }
 
 func (m *MockWGIface) GetInterfaceGUIDString() (string, error) {
@@ -103,3 +105,8 @@ func (m *MockWGIface) GetDevice() *device.FilteredDevice {
 func (m *MockWGIface) GetStats(peerKey string) (configurer.WGStats, error) {
 	return m.GetStatsFunc(peerKey)
 }
+
+func (m *MockWGIface) GetProxy() wgproxy.Proxy {
+	//TODO implement me
+	panic("implement me")
+}

client/iface/iface_new_android.go

@@ -0,0 +1,24 @@
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	wgIFace := &WGIface{
+		userspaceBind:  true,
+		tun:            device.NewTunDevice(wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, opts.MobileArgs.TunAdapter),
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+}

client/iface/iface_new_darwin.go

@@ -0,0 +1,34 @@
+//go:build !ios
+
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	var tun WGTunDevice
+	if netstack.IsEnabled() {
+		tun = device.NewNetstackDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, netstack.ListenAddr())
+	} else {
+		tun = device.NewTunDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind)
+	}
+
+	wgIFace := &WGIface{
+		userspaceBind:  true,
+		tun:            tun,
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+}

client/iface/iface_new_ios.go

@@ -0,0 +1,26 @@
+//go:build ios
+
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	wgIFace := &WGIface{
+		tun:            device.NewTunDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, iceBind, opts.MobileArgs.TunFd),
+		userspaceBind:  true,
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+}

client/iface/iface_new_unix.go

@@ -0,0 +1,45 @@
+//go:build (linux && !android) || freebsd
+
+package iface
+
+import (
+	"fmt"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	wgIFace := &WGIface{}
+
+	if netstack.IsEnabled() {
+		iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+		wgIFace.tun = device.NewNetstackDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, netstack.ListenAddr())
+		wgIFace.userspaceBind = true
+		wgIFace.wgProxyFactory = wgproxy.NewUSPFactory(iceBind)
+		return wgIFace, nil
+	}
+
+	if device.WireGuardModuleIsLoaded() {
+		wgIFace.tun = device.NewKernelDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, opts.TransportNet)
+		wgIFace.wgProxyFactory = wgproxy.NewKernelFactory(opts.WGPort)
+		return wgIFace, nil
+	}
+	if device.ModuleTunIsLoaded() {
+		iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+		wgIFace.tun = device.NewUSPDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind)
+		wgIFace.userspaceBind = true
+		wgIFace.wgProxyFactory = wgproxy.NewUSPFactory(iceBind)
+		return wgIFace, nil
+	}
+
+	return nil, fmt.Errorf("couldn't check or load tun module")
+}

client/iface/iface_new_windows.go

@@ -0,0 +1,32 @@
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	var tun WGTunDevice
+	if netstack.IsEnabled() {
+		tun = device.NewNetstackDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, netstack.ListenAddr())
+	} else {
+		tun = device.NewTunDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind)
+	}
+
+	wgIFace := &WGIface{
+		userspaceBind:  true,
+		tun:            tun,
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+
+}

client/iface/iface_test.go

@@ -45,7 +45,16 @@ func TestWGIface_UpdateAddr(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, addr, wgPort, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      addr,
+		WGPort:       wgPort,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -118,7 +127,16 @@ func Test_CreateInterface(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       33100,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +171,16 @@ func Test_Close(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       wgPort,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -189,7 +216,16 @@ func TestRecreation(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil, nil)
+			opts := WGIFaceOpts{
+				IFaceName:    ifaceName,
+				Address:      wgIP,
+				WGPort:       wgPort,
+				WGPrivKey:    key,
+				MTU:          DefaultMTU,
+				TransportNet: newNet,
+			}
+
+			iface, err := NewWGIFace(opts)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -252,7 +288,15 @@ func Test_ConfigureInterface(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       wgPort,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -300,7 +344,16 @@ func Test_UpdatePeer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       33100,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -361,7 +414,16 @@ func Test_RemovePeer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       33100,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -418,7 +480,15 @@ func Test_ConnectPeers(t *testing.T) {
 	guid := fmt.Sprintf("{%s}", uuid.New().String())
 	device.CustomWindowsGUIDString = strings.ToLower(guid)
 
-	iface1, err := NewWGIFace(peer1ifaceName, peer1wgIP, peer1wgPort, peer1Key.String(), DefaultMTU, newNet, nil, nil)
+	optsPeer1 := WGIFaceOpts{
+		IFaceName:    peer1ifaceName,
+		Address:      peer1wgIP,
+		WGPort:       peer1wgPort,
+		WGPrivKey:    peer1Key.String(),
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+	iface1, err := NewWGIFace(optsPeer1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -432,7 +502,12 @@ func Test_ConnectPeers(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", peer1wgPort))
+	localIP, err := getLocalIP()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP, peer1wgPort))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -444,7 +519,17 @@ func Test_ConnectPeers(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	iface2, err := NewWGIFace(peer2ifaceName, peer2wgIP, peer2wgPort, peer2Key.String(), DefaultMTU, newNet, nil, nil)
+
+	optsPeer2 := WGIFaceOpts{
+		IFaceName:    peer2ifaceName,
+		Address:      peer2wgIP,
+		WGPort:       peer2wgPort,
+		WGPrivKey:    peer2Key.String(),
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface2, err := NewWGIFace(optsPeer2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -458,7 +543,7 @@ func Test_ConnectPeers(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", peer2wgPort))
+	peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP, peer2wgPort))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -527,3 +612,28 @@ func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
 	}
 	return wgtypes.Peer{}, fmt.Errorf("peer not found")
 }
+
+func getLocalIP() (string, error) {
+	// Get all interfaces
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "", err
+	}
+
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok {
+			continue
+		}
+		if ipNet.IP.IsLoopback() {
+			continue
+		}
+
+		if ipNet.IP.To4() == nil {
+			continue
+		}
+		return ipNet.IP.String(), nil
+	}
+
+	return "", fmt.Errorf("no local IP found")
+}

client/iface/iface_unix.go

@@ -1,49 +0,0 @@
-//go:build (linux && !android) || freebsd
-
-package iface
-
-import (
-	"fmt"
-	"runtime"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-	"github.com/netbirdio/netbird/client/iface/netstack"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{}
-
-	// move the kernel/usp/netstack preference evaluation to upper layer
-	if netstack.IsEnabled() {
-		wgIFace.tun = device.NewNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr(), filterFn)
-		wgIFace.userspaceBind = true
-		return wgIFace, nil
-	}
-
-	if device.WireGuardModuleIsLoaded() {
-		wgIFace.tun = device.NewKernelDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet)
-		wgIFace.userspaceBind = false
-		return wgIFace, nil
-	}
-
-	if !device.ModuleTunIsLoaded() {
-		return nil, fmt.Errorf("couldn't check or load tun module")
-	}
-	wgIFace.tun = device.NewUSPDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, nil)
-	wgIFace.userspaceBind = true
-	return wgIFace, nil
-}
-
-// CreateOnAndroid this function make sense on mobile only
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("CreateOnAndroid function has not implemented on %s platform", runtime.GOOS)
-}

client/iface/iface_windows.go

@@ -1,41 +0,0 @@
-package iface
-
-import (
-	"fmt"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-	"github.com/netbirdio/netbird/client/iface/netstack"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{
-		userspaceBind: true,
-	}
-
-	if netstack.IsEnabled() {
-		wgIFace.tun = device.NewNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr(), filterFn)
-		return wgIFace, nil
-	}
-
-	wgIFace.tun = device.NewTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, filterFn)
-	return wgIFace, nil
-}
-
-// CreateOnAndroid this function make sense on mobile only
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("this function has not implemented on non mobile")
-}
-
-// GetInterfaceGUIDString returns an interface GUID. This is useful on Windows only
-func (w *WGIface) GetInterfaceGUIDString() (string, error) {
-	return w.tun.(*device.TunDevice).GetInterfaceGUIDString()
-}

client/iface/iwginterface.go

@@ -11,6 +11,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type IWGIface interface {
@@ -22,6 +23,7 @@ type IWGIface interface {
 	ToInterface() *net.Interface
 	Up() (*bind.UniversalUDPMuxDefault, error)
 	UpdateAddr(newAddr string) error
+	GetProxy() wgproxy.Proxy
 	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
 	RemovePeer(peerKey string) error
 	AddAllowedIP(peerKey string, allowedIP string) error

client/iface/iwginterface_windows.go

@@ -9,6 +9,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type IWGIface interface {
@@ -20,6 +21,7 @@ type IWGIface interface {
 	ToInterface() *net.Interface
 	Up() (*bind.UniversalUDPMuxDefault, error)
 	UpdateAddr(newAddr string) error
+	GetProxy() wgproxy.Proxy
 	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
 	RemovePeer(peerKey string) error
 	AddAllowedIP(peerKey string, allowedIP string) error

client/iface/wgproxy/bind/proxy.go

@@ -0,0 +1,137 @@
+package bind
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+)
+
+type ProxyBind struct {
+	Bind *bind.ICEBind
+
+	wgAddr     *net.UDPAddr
+	wgEndpoint *bind.Endpoint
+	remoteConn net.Conn
+	ctx        context.Context
+	cancel     context.CancelFunc
+	closeMu    sync.Mutex
+	closed     bool
+
+	pausedMu  sync.Mutex
+	paused    bool
+	isStarted bool
+}
+
+// AddTurnConn adds a new connection to the bind.
+// endpoint is the NetBird address of the remote peer. The SetEndpoint return with the address what will be used in the
+// WireGuard configuration.
+func (p *ProxyBind) AddTurnConn(ctx context.Context, nbAddr *net.UDPAddr, remoteConn net.Conn) error {
+	addr, err := p.Bind.SetEndpoint(nbAddr, remoteConn)
+	if err != nil {
+		return err
+	}
+
+	p.wgAddr = addr
+	p.wgEndpoint = addrToEndpoint(addr)
+	p.remoteConn = remoteConn
+	p.ctx, p.cancel = context.WithCancel(ctx)
+	return err
+
+}
+func (p *ProxyBind) EndpointAddr() *net.UDPAddr {
+	return p.wgAddr
+}
+
+func (p *ProxyBind) Work() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = false
+	p.pausedMu.Unlock()
+
+	// Start the proxy only once
+	if !p.isStarted {
+		p.isStarted = true
+		go p.proxyToLocal(p.ctx)
+	}
+}
+
+func (p *ProxyBind) Pause() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = true
+	p.pausedMu.Unlock()
+}
+
+func (p *ProxyBind) CloseConn() error {
+	if p.cancel == nil {
+		return fmt.Errorf("proxy not started")
+	}
+	return p.close()
+}
+
+func (p *ProxyBind) close() error {
+	p.closeMu.Lock()
+	defer p.closeMu.Unlock()
+
+	if p.closed {
+		return nil
+	}
+	p.closed = true
+
+	p.cancel()
+
+	p.Bind.RemoveEndpoint(p.wgAddr)
+
+	return p.remoteConn.Close()
+}
+
+func (p *ProxyBind) proxyToLocal(ctx context.Context) {
+	defer func() {
+		if err := p.close(); err != nil {
+			log.Warnf("failed to close remote conn: %s", err)
+		}
+	}()
+
+	buf := make([]byte, 1500)
+	for {
+		n, err := p.remoteConn.Read(buf)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.RemoteAddr(), err)
+			return
+		}
+
+		p.pausedMu.Lock()
+		if p.paused {
+			p.pausedMu.Unlock()
+			continue
+		}
+
+		msg := bind.RecvMessage{
+			Endpoint: p.wgEndpoint,
+			Buffer:   buf[:n],
+		}
+		p.Bind.RecvChan <- msg
+		p.pausedMu.Unlock()
+	}
+}
+
+func addrToEndpoint(addr *net.UDPAddr) *bind.Endpoint {
+	ip, _ := netip.AddrFromSlice(addr.IP.To4())
+	addrPort := netip.AddrPortFrom(ip, uint16(addr.Port))
+	return &bind.Endpoint{AddrPort: addrPort}
+}

client/iface/wgproxy/ebpf/portlookup.go

@@ -5,9 +5,9 @@ import (
 	"net"
 )
 
-const (
+var (
 	portRangeStart = 3128
-	portRangeEnd   = 3228
+	portRangeEnd   = portRangeStart + 100
 )
 
 type portLookup struct {

client/iface/wgproxy/ebpf/portlookup_test.go

@@ -17,6 +17,9 @@ func Test_portLookup_searchFreePort(t *testing.T) {
 func Test_portLookup_on_allocated(t *testing.T) {
 	pl := portLookup{}
 
+	portRangeStart = 4128
+	portRangeEnd = portRangeStart + 100
+
 	allocatedPort, err := allocatePort(portRangeStart)
 	if err != nil {
 		t.Fatal(err)

client/iface/wgproxy/ebpf/proxy.go

@@ -5,7 +5,6 @@ package ebpf
 import (
 	"context"
 	"fmt"
-	"io"
 	"net"
 	"os"
 	"sync"
@@ -94,13 +93,12 @@ func (p *WGEBPFProxy) Listen() error {
 }
 
 // AddTurnConn add new turn connection for the proxy
-func (p *WGEBPFProxy) AddTurnConn(ctx context.Context, turnConn net.Conn) (net.Addr, error) {
+func (p *WGEBPFProxy) AddTurnConn(turnConn net.Conn) (*net.UDPAddr, error) {
 	wgEndpointPort, err := p.storeTurnConn(turnConn)
 	if err != nil {
 		return nil, err
 	}
 
-	go p.proxyToLocal(ctx, wgEndpointPort, turnConn)
 	log.Infof("turn conn added to wg proxy store: %s, endpoint port: :%d", turnConn.RemoteAddr(), wgEndpointPort)
 
 	wgEndpoint := &net.UDPAddr{
@@ -121,7 +119,7 @@ func (p *WGEBPFProxy) Free() error {
 	p.ctxCancel()
 
 	var result *multierror.Error
-	if p.conn != nil { // p.conn will be nil if we have failed to listen
+	if p.conn != nil {
 		if err := p.conn.Close(); err != nil {
 			result = multierror.Append(result, err)
 		}
@@ -137,35 +135,6 @@ func (p *WGEBPFProxy) Free() error {
 	return nberrors.FormatErrorOrNil(result)
 }
 
-func (p *WGEBPFProxy) proxyToLocal(ctx context.Context, endpointPort uint16, remoteConn net.Conn) {
-	defer p.removeTurnConn(endpointPort)
-
-	var (
-		err error
-		n   int
-	)
-	buf := make([]byte, 1500)
-	for ctx.Err() == nil {
-		n, err = remoteConn.Read(buf)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			if err != io.EOF {
-				log.Errorf("failed to read from turn conn (endpoint: :%d): %s", endpointPort, err)
-			}
-			return
-		}
-
-		if err := p.sendPkg(buf[:n], endpointPort); err != nil {
-			if ctx.Err() != nil || p.ctx.Err() != nil {
-				return
-			}
-			log.Errorf("failed to write out turn pkg to local conn: %v", err)
-		}
-	}
-}
-
 // proxyToRemote read messages from local WireGuard interface and forward it to remote conn
 // From this go routine has only one instance.
 func (p *WGEBPFProxy) proxyToRemote() {
@@ -280,7 +249,7 @@ func (p *WGEBPFProxy) prepareSenderRawSocket() (net.PacketConn, error) {
 	return packetConn, nil
 }
 
-func (p *WGEBPFProxy) sendPkg(data []byte, port uint16) error {
+func (p *WGEBPFProxy) sendPkg(data []byte, port int) error {
 	localhost := net.ParseIP("127.0.0.1")
 
 	payload := gopacket.Payload(data)

client/iface/wgproxy/ebpf/wrapper.go

@@ -0,0 +1,126 @@
+//go:build linux && !android
+
+package ebpf
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call
+type ProxyWrapper struct {
+	WgeBPFProxy *WGEBPFProxy
+
+	remoteConn net.Conn
+	ctx        context.Context
+	cancel     context.CancelFunc
+
+	wgEndpointAddr *net.UDPAddr
+
+	pausedMu  sync.Mutex
+	paused    bool
+	isStarted bool
+}
+
+func (p *ProxyWrapper) AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error {
+	addr, err := p.WgeBPFProxy.AddTurnConn(remoteConn)
+	if err != nil {
+		return fmt.Errorf("add turn conn: %w", err)
+	}
+	p.remoteConn = remoteConn
+	p.ctx, p.cancel = context.WithCancel(ctx)
+	p.wgEndpointAddr = addr
+	return err
+}
+
+func (p *ProxyWrapper) EndpointAddr() *net.UDPAddr {
+	return p.wgEndpointAddr
+}
+
+func (p *ProxyWrapper) Work() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = false
+	p.pausedMu.Unlock()
+
+	if !p.isStarted {
+		p.isStarted = true
+		go p.proxyToLocal(p.ctx)
+	}
+}
+
+func (p *ProxyWrapper) Pause() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	log.Tracef("pause proxy reading from: %s", p.remoteConn.RemoteAddr())
+	p.pausedMu.Lock()
+	p.paused = true
+	p.pausedMu.Unlock()
+}
+
+// CloseConn close the remoteConn and automatically remove the conn instance from the map
+func (e *ProxyWrapper) CloseConn() error {
+	if e.cancel == nil {
+		return fmt.Errorf("proxy not started")
+	}
+
+	e.cancel()
+
+	if err := e.remoteConn.Close(); err != nil {
+		return fmt.Errorf("failed to close remote conn: %w", err)
+	}
+	return nil
+}
+
+func (p *ProxyWrapper) proxyToLocal(ctx context.Context) {
+	defer p.WgeBPFProxy.removeTurnConn(uint16(p.wgEndpointAddr.Port))
+
+	buf := make([]byte, 1500)
+	for {
+		n, err := p.readFromRemote(ctx, buf)
+		if err != nil {
+			return
+		}
+
+		p.pausedMu.Lock()
+		if p.paused {
+			p.pausedMu.Unlock()
+			continue
+		}
+
+		err = p.WgeBPFProxy.sendPkg(buf[:n], p.wgEndpointAddr.Port)
+		p.pausedMu.Unlock()
+
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Errorf("failed to write out turn pkg to local conn: %v", err)
+		}
+	}
+}
+
+func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, error) {
+	n, err := p.remoteConn.Read(buf)
+	if err != nil {
+		if ctx.Err() != nil {
+			return 0, ctx.Err()
+		}
+		if !errors.Is(err, io.EOF) {
+			log.Errorf("failed to read from turn conn (endpoint: :%d): %s", p.wgEndpointAddr.Port, err)
+		}
+		return 0, err
+	}
+	return n, nil
+}

client/iface/wgproxy/factory_kernel.go

@@ -0,0 +1,49 @@
+//go:build linux && !android
+
+package wgproxy
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
+	udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
+)
+
+type KernelFactory struct {
+	wgPort int
+
+	ebpfProxy *ebpf.WGEBPFProxy
+}
+
+func NewKernelFactory(wgPort int) *KernelFactory {
+	f := &KernelFactory{
+		wgPort: wgPort,
+	}
+
+	ebpfProxy := ebpf.NewWGEBPFProxy(wgPort)
+	if err := ebpfProxy.Listen(); err != nil {
+		log.Infof("WireGuard Proxy Factory will produce UDP proxy")
+		log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err)
+		return f
+	}
+	log.Infof("WireGuard Proxy Factory will produce eBPF proxy")
+	f.ebpfProxy = ebpfProxy
+	return f
+}
+
+func (w *KernelFactory) GetProxy() Proxy {
+	if w.ebpfProxy == nil {
+		return udpProxy.NewWGUDPProxy(w.wgPort)
+	}
+
+	return &ebpf.ProxyWrapper{
+		WgeBPFProxy: w.ebpfProxy,
+	}
+}
+
+func (w *KernelFactory) Free() error {
+	if w.ebpfProxy == nil {
+		return nil
+	}
+	return w.ebpfProxy.Free()
+}

client/iface/wgproxy/factory_kernel_freebsd.go

@@ -0,0 +1,29 @@
+package wgproxy
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
+)
+
+// KernelFactory todo: check eBPF support on FreeBSD
+type KernelFactory struct {
+	wgPort int
+}
+
+func NewKernelFactory(wgPort int) *KernelFactory {
+	log.Infof("WireGuard Proxy Factory will produce UDP proxy")
+	f := &KernelFactory{
+		wgPort: wgPort,
+	}
+
+	return f
+}
+
+func (w *KernelFactory) GetProxy() Proxy {
+	return udpProxy.NewWGUDPProxy(w.wgPort)
+}
+
+func (w *KernelFactory) Free() error {
+	return nil
+}

client/iface/wgproxy/factory_usp.go

@@ -0,0 +1,30 @@
+package wgproxy
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+	proxyBind "github.com/netbirdio/netbird/client/iface/wgproxy/bind"
+)
+
+type USPFactory struct {
+	bind *bind.ICEBind
+}
+
+func NewUSPFactory(iceBind *bind.ICEBind) *USPFactory {
+	log.Infof("WireGuard Proxy Factory will produce bind proxy")
+	f := &USPFactory{
+		bind: iceBind,
+	}
+	return f
+}
+
+func (w *USPFactory) GetProxy() Proxy {
+	return &proxyBind.ProxyBind{
+		Bind: w.bind,
+	}
+}
+
+func (w *USPFactory) Free() error {
+	return nil
+}

client/iface/wgproxy/proxy.go

@@ -0,0 +1,15 @@
+package wgproxy
+
+import (
+	"context"
+	"net"
+)
+
+// Proxy is a transfer layer between the relayed connection and the WireGuard
+type Proxy interface {
+	AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error
+	EndpointAddr() *net.UDPAddr // EndpointAddr returns the address of the WireGuard peer endpoint
+	Work()                      // Work start or resume the proxy
+	Pause()                     // Pause to forward the packages from remote connection to WireGuard. The opposite way still works.
+	CloseConn() error
+}

client/iface/wgproxy/proxy_linux_test.go

@@ -0,0 +1,56 @@
+//go:build linux && !android
+
+package wgproxy
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
+)
+
+func TestProxyCloseByRemoteConnEBPF(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") != "true" {
+		t.Skip("Skipping test as it requires root privileges")
+	}
+	ctx := context.Background()
+
+	ebpfProxy := ebpf.NewWGEBPFProxy(51831)
+	if err := ebpfProxy.Listen(); err != nil {
+		t.Fatalf("failed to initialize ebpf proxy: %s", err)
+	}
+
+	defer func() {
+		if err := ebpfProxy.Free(); err != nil {
+			t.Errorf("failed to free ebpf proxy: %s", err)
+		}
+	}()
+
+	tests := []struct {
+		name  string
+		proxy Proxy
+	}{
+		{
+			name: "ebpf proxy",
+			proxy: &ebpf.ProxyWrapper{
+				WgeBPFProxy: ebpfProxy,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			relayedConn := newMockConn()
+			err := tt.proxy.AddTurnConn(ctx, nil, relayedConn)
+			if err != nil {
+				t.Errorf("error: %v", err)
+			}
+
+			_ = relayedConn.Close()
+			if err := tt.proxy.CloseConn(); err != nil {
+				t.Errorf("error: %v", err)
+			}
+		})
+	}
+}

client/iface/wgproxy/proxy_test.go

@@ -11,8 +11,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/netbirdio/netbird/client/internal/wgproxy/ebpf"
-	"github.com/netbirdio/netbird/client/internal/wgproxy/usp"
+	"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
+	udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -84,7 +84,7 @@ func TestProxyCloseByRemoteConn(t *testing.T) {
 	}{
 		{
 			name:  "userspace proxy",
-			proxy: usp.NewWGUserSpaceProxy(51830),
+			proxy: udpProxy.NewWGUDPProxy(51830),
 		},
 	}
 
@@ -114,7 +114,7 @@ func TestProxyCloseByRemoteConn(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			relayedConn := newMockConn()
-			_, err := tt.proxy.AddTurnConn(ctx, relayedConn)
+			err := tt.proxy.AddTurnConn(ctx, nil, relayedConn)
 			if err != nil {
 				t.Errorf("error: %v", err)
 			}

client/iface/wgproxy/udp/proxy.go

@@ -0,0 +1,207 @@
+package udp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+
+	cerrors "github.com/netbirdio/netbird/client/errors"
+)
+
+// WGUDPProxy proxies
+type WGUDPProxy struct {
+	localWGListenPort int
+
+	remoteConn net.Conn
+	localConn  net.Conn
+	ctx        context.Context
+	cancel     context.CancelFunc
+	closeMu    sync.Mutex
+	closed     bool
+
+	pausedMu  sync.Mutex
+	paused    bool
+	isStarted bool
+}
+
+// NewWGUDPProxy instantiate a UDP based WireGuard proxy. This is not a thread safe implementation
+func NewWGUDPProxy(wgPort int) *WGUDPProxy {
+	log.Debugf("Initializing new user space proxy with port %d", wgPort)
+	p := &WGUDPProxy{
+		localWGListenPort: wgPort,
+	}
+	return p
+}
+
+// AddTurnConn
+// The provided Context must be non-nil. If the context expires before
+// the connection is complete, an error is returned. Once successfully
+// connected, any expiration of the context will not affect the
+// connection.
+func (p *WGUDPProxy) AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error {
+	dialer := net.Dialer{}
+	localConn, err := dialer.DialContext(ctx, "udp", fmt.Sprintf(":%d", p.localWGListenPort))
+	if err != nil {
+		log.Errorf("failed dialing to local Wireguard port %s", err)
+		return err
+	}
+
+	p.ctx, p.cancel = context.WithCancel(ctx)
+	p.localConn = localConn
+	p.remoteConn = remoteConn
+
+	return err
+}
+
+func (p *WGUDPProxy) EndpointAddr() *net.UDPAddr {
+	if p.localConn == nil {
+		return nil
+	}
+	endpointUdpAddr, _ := net.ResolveUDPAddr(p.localConn.LocalAddr().Network(), p.localConn.LocalAddr().String())
+	return endpointUdpAddr
+}
+
+// Work starts the proxy or resumes it if it was paused
+func (p *WGUDPProxy) Work() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = false
+	p.pausedMu.Unlock()
+
+	if !p.isStarted {
+		p.isStarted = true
+		go p.proxyToRemote(p.ctx)
+		go p.proxyToLocal(p.ctx)
+	}
+}
+
+// Pause pauses the proxy from receiving data from the remote peer
+func (p *WGUDPProxy) Pause() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = true
+	p.pausedMu.Unlock()
+}
+
+// CloseConn close the localConn
+func (p *WGUDPProxy) CloseConn() error {
+	if p.cancel == nil {
+		return fmt.Errorf("proxy not started")
+	}
+	return p.close()
+}
+
+func (p *WGUDPProxy) close() error {
+	p.closeMu.Lock()
+	defer p.closeMu.Unlock()
+
+	// prevent double close
+	if p.closed {
+		return nil
+	}
+	p.closed = true
+
+	p.cancel()
+
+	var result *multierror.Error
+	if err := p.remoteConn.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("remote conn: %s", err))
+	}
+
+	if err := p.localConn.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("local conn: %s", err))
+	}
+	return cerrors.FormatErrorOrNil(result)
+}
+
+// proxyToRemote proxies from Wireguard to the RemoteKey
+func (p *WGUDPProxy) proxyToRemote(ctx context.Context) {
+	defer func() {
+		if err := p.close(); err != nil {
+			log.Warnf("error in proxy to remote loop: %s", err)
+		}
+	}()
+
+	buf := make([]byte, 1500)
+	for ctx.Err() == nil {
+		n, err := p.localConn.Read(buf)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Debugf("failed to read from wg interface conn: %s", err)
+			return
+		}
+
+		_, err = p.remoteConn.Write(buf[:n])
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+
+			log.Debugf("failed to write to remote conn: %s", err)
+			return
+		}
+	}
+}
+
+// proxyToLocal proxies from the Remote peer to local WireGuard
+// if the proxy is paused it will drain the remote conn and drop the packets
+func (p *WGUDPProxy) proxyToLocal(ctx context.Context) {
+	defer func() {
+		if err := p.close(); err != nil {
+			if !errors.Is(err, io.EOF) {
+				log.Warnf("error in proxy to local loop: %s", err)
+			}
+		}
+	}()
+
+	buf := make([]byte, 1500)
+	for {
+		n, err := p.remoteConnRead(ctx, buf)
+		if err != nil {
+			return
+		}
+
+		p.pausedMu.Lock()
+		if p.paused {
+			p.pausedMu.Unlock()
+			continue
+		}
+
+		_, err = p.localConn.Write(buf[:n])
+		p.pausedMu.Unlock()
+
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Debugf("failed to write to wg interface conn: %s", err)
+			continue
+		}
+	}
+}
+
+func (p *WGUDPProxy) remoteConnRead(ctx context.Context, buf []byte) (n int, err error) {
+	n, err = p.remoteConn.Read(buf)
+	if err != nil {
+		if ctx.Err() != nil {
+			return
+		}
+		log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.LocalAddr(), err)
+		return
+	}
+	return
+}

client/internal/acl/id/id.go

@@ -1,8 +1,11 @@
 package id
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net/netip"
+	"strconv"
 
 	"github.com/netbirdio/netbird/client/firewall/manager"
 )
@@ -21,5 +24,41 @@ func GenerateRouteRuleKey(
 	dPort *manager.Port,
 	action manager.Action,
 ) RuleID {
-	return RuleID(fmt.Sprintf("%s-%s-%s-%s-%s-%d", sources, destination, proto, sPort, dPort, action))
+	manager.SortPrefixes(sources)
+
+	h := sha256.New()
+
+	// Write all fields to the hasher, with delimiters
+	h.Write([]byte("sources:"))
+	for _, src := range sources {
+		h.Write([]byte(src.String()))
+		h.Write([]byte(","))
+	}
+
+	h.Write([]byte("destination:"))
+	h.Write([]byte(destination.String()))
+
+	h.Write([]byte("proto:"))
+	h.Write([]byte(proto))
+
+	h.Write([]byte("sPort:"))
+	if sPort != nil {
+		h.Write([]byte(sPort.String()))
+	} else {
+		h.Write([]byte("<nil>"))
+	}
+
+	h.Write([]byte("dPort:"))
+	if dPort != nil {
+		h.Write([]byte(dPort.String()))
+	} else {
+		h.Write([]byte("<nil>"))
+	}
+
+	h.Write([]byte("action:"))
+	h.Write([]byte(strconv.Itoa(int(action))))
+	hash := hex.EncodeToString(h.Sum(nil))
+
+	// prepend destination prefix to be able to identify the rule
+	return RuleID(fmt.Sprintf("%s-%s", destination.String(), hash[:16]))
 }

client/internal/acl/manager.go

@@ -3,6 +3,7 @@ package acl
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"net"
 	"net/netip"
@@ -10,14 +11,18 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl/id"
 	"github.com/netbirdio/netbird/client/ssh"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
 )
 
+var ErrSourceRangesEmpty = errors.New("sources range is empty")
+
 // Manager is a ACL rules manager
 type Manager interface {
 	ApplyFiltering(networkMap *mgmProto.NetworkMap)
@@ -167,31 +172,40 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) {
 }
 
 func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule) error {
-	var newRouteRules = make(map[id.RuleID]struct{})
+	newRouteRules := make(map[id.RuleID]struct{}, len(rules))
+	var merr *multierror.Error
+
+	// Apply new rules - firewall manager will return existing rule ID if already present
 	for _, rule := range rules {
 		id, err := d.applyRouteACL(rule)
 		if err != nil {
-			return fmt.Errorf("apply route ACL: %w", err)
+			if errors.Is(err, ErrSourceRangesEmpty) {
+				log.Debugf("skipping empty rule with destination %s: %v", rule.Destination, err)
+			} else {
+				merr = multierror.Append(merr, fmt.Errorf("add route rule: %w", err))
+			}
+			continue
 		}
 		newRouteRules[id] = struct{}{}
 	}
 
+	// Clean up old firewall rules
 	for id := range d.routeRules {
-		if _, ok := newRouteRules[id]; !ok {
+		if _, exists := newRouteRules[id]; !exists {
 			if err := d.firewall.DeleteRouteRule(id); err != nil {
-				log.Errorf("failed to delete route firewall rule: %v", err)
-				continue
+				merr = multierror.Append(merr, fmt.Errorf("delete route rule: %w", err))
 			}
-			delete(d.routeRules, id)
+			// implicitly deleted from the map
 		}
 	}
+
 	d.routeRules = newRouteRules
-	return nil
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule) (id.RuleID, error) {
 	if len(rule.SourceRanges) == 0 {
-		return "", fmt.Errorf("source ranges is empty")
+		return "", ErrSourceRangesEmpty
 	}
 
 	var sources []netip.Prefix

client/internal/acl/manager_test.go

@@ -1,7 +1,6 @@
 package acl
 
 import (
-	"context"
 	"net"
 	"testing"
 
@@ -52,13 +51,13 @@ func TestDefaultManager(t *testing.T) {
 	}).AnyTimes()
 
 	// we receive one rule from the management so for testing purposes ignore it
-	fw, err := firewall.NewFirewall(context.Background(), ifaceMock)
+	fw, err := firewall.NewFirewall(ifaceMock, nil)
 	if err != nil {
 		t.Errorf("create firewall: %v", err)
 		return
 	}
 	defer func(fw manager.Manager) {
-		_ = fw.Reset()
+		_ = fw.Reset(nil)
 	}(fw)
 	acl := NewDefaultManager(fw)
 
@@ -345,13 +344,13 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) {
 	}).AnyTimes()
 
 	// we receive one rule from the management so for testing purposes ignore it
-	fw, err := firewall.NewFirewall(context.Background(), ifaceMock)
+	fw, err := firewall.NewFirewall(ifaceMock, nil)
 	if err != nil {
 		t.Errorf("create firewall: %v", err)
 		return
 	}
 	defer func(fw manager.Manager) {
-		_ = fw.Reset()
+		_ = fw.Reset(nil)
 	}(fw)
 	acl := NewDefaultManager(fw)
 

client/internal/connect.go

@@ -62,10 +62,7 @@ func (c *ConnectClient) Run() error {
 }
 
 // RunWithProbes runs the client's main logic with probes attached
-func (c *ConnectClient) RunWithProbes(
-	probes *ProbeHolder,
-	runningChan chan error,
-) error {
+func (c *ConnectClient) RunWithProbes(probes *ProbeHolder, runningChan chan error) error {
 	return c.run(MobileDependency{}, probes, runningChan)
 }
 
@@ -104,11 +101,7 @@ func (c *ConnectClient) RunOniOS(
 	return c.run(mobileDependency, nil, nil)
 }
 
-func (c *ConnectClient) run(
-	mobileDependency MobileDependency,
-	probes *ProbeHolder,
-	runningChan chan error,
-) error {
+func (c *ConnectClient) run(mobileDependency MobileDependency, probes *ProbeHolder, runningChan chan error) error {
 	defer func() {
 		if r := recover(); r != nil {
 			log.Panicf("Panic occurred: %v, stack trace: %s", r, string(debug.Stack()))
@@ -117,12 +110,6 @@ func (c *ConnectClient) run(
 
 	log.Infof("starting NetBird client version %s on %s/%s", version.NetbirdVersion(), runtime.GOOS, runtime.GOARCH)
 
-	// Check if client was not shut down in a clean way and restore DNS config if required.
-	// Otherwise, we might not be able to connect to the management server to retrieve new config.
-	if err := dns.CheckUncleanShutdown(c.config.WgIface); err != nil {
-		log.Errorf("checking unclean shutdown error: %s", err)
-	}
-
 	backOff := &backoff.ExponentialBackOff{
 		InitialInterval:     time.Second,
 		RandomizationFactor: 1,
@@ -358,7 +345,11 @@ func (c *ConnectClient) Stop() error {
 	if c.engine == nil {
 		return nil
 	}
-	return c.engine.Stop()
+	if err := c.engine.Stop(); err != nil {
+		return fmt.Errorf("stop engine: %w", err)
+	}
+
+	return nil
 }
 
 func (c *ConnectClient) isContextCancelled() bool {

client/internal/dns/consts_freebsd.go

@@ -1,6 +1,5 @@
 package dns
 
 const (
-	fileUncleanShutdownResolvConfLocation  = "/var/db/netbird/resolv.conf"
-	fileUncleanShutdownManagerTypeLocation = "/var/db/netbird/manager"
+	fileUncleanShutdownResolvConfLocation = "/var/db/netbird/resolv.conf"
 )

client/internal/dns/consts_linux.go

@@ -3,6 +3,5 @@
 package dns
 
 const (
-	fileUncleanShutdownResolvConfLocation  = "/var/lib/netbird/resolv.conf"
-	fileUncleanShutdownManagerTypeLocation = "/var/lib/netbird/manager"
+	fileUncleanShutdownResolvConfLocation = "/var/lib/netbird/resolv.conf"
 )

client/internal/dns/file_repair_unix.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/fsnotify/fsnotify"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 var (
@@ -20,7 +22,7 @@ var (
 	}
 )
 
-type repairConfFn func([]string, string, *resolvConf) error
+type repairConfFn func([]string, string, *resolvConf, *statemanager.Manager) error
 
 type repair struct {
 	operationFile string
@@ -40,7 +42,7 @@ func newRepair(operationFile string, updateFn repairConfFn) *repair {
 	}
 }
 
-func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP string) {
+func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP string, stateManager *statemanager.Manager) {
 	if f.inotify != nil {
 		return
 	}
@@ -81,7 +83,7 @@ func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP strin
 				log.Errorf("failed to rm inotify watch for resolv.conf: %s", err)
 			}
 
-			err = f.updateFn(nbSearchDomains, nbNameserverIP, rConf)
+			err = f.updateFn(nbSearchDomains, nbNameserverIP, rConf, stateManager)
 			if err != nil {
 				log.Errorf("failed to repair resolv.conf: %v", err)
 			}

client/internal/dns/file_repair_unix_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -104,14 +105,14 @@ nameserver 8.8.8.8`,
 
 			var changed bool
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-			updateFn := func([]string, string, *resolvConf) error {
+			updateFn := func([]string, string, *resolvConf, *statemanager.Manager) error {
 				changed = true
 				cancel()
 				return nil
 			}
 
 			r := newRepair(operationFile, updateFn)
-			r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1")
+			r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1", nil)
 
 			err = os.WriteFile(operationFile, []byte(tt.touchedConfContent), 0755)
 			if err != nil {
@@ -151,14 +152,14 @@ searchdomain netbird.cloud something`
 
 	var changed bool
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	updateFn := func([]string, string, *resolvConf) error {
+	updateFn := func([]string, string, *resolvConf, *statemanager.Manager) error {
 		changed = true
 		cancel()
 		return nil
 	}
 
 	r := newRepair(tmpLink, updateFn)
-	r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1")
+	r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1", nil)
 
 	err = os.WriteFile(tmpLink, []byte(modifyContent), 0755)
 	if err != nil {

client/internal/dns/file_unix.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -36,7 +38,7 @@ type fileConfigurator struct {
 	nbNameserverIP string
 }
 
-func newFileConfigurator() (hostManager, error) {
+func newFileConfigurator() (*fileConfigurator, error) {
 	fc := &fileConfigurator{}
 	fc.repair = newRepair(defaultResolvConfPath, fc.updateConfig)
 	return fc, nil
@@ -46,7 +48,7 @@ func (f *fileConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	backupFileExist := f.isBackupFileExist()
 	if !config.RouteAll {
 		if backupFileExist {
@@ -76,15 +78,15 @@ func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig) error {
 
 	f.repair.stopWatchFileChanges()
 
-	err = f.updateConfig(nbSearchDomains, f.nbNameserverIP, resolvConf)
+	err = f.updateConfig(nbSearchDomains, f.nbNameserverIP, resolvConf, stateManager)
 	if err != nil {
 		return err
 	}
-	f.repair.watchFileChanges(nbSearchDomains, f.nbNameserverIP)
+	f.repair.watchFileChanges(nbSearchDomains, f.nbNameserverIP, stateManager)
 	return nil
 }
 
-func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP string, cfg *resolvConf) error {
+func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP string, cfg *resolvConf, stateManager *statemanager.Manager) error {
 	searchDomainList := mergeSearchDomains(nbSearchDomains, cfg.searchDomains)
 	nameServers := generateNsList(nbNameserverIP, cfg)
 
@@ -107,7 +109,7 @@ func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP
 	log.Infof("created a NetBird managed %s file with the DNS settings. Added %d search domains. Search list: %s", defaultResolvConfPath, len(searchDomainList), searchDomainList)
 
 	// create another backup for unclean shutdown detection right after overwriting the original resolv.conf
-	if err := createUncleanShutdownIndicator(fileDefaultResolvConfBackupLocation, fileManager, nbNameserverIP); err != nil {
+	if err := createUncleanShutdownIndicator(fileDefaultResolvConfBackupLocation, nbNameserverIP, stateManager); err != nil {
 		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
 	}
 
@@ -145,10 +147,6 @@ func (f *fileConfigurator) restore() error {
 		return fmt.Errorf("restoring %s from %s: %w", defaultResolvConfPath, fileDefaultResolvConfBackupLocation, err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return os.RemoveAll(fileDefaultResolvConfBackupLocation)
 }
 
@@ -176,7 +174,7 @@ func (f *fileConfigurator) restoreUncleanShutdownDNS(storedDNSAddress *netip.Add
 		return restoreResolvConfFile()
 	}
 
-	log.Info("restoring unclean shutdown: first current nameserver differs from saved nameserver pre-netbird: not restoring")
+	log.Infof("restoring unclean shutdown: first current nameserver differs from saved nameserver pre-netbird: %s (current) vs %s (stored): not restoring", currentDNSAddress, storedDNSAddress)
 	return nil
 }
 
@@ -192,10 +190,6 @@ func restoreResolvConfFile() error {
 		return fmt.Errorf("restoring %s from %s: %w", defaultResolvConfPath, fileUncleanShutdownResolvConfLocation, err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf file: %s", err)
-	}
-
 	return nil
 }
 

client/internal/dns/host.go

@@ -5,14 +5,14 @@ import (
 	"net/netip"
 	"strings"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
 type hostManager interface {
-	applyDNSConfig(config HostDNSConfig) error
+	applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error
 	restoreHostDNS() error
 	supportCustomPort() bool
-	restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error
 }
 
 type SystemDNSSettings struct {
@@ -35,15 +35,15 @@ type DomainConfig struct {
 }
 
 type mockHostConfigurator struct {
-	applyDNSConfigFunc            func(config HostDNSConfig) error
+	applyDNSConfigFunc            func(config HostDNSConfig, stateManager *statemanager.Manager) error
 	restoreHostDNSFunc            func() error
 	supportCustomPortFunc         func() bool
 	restoreUncleanShutdownDNSFunc func(*netip.Addr) error
 }
 
-func (m *mockHostConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (m *mockHostConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	if m.applyDNSConfigFunc != nil {
-		return m.applyDNSConfigFunc(config)
+		return m.applyDNSConfigFunc(config, stateManager)
 	}
 	return fmt.Errorf("method applyDNSSettings is not implemented")
 }
@@ -62,16 +62,9 @@ func (m *mockHostConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (m *mockHostConfigurator) restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error {
-	if m.restoreUncleanShutdownDNSFunc != nil {
-		return m.restoreUncleanShutdownDNSFunc(storedDNSAddress)
-	}
-	return fmt.Errorf("method restoreUncleanShutdownDNS is not implemented")
-}
-
 func newNoopHostMocker() hostManager {
 	return &mockHostConfigurator{
-		applyDNSConfigFunc:            func(config HostDNSConfig) error { return nil },
+		applyDNSConfigFunc:            func(config HostDNSConfig, stateManager *statemanager.Manager) error { return nil },
 		restoreHostDNSFunc:            func() error { return nil },
 		supportCustomPortFunc:         func() bool { return true },
 		restoreUncleanShutdownDNSFunc: func(*netip.Addr) error { return nil },

client/internal/dns/host_android.go

@@ -1,15 +1,17 @@
 package dns
 
-import "net/netip"
+import (
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
 
 type androidHostManager struct {
 }
 
-func newHostManager() (hostManager, error) {
+func newHostManager() (*androidHostManager, error) {
 	return &androidHostManager{}, nil
 }
 
-func (a androidHostManager) applyDNSConfig(config HostDNSConfig) error {
+func (a androidHostManager) applyDNSConfig(HostDNSConfig, *statemanager.Manager) error {
 	return nil
 }
 
@@ -20,7 +22,3 @@ func (a androidHostManager) restoreHostDNS() error {
 func (a androidHostManager) supportCustomPort() bool {
 	return false
 }
-
-func (a androidHostManager) restoreUncleanShutdownDNS(*netip.Addr) error {
-	return nil
-}

client/internal/dns/host_darwin.go

@@ -8,12 +8,13 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"net/netip"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -37,7 +38,7 @@ type systemConfigurator struct {
 	systemDNSSettings SystemDNSSettings
 }
 
-func newHostManager() (hostManager, error) {
+func newHostManager() (*systemConfigurator, error) {
 	return &systemConfigurator{
 		createdKeys: make(map[string]struct{}),
 	}, nil
@@ -47,12 +48,11 @@ func (s *systemConfigurator) supportCustomPort() bool {
 	return true
 }
 
-func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 
-	// create a file for unclean shutdown detection
-	if err := createUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to create unclean shutdown file: %s", err)
+	if err := stateManager.UpdateState(&ShutdownState{}); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	var (
@@ -123,10 +123,6 @@ func (s *systemConfigurator) restoreHostDNS() error {
 		}
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown file: %s", err)
-	}
-
 	return nil
 }
 
@@ -320,7 +316,7 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) {
 	return primaryService, router, nil
 }
 
-func (s *systemConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
+func (s *systemConfigurator) restoreUncleanShutdownDNS() error {
 	if err := s.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via scutil: %w", err)
 	}

client/internal/dns/host_ios.go

@@ -3,9 +3,10 @@ package dns
 import (
 	"encoding/json"
 	"fmt"
-	"net/netip"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 type iosHostManager struct {
@@ -13,13 +14,13 @@ type iosHostManager struct {
 	config     HostDNSConfig
 }
 
-func newHostManager(dnsManager IosDnsManager) (hostManager, error) {
+func newHostManager(dnsManager IosDnsManager) (*iosHostManager, error) {
 	return &iosHostManager{
 		dnsManager: dnsManager,
 	}, nil
 }
 
-func (a iosHostManager) applyDNSConfig(config HostDNSConfig) error {
+func (a iosHostManager) applyDNSConfig(config HostDNSConfig, _ *statemanager.Manager) error {
 	jsonData, err := json.Marshal(config)
 	if err != nil {
 		return fmt.Errorf("marshal: %w", err)
@@ -37,7 +38,3 @@ func (a iosHostManager) restoreHostDNS() error {
 func (a iosHostManager) supportCustomPort() bool {
 	return false
 }
-
-func (a iosHostManager) restoreUncleanShutdownDNS(*netip.Addr) error {
-	return nil
-}

client/internal/dns/host_unix.go

@@ -4,9 +4,9 @@ package dns
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
+	"net/netip"
 	"os"
 	"strings"
 
@@ -21,27 +21,8 @@ const (
 	resolvConfManager
 )
 
-var ErrUnknownOsManagerType = errors.New("unknown os manager type")
-
 type osManagerType int
 
-func newOsManagerType(osManager string) (osManagerType, error) {
-	switch osManager {
-	case "netbird":
-		return fileManager, nil
-	case "file":
-		return netbirdManager, nil
-	case "networkManager":
-		return networkManager, nil
-	case "systemd":
-		return systemdManager, nil
-	case "resolvconf":
-		return resolvConfManager, nil
-	default:
-		return 0, ErrUnknownOsManagerType
-	}
-}
-
 func (t osManagerType) String() string {
 	switch t {
 	case netbirdManager:
@@ -59,6 +40,11 @@ func (t osManagerType) String() string {
 	}
 }
 
+type restoreHostManager interface {
+	hostManager
+	restoreUncleanShutdownDNS(*netip.Addr) error
+}
+
 func newHostManager(wgInterface string) (hostManager, error) {
 	osManager, err := getOSDNSManagerType()
 	if err != nil {
@@ -69,7 +55,7 @@ func newHostManager(wgInterface string) (hostManager, error) {
 	return newHostManagerFromType(wgInterface, osManager)
 }
 
-func newHostManagerFromType(wgInterface string, osManager osManagerType) (hostManager, error) {
+func newHostManagerFromType(wgInterface string, osManager osManagerType) (restoreHostManager, error) {
 	switch osManager {
 	case networkManager:
 		return newNetworkManagerDbusConfigurator(wgInterface)

client/internal/dns/host_windows.go

@@ -3,11 +3,12 @@ package dns
 import (
 	"fmt"
 	"io"
-	"net/netip"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/registry"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -31,7 +32,7 @@ type registryConfigurator struct {
 	routingAll bool
 }
 
-func newHostManager(wgInterface WGIface) (hostManager, error) {
+func newHostManager(wgInterface WGIface) (*registryConfigurator, error) {
 	guid, err := wgInterface.GetInterfaceGUIDString()
 	if err != nil {
 		return nil, err
@@ -39,7 +40,7 @@ func newHostManager(wgInterface WGIface) (hostManager, error) {
 	return newHostManagerWithGuid(guid)
 }
 
-func newHostManagerWithGuid(guid string) (hostManager, error) {
+func newHostManagerWithGuid(guid string) (*registryConfigurator, error) {
 	return &registryConfigurator{
 		guid: guid,
 	}, nil
@@ -49,7 +50,7 @@ func (r *registryConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 	if config.RouteAll {
 		err = r.addDNSSetupForAll(config.ServerIP)
@@ -65,9 +66,8 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig) error {
 		log.Infof("removed %s as main DNS forwarder for this peer", config.ServerIP)
 	}
 
-	// create a file for unclean shutdown detection
-	if err := createUncleanShutdownIndicator(r.guid); err != nil {
-		log.Errorf("failed to create unclean shutdown file: %s", err)
+	if err := stateManager.UpdateState(&ShutdownState{Guid: r.guid}); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	var (
@@ -160,10 +160,6 @@ func (r *registryConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("remove interface registry key: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown file: %s", err)
-	}
-
 	return nil
 }
 
@@ -221,7 +217,7 @@ func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) {
 	return regKey, nil
 }
 
-func (r *registryConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
+func (r *registryConfigurator) restoreUncleanShutdownDNS() error {
 	if err := r.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via registry: %w", err)
 	}

client/internal/dns/network_manager_unix.go

@@ -16,6 +16,7 @@ import (
 	"github.com/miekg/dns"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbversion "github.com/netbirdio/netbird/version"
 )
 
@@ -53,6 +54,7 @@ var supportedNetworkManagerVersionConstraints = []string{
 type networkManagerDbusConfigurator struct {
 	dbusLinkObject dbus.ObjectPath
 	routingAll     bool
+	ifaceName      string
 }
 
 // the types below are based on dbus specification, each field is mapped to a dbus type
@@ -77,7 +79,7 @@ func (s networkManagerConnSettings) cleanDeprecatedSettings() {
 	}
 }
 
-func newNetworkManagerDbusConfigurator(wgInterface string) (hostManager, error) {
+func newNetworkManagerDbusConfigurator(wgInterface string) (*networkManagerDbusConfigurator, error) {
 	obj, closeConn, err := getDbusObject(networkManagerDest, networkManagerDbusObjectNode)
 	if err != nil {
 		return nil, fmt.Errorf("get nm dbus: %w", err)
@@ -93,6 +95,7 @@ func newNetworkManagerDbusConfigurator(wgInterface string) (hostManager, error)
 
 	return &networkManagerDbusConfigurator{
 		dbusLinkObject: dbus.ObjectPath(s),
+		ifaceName:      wgInterface,
 	}, nil
 }
 
@@ -100,7 +103,7 @@ func (n *networkManagerDbusConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	connSettings, configVersion, err := n.getAppliedConnectionSettings()
 	if err != nil {
 		return fmt.Errorf("retrieving the applied connection settings, error: %w", err)
@@ -151,10 +154,12 @@ func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig) er
 	connSettings[networkManagerDbusIPv4Key][networkManagerDbusDNSPriorityKey] = dbus.MakeVariant(priority)
 	connSettings[networkManagerDbusIPv4Key][networkManagerDbusDNSSearchKey] = dbus.MakeVariant(newDomainList)
 
-	// create a backup for unclean shutdown detection before adding domains, as these might end up in the resolv.conf file.
-	// The file content itself is not important for network-manager restoration
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, networkManager, dnsIP.String()); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: networkManager,
+		WgIface:     n.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	log.Infof("adding %d search domains and %d match domains. Search list: %s , Match list: %s", len(searchDomains), len(matchDomains), searchDomains, matchDomains)
@@ -171,10 +176,6 @@ func (n *networkManagerDbusConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("delete connection settings: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return nil
 }
 

client/internal/dns/resolvconf_unix.go

@@ -9,6 +9,8 @@ import (
 	"os/exec"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const resolvconfCommand = "resolvconf"
@@ -22,7 +24,7 @@ type resolvconf struct {
 }
 
 // supported "openresolv" only
-func newResolvConfConfigurator(wgInterface string) (hostManager, error) {
+func newResolvConfConfigurator(wgInterface string) (*resolvconf, error) {
 	resolvConfEntries, err := parseDefaultResolvConf()
 	if err != nil {
 		log.Errorf("could not read original search domains from %s: %s", defaultResolvConfPath, err)
@@ -40,7 +42,7 @@ func (r *resolvconf) supportCustomPort() bool {
 	return false
 }
 
-func (r *resolvconf) applyDNSConfig(config HostDNSConfig) error {
+func (r *resolvconf) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 	if !config.RouteAll {
 		err = r.restoreHostDNS()
@@ -60,9 +62,12 @@ func (r *resolvconf) applyDNSConfig(config HostDNSConfig) error {
 		append([]string{config.ServerIP}, r.originalNameServers...),
 		options)
 
-	// create a backup for unclean shutdown detection before the resolv.conf is changed
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, resolvConfManager, config.ServerIP); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: resolvConfManager,
+		WgIface:     r.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	err = r.applyConfig(buf)
@@ -79,11 +84,7 @@ func (r *resolvconf) restoreHostDNS() error {
 	cmd := exec.Command(resolvconfCommand, "-f", "-d", r.ifaceName)
 	_, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("removing resolvconf configuration for %s interface, error: %w", r.ifaceName, err)
-	}
-
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
+		return fmt.Errorf("removing resolvconf configuration for %s interface: %w", r.ifaceName, err)
 	}
 
 	return nil
@@ -95,7 +96,7 @@ func (r *resolvconf) applyConfig(content bytes.Buffer) error {
 	cmd.Stdin = &content
 	_, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("applying resolvconf configuration for %s interface, error: %w", r.ifaceName, err)
+		return fmt.Errorf("applying resolvconf configuration for %s interface: %w", r.ifaceName, err)
 	}
 	return nil
 }

client/internal/dns/server.go

@@ -7,6 +7,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/miekg/dns"
 	"github.com/mitchellh/hashstructure/v2"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/netbirdio/netbird/client/internal/listener"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
@@ -63,6 +65,7 @@ type DefaultServer struct {
 	iosDnsManager        IosDnsManager
 
 	statusRecorder *peer.Status
+	stateManager   *statemanager.Manager
 }
 
 type handlerWithStop interface {
@@ -77,12 +80,7 @@ type muxUpdate struct {
 }
 
 // NewDefaultServer returns a new dns server
-func NewDefaultServer(
-	ctx context.Context,
-	wgInterface WGIface,
-	customAddress string,
-	statusRecorder *peer.Status,
-) (*DefaultServer, error) {
+func NewDefaultServer(ctx context.Context, wgInterface WGIface, customAddress string, statusRecorder *peer.Status, stateManager *statemanager.Manager) (*DefaultServer, error) {
 	var addrPort *netip.AddrPort
 	if customAddress != "" {
 		parsedAddrPort, err := netip.ParseAddrPort(customAddress)
@@ -99,7 +97,7 @@ func NewDefaultServer(
 		dnsService = newServiceViaListener(wgInterface, addrPort)
 	}
 
-	return newDefaultServer(ctx, wgInterface, dnsService, statusRecorder), nil
+	return newDefaultServer(ctx, wgInterface, dnsService, statusRecorder, stateManager), nil
 }
 
 // NewDefaultServerPermanentUpstream returns a new dns server. It optimized for mobile systems
@@ -112,7 +110,7 @@ func NewDefaultServerPermanentUpstream(
 	statusRecorder *peer.Status,
 ) *DefaultServer {
 	log.Debugf("host dns address list is: %v", hostsDnsList)
-	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil)
 	ds.hostsDNSHolder.set(hostsDnsList)
 	ds.permanent = true
 	ds.addHostRootZone()
@@ -130,12 +128,12 @@ func NewDefaultServerIos(
 	iosDnsManager IosDnsManager,
 	statusRecorder *peer.Status,
 ) *DefaultServer {
-	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil)
 	ds.iosDnsManager = iosDnsManager
 	return ds
 }
 
-func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService service, statusRecorder *peer.Status) *DefaultServer {
+func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService service, statusRecorder *peer.Status, stateManager *statemanager.Manager) *DefaultServer {
 	ctx, stop := context.WithCancel(ctx)
 	defaultServer := &DefaultServer{
 		ctx:       ctx,
@@ -147,6 +145,7 @@ func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService servi
 		},
 		wgInterface:    wgInterface,
 		statusRecorder: statusRecorder,
+		stateManager:   stateManager,
 		hostsDNSHolder: newHostsDNSHolder(),
 	}
 
@@ -169,6 +168,7 @@ func (s *DefaultServer) Initialize() (err error) {
 		}
 	}
 
+	s.stateManager.RegisterState(&ShutdownState{})
 	s.hostManager, err = s.initialize()
 	if err != nil {
 		return fmt.Errorf("initialize: %w", err)
@@ -191,9 +191,10 @@ func (s *DefaultServer) Stop() {
 	s.ctxCancel()
 
 	if s.hostManager != nil {
-		err := s.hostManager.restoreHostDNS()
-		if err != nil {
-			log.Error(err)
+		if err := s.hostManager.restoreHostDNS(); err != nil {
+			log.Error("failed to restore host DNS settings: ", err)
+		} else if err := s.stateManager.DeleteState(&ShutdownState{}); err != nil {
+			log.Errorf("failed to delete shutdown dns state: %v", err)
 		}
 	}
 
@@ -318,10 +319,17 @@ func (s *DefaultServer) applyConfiguration(update nbdns.Config) error {
 		hostUpdate.RouteAll = false
 	}
 
-	if err = s.hostManager.applyDNSConfig(hostUpdate); err != nil {
+	if err = s.hostManager.applyDNSConfig(hostUpdate, s.stateManager); err != nil {
 		log.Error(err)
 	}
 
+	// persist dns state right away
+	ctx, cancel := context.WithTimeout(s.ctx, 3*time.Second)
+	defer cancel()
+	if err := s.stateManager.PersistState(ctx); err != nil {
+		log.Errorf("Failed to persist dns state: %v", err)
+	}
+
 	if s.searchDomainNotifier != nil {
 		s.searchDomainNotifier.onNewSearchDomains(s.SearchDomains())
 	}
@@ -521,10 +529,17 @@ func (s *DefaultServer) upstreamCallbacks(
 			}
 		}
 
-		if err := s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+		if err := s.hostManager.applyDNSConfig(s.currentConfig, s.stateManager); err != nil {
 			l.Errorf("Failed to apply nameserver deactivation on the host: %v", err)
 		}
 
+		// persist dns state right away
+		ctx, cancel := context.WithTimeout(s.ctx, 3*time.Second)
+		defer cancel()
+		if err := s.stateManager.PersistState(ctx); err != nil {
+			l.Errorf("Failed to persist dns state: %v", err)
+		}
+
 		if runtime.GOOS == "android" && nsGroup.Primary && len(s.hostsDNSHolder.get()) > 0 {
 			s.addHostRootZone()
 		}
@@ -551,7 +566,7 @@ func (s *DefaultServer) upstreamCallbacks(
 			s.currentConfig.RouteAll = true
 			s.service.RegisterMux(nbdns.RootZone, handler)
 		}
-		if err := s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+		if err := s.hostManager.applyDNSConfig(s.currentConfig, s.stateManager); err != nil {
 			l.WithError(err).Error("reactivate temporary disabled nameserver group, DNS update apply")
 		}
 

client/internal/dns/server_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/device"
 	pfmock "github.com/netbirdio/netbird/client/iface/mocks"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/formatter"
@@ -267,7 +268,17 @@ func TestUpdateDNSServer(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+
+			opts := iface.WGIFaceOpts{
+				IFaceName:    fmt.Sprintf("utun230%d", n),
+				Address:      fmt.Sprintf("100.66.100.%d/32", n+1),
+				WGPort:       33100,
+				WGPrivKey:    privKey.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+
+			wgIface, err := iface.NewWGIFace(opts)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -281,7 +292,7 @@ func TestUpdateDNSServer(t *testing.T) {
 					t.Log(err)
 				}
 			}()
-			dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{})
+			dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{}, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -345,7 +356,15 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 	}
 
 	privKey, _ := wgtypes.GeneratePrivateKey()
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+	opts := iface.WGIFaceOpts{
+		IFaceName:    "utun2301",
+		Address:      "100.66.100.1/32",
+		WGPort:       33100,
+		WGPrivKey:    privKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+	wgIface, err := iface.NewWGIFace(opts)
 	if err != nil {
 		t.Errorf("build interface wireguard: %v", err)
 		return
@@ -382,7 +401,7 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 		return
 	}
 
-	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{})
+	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{}, nil)
 	if err != nil {
 		t.Errorf("create DNS server: %v", err)
 		return
@@ -477,7 +496,7 @@ func TestDNSServerStartStop(t *testing.T) {
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			dnsServer, err := NewDefaultServer(context.Background(), &mocWGIface{}, testCase.addrPort, &peer.Status{})
+			dnsServer, err := NewDefaultServer(context.Background(), &mocWGIface{}, testCase.addrPort, &peer.Status{}, nil)
 			if err != nil {
 				t.Fatalf("%v", err)
 			}
@@ -536,6 +555,7 @@ func TestDNSServerStartStop(t *testing.T) {
 func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	hostManager := &mockHostConfigurator{}
 	server := DefaultServer{
+		ctx:     context.Background(),
 		service: NewServiceViaMemory(&mocWGIface{}),
 		localResolver: &localResolver{
 			registeredMap: make(registrationMap),
@@ -552,7 +572,7 @@ func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	}
 
 	var domainsUpdate string
-	hostManager.applyDNSConfigFunc = func(config HostDNSConfig) error {
+	hostManager.applyDNSConfigFunc = func(config HostDNSConfig, statemanager *statemanager.Manager) error {
 		domains := []string{}
 		for _, item := range config.Domains {
 			if item.Disabled {
@@ -803,7 +823,17 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) {
 	}
 
 	privKey, _ := wgtypes.GeneratePrivateKey()
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+
+	opts := iface.WGIFaceOpts{
+		IFaceName:    "utun2301",
+		Address:      "100.66.100.2/24",
+		WGPort:       33100,
+		WGPrivKey:    privKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	wgIface, err := iface.NewWGIFace(opts)
 	if err != nil {
 		t.Fatalf("build interface wireguard: %v", err)
 		return nil, err

client/internal/dns/server_windows.go

@@ -1,5 +1,5 @@
 package dns
 
-func (s *DefaultServer) initialize() (manager hostManager, err error) {
+func (s *DefaultServer) initialize() (hostManager, error) {
 	return newHostManager(s.wgInterface)
 }

client/internal/dns/systemd_freebsd.go

@@ -7,7 +7,7 @@ import (
 
 var errNotImplemented = errors.New("not implemented")
 
-func newSystemdDbusConfigurator(wgInterface string) (hostManager, error) {
+func newSystemdDbusConfigurator(string) (restoreHostManager, error) {
 	return nil, fmt.Errorf("systemd dns management: %w on freebsd", errNotImplemented)
 }
 

client/internal/dns/systemd_linux.go

@@ -15,6 +15,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
@@ -38,6 +39,7 @@ const (
 type systemdDbusConfigurator struct {
 	dbusLinkObject dbus.ObjectPath
 	routingAll     bool
+	ifaceName      string
 }
 
 // the types below are based on dbus specification, each field is mapped to a dbus type
@@ -55,7 +57,7 @@ type systemdDbusLinkDomainsInput struct {
 	MatchOnly bool
 }
 
-func newSystemdDbusConfigurator(wgInterface string) (hostManager, error) {
+func newSystemdDbusConfigurator(wgInterface string) (*systemdDbusConfigurator, error) {
 	iface, err := net.InterfaceByName(wgInterface)
 	if err != nil {
 		return nil, fmt.Errorf("get interface: %w", err)
@@ -77,6 +79,7 @@ func newSystemdDbusConfigurator(wgInterface string) (hostManager, error) {
 
 	return &systemdDbusConfigurator{
 		dbusLinkObject: dbus.ObjectPath(s),
+		ifaceName:      wgInterface,
 	}, nil
 }
 
@@ -84,7 +87,7 @@ func (s *systemdDbusConfigurator) supportCustomPort() bool {
 	return true
 }
 
-func (s *systemdDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (s *systemdDbusConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	parsedIP, err := netip.ParseAddr(config.ServerIP)
 	if err != nil {
 		return fmt.Errorf("unable to parse ip address, error: %w", err)
@@ -135,10 +138,12 @@ func (s *systemdDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
 		log.Infof("removing %s:%d as main DNS forwarder for this peer", config.ServerIP, config.ServerPort)
 	}
 
-	// create a backup for unclean shutdown detection before adding domains, as these might end up in the resolv.conf file.
-	// The file content itself is not important for systemd restoration
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, systemdManager, parsedIP.String()); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: systemdManager,
+		WgIface:     s.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	log.Infof("adding %d search domains and %d match domains. Search list: %s , Match list: %s", len(searchDomains), len(matchDomains), searchDomains, matchDomains)
@@ -174,10 +179,6 @@ func (s *systemdDbusConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("unable to revert link configuration, got error: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return s.flushCaches()
 }
 

client/internal/dns/unclean_shutdown_android.go

@@ -1,5 +0,0 @@
-package dns
-
-func CheckUncleanShutdown(string) error {
-	return nil
-}

client/internal/dns/unclean_shutdown_darwin.go

@@ -3,57 +3,25 @@
 package dns
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-
-	log "github.com/sirupsen/logrus"
 )
 
-const fileUncleanShutdownFileLocation = "/var/lib/netbird/unclean_shutdown_dns"
+type ShutdownState struct {
+}
 
-func CheckUncleanShutdown(string) error {
-	if _, err := os.Stat(fileUncleanShutdownFileLocation); err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// no file -> clean shutdown
-			return nil
-		} else {
-			return fmt.Errorf("state: %w", err)
-		}
-	}
-
-	log.Warnf("detected unclean shutdown, file %s exists. Restoring unclean shutdown dns settings.", fileUncleanShutdownFileLocation)
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
 
+func (s *ShutdownState) Cleanup() error {
 	manager, err := newHostManager()
 	if err != nil {
 		return fmt.Errorf("create host manager: %w", err)
 	}
 
-	if err := manager.restoreUncleanShutdownDNS(nil); err != nil {
-		return fmt.Errorf("restore unclean shutdown backup: %w", err)
+	if err := manager.restoreUncleanShutdownDNS(); err != nil {
+		return fmt.Errorf("restore unclean shutdown dns: %w", err)
 	}
 
 	return nil
 }
-
-func createUncleanShutdownIndicator() error {
-	dir := filepath.Dir(fileUncleanShutdownFileLocation)
-	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
-		return fmt.Errorf("create dir %s: %w", dir, err)
-	}
-
-	if err := os.WriteFile(fileUncleanShutdownFileLocation, nil, 0644); err != nil { //nolint:gosec
-		return fmt.Errorf("create %s: %w", fileUncleanShutdownFileLocation, err)
-	}
-
-	return nil
-}
-
-func removeUncleanShutdownIndicator() error {
-	if err := os.Remove(fileUncleanShutdownFileLocation); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", fileUncleanShutdownFileLocation, err)
-	}
-	return nil
-}

client/internal/dns/unclean_shutdown_ios.go

@@ -1,5 +0,0 @@
-package dns
-
-func CheckUncleanShutdown(string) error {
-	return nil
-}

client/internal/dns/unclean_shutdown_mobile.go

@@ -0,0 +1,14 @@
+//go:build ios || android
+
+package dns
+
+type ShutdownState struct {
+}
+
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	return nil
+}

client/internal/dns/unclean_shutdown_unix.go

@@ -3,66 +3,44 @@
 package dns
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
 	"net/netip"
 	"os"
 	"path/filepath"
-	"strings"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
-func CheckUncleanShutdown(wgIface string) error {
-	if _, err := os.Stat(fileUncleanShutdownResolvConfLocation); err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// no file -> clean shutdown
-			return nil
-		} else {
-			return fmt.Errorf("state: %w", err)
-		}
-	}
+type ShutdownState struct {
+	ManagerType osManagerType
+	DNSAddress  netip.Addr
+	WgIface     string
+}
 
-	log.Warnf("detected unclean shutdown, file %s exists", fileUncleanShutdownResolvConfLocation)
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
 
-	managerData, err := os.ReadFile(fileUncleanShutdownManagerTypeLocation)
-	if err != nil {
-		return fmt.Errorf("read %s: %w", fileUncleanShutdownManagerTypeLocation, err)
-	}
-
-	managerFields := strings.Split(string(managerData), ",")
-	if len(managerFields) < 2 {
-		return errors.New("split manager data: insufficient number of fields")
-	}
-	osManagerTypeStr, dnsAddressStr := managerFields[0], managerFields[1]
-
-	dnsAddress, err := netip.ParseAddr(dnsAddressStr)
-	if err != nil {
-		return fmt.Errorf("parse dns address %s failed: %w", dnsAddressStr, err)
-	}
-
-	log.Warnf("restoring unclean shutdown dns settings via previously detected manager: %s", osManagerTypeStr)
-
-	// determine os manager type, so we can invoke the respective restore action
-	osManagerType, err := newOsManagerType(osManagerTypeStr)
-	if err != nil {
-		return fmt.Errorf("detect previous host manager: %w", err)
-	}
-
-	manager, err := newHostManagerFromType(wgIface, osManagerType)
+func (s *ShutdownState) Cleanup() error {
+	manager, err := newHostManagerFromType(s.WgIface, s.ManagerType)
 	if err != nil {
 		return fmt.Errorf("create previous host manager: %w", err)
 	}
 
-	if err := manager.restoreUncleanShutdownDNS(&dnsAddress); err != nil {
-		return fmt.Errorf("restore unclean shutdown backup: %w", err)
+	if err := manager.restoreUncleanShutdownDNS(&s.DNSAddress); err != nil {
+		return fmt.Errorf("restore unclean shutdown dns: %w", err)
 	}
 
 	return nil
 }
 
-func createUncleanShutdownIndicator(sourcePath string, managerType osManagerType, dnsAddress string) error {
+// TODO: move file contents to state manager
+func createUncleanShutdownIndicator(sourcePath string, dnsAddressStr string, stateManager *statemanager.Manager) error {
+	dnsAddress, err := netip.ParseAddr(dnsAddressStr)
+	if err != nil {
+		return fmt.Errorf("parse dns address %s: %w", dnsAddressStr, err)
+	}
+
 	dir := filepath.Dir(fileUncleanShutdownResolvConfLocation)
 	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
 		return fmt.Errorf("create dir %s: %w", dir, err)
@@ -72,20 +50,13 @@ func createUncleanShutdownIndicator(sourcePath string, managerType osManagerType
 		return fmt.Errorf("create %s: %w", sourcePath, err)
 	}
 
-	managerData := fmt.Sprintf("%s,%s", managerType, dnsAddress)
-
-	if err := os.WriteFile(fileUncleanShutdownManagerTypeLocation, []byte(managerData), 0644); err != nil { //nolint:gosec
-		return fmt.Errorf("create %s: %w", fileUncleanShutdownManagerTypeLocation, err)
-	}
-	return nil
-}
-
-func removeUncleanShutdownIndicator() error {
-	if err := os.Remove(fileUncleanShutdownResolvConfLocation); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", fileUncleanShutdownResolvConfLocation, err)
-	}
-	if err := os.Remove(fileUncleanShutdownManagerTypeLocation); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", fileUncleanShutdownManagerTypeLocation, err)
+	state := &ShutdownState{
+		ManagerType: fileManager,
+		DNSAddress:  dnsAddress,
 	}
+	if err := stateManager.UpdateState(state); err != nil {
+		return fmt.Errorf("update state: %w", err)
+	}
+
 	return nil
 }

client/internal/dns/unclean_shutdown_windows.go

@@ -1,75 +1,26 @@
 package dns
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-
-	"github.com/sirupsen/logrus"
 )
 
-const (
-	netbirdProgramDataLocation = "Netbird"
-	fileUncleanShutdownFile    = "unclean_shutdown_dns.txt"
-)
+type ShutdownState struct {
+	Guid string
+}
 
-func CheckUncleanShutdown(string) error {
-	file := getUncleanShutdownFile()
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
 
-	if _, err := os.Stat(file); err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// no file -> clean shutdown
-			return nil
-		} else {
-			return fmt.Errorf("state: %w", err)
-		}
-	}
-
-	logrus.Warnf("detected unclean shutdown, file %s exists. Restoring unclean shutdown dns settings.", file)
-
-	guid, err := os.ReadFile(file)
-	if err != nil {
-		return fmt.Errorf("read %s: %w", file, err)
-	}
-
-	manager, err := newHostManagerWithGuid(string(guid))
+func (s *ShutdownState) Cleanup() error {
+	manager, err := newHostManagerWithGuid(s.Guid)
 	if err != nil {
 		return fmt.Errorf("create host manager: %w", err)
 	}
 
-	if err := manager.restoreUncleanShutdownDNS(nil); err != nil {
-		return fmt.Errorf("restore unclean shutdown backup: %w", err)
+	if err := manager.restoreUncleanShutdownDNS(); err != nil {
+		return fmt.Errorf("restore unclean shutdown dns: %w", err)
 	}
 
 	return nil
 }
-
-func createUncleanShutdownIndicator(guid string) error {
-	file := getUncleanShutdownFile()
-
-	dir := filepath.Dir(file)
-	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
-		return fmt.Errorf("create dir %s: %w", dir, err)
-	}
-
-	if err := os.WriteFile(file, []byte(guid), 0600); err != nil {
-		return fmt.Errorf("create %s: %w", file, err)
-	}
-
-	return nil
-}
-
-func removeUncleanShutdownIndicator() error {
-	file := getUncleanShutdownFile()
-
-	if err := os.Remove(file); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", file, err)
-	}
-	return nil
-}
-
-func getUncleanShutdownFile() string {
-	return filepath.Join(os.Getenv("PROGRAMDATA"), netbirdProgramDataLocation, fileUncleanShutdownFile)
-}

client/internal/engine.go

@@ -23,19 +23,22 @@ import (
 
 	"github.com/netbirdio/netbird/client/firewall"
 	"github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/device"
 	"github.com/netbirdio/netbird/client/internal/acl"
 	"github.com/netbirdio/netbird/client/internal/dns"
-
-	"github.com/netbirdio/netbird/client/iface"
-	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/internal/networkmonitor"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/relay"
 	"github.com/netbirdio/netbird/client/internal/rosenpass"
 	"github.com/netbirdio/netbird/client/internal/routemanager"
 	"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
-	"github.com/netbirdio/netbird/client/internal/wgproxy"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+
+
 	nbssh "github.com/netbirdio/netbird/client/ssh"
 	"github.com/netbirdio/netbird/client/system"
 	nbdns "github.com/netbirdio/netbird/dns"
@@ -141,8 +144,7 @@ type Engine struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 
-	wgInterface    iface.IWGIface
-	wgProxyFactory *wgproxy.Factory
+	wgInterface iface.IWGIface
 
 	udpMux *bind.UniversalUDPMuxDefault
 
@@ -168,6 +170,8 @@ type Engine struct {
 	checks []*mgmProto.Checks
 
 	relayManager *relayClient.Manager
+	stateManager *statemanager.Manager
+	srWatcher *guard.SRWatcher
 }
 
 // Peer is an instance of the Connection Peer
@@ -215,7 +219,7 @@ func NewEngineWithProbes(
 	probes *ProbeHolder,
 	checks []*mgmProto.Checks,
 ) *Engine {
-	return &Engine{
+	engine := &Engine{
 		clientCtx:      clientCtx,
 		clientCancel:   clientCancel,
 		signal:         signalClient,
@@ -234,6 +238,11 @@ func NewEngineWithProbes(
 		probes:         probes,
 		checks:         checks,
 	}
+	if path := statemanager.GetDefaultStatePath(); path != "" {
+		engine.stateManager = statemanager.New(path)
+	}
+
+	return engine
 }
 
 func (e *Engine) Stop() error {
@@ -255,7 +264,11 @@ func (e *Engine) Stop() error {
 	e.stopDNSServer()
 
 	if e.routeManager != nil {
-		e.routeManager.Stop()
+		e.routeManager.Stop(e.stateManager)
+	}
+
+	if e.srWatcher != nil {
+		e.srWatcher.Close()
 	}
 
 	err := e.removeAllPeers()
@@ -277,6 +290,17 @@ func (e *Engine) Stop() error {
 
 	e.close()
 	log.Infof("stopped Netbird Engine")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	if err := e.stateManager.Stop(ctx); err != nil {
+		return fmt.Errorf("failed to stop state manager: %w", err)
+	}
+	if err := e.stateManager.PersistState(ctx); err != nil {
+		log.Errorf("failed to persist state: %v", err)
+	}
+
 	return nil
 }
 
@@ -299,9 +323,6 @@ func (e *Engine) Start() error {
 	}
 	e.wgInterface = wgIface
 
-	userspace := e.wgInterface.IsUserspaceBind()
-	e.wgProxyFactory = wgproxy.NewFactory(userspace, e.config.WgPort)
-
 	if e.config.RosenpassEnabled {
 		log.Infof("rosenpass is enabled")
 		if e.config.RosenpassPermissive {
@@ -319,6 +340,8 @@ func (e *Engine) Start() error {
 		}
 	}
 
+	e.stateManager.Start()
+
 	initialRoutes, dnsServer, err := e.newDnsServer()
 	if err != nil {
 		e.close()
@@ -327,7 +350,7 @@ func (e *Engine) Start() error {
 	e.dnsServer = dnsServer
 
 	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, e.relayManager, initialRoutes)
-	beforePeerHook, afterPeerHook, err := e.routeManager.Init()
+	beforePeerHook, afterPeerHook, err := e.routeManager.Init(e.stateManager)
 	if err != nil {
 		log.Errorf("Failed to initialize route manager: %s", err)
 	} else {
@@ -344,7 +367,7 @@ func (e *Engine) Start() error {
 		return fmt.Errorf("create wg interface: %w", err)
 	}
 
-	e.firewall, err = firewall.NewFirewall(e.ctx, e.wgInterface)
+	e.firewall, err = firewall.NewFirewall(e.wgInterface, e.stateManager)
 	if err != nil {
 		log.Errorf("failed creating firewall manager: %s", err)
 	}
@@ -374,6 +397,18 @@ func (e *Engine) Start() error {
 		return fmt.Errorf("initialize dns server: %w", err)
 	}
 
+	iceCfg := icemaker.Config{
+		StunTurn:             &e.stunTurn,
+		InterfaceBlackList:   e.config.IFaceBlackList,
+		DisableIPv6Discovery: e.config.DisableIPv6Discovery,
+		UDPMux:               e.udpMux.UDPMuxDefault,
+		UDPMuxSrflx:          e.udpMux,
+		NATExternalIPs:       e.parseNATExternalIPMappings(),
+	}
+
+	e.srWatcher = guard.NewSRWatcher(e.signal, e.relayManager, e.mobileDep.IFaceDiscover, iceCfg)
+	e.srWatcher.Start()
+
 	e.receiveSignalEvents()
 	e.receiveManagementEvents()
 	e.receiveProbeEvents()
@@ -956,7 +991,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 		LocalWgPort:     e.config.WgPort,
 		RosenpassPubKey: e.getRosenpassPubKey(),
 		RosenpassAddr:   e.getRosenpassAddr(),
-		ICEConfig: peer.ICEConfig{
+		ICEConfig: icemaker.Config{
 			StunTurn:             &e.stunTurn,
 			InterfaceBlackList:   e.config.IFaceBlackList,
 			DisableIPv6Discovery: e.config.DisableIPv6Discovery,
@@ -966,7 +1001,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 		},
 	}
 
-	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager)
+	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager, e.srWatcher)
 	if err != nil {
 		return nil, err
 	}
@@ -1117,12 +1152,6 @@ func (e *Engine) parseNATExternalIPMappings() []string {
 }
 
 func (e *Engine) close() {
-	if e.wgProxyFactory != nil {
-		if err := e.wgProxyFactory.Free(); err != nil {
-			log.Errorf("failed closing ebpf proxy: %s", err)
-		}
-	}
-
 	log.Debugf("removing Netbird interface %s", e.config.WgIfaceName)
 	if e.wgInterface != nil {
 		if err := e.wgInterface.Close(); err != nil {
@@ -1139,7 +1168,7 @@ func (e *Engine) close() {
 	}
 
 	if e.firewall != nil {
-		err := e.firewall.Reset()
+		err := e.firewall.Reset(e.stateManager)
 		if err != nil {
 			log.Warnf("failed to reset firewall: %s", err)
 		}
@@ -1167,21 +1196,29 @@ func (e *Engine) newWgIface() (*iface.WGIface, error) {
 		log.Errorf("failed to create pion's stdnet: %s", err)
 	}
 
-	var mArgs *device.MobileIFaceArguments
+	opts := iface.WGIFaceOpts{
+		IFaceName:    e.config.WgIfaceName,
+		Address:      e.config.WgAddr,
+		WGPort:       e.config.WgPort,
+		WGPrivKey:    e.config.WgPrivateKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: transportNet,
+		FilterFn:     e.addrViaRoutes,
+	}
+
 	switch runtime.GOOS {
 	case "android":
-		mArgs = &device.MobileIFaceArguments{
+		opts.MobileArgs = &device.MobileIFaceArguments{
 			TunAdapter: e.mobileDep.TunAdapter,
 			TunFd:      int(e.mobileDep.FileDescriptor),
 		}
 	case "ios":
-		mArgs = &device.MobileIFaceArguments{
+		opts.MobileArgs = &device.MobileIFaceArguments{
 			TunFd: int(e.mobileDep.FileDescriptor),
 		}
-	default:
 	}
 
-	return iface.NewWGIFace(e.config.WgIfaceName, e.config.WgAddr, e.config.WgPort, e.config.WgPrivateKey.String(), iface.DefaultMTU, transportNet, mArgs, e.addrViaRoutes)
+	return iface.NewWGIFace(opts)
 }
 
 func (e *Engine) wgInterfaceCreate() (err error) {
@@ -1222,10 +1259,11 @@ func (e *Engine) newDnsServer() ([]*route.Route, dns.Server, error) {
 		dnsServer := dns.NewDefaultServerIos(e.ctx, e.wgInterface, e.mobileDep.DnsManager, e.statusRecorder)
 		return nil, dnsServer, nil
 	default:
-		dnsServer, err := dns.NewDefaultServer(e.ctx, e.wgInterface, e.config.CustomDNSAddress, e.statusRecorder)
+		dnsServer, err := dns.NewDefaultServer(e.ctx, e.wgInterface, e.config.CustomDNSAddress, e.statusRecorder, e.stateManager)
 		if err != nil {
 			return nil, nil, err
 		}
+
 		return nil, dnsServer, nil
 	}
 }

client/internal/engine_test.go

@@ -29,6 +29,8 @@ import (
 	"github.com/netbirdio/netbird/client/iface/device"
 	"github.com/netbirdio/netbird/client/internal/dns"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/routemanager"
 	"github.com/netbirdio/netbird/client/ssh"
 	"github.com/netbirdio/netbird/client/system"
@@ -258,6 +260,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 	}
 	engine.udpMux = bind.NewUniversalUDPMuxDefault(bind.UniversalUDPMuxParams{UDPConn: conn})
 	engine.ctx = ctx
+	engine.srWatcher = guard.NewSRWatcher(nil, nil, nil, icemaker.Config{})
 
 	type testCase struct {
 		name       string
@@ -602,7 +605,16 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil, nil)
+
+			opts := iface.WGIFaceOpts{
+				IFaceName:    wgIfaceName,
+				Address:      wgAddr,
+				WGPort:       engine.config.WgPort,
+				WGPrivKey:    key.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			engine.wgInterface, err = iface.NewWGIFace(opts)
 			assert.NoError(t, err, "shouldn't return error")
 			input := struct {
 				inputSerial uint64
@@ -774,7 +786,15 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, 33100, key.String(), iface.DefaultMTU, newNet, nil, nil)
+			opts := iface.WGIFaceOpts{
+				IFaceName:    wgIfaceName,
+				Address:      wgAddr,
+				WGPort:       33100,
+				WGPrivKey:    key.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			engine.wgInterface, err = iface.NewWGIFace(opts)
 			assert.NoError(t, err, "shouldn't return error")
 
 			mockRouteManager := &routemanager.MockManager{
@@ -832,7 +852,7 @@ func TestEngine_MultiplePeers(t *testing.T) {
 		return
 	}
 	defer sigServer.Stop()
-	mgmtServer, mgmtAddr, err := startManagement(t, t.TempDir(), "../testdata/store.sqlite")
+	mgmtServer, mgmtAddr, err := startManagement(t, t.TempDir(), "../testdata/store.sql")
 	if err != nil {
 		t.Fatal(err)
 		return
@@ -1080,7 +1100,7 @@ func startManagement(t *testing.T, dataDir, testFile string) (*grpc.Server, stri
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
 
-	store, cleanUp, err := server.NewTestStoreFromSqlite(context.Background(), testFile, config.Datadir)
+	store, cleanUp, err := server.NewTestStoreFromSQL(context.Background(), testFile, config.Datadir)
 	if err != nil {
 		return nil, "", err
 	}

client/internal/peer/conn.go

@@ -10,15 +10,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cenkalti/backoff/v4"
 	"github.com/pion/ice/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/configurer"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
-	"github.com/netbirdio/netbird/client/internal/wgproxy"
 	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	nbnet "github.com/netbirdio/netbird/util/net"
@@ -61,7 +62,7 @@ type ConnConfig struct {
 	RosenpassAddr string
 
 	// ICEConfig ICE protocol configuration
-	ICEConfig ICEConfig
+	ICEConfig icemaker.Config
 }
 
 type WorkerCallbacks struct {
@@ -79,12 +80,10 @@ type Conn struct {
 	ctxCancel      context.CancelFunc
 	config         ConnConfig
 	statusRecorder *Status
-	wgProxyFactory *wgproxy.Factory
-	wgProxyICE     wgproxy.Proxy
-	wgProxyRelay   wgproxy.Proxy
 	signaler       *Signaler
 	relayManager   *relayClient.Manager
-	allowedIPsIP   string
+	allowedIP      net.IP
+	allowedNet     string
 	handshaker     *Handshaker
 
 	onConnected    func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)
@@ -103,17 +102,16 @@ type Conn struct {
 	beforeAddPeerHooks   []nbnet.AddHookFunc
 	afterRemovePeerHooks []nbnet.RemoveHookFunc
 
-	endpointRelay *net.UDPAddr
+	wgProxyICE   wgproxy.Proxy
+	wgProxyRelay wgproxy.Proxy
 
-	// for reconnection operations
-	iCEDisconnected   chan bool
-	relayDisconnected chan bool
+	guard *guard.Guard
 }
 
 // NewConn creates a new not opened Conn to the remote peer.
 // To establish a connection run Conn.Open
-func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) {
-	_, allowedIPsIP, err := net.ParseCIDR(config.WgConfig.AllowedIps)
+func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager, srWatcher *guard.SRWatcher) (*Conn, error) {
+	allowedIP, allowedNet, err := net.ParseCIDR(config.WgConfig.AllowedIps)
 	if err != nil {
 		log.Errorf("failed to parse allowedIPS: %v", err)
 		return nil, err
@@ -123,19 +121,17 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 	connLog := log.WithField("peer", config.Key)
 
 	var conn = &Conn{
-		log:               connLog,
-		ctx:               ctx,
-		ctxCancel:         ctxCancel,
-		config:            config,
-		statusRecorder:    statusRecorder,
-		wgProxyFactory:    wgProxyFactory,
-		signaler:          signaler,
-		relayManager:      relayManager,
-		allowedIPsIP:      allowedIPsIP.String(),
-		statusRelay:       NewAtomicConnStatus(),
-		statusICE:         NewAtomicConnStatus(),
-		iCEDisconnected:   make(chan bool, 1),
-		relayDisconnected: make(chan bool, 1),
+		log:            connLog,
+		ctx:            ctx,
+		ctxCancel:      ctxCancel,
+		config:         config,
+		statusRecorder: statusRecorder,
+		signaler:       signaler,
+		relayManager:   relayManager,
+		allowedIP:      allowedIP,
+		allowedNet:     allowedNet.String(),
+		statusRelay:    NewAtomicConnStatus(),
+		statusICE:      NewAtomicConnStatus(),
 	}
 
 	rFns := WorkerRelayCallbacks{
@@ -148,7 +144,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 		OnStatusChanged: conn.onWorkerICEStateDisconnected,
 	}
 
-	conn.workerRelay = NewWorkerRelay(connLog, config, relayManager, rFns)
+	ctrl := isController(config)
+	conn.workerRelay = NewWorkerRelay(connLog, ctrl, config, relayManager, rFns)
 
 	relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally()
 	conn.workerICE, err = NewWorkerICE(ctx, connLog, config, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns)
@@ -163,6 +160,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 		conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer)
 	}
 
+	conn.guard = guard.NewGuard(connLog, ctrl, conn.isConnectedOnAllWay, config.Timeout, srWatcher)
+
 	go conn.handshaker.Listen()
 
 	return conn, nil
@@ -173,6 +172,7 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 // be used.
 func (conn *Conn) Open() {
 	conn.log.Debugf("open connection to peer")
+
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 	conn.opened = true
@@ -189,22 +189,19 @@ func (conn *Conn) Open() {
 		conn.log.Warnf("error while updating the state err: %v", err)
 	}
 
-	go conn.startHandshakeAndReconnect()
+	go conn.startHandshakeAndReconnect(conn.ctx)
 }
 
-func (conn *Conn) startHandshakeAndReconnect() {
-	conn.waitInitialRandomSleepTime()
+func (conn *Conn) startHandshakeAndReconnect(ctx context.Context) {
+	conn.waitInitialRandomSleepTime(ctx)
 
 	err := conn.handshaker.sendOffer()
 	if err != nil {
 		conn.log.Errorf("failed to send initial offer: %v", err)
 	}
 
-	if conn.workerRelay.IsController() {
-		conn.reconnectLoopWithRetry()
-	} else {
-		conn.reconnectLoopForOnDisconnectedEvent()
-	}
+	go conn.guard.Start(ctx)
+	go conn.listenGuardEvent(ctx)
 }
 
 // Close closes this peer Conn issuing a close event to the Conn closeCh
@@ -240,8 +237,7 @@ func (conn *Conn) Close() {
 		conn.wgProxyICE = nil
 	}
 
-	err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey)
-	if err != nil {
+	if err := conn.removeWgPeer(); err != nil {
 		conn.log.Errorf("failed to remove wg endpoint: %v", err)
 	}
 
@@ -304,111 +300,6 @@ func (conn *Conn) GetKey() string {
 	return conn.config.Key
 }
 
-func (conn *Conn) reconnectLoopWithRetry() {
-	// Give chance to the peer to establish the initial connection.
-	// With it, we can decrease to send necessary offer
-	select {
-	case <-conn.ctx.Done():
-	case <-time.After(3 * time.Second):
-	}
-
-	ticker := conn.prepareExponentTicker()
-	defer ticker.Stop()
-	time.Sleep(1 * time.Second)
-	for {
-		select {
-		case t := <-ticker.C:
-			if t.IsZero() {
-				// in case if the ticker has been canceled by context then avoid the temporary loop
-				return
-			}
-
-			if conn.workerRelay.IsRelayConnectionSupportedWithPeer() {
-				if conn.statusRelay.Get() == StatusDisconnected || conn.statusICE.Get() == StatusDisconnected {
-					conn.log.Tracef("connectivity guard timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE)
-				}
-			} else {
-				if conn.statusICE.Get() == StatusDisconnected {
-					conn.log.Tracef("connectivity guard timedout, ice state: %s", conn.statusICE)
-				}
-			}
-
-			// checks if there is peer connection is established via relay or ice
-			if conn.isConnected() {
-				continue
-			}
-
-			err := conn.handshaker.sendOffer()
-			if err != nil {
-				conn.log.Errorf("failed to do handshake: %v", err)
-			}
-		case changed := <-conn.relayDisconnected:
-			if !changed {
-				continue
-			}
-			conn.log.Debugf("Relay state changed, reset reconnect timer")
-			ticker.Stop()
-			ticker = conn.prepareExponentTicker()
-		case changed := <-conn.iCEDisconnected:
-			if !changed {
-				continue
-			}
-			conn.log.Debugf("ICE state changed, reset reconnect timer")
-			ticker.Stop()
-			ticker = conn.prepareExponentTicker()
-		case <-conn.ctx.Done():
-			conn.log.Debugf("context is done, stop reconnect loop")
-			return
-		}
-	}
-}
-
-func (conn *Conn) prepareExponentTicker() *backoff.Ticker {
-	bo := backoff.WithContext(&backoff.ExponentialBackOff{
-		InitialInterval:     800 * time.Millisecond,
-		RandomizationFactor: 0.01,
-		Multiplier:          2,
-		MaxInterval:         conn.config.Timeout,
-		MaxElapsedTime:      0,
-		Stop:                backoff.Stop,
-		Clock:               backoff.SystemClock,
-	}, conn.ctx)
-
-	ticker := backoff.NewTicker(bo)
-	<-ticker.C // consume the initial tick what is happening right after the ticker has been created
-
-	return ticker
-}
-
-// reconnectLoopForOnDisconnectedEvent is used when the peer is not a controller and it should reconnect to the peer
-// when the connection is lost. It will try to establish a connection only once time if before the connection was established
-// It track separately the ice and relay connection status. Just because a lover priority connection reestablished it does not
-// mean that to switch to it. We always force to use the higher priority connection.
-func (conn *Conn) reconnectLoopForOnDisconnectedEvent() {
-	for {
-		select {
-		case changed := <-conn.relayDisconnected:
-			if !changed {
-				continue
-			}
-			conn.log.Debugf("Relay state changed, try to send new offer")
-		case changed := <-conn.iCEDisconnected:
-			if !changed {
-				continue
-			}
-			conn.log.Debugf("ICE state changed, try to send new offer")
-		case <-conn.ctx.Done():
-			conn.log.Debugf("context is done, stop reconnect loop")
-			return
-		}
-
-		err := conn.handshaker.SendOffer()
-		if err != nil {
-			conn.log.Errorf("failed to do handshake: %v", err)
-		}
-	}
-}
-
 // configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected
 func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICEConnInfo) {
 	conn.mu.Lock()
@@ -420,54 +311,62 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon
 
 	conn.log.Debugf("ICE connection is ready")
 
-	conn.statusICE.Set(StatusConnected)
-
-	defer conn.updateIceState(iceConnInfo)
-
 	if conn.currentConnPriority > priority {
+		conn.statusICE.Set(StatusConnected)
+		conn.updateIceState(iceConnInfo)
 		return
 	}
 
 	conn.log.Infof("set ICE to active connection")
 
-	endpoint, wgProxy, err := conn.getEndpointForICEConnInfo(iceConnInfo)
-	if err != nil {
-		return
+	var (
+		ep      *net.UDPAddr
+		wgProxy wgproxy.Proxy
+		err     error
+	)
+	if iceConnInfo.RelayedOnLocal {
+		wgProxy, err = conn.newProxy(iceConnInfo.RemoteConn)
+		if err != nil {
+			conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
+			return
+		}
+		ep = wgProxy.EndpointAddr()
+		conn.wgProxyICE = wgProxy
+	} else {
+		conn.log.Infof("direct iceConnInfo: %v", iceConnInfo.RemoteConn)
+		agentCheck(conn.log, iceConnInfo.Agent)
+		nilCheck(conn.log, iceConnInfo.RemoteConn)
+		directEp, err := net.ResolveUDPAddr("udp", iceConnInfo.RemoteConn.RemoteAddr().String())
+		if err != nil {
+			conn.log.Errorf("failed to resolveUDPaddr")
+			conn.handleConfigurationFailure(err, nil)
+			return
+		}
+		ep = directEp
 	}
 
-	endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String())
-	conn.log.Debugf("Conn resolved IP is %s for endopint %s", endpoint, endpointUdpAddr.IP)
-
-	conn.connIDICE = nbnet.GenerateConnID()
-	for _, hook := range conn.beforeAddPeerHooks {
-		if err := hook(conn.connIDICE, endpointUdpAddr.IP); err != nil {
-			conn.log.Errorf("Before add peer hook failed: %v", err)
-		}
+	if err := conn.runBeforeAddPeerHooks(ep.IP); err != nil {
+		conn.log.Errorf("Before add peer hook failed: %v", err)
 	}
 
 	conn.workerRelay.DisableWgWatcher()
 
-	err = conn.configureWGEndpoint(endpointUdpAddr)
-	if err != nil {
-		if wgProxy != nil {
-			if err := wgProxy.CloseConn(); err != nil {
-				conn.log.Warnf("Failed to close turn connection: %v", err)
-			}
-		}
-		conn.log.Warnf("Failed to update wg peer configuration: %v", err)
+	if conn.wgProxyRelay != nil {
+		conn.wgProxyRelay.Pause()
+	}
+
+	if wgProxy != nil {
+		wgProxy.Work()
+	}
+
+	if err = conn.configureWGEndpoint(ep); err != nil {
+		conn.handleConfigurationFailure(err, wgProxy)
 		return
 	}
 	wgConfigWorkaround()
-
-	if conn.wgProxyICE != nil {
-		if err := conn.wgProxyICE.CloseConn(); err != nil {
-			conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err)
-		}
-	}
-	conn.wgProxyICE = wgProxy
-
 	conn.currentConnPriority = priority
-
+	conn.statusICE.Set(StatusConnected)
+	conn.updateIceState(iceConnInfo)
 	conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr)
 }
 
@@ -482,11 +381,18 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) {
 
 	conn.log.Tracef("ICE connection state changed to %s", newState)
 
+	if conn.wgProxyICE != nil {
+		if err := conn.wgProxyICE.CloseConn(); err != nil {
+			conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err)
+		}
+	}
+
 	// switch back to relay connection
-	if conn.endpointRelay != nil && conn.currentConnPriority != connPriorityRelay {
+	if conn.isReadyToUpgrade() {
 		conn.log.Debugf("ICE disconnected, set Relay to active connection")
-		err := conn.configureWGEndpoint(conn.endpointRelay)
-		if err != nil {
+		conn.wgProxyRelay.Work()
+
+		if err := conn.configureWGEndpoint(conn.wgProxyRelay.EndpointAddr()); err != nil {
 			conn.log.Errorf("failed to switch to relay conn: %v", err)
 		}
 		conn.workerRelay.EnableWgWatcher(conn.ctx)
@@ -496,10 +402,7 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) {
 	changed := conn.statusICE.Get() != newState && newState != StatusConnecting
 	conn.statusICE.Set(newState)
 
-	select {
-	case conn.iCEDisconnected <- changed:
-	default:
-	}
+	conn.guard.SetICEConnDisconnected(changed)
 
 	peerState := State{
 		PubKey:           conn.config.Key,
@@ -520,61 +423,48 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) {
 
 	if conn.ctx.Err() != nil {
 		if err := rci.relayedConn.Close(); err != nil {
-			log.Warnf("failed to close unnecessary relayed connection: %v", err)
+			conn.log.Warnf("failed to close unnecessary relayed connection: %v", err)
 		}
 		return
 	}
 
-	conn.log.Debugf("Relay connection is ready to use")
-	conn.statusRelay.Set(StatusConnected)
+	conn.log.Debugf("Relay connection has been established, setup the WireGuard")
 
-	wgProxy := conn.wgProxyFactory.GetProxy()
-	endpoint, err := wgProxy.AddTurnConn(conn.ctx, rci.relayedConn)
+	wgProxy, err := conn.newProxy(rci.relayedConn)
 	if err != nil {
 		conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
 		return
 	}
-	conn.log.Infof("created new wgProxy for relay connection: %s", endpoint)
 
-	endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String())
-	conn.endpointRelay = endpointUdpAddr
-	conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP)
+	conn.log.Infof("created new wgProxy for relay connection: %s", wgProxy.EndpointAddr().String())
 
-	defer conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey)
-
-	if conn.currentConnPriority > connPriorityRelay {
-		if conn.statusICE.Get() == StatusConnected {
-			log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnPriority)
-			return
-		}
+	if conn.iceP2PIsActive() {
+		conn.log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnPriority)
+		conn.wgProxyRelay = wgProxy
+		conn.statusRelay.Set(StatusConnected)
+		conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey)
+		return
 	}
 
-	conn.connIDRelay = nbnet.GenerateConnID()
-	for _, hook := range conn.beforeAddPeerHooks {
-		if err := hook(conn.connIDRelay, endpointUdpAddr.IP); err != nil {
-			conn.log.Errorf("Before add peer hook failed: %v", err)
-		}
+	if err := conn.runBeforeAddPeerHooks(wgProxy.EndpointAddr().IP); err != nil {
+		conn.log.Errorf("Before add peer hook failed: %v", err)
 	}
 
-	err = conn.configureWGEndpoint(endpointUdpAddr)
-	if err != nil {
+	wgProxy.Work()
+	if err := conn.configureWGEndpoint(wgProxy.EndpointAddr()); err != nil {
 		if err := wgProxy.CloseConn(); err != nil {
 			conn.log.Warnf("Failed to close relay connection: %v", err)
 		}
-		conn.log.Errorf("Failed to update wg peer configuration: %v", err)
+		conn.log.Errorf("Failed to update WireGuard peer configuration: %v", err)
 		return
 	}
 	conn.workerRelay.EnableWgWatcher(conn.ctx)
+
 	wgConfigWorkaround()
-
-	if conn.wgProxyRelay != nil {
-		if err := conn.wgProxyRelay.CloseConn(); err != nil {
-			conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err)
-		}
-	}
-	conn.wgProxyRelay = wgProxy
 	conn.currentConnPriority = connPriorityRelay
-
+	conn.statusRelay.Set(StatusConnected)
+	conn.wgProxyRelay = wgProxy
+	conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey)
 	conn.log.Infof("start to communicate with peer via relay")
 	conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr)
 }
@@ -587,29 +477,23 @@ func (conn *Conn) onWorkerRelayStateDisconnected() {
 		return
 	}
 
-	log.Debugf("relay connection is disconnected")
+	conn.log.Debugf("relay connection is disconnected")
 
 	if conn.currentConnPriority == connPriorityRelay {
-		log.Debugf("clean up WireGuard config")
-		err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey)
-		if err != nil {
+		conn.log.Debugf("clean up WireGuard config")
+		if err := conn.removeWgPeer(); err != nil {
 			conn.log.Errorf("failed to remove wg endpoint: %v", err)
 		}
 	}
 
 	if conn.wgProxyRelay != nil {
-		conn.endpointRelay = nil
 		_ = conn.wgProxyRelay.CloseConn()
 		conn.wgProxyRelay = nil
 	}
 
 	changed := conn.statusRelay.Get() != StatusDisconnected
 	conn.statusRelay.Set(StatusDisconnected)
-
-	select {
-	case conn.relayDisconnected <- changed:
-	default:
-	}
+	conn.guard.SetRelayedConnDisconnected(changed)
 
 	peerState := State{
 		PubKey:           conn.config.Key,
@@ -617,13 +501,25 @@ func (conn *Conn) onWorkerRelayStateDisconnected() {
 		Relayed:          conn.isRelayed(),
 		ConnStatusUpdate: time.Now(),
 	}
-
-	err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState)
-	if err != nil {
+	if err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState); err != nil {
 		conn.log.Warnf("unable to save peer's state to Relay disconnected, got error: %v", err)
 	}
 }
 
+func (conn *Conn) listenGuardEvent(ctx context.Context) {
+	for {
+		select {
+		case <-conn.guard.Reconnect:
+			conn.log.Debugf("send offer to peer")
+			if err := conn.handshaker.SendOffer(); err != nil {
+				conn.log.Errorf("failed to send offer: %v", err)
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error {
 	return conn.config.WgConfig.WgInterface.UpdatePeer(
 		conn.config.WgConfig.RemoteKey,
@@ -696,11 +592,11 @@ func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAdd
 	}
 
 	if conn.onConnected != nil {
-		conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIPsIP, remoteRosenpassAddr)
+		conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedNet, remoteRosenpassAddr)
 	}
 }
 
-func (conn *Conn) waitInitialRandomSleepTime() {
+func (conn *Conn) waitInitialRandomSleepTime(ctx context.Context) {
 	minWait := 100
 	maxWait := 800
 	duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond
@@ -709,7 +605,7 @@ func (conn *Conn) waitInitialRandomSleepTime() {
 	defer timeout.Stop()
 
 	select {
-	case <-conn.ctx.Done():
+	case <-ctx.Done():
 	case <-timeout.C:
 	}
 }
@@ -738,11 +634,17 @@ func (conn *Conn) evalStatus() ConnStatus {
 	return StatusDisconnected
 }
 
-func (conn *Conn) isConnected() bool {
+func (conn *Conn) isConnectedOnAllWay() (connected bool) {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 
-	if conn.statusICE.Get() != StatusConnected && conn.statusICE.Get() != StatusConnecting {
+	defer func() {
+		if !connected {
+			conn.logTraceConnState()
+		}
+	}()
+
+	if conn.statusICE.Get() == StatusDisconnected {
 		return false
 	}
 
@@ -755,6 +657,16 @@ func (conn *Conn) isConnected() bool {
 	return true
 }
 
+func (conn *Conn) runBeforeAddPeerHooks(ip net.IP) error {
+	conn.connIDICE = nbnet.GenerateConnID()
+	for _, hook := range conn.beforeAddPeerHooks {
+		if err := hook(conn.connIDICE, ip); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (conn *Conn) freeUpConnID() {
 	if conn.connIDRelay != "" {
 		for _, hook := range conn.afterRemovePeerHooks {
@@ -775,21 +687,55 @@ func (conn *Conn) freeUpConnID() {
 	}
 }
 
-func (conn *Conn) getEndpointForICEConnInfo(iceConnInfo ICEConnInfo) (net.Addr, wgproxy.Proxy, error) {
-	if !iceConnInfo.RelayedOnLocal {
-		return iceConnInfo.RemoteConn.RemoteAddr(), nil, nil
+func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
+	conn.log.Debugf("setup proxied WireGuard connection")
+	udpAddr := &net.UDPAddr{
+		IP:   conn.allowedIP,
+		Port: conn.config.WgConfig.WgListenPort,
 	}
-	conn.log.Debugf("setup ice turn connection")
-	wgProxy := conn.wgProxyFactory.GetProxy()
-	ep, err := wgProxy.AddTurnConn(conn.ctx, iceConnInfo.RemoteConn)
-	if err != nil {
+
+	wgProxy := conn.config.WgConfig.WgInterface.GetProxy()
+	if err := wgProxy.AddTurnConn(conn.ctx, udpAddr, remoteConn); err != nil {
 		conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
-		if errClose := wgProxy.CloseConn(); errClose != nil {
-			conn.log.Warnf("failed to close turn proxy connection: %v", errClose)
-		}
-		return nil, nil, err
+		return nil, err
 	}
-	return ep, wgProxy, nil
+	return wgProxy, nil
+}
+
+func (conn *Conn) isReadyToUpgrade() bool {
+	return conn.wgProxyRelay != nil && conn.currentConnPriority != connPriorityRelay
+}
+
+func (conn *Conn) iceP2PIsActive() bool {
+	return conn.currentConnPriority == connPriorityICEP2P && conn.statusICE.Get() == StatusConnected
+}
+
+func (conn *Conn) removeWgPeer() error {
+	return conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey)
+}
+
+func (conn *Conn) handleConfigurationFailure(err error, wgProxy wgproxy.Proxy) {
+	conn.log.Warnf("Failed to update wg peer configuration: %v", err)
+	if wgProxy != nil {
+		if ierr := wgProxy.CloseConn(); ierr != nil {
+			conn.log.Warnf("Failed to close wg proxy: %v", ierr)
+		}
+	}
+	if conn.wgProxyRelay != nil {
+		conn.wgProxyRelay.Work()
+	}
+}
+
+func (conn *Conn) logTraceConnState() {
+	if conn.workerRelay.IsRelayConnectionSupportedWithPeer() {
+		conn.log.Tracef("connectivity guard check, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE)
+	} else {
+		conn.log.Tracef("connectivity guard check, ice state: %s", conn.statusICE)
+	}
+}
+
+func isController(config ConnConfig) bool {
+	return config.LocalKey > config.Key
 }
 
 func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool {