Merge branch 'main' into debug-dns

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [netbirdio]

.github/workflows/golang-test-darwin.yml

@@ -42,4 +42,4 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...
+        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...

.github/workflows/golang-test-linux.yml

@@ -13,6 +13,7 @@ concurrency:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
         arch: [ '386','amd64' ]
         store: [ 'sqlite', 'postgres']
@@ -49,7 +50,48 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 6m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
+
+  benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ '386','amd64' ]
+        store: [ 'sqlite', 'postgres' ]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: matrix.arch == '386'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
+
+      - name: Install modules
+        run: go mod tidy
+
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
+      - name: Test
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
 
   test_client_on_docker:
     runs-on: ubuntu-20.04
@@ -79,9 +121,6 @@ jobs:
       - name: check git status
         run: git --no-pager diff --exit-code
 
-      - name: Generate Iface Test bin
-        run: CGO_ENABLED=0 go test -c -o iface-testing.bin ./client/iface/
-
       - name: Generate Shared Sock Test bin
         run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock
 
@@ -98,7 +137,7 @@ jobs:
         run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal
 
       - name: Generate Peer Test bin
-        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/...
+        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/
 
       - run: chmod +x *testing.bin
 
@@ -106,7 +145,7 @@ jobs:
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/sharedsock --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/sharedsock-testing.bin -test.timeout 5m -test.parallel 1
 
       - name: Run Iface tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/iface --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/iface-testing.bin -test.timeout 5m -test.parallel 1
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/netbird -v /tmp/cache:/tmp/cache -v /tmp/modcache:/tmp/modcache -w /netbird -e GOCACHE=/tmp/cache -e GOMODCACHE=/tmp/modcache -e CGO_ENABLED=0 golang:1.23-alpine go test  -test.timeout 5m -test.parallel 1 ./client/iface/...
 
       - name: Run RouteManager tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/routemanager-testing.bin  -test.timeout 5m -test.parallel 1

.github/workflows/golangci-lint.yml

@@ -19,7 +19,7 @@ jobs:
       - name: codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          ignore_words_list: erro,clienta,hastable,iif
+          ignore_words_list: erro,clienta,hastable,iif,groupd
           skip: go.mod,go.sum
           only_warn: 1
   golangci:

.github/workflows/release.yml

@@ -9,7 +9,7 @@ on:
   pull_request:
 
 env:
-  SIGN_PIPE_VER: "v0.0.14"
+  SIGN_PIPE_VER: "v0.0.17"
   GORELEASER_VER: "v2.3.2"
   PRODUCT_NAME: "NetBird"
   COPYRIGHT: "Wiretrustee UG (haftungsbeschreankt)"
@@ -223,4 +223,4 @@ jobs:
           repo: netbirdio/sign-pipelines
           ref: ${{ env.SIGN_PIPE_VER }}
           token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'
+          inputs: '{ "tag": "${{ github.ref }}", "skipRelease": false }'

.goreleaser.yaml

@@ -96,6 +96,9 @@ builds:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
     mod_timestamp: "{{ .CommitTimestamp }}"
 
+universal_binaries:
+  - id: netbird
+
 archives:
   - builds:
       - netbird

.goreleaser_ui_darwin.yaml

@@ -23,6 +23,9 @@ builds:
     tags:
       - load_wgnt_from_rsrc
 
+universal_binaries:
+  - id: netbird-ui-darwin
+
 archives:
   - builds:
       - netbird-ui-darwin

README.md

@@ -17,8 +17,12 @@
        <img src="https://img.shields.io/badge/license-BSD--3-blue" />
      </a> 
     <br>
-    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2p5zwhm4g-8fHollzrQa5y4PZF5AEpvQ">
+    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2utg2ncdz-W7LEB6toRBLE1Jca37dYpg">
         <img src="https://img.shields.io/badge/slack-@netbird-red.svg?logo=slack"/>
+     </a>  
+     <br>
+    <a href="https://gurubase.io/g/netbird">
+        <img src="https://img.shields.io/badge/Gurubase-Ask%20NetBird%20Guru-006BFF"/>
      </a>    
   </p>
 </div>
@@ -30,7 +34,7 @@
   <br/>
   See <a href="https://netbird.io/docs/">Documentation</a>
   <br/>
-   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2p5zwhm4g-8fHollzrQa5y4PZF5AEpvQ">Slack channel</a>
+   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2utg2ncdz-W7LEB6toRBLE1Jca37dYpg">Slack channel</a>
   <br/>
  
 </strong>

client/anonymize/anonymize.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 )
 
+const anonTLD = ".domain"
+
 type Anonymizer struct {
 	ipAnonymizer     map[netip.Addr]netip.Addr
 	domainAnonymizer map[string]string
@@ -83,29 +85,39 @@ func (a *Anonymizer) AnonymizeIPString(ip string) string {
 }
 
 func (a *Anonymizer) AnonymizeDomain(domain string) string {
-	if strings.HasSuffix(domain, "netbird.io") ||
-		strings.HasSuffix(domain, "netbird.selfhosted") ||
-		strings.HasSuffix(domain, "netbird.cloud") ||
-		strings.HasSuffix(domain, "netbird.stage") ||
-		strings.HasSuffix(domain, ".domain") {
+	baseDomain := domain
+	hasDot := strings.HasSuffix(domain, ".")
+	if hasDot {
+		baseDomain = domain[:len(domain)-1]
+	}
+
+	if strings.HasSuffix(baseDomain, "netbird.io") ||
+		strings.HasSuffix(baseDomain, "netbird.selfhosted") ||
+		strings.HasSuffix(baseDomain, "netbird.cloud") ||
+		strings.HasSuffix(baseDomain, "netbird.stage") ||
+		strings.HasSuffix(baseDomain, anonTLD) {
 		return domain
 	}
 
-	parts := strings.Split(domain, ".")
+	parts := strings.Split(baseDomain, ".")
 	if len(parts) < 2 {
 		return domain
 	}
 
-	baseDomain := parts[len(parts)-2] + "." + parts[len(parts)-1]
+	baseForLookup := parts[len(parts)-2] + "." + parts[len(parts)-1]
 
-	anonymized, ok := a.domainAnonymizer[baseDomain]
+	anonymized, ok := a.domainAnonymizer[baseForLookup]
 	if !ok {
-		anonymizedBase := "anon-" + generateRandomString(5) + ".domain"
-		a.domainAnonymizer[baseDomain] = anonymizedBase
+		anonymizedBase := "anon-" + generateRandomString(5) + anonTLD
+		a.domainAnonymizer[baseForLookup] = anonymizedBase
 		anonymized = anonymizedBase
 	}
 
-	return strings.Replace(domain, baseDomain, anonymized, 1)
+	result := strings.Replace(baseDomain, baseForLookup, anonymized, 1)
+	if hasDot {
+		result += "."
+	}
+	return result
 }
 
 func (a *Anonymizer) AnonymizeURI(uri string) string {
@@ -152,9 +164,9 @@ func (a *Anonymizer) AnonymizeString(str string) string {
 	return str
 }
 
-// AnonymizeSchemeURI finds and anonymizes URIs with stun, stuns, turn, and turns schemes.
+// AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes.
 func (a *Anonymizer) AnonymizeSchemeURI(text string) string {
-	re := regexp.MustCompile(`(?i)\b(stuns?:|turns?:|https?://)\S+\b`)
+	re := regexp.MustCompile(`(?i)\b(wss?://|rels?://|stuns?:|turns?:|https?://)\S+\b`)
 
 	return re.ReplaceAllStringFunc(text, a.AnonymizeURI)
 }
@@ -168,10 +180,10 @@ func (a *Anonymizer) AnonymizeDNSLogLine(logEntry string) string {
 		parts := strings.Split(match, `"`)
 		if len(parts) >= 2 {
 			domain := parts[1]
-			if strings.HasSuffix(domain, ".domain") {
+			if strings.HasSuffix(domain, anonTLD) {
 				return match
 			}
-			randomDomain := generateRandomString(10) + ".domain"
+			randomDomain := generateRandomString(10) + anonTLD
 			return strings.Replace(match, domain, randomDomain, 1)
 		}
 		return match
@@ -201,6 +213,8 @@ func isWellKnown(addr netip.Addr) bool {
 		"2606:4700:4700::1111", "2606:4700:4700::1001", // Cloudflare DNS IPv6
 		"9.9.9.9", "149.112.112.112", // Quad9 DNS IPv4
 		"2620:fe::fe", "2620:fe::9", // Quad9 DNS IPv6
+
+		"128.0.0.0", "8000::", // 2nd split subnet for default routes
 	}
 
 	if slices.Contains(wellKnown, addr.String()) {

client/anonymize/anonymize_test.go

@@ -67,18 +67,36 @@ func TestAnonymizeDomain(t *testing.T) {
 			`^anon-[a-zA-Z0-9]+\.domain$`,
 			true,
 		},
+		{
+			"Domain with Trailing Dot",
+			"example.com.",
+			`^anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
 		{
 			"Subdomain",
 			"sub.example.com",
 			`^sub\.anon-[a-zA-Z0-9]+\.domain$`,
 			true,
 		},
+		{
+			"Subdomain with Trailing Dot",
+			"sub.example.com.",
+			`^sub\.anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
 		{
 			"Protected Domain",
 			"netbird.io",
 			`^netbird\.io$`,
 			false,
 		},
+		{
+			"Protected Domain with Trailing Dot",
+			"netbird.io.",
+			`^netbird\.io.$`,
+			false,
+		},
 	}
 
 	for _, tc := range tests {
@@ -140,8 +158,16 @@ func TestAnonymizeSchemeURI(t *testing.T) {
 		expect string
 	}{
 		{"STUN URI in text", "Connection made via stun:example.com", `Connection made via stun:anon-[a-zA-Z0-9]+\.domain`},
+		{"STUNS URI in message", "Secure connection to stuns:example.com:443", `Secure connection to stuns:anon-[a-zA-Z0-9]+\.domain:443`},
 		{"TURN URI in log", "Failed attempt turn:some.example.com:3478?transport=tcp: retrying", `Failed attempt turn:some.anon-[a-zA-Z0-9]+\.domain:3478\?transport=tcp: retrying`},
+		{"TURNS URI in message", "Secure connection to turns:example.com:5349", `Secure connection to turns:anon-[a-zA-Z0-9]+\.domain:5349`},
+		{"HTTP URI in text", "Visit http://example.com for more", `Visit http://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"HTTPS URI in CAPS", "Visit HTTPS://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
 		{"HTTPS URI in message", "Visit https://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"WS URI in log", "Connection established to ws://example.com:8080", `Connection established to ws://anon-[a-zA-Z0-9]+\.domain:8080`},
+		{"WSS URI in message", "Secure connection to wss://example.com", `Secure connection to wss://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rel URI in text", "Relaying to rel://example.com", `Relaying to rel://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rels URI in message", "Relaying to rels://example.com", `Relaying to rels://anon-[a-zA-Z0-9]+\.domain`},
 	}
 
 	for _, tc := range tests {

client/cmd/debug.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -61,6 +62,15 @@ var forCmd = &cobra.Command{
 	RunE:    runForDuration,
 }
 
+var persistenceCmd = &cobra.Command{
+	Use:     "persistence [on|off]",
+	Short:   "Set network map memory persistence",
+	Long:    `Configure whether the latest network map should persist in memory. When enabled, the last known network map will be kept in memory.`,
+	Example: "  netbird debug persistence on",
+	Args:    cobra.ExactArgs(1),
+	RunE:    setNetworkMapPersistence,
+}
+
 func debugBundle(cmd *cobra.Command, _ []string) error {
 	conn, err := getClient(cmd)
 	if err != nil {
@@ -171,6 +181,13 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 
 	time.Sleep(1 * time.Second)
 
+	// Enable network map persistence before bringing the service up
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: true,
+	}); err != nil {
+		return fmt.Errorf("failed to enable network map persistence: %v", status.Convert(err).Message())
+	}
+
 	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
 		return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
 	}
@@ -200,6 +217,13 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
 	}
 
+	// Disable network map persistence after creating the debug bundle
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: false,
+	}); err != nil {
+		return fmt.Errorf("failed to disable network map persistence: %v", status.Convert(err).Message())
+	}
+
 	if stateWasDown {
 		if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
 			return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
@@ -219,6 +243,34 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func setNetworkMapPersistence(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	persistence := strings.ToLower(args[0])
+	if persistence != "on" && persistence != "off" {
+		return fmt.Errorf("invalid persistence value: %s. Use 'on' or 'off'", args[0])
+	}
+
+	client := proto.NewDaemonServiceClient(conn)
+	_, err = client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: persistence == "on",
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set network map persistence: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("Network map persistence set to: %s\n", persistence)
+	return nil
+}
+
 func getStatusOutput(cmd *cobra.Command) string {
 	var statusOutputString string
 	statusResp, err := getStatus(cmd.Context())

client/cmd/pprof.go

@@ -0,0 +1,33 @@
+//go:build pprof
+// +build pprof
+
+package cmd
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	addr := pprofAddr()
+	go pprof(addr)
+}
+
+func pprofAddr() string {
+	listenAddr := os.Getenv("NB_PPROF_ADDR")
+	if listenAddr == "" {
+		return "localhost:6969"
+	}
+
+	return listenAddr
+}
+
+func pprof(listenAddr string) {
+	log.Infof("listening pprof on: %s\n", listenAddr)
+	if err := http.ListenAndServe(listenAddr, nil); err != nil {
+		log.Fatalf("Failed to start pprof: %v", err)
+	}
+}

client/cmd/root.go

@@ -155,6 +155,7 @@ func init() {
 	debugCmd.AddCommand(logCmd)
 	logCmd.AddCommand(logLevelCmd)
 	debugCmd.AddCommand(forCmd)
+	debugCmd.AddCommand(persistenceCmd)
 
 	upCmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
 		`Sets external IPs maps between local addresses and interfaces.`+

client/cmd/service.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"sync"
 
 	"github.com/kardianos/service"
 	log "github.com/sirupsen/logrus"
@@ -13,10 +14,11 @@ import (
 )
 
 type program struct {
-	ctx            context.Context
-	cancel         context.CancelFunc
-	serv           *grpc.Server
-	serverInstance *server.Server
+	ctx              context.Context
+	cancel           context.CancelFunc
+	serv             *grpc.Server
+	serverInstance   *server.Server
+	serverInstanceMu sync.Mutex
 }
 
 func newProgram(ctx context.Context, cancel context.CancelFunc) *program {

client/cmd/service_controller.go

@@ -61,7 +61,9 @@ func (p *program) Start(svc service.Service) error {
 		}
 		proto.RegisterDaemonServiceServer(p.serv, serverInstance)
 
+		p.serverInstanceMu.Lock()
 		p.serverInstance = serverInstance
+		p.serverInstanceMu.Unlock()
 
 		log.Printf("started daemon server: %v", split[1])
 		if err := p.serv.Serve(listen); err != nil {
@@ -72,6 +74,7 @@ func (p *program) Start(svc service.Service) error {
 }
 
 func (p *program) Stop(srv service.Service) error {
+	p.serverInstanceMu.Lock()
 	if p.serverInstance != nil {
 		in := new(proto.DownRequest)
 		_, err := p.serverInstance.Down(p.ctx, in)
@@ -79,6 +82,7 @@ func (p *program) Stop(srv service.Service) error {
 			log.Errorf("failed to stop daemon: %v", err)
 		}
 	}
+	p.serverInstanceMu.Unlock()
 
 	p.cancel()
 

client/cmd/state.go

@@ -0,0 +1,181 @@
+package cmd
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/proto"
+)
+
+var (
+	allFlag bool
+)
+
+var stateCmd = &cobra.Command{
+	Use:   "state",
+	Short: "Manage daemon state",
+	Long:  "Provides commands for managing and inspecting the Netbird daemon state.",
+}
+
+var stateListCmd = &cobra.Command{
+	Use:     "list",
+	Aliases: []string{"ls"},
+	Short:   "List all stored states",
+	Long:    "Lists all registered states with their status and basic information.",
+	Example: "  netbird state list",
+	RunE:    stateList,
+}
+
+var stateCleanCmd = &cobra.Command{
+	Use:   "clean [state-name]",
+	Short: "Clean stored states",
+	Long: `Clean specific state or all states. The daemon must not be running.
+This will perform cleanup operations and remove the state.`,
+	Example: `  netbird state clean dns_state
+  netbird state clean --all`,
+	RunE: stateClean,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Check mutual exclusivity between --all flag and state-name argument
+		if allFlag && len(args) > 0 {
+			return fmt.Errorf("cannot specify both --all flag and state name")
+		}
+		if !allFlag && len(args) != 1 {
+			return fmt.Errorf("requires a state name argument or --all flag")
+		}
+		return nil
+	},
+}
+
+var stateDeleteCmd = &cobra.Command{
+	Use:   "delete [state-name]",
+	Short: "Delete stored states",
+	Long: `Delete specific state or all states from storage. The daemon must not be running.
+This will remove the state without performing any cleanup operations.`,
+	Example: `  netbird state delete dns_state
+  netbird state delete --all`,
+	RunE: stateDelete,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Check mutual exclusivity between --all flag and state-name argument
+		if allFlag && len(args) > 0 {
+			return fmt.Errorf("cannot specify both --all flag and state name")
+		}
+		if !allFlag && len(args) != 1 {
+			return fmt.Errorf("requires a state name argument or --all flag")
+		}
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(stateCmd)
+	stateCmd.AddCommand(stateListCmd, stateCleanCmd, stateDeleteCmd)
+
+	stateCleanCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Clean all states")
+	stateDeleteCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Delete all states")
+}
+
+func stateList(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.ListStates(cmd.Context(), &proto.ListStatesRequest{})
+	if err != nil {
+		return fmt.Errorf("failed to list states: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("\nStored states:\n\n")
+	for _, state := range resp.States {
+		cmd.Printf("- %s\n", state.Name)
+	}
+
+	return nil
+}
+
+func stateClean(cmd *cobra.Command, args []string) error {
+	var stateName string
+	if !allFlag {
+		stateName = args[0]
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.CleanState(cmd.Context(), &proto.CleanStateRequest{
+		StateName: stateName,
+		All:       allFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to clean state: %v", status.Convert(err).Message())
+	}
+
+	if resp.CleanedStates == 0 {
+		cmd.Println("No states were cleaned")
+		return nil
+	}
+
+	if allFlag {
+		cmd.Printf("Successfully cleaned %d states\n", resp.CleanedStates)
+	} else {
+		cmd.Printf("Successfully cleaned state %q\n", stateName)
+	}
+
+	return nil
+}
+
+func stateDelete(cmd *cobra.Command, args []string) error {
+	var stateName string
+	if !allFlag {
+		stateName = args[0]
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.DeleteState(cmd.Context(), &proto.DeleteStateRequest{
+		StateName: stateName,
+		All:       allFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete state: %v", status.Convert(err).Message())
+	}
+
+	if resp.DeletedStates == 0 {
+		cmd.Println("No states were deleted")
+		return nil
+	}
+
+	if allFlag {
+		cmd.Printf("Successfully deleted %d states\n", resp.DeletedStates)
+	} else {
+		cmd.Printf("Successfully deleted state %q\n", stateName)
+	}
+
+	return nil
+}

client/cmd/status.go

@@ -680,7 +680,7 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 func skipDetailByFilters(peerState *proto.PeerState, isConnected bool) bool {
 	statusEval := false
 	ipEval := false
-	nameEval := false
+	nameEval := true
 
 	if statusFilter != "" {
 		lowerStatusFilter := strings.ToLower(statusFilter)
@@ -700,11 +700,13 @@ func skipDetailByFilters(peerState *proto.PeerState, isConnected bool) bool {
 
 	if len(prefixNamesFilter) > 0 {
 		for prefixNameFilter := range prefixNamesFilterMap {
-			if !strings.HasPrefix(peerState.Fqdn, prefixNameFilter) {
-				nameEval = true
+			if strings.HasPrefix(peerState.Fqdn, prefixNameFilter) {
+				nameEval = false
 				break
 			}
 		}
+	} else {
+		nameEval = false
 	}
 
 	return statusEval || ipEval || nameEval

client/cmd/testutil_test.go

@@ -38,7 +38,7 @@ func startTestingServices(t *testing.T) string {
 	signalAddr := signalLis.Addr().String()
 	config.Signal.URI = signalAddr
 
-	_, mgmLis := startManagement(t, config, "../testdata/store.sqlite")
+	_, mgmLis := startManagement(t, config, "../testdata/store.sql")
 	mgmAddr := mgmLis.Addr().String()
 	return mgmAddr
 }
@@ -71,7 +71,7 @@ func startManagement(t *testing.T, config *mgmt.Config, testFile string) (*grpc.
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, cleanUp, err := mgmt.NewTestStoreFromSqlite(context.Background(), testFile, t.TempDir())
+	store, cleanUp, err := mgmt.NewTestStoreFromSQL(context.Background(), testFile, t.TempDir())
 	if err != nil {
 		t.Fatal(err)
 	}

client/firewall/create.go

@@ -3,7 +3,6 @@
 package firewall
 
 import (
-	"context"
 	"fmt"
 	"runtime"
 
@@ -11,10 +10,11 @@ import (
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 // NewFirewall creates a firewall manager instance
-func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, _ *statemanager.Manager) (firewall.Manager, error) {
 	if !iface.IsUserspaceBind() {
 		return nil, fmt.Errorf("not implemented for this OS: %s", runtime.GOOS)
 	}

client/firewall/create_linux.go

@@ -3,7 +3,7 @@
 package firewall
 
 import (
-	"context"
+	"errors"
 	"fmt"
 	"os"
 
@@ -15,6 +15,7 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	nbnftables "github.com/netbirdio/netbird/client/firewall/nftables"
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -32,54 +33,65 @@ const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK"
 // FWType is the type for the firewall type
 type FWType int
 
-func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
 	// on the linux system we try to user nftables or iptables
 	// in any case, because we need to allow netbird interface traffic
 	// so we use AllowNetbird traffic from these firewall managers
 	// for the userspace packet filtering firewall
-	var fm firewall.Manager
-	var errFw error
+	fm, err := createNativeFirewall(iface, stateManager)
 
+	if !iface.IsUserspaceBind() {
+		return fm, err
+	}
+
+	if err != nil {
+		log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err)
+	}
+	return createUserspaceFirewall(iface, fm)
+}
+
+func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+	fm, err := createFW(iface)
+	if err != nil {
+		return nil, fmt.Errorf("create firewall: %s", err)
+	}
+
+	if err = fm.Init(stateManager); err != nil {
+		return nil, fmt.Errorf("init firewall: %s", err)
+	}
+
+	return fm, nil
+}
+
+func createFW(iface IFaceMapper) (firewall.Manager, error) {
 	switch check() {
 	case IPTABLES:
 		log.Info("creating an iptables firewall manager")
-		fm, errFw = nbiptables.Create(context, iface)
-		if errFw != nil {
-			log.Errorf("failed to create iptables manager: %s", errFw)
-		}
+		return nbiptables.Create(iface)
 	case NFTABLES:
 		log.Info("creating an nftables firewall manager")
-		fm, errFw = nbnftables.Create(context, iface)
-		if errFw != nil {
-			log.Errorf("failed to create nftables manager: %s", errFw)
-		}
+		return nbnftables.Create(iface)
 	default:
-		errFw = fmt.Errorf("no firewall manager found")
 		log.Info("no firewall manager found, trying to use userspace packet filtering firewall")
+		return nil, errors.New("no firewall manager found")
+	}
+}
+
+func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager) (firewall.Manager, error) {
+	var errUsp error
+	if fm != nil {
+		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
+	} else {
+		fm, errUsp = uspfilter.Create(iface)
 	}
 
-	if iface.IsUserspaceBind() {
-		var errUsp error
-		if errFw == nil {
-			fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
-		} else {
-			fm, errUsp = uspfilter.Create(iface)
-		}
-		if errUsp != nil {
-			log.Debugf("failed to create userspace filtering firewall: %s", errUsp)
-			return nil, errUsp
-		}
-
-		if err := fm.AllowNetbird(); err != nil {
-			log.Errorf("failed to allow netbird interface traffic: %v", err)
-		}
-		return fm, nil
+	if errUsp != nil {
+		return nil, fmt.Errorf("create userspace firewall: %s", errUsp)
 	}
 
-	if errFw != nil {
-		return nil, errFw
+	if err := fm.AllowNetbird(); err != nil {
+		log.Errorf("failed to allow netbird interface traffic: %v", err)
 	}
-
 	return fm, nil
 }
 

client/firewall/iptables/acl_linux.go

@@ -11,6 +11,8 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
@@ -21,13 +23,23 @@ const (
 	chainNameOutputRules = "NETBIRD-ACL-OUTPUT"
 )
 
+type aclEntries map[string][][]string
+
+type entry struct {
+	spec     []string
+	position int
+}
+
 type aclManager struct {
 	iptablesClient     *iptables.IPTables
 	wgIface            iFaceMapper
 	routingFwChainName string
 
-	entries    map[string][][]string
-	ipsetStore *ipsetStore
+	entries         aclEntries
+	optionalEntries map[string][]entry
+	ipsetStore      *ipsetStore
+
+	stateManager *statemanager.Manager
 }
 
 func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routingFwChainName string) (*aclManager, error) {
@@ -36,27 +48,35 @@ func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routi
 		wgIface:            wgIface,
 		routingFwChainName: routingFwChainName,
 
-		entries:    make(map[string][][]string),
-		ipsetStore: newIpsetStore(),
+		entries:         make(map[string][][]string),
+		optionalEntries: make(map[string][]entry),
+		ipsetStore:      newIpsetStore(),
 	}
 
-	err := ipset.Init()
-	if err != nil {
-		return nil, fmt.Errorf("failed to init ipset: %w", err)
+	if err := ipset.Init(); err != nil {
+		return nil, fmt.Errorf("init ipset: %w", err)
 	}
 
+	return m, nil
+}
+
+func (m *aclManager) init(stateManager *statemanager.Manager) error {
+	m.stateManager = stateManager
+
 	m.seedInitialEntries()
+	m.seedInitialOptionalEntries()
 
-	err = m.cleanChains()
-	if err != nil {
-		return nil, err
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
 	}
 
-	err = m.createDefaultChains()
-	if err != nil {
-		return nil, err
+	if err := m.createDefaultChains(); err != nil {
+		return fmt.Errorf("create default chains: %w", err)
 	}
-	return m, nil
+
+	m.updateState()
+
+	return nil
 }
 
 func (m *aclManager) AddPeerFiltering(
@@ -137,6 +157,8 @@ func (m *aclManager) AddPeerFiltering(
 		chain:     chain,
 	}
 
+	m.updateState()
+
 	return []firewall.Rule{rule}, nil
 }
 
@@ -171,15 +193,23 @@ func (m *aclManager) DeletePeerRule(rule firewall.Rule) error {
 		}
 	}
 
-	err := m.iptablesClient.Delete(tableName, r.chain, r.specs...)
-	if err != nil {
-		log.Debugf("failed to delete rule, %s, %v: %s", r.chain, r.specs, err)
+	if err := m.iptablesClient.Delete(tableName, r.chain, r.specs...); err != nil {
+		return fmt.Errorf("failed to delete rule: %s, %v: %w", r.chain, r.specs, err)
 	}
-	return err
+
+	m.updateState()
+
+	return nil
 }
 
 func (m *aclManager) Reset() error {
-	return m.cleanChains()
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
+	}
+
+	m.updateState()
+
+	return nil
 }
 
 // todo write less destructive cleanup mechanism
@@ -232,6 +262,19 @@ func (m *aclManager) cleanChains() error {
 		}
 	}
 
+	ok, err = m.iptablesClient.ChainExists("mangle", "PREROUTING")
+	if err != nil {
+		return fmt.Errorf("list chains: %w", err)
+	}
+	if ok {
+		for _, rule := range m.entries["PREROUTING"] {
+			err := m.iptablesClient.DeleteIfExists("mangle", "PREROUTING", rule...)
+			if err != nil {
+				log.Errorf("failed to delete rule: %v, %s", rule, err)
+			}
+		}
+	}
+
 	for _, ipsetName := range m.ipsetStore.ipsetNames() {
 		if err := ipset.Flush(ipsetName); err != nil {
 			log.Errorf("flush ipset %q during reset: %v", ipsetName, err)
@@ -267,6 +310,17 @@ func (m *aclManager) createDefaultChains() error {
 		}
 	}
 
+	for chainName, entries := range m.optionalEntries {
+		for _, entry := range entries {
+			if err := m.iptablesClient.InsertUnique(tableName, chainName, entry.position, entry.spec...); err != nil {
+				log.Errorf("failed to insert optional entry %v: %v", entry.spec, err)
+				continue
+			}
+			m.entries[chainName] = append(m.entries[chainName], entry.spec)
+		}
+	}
+	clear(m.optionalEntries)
+
 	return nil
 }
 
@@ -295,10 +349,52 @@ func (m *aclManager) seedInitialEntries() {
 	m.appendToEntries("FORWARD", append([]string{"-o", m.wgIface.Name()}, established...))
 }
 
+func (m *aclManager) seedInitialOptionalEntries() {
+	m.optionalEntries["FORWARD"] = []entry{
+		{
+			spec:     []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", chainNameInputRules},
+			position: 2,
+		},
+	}
+
+	m.optionalEntries["PREROUTING"] = []entry{
+		{
+			spec:     []string{"-t", "mangle", "-i", m.wgIface.Name(), "-m", "addrtype", "--dst-type", "LOCAL", "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected)},
+			position: 1,
+		},
+	}
+}
+
 func (m *aclManager) appendToEntries(chainName string, spec []string) {
 	m.entries[chainName] = append(m.entries[chainName], spec)
 }
 
+func (m *aclManager) updateState() {
+	if m.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := m.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.ACLEntries = m.entries
+	currentState.ACLIPsetStore = m.ipsetStore
+
+	if err := m.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
 // filterRuleSpecs returns the specs of a filtering rule
 func filterRuleSpecs(
 	ip net.IP, protocol string, sPort, dPort string, direction firewall.RuleDirection, action firewall.Action, ipsetName string,

client/firewall/iptables/manager_linux.go

@@ -8,10 +8,13 @@ import (
 	"sync"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 // Manager of iptables firewall
@@ -33,10 +36,10 @@ type iFaceMapper interface {
 }
 
 // Create iptables firewall manager
-func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	if err != nil {
-		return nil, fmt.Errorf("iptables is not installed in the system or not supported")
+		return nil, fmt.Errorf("init iptables: %w", err)
 	}
 
 	m := &Manager{
@@ -44,20 +47,51 @@ func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
 		ipv4Client: iptablesClient,
 	}
 
-	m.router, err = newRouter(context, iptablesClient, wgIface)
+	m.router, err = newRouter(iptablesClient, wgIface)
 	if err != nil {
-		log.Debugf("failed to initialize route related chains: %s", err)
-		return nil, err
+		return nil, fmt.Errorf("create router: %w", err)
 	}
+
 	m.aclMgr, err = newAclManager(iptablesClient, wgIface, chainRTFWD)
 	if err != nil {
-		log.Debugf("failed to initialize ACL manager: %s", err)
-		return nil, err
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	state := &ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}
+	stateManager.RegisterState(state)
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	if err := m.router.init(stateManager); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclMgr.init(stateManager); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	// persist early to ensure cleanup of chains
+	go func() {
+		if err := stateManager.PersistState(context.Background()); err != nil {
+			log.Errorf("failed to persist state: %v", err)
+		}
+	}()
+
+	return nil
+}
+
 // AddPeerFiltering adds a rule to the firewall
 //
 // Comment will be ignored because some system this feature is not supported
@@ -78,7 +112,7 @@ func (m *Manager) AddPeerFiltering(
 }
 
 func (m *Manager) AddRouteFiltering(
-	sources [] netip.Prefix,
+	sources []netip.Prefix,
 	destination netip.Prefix,
 	proto firewall.Protocol,
 	sPort *firewall.Port,
@@ -133,20 +167,27 @@ func (m *Manager) SetLegacyManagement(isLegacy bool) error {
 }
 
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	errAcl := m.aclMgr.Reset()
-	if errAcl != nil {
-		log.Errorf("failed to clean up ACL rules from firewall: %s", errAcl)
+	var merr *multierror.Error
+
+	if err := m.aclMgr.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset acl manager: %w", err))
 	}
-	errMgr := m.router.Reset()
-	if errMgr != nil {
-		log.Errorf("failed to clean up router rules from firewall: %s", errMgr)
-		return errMgr
+	if err := m.router.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset router: %w", err))
 	}
-	return errAcl
+
+	// attempt to delete state only if all other operations succeeded
+	if merr == nil {
+		if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("delete state: %w", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 // AllowNetbird allows netbird interface traffic

client/firewall/iptables/manager_linux_test.go

@@ -1,7 +1,6 @@
 package iptables
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"testing"
@@ -56,13 +55,14 @@ func TestIptablesManager(t *testing.T) {
 	require.NoError(t, err)
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), ifaceMock)
+	manager, err := Create(ifaceMock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 
 	time.Sleep(time.Second)
 
 	defer func() {
-		err := manager.Reset()
+		err := manager.Reset(nil)
 		require.NoError(t, err, "clear the manager state")
 
 		time.Sleep(time.Second)
@@ -122,7 +122,7 @@ func TestIptablesManager(t *testing.T) {
 		_, err = manager.AddPeerFiltering(ip, "udp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept Fake DNS traffic")
 		require.NoError(t, err, "failed to add rule")
 
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 
 		ok, err := ipv4Client.ChainExists("filter", chainNameInputRules)
@@ -154,13 +154,14 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	}
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), mock)
+	manager, err := Create(mock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 
 	time.Sleep(time.Second)
 
 	defer func() {
-		err := manager.Reset()
+		err := manager.Reset(nil)
 		require.NoError(t, err, "clear the manager state")
 
 		time.Sleep(time.Second)
@@ -219,7 +220,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	})
 
 	t.Run("reset check", func(t *testing.T) {
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 	})
 }
@@ -251,12 +252,13 @@ func TestIptablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(context.Background(), mock)
+			manager, err := Create(mock)
 			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second)
 
 			defer func() {
-				err := manager.Reset()
+				err := manager.Reset(nil)
 				require.NoError(t, err, "clear the manager state")
 
 				time.Sleep(time.Second)

client/firewall/iptables/router_linux.go

@@ -3,7 +3,6 @@
 package iptables
 
 import (
-	"context"
 	"fmt"
 	"net/netip"
 	"strconv"
@@ -18,22 +17,25 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl/id"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
-)
-
-const (
-	ipv4Nat = "netbird-rt-nat"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 // constants needed to manage and create iptable rules
 const (
 	tableFilter             = "filter"
 	tableNat                = "nat"
+	tableMangle             = "mangle"
 	chainPOSTROUTING        = "POSTROUTING"
+	chainPREROUTING         = "PREROUTING"
 	chainRTNAT              = "NETBIRD-RT-NAT"
 	chainRTFWD              = "NETBIRD-RT-FWD"
+	chainRTPRE              = "NETBIRD-RT-PRE"
 	routingFinalForwardJump = "ACCEPT"
 	routingFinalNatJump     = "MASQUERADE"
 
+	jumpPre  = "jump-pre"
+	jumpNat  = "jump-nat"
 	matchSet = "--match-set"
 )
 
@@ -48,28 +50,31 @@ type routeFilteringRuleParams struct {
 	SetName     string
 }
 
+type routeRules map[string][]string
+
+type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}]
+
 type router struct {
-	ctx              context.Context
-	stop             context.CancelFunc
 	iptablesClient   *iptables.IPTables
-	rules            map[string][]string
-	ipsetCounter     *refcounter.Counter[string, []netip.Prefix, struct{}]
+	rules            routeRules
+	ipsetCounter     *ipsetCounter
 	wgIface          iFaceMapper
 	legacyManagement bool
+
+	stateManager *statemanager.Manager
 }
 
-func newRouter(parentCtx context.Context, iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
-	ctx, cancel := context.WithCancel(parentCtx)
+func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
 	r := &router{
-		ctx:            ctx,
-		stop:           cancel,
 		iptablesClient: iptablesClient,
 		rules:          make(map[string][]string),
 		wgIface:        wgIface,
 	}
 
 	r.ipsetCounter = refcounter.New(
-		r.createIpSet,
+		func(name string, sources []netip.Prefix) (struct{}, error) {
+			return struct{}{}, r.createIpSet(name, sources)
+		},
 		func(name string, _ struct{}) error {
 			return r.deleteIpSet(name)
 		},
@@ -79,16 +84,23 @@ func newRouter(parentCtx context.Context, iptablesClient *iptables.IPTables, wgI
 		return nil, fmt.Errorf("init ipset: %w", err)
 	}
 
-	err := r.cleanUpDefaultForwardRules()
-	if err != nil {
-		log.Errorf("cleanup routing rules: %s", err)
-		return nil, err
+	return r, nil
+}
+
+func (r *router) init(stateManager *statemanager.Manager) error {
+	r.stateManager = stateManager
+
+	if err := r.cleanUpDefaultForwardRules(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
-	err = r.createContainers()
-	if err != nil {
-		log.Errorf("create containers for route: %s", err)
+
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
 	}
-	return r, err
+
+	r.updateState()
+
+	return nil
 }
 
 func (r *router) AddRouteFiltering(
@@ -129,6 +141,8 @@ func (r *router) AddRouteFiltering(
 
 	r.rules[string(ruleKey)] = rule
 
+	r.updateState()
+
 	return ruleKey, nil
 }
 
@@ -152,6 +166,8 @@ func (r *router) DeleteRouteRule(rule firewall.Rule) error {
 		log.Debugf("route rule %s not found", ruleKey)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
@@ -164,18 +180,18 @@ func (r *router) findSetNameInRule(rule []string) string {
 	return ""
 }
 
-func (r *router) createIpSet(setName string, sources []netip.Prefix) (struct{}, error) {
+func (r *router) createIpSet(setName string, sources []netip.Prefix) error {
 	if err := ipset.Create(setName, ipset.OptTimeout(0)); err != nil {
-		return struct{}{}, fmt.Errorf("create set %s: %w", setName, err)
+		return fmt.Errorf("create set %s: %w", setName, err)
 	}
 
 	for _, prefix := range sources {
 		if err := ipset.AddPrefix(setName, prefix); err != nil {
-			return struct{}{}, fmt.Errorf("add element to set %s: %w", setName, err)
+			return fmt.Errorf("add element to set %s: %w", setName, err)
 		}
 	}
 
-	return struct{}{}, nil
+	return nil
 }
 
 func (r *router) deleteIpSet(setName string) error {
@@ -206,6 +222,8 @@ func (r *router) AddNatRule(pair firewall.RouterPair) error {
 		return fmt.Errorf("add inverse nat rule: %w", err)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
@@ -223,6 +241,8 @@ func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
 		return fmt.Errorf("remove legacy routing rule: %w", err)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
@@ -278,8 +298,13 @@ func (r *router) RemoveAllLegacyRouteRules() error {
 		}
 		if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWD, rule...); err != nil {
 			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
 		}
 	}
+
+	r.updateState()
+
 	return nberrors.FormatErrorOrNil(merr)
 }
 
@@ -294,31 +319,31 @@ func (r *router) Reset() error {
 		merr = multierror.Append(merr, err)
 	}
 
+	r.updateState()
+
 	return nberrors.FormatErrorOrNil(merr)
 }
 
 func (r *router) cleanUpDefaultForwardRules() error {
-	err := r.cleanJumpRules()
-	if err != nil {
-		return err
+	if err := r.cleanJumpRules(); err != nil {
+		return fmt.Errorf("clean jump rules: %w", err)
 	}
 
 	log.Debug("flushing routing related tables")
-	for _, chain := range []string{chainRTFWD, chainRTNAT} {
-		table := tableFilter
-		if chain == chainRTNAT {
-			table = tableNat
-		}
-
-		ok, err := r.iptablesClient.ChainExists(table, chain)
+	for _, chainInfo := range []struct {
+		chain string
+		table string
+	}{
+		{chainRTFWD, tableFilter},
+		{chainRTNAT, tableNat},
+		{chainRTPRE, tableMangle},
+	} {
+		ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain)
 		if err != nil {
-			log.Errorf("failed check chain %s, error: %v", chain, err)
-			return err
+			return fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
 		} else if ok {
-			err = r.iptablesClient.ClearAndDeleteChain(table, chain)
-			if err != nil {
-				log.Errorf("failed cleaning chain %s, error: %v", chain, err)
-				return err
+			if err = r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil {
+				return fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
 			}
 		}
 	}
@@ -327,17 +352,58 @@ func (r *router) cleanUpDefaultForwardRules() error {
 }
 
 func (r *router) createContainers() error {
-	for _, chain := range []string{chainRTFWD, chainRTNAT} {
-		if err := r.createAndSetupChain(chain); err != nil {
-			return fmt.Errorf("create chain %s: %v", chain, err)
+	for _, chainInfo := range []struct {
+		chain string
+		table string
+	}{
+		{chainRTFWD, tableFilter},
+		{chainRTPRE, tableMangle},
+		{chainRTNAT, tableNat},
+	} {
+		if err := r.createAndSetupChain(chainInfo.chain); err != nil {
+			return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
 		}
 	}
 
 	if err := r.insertEstablishedRule(chainRTFWD); err != nil {
-		return fmt.Errorf("insert established rule: %v", err)
+		return fmt.Errorf("insert established rule: %w", err)
 	}
 
-	return r.addJumpRules()
+	if err := r.addPostroutingRules(); err != nil {
+		return fmt.Errorf("add static nat rules: %w", err)
+	}
+
+	if err := r.addJumpRules(); err != nil {
+		return fmt.Errorf("add jump rules: %w", err)
+	}
+
+	return nil
+}
+
+func (r *router) addPostroutingRules() error {
+	// First rule for outbound masquerade
+	rule1 := []string{
+		"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+		"!", "-o", "lo",
+		"-j", routingFinalNatJump,
+	}
+	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil {
+		return fmt.Errorf("add outbound masquerade rule: %v", err)
+	}
+	r.rules["static-nat-outbound"] = rule1
+
+	// Second rule for return traffic masquerade
+	rule2 := []string{
+		"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+		"-o", r.wgIface.Name(),
+		"-j", routingFinalNatJump,
+	}
+	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil {
+		return fmt.Errorf("add return masquerade rule: %v", err)
+	}
+	r.rules["static-nat-return"] = rule2
+
+	return nil
 }
 
 func (r *router) createAndSetupChain(chain string) error {
@@ -351,10 +417,14 @@ func (r *router) createAndSetupChain(chain string) error {
 }
 
 func (r *router) getTableForChain(chain string) string {
-	if chain == chainRTNAT {
+	switch chain {
+	case chainRTNAT:
 		return tableNat
+	case chainRTPRE:
+		return tableMangle
+	default:
+		return tableFilter
 	}
-	return tableFilter
 }
 
 func (r *router) insertEstablishedRule(chain string) error {
@@ -372,25 +442,39 @@ func (r *router) insertEstablishedRule(chain string) error {
 }
 
 func (r *router) addJumpRules() error {
-	rule := []string{"-j", chainRTNAT}
-	err := r.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, rule...)
-	if err != nil {
-		return err
+	// Jump to NAT chain
+	natRule := []string{"-j", chainRTNAT}
+	if err := r.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, natRule...); err != nil {
+		return fmt.Errorf("add nat jump rule: %v", err)
 	}
-	r.rules[ipv4Nat] = rule
+	r.rules[jumpNat] = natRule
+
+	// Jump to prerouting chain
+	preRule := []string{"-j", chainRTPRE}
+	if err := r.iptablesClient.Insert(tableMangle, chainPREROUTING, 1, preRule...); err != nil {
+		return fmt.Errorf("add prerouting jump rule: %v", err)
+	}
+	r.rules[jumpPre] = preRule
 
 	return nil
 }
 
 func (r *router) cleanJumpRules() error {
-	rule, found := r.rules[ipv4Nat]
-	if found {
-		err := r.iptablesClient.DeleteIfExists(tableNat, chainPOSTROUTING, rule...)
-		if err != nil {
-			return fmt.Errorf("failed cleaning rule from chain %s, err: %v", chainPOSTROUTING, err)
+	for _, ruleKey := range []string{jumpNat, jumpPre} {
+		if rule, exists := r.rules[ruleKey]; exists {
+			table := tableNat
+			chain := chainPOSTROUTING
+			if ruleKey == jumpPre {
+				table = tableMangle
+				chain = chainPREROUTING
+			}
+
+			if err := r.iptablesClient.DeleteIfExists(table, chain, rule...); err != nil {
+				return fmt.Errorf("delete rule from chain %s in table %s, err: %v", chain, table, err)
+			}
+			delete(r.rules, ruleKey)
 		}
 	}
-
 	return nil
 }
 
@@ -398,19 +482,35 @@ func (r *router) addNatRule(pair firewall.RouterPair) error {
 	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
 
 	if rule, exists := r.rules[ruleKey]; exists {
-		if err := r.iptablesClient.DeleteIfExists(tableNat, chainRTNAT, rule...); err != nil {
-			return fmt.Errorf("error while removing existing NAT rule for %s: %v", pair.Destination, err)
+		if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil {
+			return fmt.Errorf("error while removing existing marking rule for %s: %v", pair.Destination, err)
 		}
 		delete(r.rules, ruleKey)
 	}
 
-	rule := genRuleSpec(routingFinalNatJump, pair.Source, pair.Destination, r.wgIface.Name(), pair.Inverse)
-	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule...); err != nil {
-		return fmt.Errorf("error while appending new NAT rule for %s: %v", pair.Destination, err)
+	markValue := nbnet.PreroutingFwmarkMasquerade
+	if pair.Inverse {
+		markValue = nbnet.PreroutingFwmarkMasqueradeReturn
+	}
+
+	rule := []string{"-i", r.wgIface.Name()}
+	if pair.Inverse {
+		rule = []string{"!", "-i", r.wgIface.Name()}
+	}
+
+	rule = append(rule,
+		"-m", "conntrack",
+		"--ctstate", "NEW",
+		"-s", pair.Source.String(),
+		"-d", pair.Destination.String(),
+		"-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue),
+	)
+
+	if err := r.iptablesClient.Append(tableMangle, chainRTPRE, rule...); err != nil {
+		return fmt.Errorf("error while adding marking rule for %s: %v", pair.Destination, err)
 	}
 
 	r.rules[ruleKey] = rule
-
 	return nil
 }
 
@@ -418,24 +518,41 @@ func (r *router) removeNatRule(pair firewall.RouterPair) error {
 	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
 
 	if rule, exists := r.rules[ruleKey]; exists {
-		if err := r.iptablesClient.DeleteIfExists(tableNat, chainRTNAT, rule...); err != nil {
-			return fmt.Errorf("error while removing existing nat rule for %s: %v", pair.Destination, err)
+		if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil {
+			return fmt.Errorf("error while removing marking rule for %s: %v", pair.Destination, err)
 		}
-
 		delete(r.rules, ruleKey)
 	} else {
-		log.Debugf("nat rule %s not found", ruleKey)
+		log.Debugf("marking rule %s not found", ruleKey)
 	}
 
 	return nil
 }
 
-func genRuleSpec(jump string, source, destination netip.Prefix, intf string, inverse bool) []string {
-	intdir := "-i"
-	if inverse {
-		intdir = "-o"
+func (r *router) updateState() {
+	if r.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := r.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.RouteRules = r.rules
+	currentState.RouteIPsetCounter = r.ipsetCounter
+
+	if err := r.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
 	}
-	return []string{intdir, intf, "-s", source.String(), "-d", destination.String(), "-j", jump}
 }
 
 func genRouteFilteringRuleSpec(params routeFilteringRuleParams) []string {

client/firewall/iptables/router_linux_test.go

@@ -3,18 +3,18 @@
 package iptables
 
 import (
-	"context"
+	"fmt"
 	"net/netip"
 	"os/exec"
 	"testing"
 
 	"github.com/coreos/go-iptables/iptables"
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/firewall/test"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 func isIptablesSupported() bool {
@@ -30,18 +30,29 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "failed to init iptables client")
 
-	manager, err := newRouter(context.TODO(), iptablesClient, ifaceMock)
+	manager, err := newRouter(iptablesClient, ifaceMock)
 	require.NoError(t, err, "should return a valid iptables manager")
+	require.NoError(t, manager.init(nil))
 
 	defer func() {
-		_ = manager.Reset()
+		assert.NoError(t, manager.Reset(), "shouldn't return error")
 	}()
 
-	require.Len(t, manager.rules, 2, "should have created rules map")
+	// Now 5 rules:
+	// 1. established rule in forward chain
+	// 2. jump rule to NAT chain
+	// 3. jump rule to PRE chain
+	// 4. static outbound masquerade rule
+	// 5. static return masquerade rule
+	require.Len(t, manager.rules, 5, "should have created rules map")
 
-	exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, manager.rules[ipv4Nat]...)
+	exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, "-j", chainRTNAT)
 	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPOSTROUTING)
-	require.True(t, exists, "postrouting rule should exist")
+	require.True(t, exists, "postrouting jump rule should exist")
+
+	exists, err = manager.iptablesClient.Exists(tableMangle, chainPREROUTING, "-j", chainRTPRE)
+	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPREROUTING)
+	require.True(t, exists, "prerouting jump rule should exist")
 
 	pair := firewall.RouterPair{
 		ID:          "abc",
@@ -49,22 +60,15 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
 		Destination: netip.MustParsePrefix("100.100.100.0/24"),
 		Masquerade:  true,
 	}
-	forward4Rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", routingFinalForwardJump}
 
-	err = manager.iptablesClient.Insert(tableFilter, chainRTFWD, 1, forward4Rule...)
-	require.NoError(t, err, "inserting rule should not return error")
-
-	nat4Rule := genRuleSpec(routingFinalNatJump, pair.Source, pair.Destination, ifaceMock.Name(), false)
-
-	err = manager.iptablesClient.Insert(tableNat, chainRTNAT, 1, nat4Rule...)
-	require.NoError(t, err, "inserting rule should not return error")
+	err = manager.AddNatRule(pair)
+	require.NoError(t, err, "adding NAT rule should not return error")
 
 	err = manager.Reset()
 	require.NoError(t, err, "shouldn't return error")
 }
 
 func TestIptablesManager_AddNatRule(t *testing.T) {
-
 	if !isIptablesSupported() {
 		t.SkipNow()
 	}
@@ -74,56 +78,71 @@ func TestIptablesManager_AddNatRule(t *testing.T) {
 			iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 			require.NoError(t, err, "failed to init iptables client")
 
-			manager, err := newRouter(context.TODO(), iptablesClient, ifaceMock)
+			manager, err := newRouter(iptablesClient, ifaceMock)
 			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
 
 			defer func() {
-				err := manager.Reset()
-				if err != nil {
-					log.Errorf("failed to reset iptables manager: %s", err)
-				}
+				assert.NoError(t, manager.Reset(), "shouldn't return error")
 			}()
 
 			err = manager.AddNatRule(testCase.InputPair)
-			require.NoError(t, err, "forwarding pair should be inserted")
+			require.NoError(t, err, "marking rule should be inserted")
 
 			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
-			natRule := genRuleSpec(routingFinalNatJump, testCase.InputPair.Source, testCase.InputPair.Destination, ifaceMock.Name(), false)
-
-			exists, err := iptablesClient.Exists(tableNat, chainRTNAT, natRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
-			if testCase.InputPair.Masquerade {
-				require.True(t, exists, "nat rule should be created")
-				foundNatRule, foundNat := manager.rules[natRuleKey]
-				require.True(t, foundNat, "nat rule should exist in the map")
-				require.Equal(t, natRule[:4], foundNatRule[:4], "stored nat rule should match")
-			} else {
-				require.False(t, exists, "nat rule should not be created")
-				_, foundNat := manager.rules[natRuleKey]
-				require.False(t, foundNat, "nat rule should not exist in the map")
+			markingRule := []string{
+				"-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", testCase.InputPair.Source.String(),
+				"-d", testCase.InputPair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
 			}
 
-			inNatRuleKey := firewall.GenKey(firewall.NatFormat, firewall.GetInversePair(testCase.InputPair))
-			inNatRule := genRuleSpec(routingFinalNatJump, firewall.GetInversePair(testCase.InputPair).Source, firewall.GetInversePair(testCase.InputPair).Destination, ifaceMock.Name(), true)
-
-			exists, err = iptablesClient.Exists(tableNat, chainRTNAT, inNatRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
+			exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
 			if testCase.InputPair.Masquerade {
-				require.True(t, exists, "income nat rule should be created")
-				foundNatRule, foundNat := manager.rules[inNatRuleKey]
-				require.True(t, foundNat, "income nat rule should exist in the map")
-				require.Equal(t, inNatRule[:4], foundNatRule[:4], "stored income nat rule should match")
+				require.True(t, exists, "marking rule should be created")
+				foundRule, found := manager.rules[natRuleKey]
+				require.True(t, found, "marking rule should exist in the map")
+				require.Equal(t, markingRule, foundRule, "stored marking rule should match")
 			} else {
-				require.False(t, exists, "nat rule should not be created")
-				_, foundNat := manager.rules[inNatRuleKey]
-				require.False(t, foundNat, "income nat rule should not exist in the map")
+				require.False(t, exists, "marking rule should not be created")
+				_, found := manager.rules[natRuleKey]
+				require.False(t, found, "marking rule should not exist in the map")
+			}
+
+			// Check inverse rule
+			inversePair := firewall.GetInversePair(testCase.InputPair)
+			inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair)
+			inverseMarkingRule := []string{
+				"!", "-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", inversePair.Source.String(),
+				"-d", inversePair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+			}
+
+			exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			if testCase.InputPair.Masquerade {
+				require.True(t, exists, "inverse marking rule should be created")
+				foundRule, found := manager.rules[inverseRuleKey]
+				require.True(t, found, "inverse marking rule should exist in the map")
+				require.Equal(t, inverseMarkingRule, foundRule, "stored inverse marking rule should match")
+			} else {
+				require.False(t, exists, "inverse marking rule should not be created")
+				_, found := manager.rules[inverseRuleKey]
+				require.False(t, found, "inverse marking rule should not exist in the map")
 			}
 		})
 	}
 }
 
 func TestIptablesManager_RemoveNatRule(t *testing.T) {
-
 	if !isIptablesSupported() {
 		t.SkipNow()
 	}
@@ -132,45 +151,56 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) {
 		t.Run(testCase.Name, func(t *testing.T) {
 			iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 
-			manager, err := newRouter(context.TODO(), iptablesClient, ifaceMock)
+			manager, err := newRouter(iptablesClient, ifaceMock)
 			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
 			defer func() {
-				_ = manager.Reset()
+				assert.NoError(t, manager.Reset(), "shouldn't return error")
 			}()
 
-			require.NoError(t, err, "shouldn't return error")
-
-			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
-			natRule := genRuleSpec(routingFinalNatJump, testCase.InputPair.Source, testCase.InputPair.Destination, ifaceMock.Name(), false)
-
-			err = iptablesClient.Insert(tableNat, chainRTNAT, 1, natRule...)
-			require.NoError(t, err, "inserting rule should not return error")
-
-			inNatRuleKey := firewall.GenKey(firewall.NatFormat, firewall.GetInversePair(testCase.InputPair))
-			inNatRule := genRuleSpec(routingFinalNatJump, firewall.GetInversePair(testCase.InputPair).Source, firewall.GetInversePair(testCase.InputPair).Destination, ifaceMock.Name(), true)
-
-			err = iptablesClient.Insert(tableNat, chainRTNAT, 1, inNatRule...)
-			require.NoError(t, err, "inserting rule should not return error")
-
-			err = manager.Reset()
-			require.NoError(t, err, "shouldn't return error")
+			err = manager.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "should add NAT rule without error")
 
 			err = manager.RemoveNatRule(testCase.InputPair)
 			require.NoError(t, err, "shouldn't return error")
 
-			exists, err := iptablesClient.Exists(tableNat, chainRTNAT, natRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
-			require.False(t, exists, "nat rule should not exist")
+			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
+			markingRule := []string{
+				"-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", testCase.InputPair.Source.String(),
+				"-d", testCase.InputPair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+			}
+
+			exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			require.False(t, exists, "marking rule should not exist")
 
 			_, found := manager.rules[natRuleKey]
-			require.False(t, found, "nat rule should exist in the manager map")
+			require.False(t, found, "marking rule should not exist in the manager map")
 
-			exists, err = iptablesClient.Exists(tableNat, chainRTNAT, inNatRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
-			require.False(t, exists, "income nat rule should not exist")
+			// Check inverse rule removal
+			inversePair := firewall.GetInversePair(testCase.InputPair)
+			inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair)
+			inverseMarkingRule := []string{
+				"!", "-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", inversePair.Source.String(),
+				"-d", inversePair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+			}
 
-			_, found = manager.rules[inNatRuleKey]
-			require.False(t, found, "income nat rule should exist in the manager map")
+			exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			require.False(t, exists, "inverse marking rule should not exist")
+
+			_, found = manager.rules[inverseRuleKey]
+			require.False(t, found, "inverse marking rule should not exist in the map")
 		})
 	}
 }
@@ -183,8 +213,9 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "Failed to create iptables client")
 
-	r, err := newRouter(context.Background(), iptablesClient, ifaceMock)
+	r, err := newRouter(iptablesClient, ifaceMock)
 	require.NoError(t, err, "Failed to create router manager")
+	require.NoError(t, r.init(nil))
 
 	defer func() {
 		err := r.Reset()

client/firewall/iptables/rulestore_linux.go

@@ -1,14 +1,16 @@
 package iptables
 
+import "encoding/json"
+
 type ipList struct {
 	ips map[string]struct{}
 }
 
-func newIpList(ip string) ipList {
+func newIpList(ip string) *ipList {
 	ips := make(map[string]struct{})
 	ips[ip] = struct{}{}
 
-	return ipList{
+	return &ipList{
 		ips: ips,
 	}
 }
@@ -17,27 +19,52 @@ func (s *ipList) addIP(ip string) {
 	s.ips[ip] = struct{}{}
 }
 
+// MarshalJSON implements json.Marshaler
+func (s *ipList) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{
+		IPs: s.ips,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipList) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ips = temp.IPs
+
+	if temp.IPs == nil {
+		temp.IPs = make(map[string]struct{})
+	}
+
+	return nil
+}
+
 type ipsetStore struct {
-	ipsets map[string]ipList // ipsetName -> ruleset
+	ipsets map[string]*ipList
 }
 
 func newIpsetStore() *ipsetStore {
 	return &ipsetStore{
-		ipsets: make(map[string]ipList),
+		ipsets: make(map[string]*ipList),
 	}
 }
 
-func (s *ipsetStore) ipset(ipsetName string) (ipList, bool) {
+func (s *ipsetStore) ipset(ipsetName string) (*ipList, bool) {
 	r, ok := s.ipsets[ipsetName]
 	return r, ok
 }
 
-func (s *ipsetStore) addIpList(ipsetName string, list ipList) {
+func (s *ipsetStore) addIpList(ipsetName string, list *ipList) {
 	s.ipsets[ipsetName] = list
 }
 
 func (s *ipsetStore) deleteIpset(ipsetName string) {
-	s.ipsets[ipsetName] = ipList{}
 	delete(s.ipsets, ipsetName)
 }
 
@@ -48,3 +75,29 @@ func (s *ipsetStore) ipsetNames() []string {
 	}
 	return names
 }
+
+// MarshalJSON implements json.Marshaler
+func (s *ipsetStore) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{
+		IPSets: s.ipsets,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipsetStore) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ipsets = temp.IPSets
+
+	if temp.IPSets == nil {
+		temp.IPSets = make(map[string]*ipList)
+	}
+
+	return nil
+}

client/firewall/iptables/state_linux.go

@@ -0,0 +1,70 @@
+package iptables
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	sync.Mutex
+
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+
+	RouteRules        routeRules    `json:"route_rules,omitempty"`
+	RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"`
+
+	ACLEntries    aclEntries  `json:"acl_entries,omitempty"`
+	ACLIPsetStore *ipsetStore `json:"acl_ipset_store,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "iptables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	ipt, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create iptables manager: %w", err)
+	}
+
+	if s.RouteRules != nil {
+		ipt.router.rules = s.RouteRules
+	}
+	if s.RouteIPsetCounter != nil {
+		ipt.router.ipsetCounter.LoadData(s.RouteIPsetCounter)
+	}
+
+	if s.ACLEntries != nil {
+		ipt.aclMgr.entries = s.ACLEntries
+	}
+	if s.ACLIPsetStore != nil {
+		ipt.aclMgr.ipsetStore = s.ACLIPsetStore
+	}
+
+	if err := ipt.Reset(nil); err != nil {
+		return fmt.Errorf("reset iptables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/manager/firewall.go

@@ -10,11 +10,14 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
 	ForwardingFormatPrefix = "netbird-fwd-"
 	ForwardingFormat       = "netbird-fwd-%s-%t"
+	PreroutingFormat       = "netbird-prerouting-%s-%t"
 	NatFormat              = "netbird-nat-%s-%t"
 )
 
@@ -52,6 +55,8 @@ const (
 // It declares methods which handle actions required by the
 // Netbird client for ACL and routing functionality
 type Manager interface {
+	Init(stateManager *statemanager.Manager) error
+
 	// AllowNetbird allows netbird interface traffic
 	AllowNetbird() error
 
@@ -91,7 +96,7 @@ type Manager interface {
 	SetLegacyManagement(legacy bool) error
 
 	// Reset firewall to the default state
-	Reset() error
+	Reset(stateManager *statemanager.Manager) error
 
 	// Flush the changes to firewall controller
 	Flush() error
@@ -132,7 +137,7 @@ func SetLegacyManagement(router LegacyManager, isLegacy bool) error {
 // GenerateSetName generates a unique name for an ipset based on the given sources.
 func GenerateSetName(sources []netip.Prefix) string {
 	// sort for consistent naming
-	sortPrefixes(sources)
+	SortPrefixes(sources)
 
 	var sourcesStr strings.Builder
 	for _, src := range sources {
@@ -170,9 +175,9 @@ func MergeIPRanges(prefixes []netip.Prefix) []netip.Prefix {
 	return merged
 }
 
-// sortPrefixes sorts the given slice of netip.Prefix in place.
+// SortPrefixes sorts the given slice of netip.Prefix in place.
 // It sorts first by IP address, then by prefix length (most specific to least specific).
-func sortPrefixes(prefixes []netip.Prefix) {
+func SortPrefixes(prefixes []netip.Prefix) {
 	sort.Slice(prefixes, func(i, j int) bool {
 		addrCmp := prefixes[i].Addr().Compare(prefixes[j].Addr())
 		if addrCmp != 0 {

client/firewall/nftables/acl_linux.go

@@ -11,12 +11,13 @@ import (
 	"time"
 
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/client/iface"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
@@ -29,6 +30,7 @@ const (
 	chainNameInputFilter   = "netbird-acl-input-filter"
 	chainNameOutputFilter  = "netbird-acl-output-filter"
 	chainNameForwardFilter = "netbird-acl-forward-filter"
+	chainNamePrerouting    = "netbird-rt-prerouting"
 
 	allowNetbirdInputRuleID = "allow Netbird incoming traffic"
 )
@@ -40,54 +42,44 @@ var (
 )
 
 type AclManager struct {
-	rConn               *nftables.Conn
-	sConn               *nftables.Conn
-	wgIface             iFaceMapper
-	routeingFwChainName string
+	rConn              *nftables.Conn
+	sConn              *nftables.Conn
+	wgIface            iFaceMapper
+	routingFwChainName string
 
 	workTable        *nftables.Table
 	chainInputRules  *nftables.Chain
 	chainOutputRules *nftables.Chain
-	chainFwFilter    *nftables.Chain
 
 	ipsetStore *ipsetStore
 	rules      map[string]*Rule
 }
 
-// iFaceMapper defines subset methods of interface required for manager
-type iFaceMapper interface {
-	Name() string
-	Address() iface.WGAddress
-	IsUserspaceBind() bool
-}
-
-func newAclManager(table *nftables.Table, wgIface iFaceMapper, routeingFwChainName string) (*AclManager, error) {
+func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) {
 	// sConn is used for creating sets and adding/removing elements from them
 	// it's differ then rConn (which does create new conn for each flush operation)
 	// and is permanent. Using same connection for both type of operations
 	// overloads netlink with high amount of rules ( > 10000)
 	sConn, err := nftables.New(nftables.AsLasting())
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create nf conn: %w", err)
 	}
 
-	m := &AclManager{
-		rConn:               &nftables.Conn{},
-		sConn:               sConn,
-		wgIface:             wgIface,
-		workTable:           table,
-		routeingFwChainName: routeingFwChainName,
+	return &AclManager{
+		rConn:              &nftables.Conn{},
+		sConn:              sConn,
+		wgIface:            wgIface,
+		workTable:          table,
+		routingFwChainName: routingFwChainName,
 
 		ipsetStore: newIpsetStore(),
 		rules:      make(map[string]*Rule),
-	}
+	}, nil
+}
 
-	err = m.createDefaultChains()
-	if err != nil {
-		return nil, err
-	}
-
-	return m, nil
+func (m *AclManager) init(workTable *nftables.Table) error {
+	m.workTable = workTable
+	return m.createDefaultChains()
 }
 
 // AddPeerFiltering rule to the firewall
@@ -462,9 +454,9 @@ func (m *AclManager) createDefaultChains() (err error) {
 	}
 
 	// netbird-acl-forward-filter
-	m.chainFwFilter = m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
-	m.addJumpRulesToRtForward() // to netbird-rt-fwd
-	m.addDropExpressions(m.chainFwFilter, expr.MetaKeyIIFNAME)
+	chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
+	m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd
+	m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME)
 
 	err = m.rConn.Flush()
 	if err != nil {
@@ -472,10 +464,96 @@ func (m *AclManager) createDefaultChains() (err error) {
 		return fmt.Errorf(flushError, err)
 	}
 
+	if err := m.allowRedirectedTraffic(chainFwFilter); err != nil {
+		log.Errorf("failed to allow redirected traffic: %s", err)
+	}
+
 	return nil
 }
 
-func (m *AclManager) addJumpRulesToRtForward() {
+// Makes redirected traffic originally destined for the host itself (now subject to the forward filter)
+// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the
+// netbird peer IP.
+func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error {
+	preroutingChain := m.rConn.AddChain(&nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    m.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	})
+
+	m.addPreroutingRule(preroutingChain)
+
+	m.addFwmarkToForward(chainFwFilter)
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (m *AclManager) addPreroutingRule(preroutingChain *nftables.Chain) {
+	m.rConn.AddRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: preroutingChain,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyIIFNAME,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     ifname(m.wgIface.Name()),
+			},
+			&expr.Fib{
+				Register:       1,
+				ResultADDRTYPE: true,
+				FlagDADDR:      true,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL),
+			},
+			&expr.Immediate{
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
+			},
+			&expr.Meta{
+				Key:            expr.MetaKeyMARK,
+				Register:       1,
+				SourceRegister: true,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) {
+	m.rConn.InsertRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: chainFwFilter,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyMARK,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
+			},
+			&expr.Verdict{
+				Kind:  expr.VerdictJump,
+				Chain: m.chainInputRules.Name,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) {
 	expressions := []expr.Any{
 		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
 		&expr.Cmp{
@@ -485,13 +563,13 @@ func (m *AclManager) addJumpRulesToRtForward() {
 		},
 		&expr.Verdict{
 			Kind:  expr.VerdictJump,
-			Chain: m.routeingFwChainName,
+			Chain: m.routingFwChainName,
 		},
 	}
 
 	_ = m.rConn.AddRule(&nftables.Rule{
 		Table: m.workTable,
-		Chain: m.chainFwFilter,
+		Chain: chainFwFilter,
 		Exprs: expressions,
 	})
 }
@@ -509,7 +587,7 @@ func (m *AclManager) createChain(name string) *nftables.Chain {
 	return chain
 }
 
-func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.ChainHook) *nftables.Chain {
+func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain {
 	polAccept := nftables.ChainPolicyAccept
 	chain := &nftables.Chain{
 		Name:     name,

client/firewall/nftables/manager_linux.go

@@ -14,6 +14,8 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -24,6 +26,13 @@ const (
 	chainNameInput  = "INPUT"
 )
 
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMapper interface {
+	Name() string
+	Address() iface.WGAddress
+	IsUserspaceBind() bool
+}
+
 // Manager of iptables firewall
 type Manager struct {
 	mutex   sync.Mutex
@@ -35,30 +44,70 @@ type Manager struct {
 }
 
 // Create nftables firewall manager
-func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
 	m := &Manager{
 		rConn:   &nftables.Conn{},
 		wgIface: wgIface,
 	}
 
-	workTable, err := m.createWorkTable()
-	if err != nil {
-		return nil, err
-	}
+	workTable := &nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4}
 
-	m.router, err = newRouter(context, workTable, wgIface)
+	var err error
+	m.router, err = newRouter(workTable, wgIface)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create router: %w", err)
 	}
 
 	m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
+// Init nftables firewall manager
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	workTable, err := m.createWorkTable()
+	if err != nil {
+		return fmt.Errorf("create work table: %w", err)
+	}
+
+	if err := m.router.init(workTable); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclManager.init(workTable); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	stateManager.RegisterState(&ShutdownState{})
+
+	// We only need to record minimal interface state for potential recreation.
+	// Unlike iptables, which requires tracking individual rules, nftables maintains
+	// a known state (our netbird table plus a few static rules). This allows for easy
+	// cleanup using Reset() without needing to store specific rules.
+	if err := stateManager.UpdateState(&ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	// persist early
+	go func() {
+		if err := stateManager.PersistState(context.Background()); err != nil {
+			log.Errorf("failed to persist state: %v", err)
+		}
+	}()
+
+	return nil
+}
+
 // AddPeerFiltering rule to the firewall
 //
 // If comment argument is empty firewall manager should set
@@ -150,7 +199,7 @@ func (m *Manager) AllowNetbird() error {
 
 	var chain *nftables.Chain
 	for _, c := range chains {
-		if c.Table.Name == tableNameFilter && c.Name == chainNameForward {
+		if c.Table.Name == tableNameFilter && c.Name == chainNameInput {
 			chain = c
 			break
 		}
@@ -183,68 +232,84 @@ func (m *Manager) AllowNetbird() error {
 
 // SetLegacyManagement sets the route manager to use legacy management
 func (m *Manager) SetLegacyManagement(isLegacy bool) error {
-	oldLegacy := m.router.legacyManagement
+	return firewall.SetLegacyManagement(m.router, isLegacy)
+}
 
-	if oldLegacy != isLegacy {
-		m.router.legacyManagement = isLegacy
-		log.Debugf("Set legacy management to %v", isLegacy)
+// Reset firewall to the default state
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if err := m.resetNetbirdInputRules(); err != nil {
+		return fmt.Errorf("reset netbird input rules: %v", err)
 	}
 
-	// client reconnected to a newer mgmt, we need to cleanup the legacy rules
-	if !isLegacy && oldLegacy {
-		if err := m.router.RemoveAllLegacyRouteRules(); err != nil {
-			return fmt.Errorf("remove legacy routing rules: %v", err)
-		}
+	if err := m.router.Reset(); err != nil {
+		return fmt.Errorf("reset router: %v", err)
+	}
 
-		log.Debugf("Legacy routing rules removed")
+	if err := m.cleanupNetbirdTables(); err != nil {
+		return fmt.Errorf("cleanup netbird tables: %v", err)
+	}
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+		return fmt.Errorf("delete state: %v", err)
 	}
 
 	return nil
 }
 
-// Reset firewall to the default state
-func (m *Manager) Reset() error {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-
+func (m *Manager) resetNetbirdInputRules() error {
 	chains, err := m.rConn.ListChains()
 	if err != nil {
-		return fmt.Errorf("list of chains: %w", err)
+		return fmt.Errorf("list chains: %w", err)
 	}
 
+	m.deleteNetbirdInputRules(chains)
+
+	return nil
+}
+
+func (m *Manager) deleteNetbirdInputRules(chains []*nftables.Chain) {
 	for _, c := range chains {
-		// delete Netbird allow input traffic rule if it exists
-		if c.Table.Name == "filter" && c.Name == "INPUT" {
+		if c.Table.Name == tableNameFilter && c.Name == chainNameInput {
 			rules, err := m.rConn.GetRules(c.Table, c)
 			if err != nil {
 				log.Errorf("get rules for chain %q: %v", c.Name, err)
 				continue
 			}
-			for _, r := range rules {
-				if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
-					if err := m.rConn.DelRule(r); err != nil {
-						log.Errorf("delete rule: %v", err)
-					}
-				}
+
+			m.deleteMatchingRules(rules)
+		}
+	}
+}
+
+func (m *Manager) deleteMatchingRules(rules []*nftables.Rule) {
+	for _, r := range rules {
+		if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
+			if err := m.rConn.DelRule(r); err != nil {
+				log.Errorf("delete rule: %v", err)
 			}
 		}
 	}
+}
 
-	if err := m.router.Reset(); err != nil {
-		return fmt.Errorf("reset forward rules: %v", err)
-	}
-
+func (m *Manager) cleanupNetbirdTables() error {
 	tables, err := m.rConn.ListTables()
 	if err != nil {
-		return fmt.Errorf("list of tables: %w", err)
+		return fmt.Errorf("list tables: %w", err)
 	}
+
 	for _, t := range tables {
 		if t.Name == tableNameNetbird {
 			m.rConn.DelTable(t)
 		}
 	}
-
-	return m.rConn.Flush()
+	return nil
 }
 
 // Flush rule/chain/set operations from the buffer
@@ -286,7 +351,9 @@ func (m *Manager) applyAllowNetbirdRules(chain *nftables.Chain) {
 				Register: 1,
 				Data:     ifname(m.wgIface.Name()),
 			},
-			&expr.Verdict{},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
 		},
 		UserData: []byte(allowNetbirdInputRuleID),
 	}
@@ -315,28 +382,33 @@ func insertReturnTrafficRule(conn *nftables.Conn, table *nftables.Table, chain *
 	rule := &nftables.Rule{
 		Table: table,
 		Chain: chain,
-		Exprs: []expr.Any{
-			&expr.Ct{
-				Key:      expr.CtKeySTATE,
-				Register: 1,
-			},
-			&expr.Bitwise{
-				SourceRegister: 1,
-				DestRegister:   1,
-				Len:            4,
-				Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
-				Xor:            binaryutil.NativeEndian.PutUint32(0),
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpNeq,
-				Register: 1,
-				Data:     []byte{0, 0, 0, 0},
-			},
-			&expr.Verdict{
-				Kind: expr.VerdictAccept,
-			},
-		},
+		Exprs: getEstablishedExprs(1),
 	}
 
 	conn.InsertRule(rule)
 }
+
+func getEstablishedExprs(register uint32) []expr.Any {
+	return []expr.Any{
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: register,
+		},
+		&expr.Bitwise{
+			SourceRegister: register,
+			DestRegister:   register,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: register,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+}

client/firewall/nftables/manager_linux_test.go

@@ -1,10 +1,11 @@
 package nftables
 
 import (
-	"context"
+	"bytes"
 	"fmt"
 	"net"
 	"net/netip"
+	"os/exec"
 	"testing"
 	"time"
 
@@ -58,12 +59,13 @@ func (i *iFaceMock) IsUserspaceBind() bool { return false }
 func TestNftablesManager(t *testing.T) {
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), ifaceMock)
+	manager, err := Create(ifaceMock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 	time.Sleep(time.Second * 3)
 
 	defer func() {
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 		time.Sleep(time.Second)
 	}()
@@ -109,6 +111,7 @@ func TestNftablesManager(t *testing.T) {
 			Register: 1,
 			Data:     []byte{0, 0, 0, 0},
 		},
+		&expr.Counter{},
 		&expr.Verdict{
 			Kind: expr.VerdictAccept,
 		},
@@ -168,7 +171,7 @@ func TestNftablesManager(t *testing.T) {
 	// established rule remains
 	require.Len(t, rules, 1, "expected 1 rules after deletion")
 
-	err = manager.Reset()
+	err = manager.Reset(nil)
 	require.NoError(t, err, "failed to reset")
 }
 
@@ -191,12 +194,13 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(context.Background(), mock)
+			manager, err := Create(mock)
 			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second * 3)
 
 			defer func() {
-				if err := manager.Reset(); err != nil {
+				if err := manager.Reset(nil); err != nil {
 					t.Errorf("clear the manager state: %v", err)
 				}
 				time.Sleep(time.Second)
@@ -223,3 +227,105 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 		})
 	}
 }
+
+func runIptablesSave(t *testing.T) (string, string) {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("iptables-save")
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	require.NoError(t, err, "iptables-save failed to run")
+
+	return stdout.String(), stderr.String()
+}
+
+func verifyIptablesOutput(t *testing.T, stdout, stderr string) {
+	t.Helper()
+	// Check for any incompatibility warnings
+	require.NotContains(t,
+		stderr,
+		"incompatible",
+		"iptables-save produced compatibility warning. Full stderr: %s",
+		stderr,
+	)
+
+	// Verify standard tables are present
+	expectedTables := []string{
+		"*filter",
+		"*nat",
+		"*mangle",
+	}
+
+	for _, table := range expectedTables {
+		require.Contains(t,
+			stdout,
+			table,
+			"iptables-save output missing expected table: %s\nFull stdout: %s",
+			table,
+			stdout,
+		)
+	}
+}
+
+func TestNftablesManagerCompatibilityWithIptables(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
+
+	if _, err := exec.LookPath("iptables-save"); err != nil {
+		t.Skipf("iptables-save not available on this system: %v", err)
+	}
+
+	// First ensure iptables-nft tables exist by running iptables-save
+	stdout, stderr := runIptablesSave(t)
+	verifyIptablesOutput(t, stdout, stderr)
+
+	manager, err := Create(ifaceMock)
+	require.NoError(t, err, "failed to create manager")
+	require.NoError(t, manager.Init(nil))
+
+	t.Cleanup(func() {
+		err := manager.Reset(nil)
+		require.NoError(t, err, "failed to reset manager state")
+
+		// Verify iptables output after reset
+		stdout, stderr := runIptablesSave(t)
+		verifyIptablesOutput(t, stdout, stderr)
+	})
+
+	ip := net.ParseIP("100.96.0.1")
+	_, err = manager.AddPeerFiltering(
+		ip,
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{80}},
+		fw.RuleDirectionIN,
+		fw.ActionAccept,
+		"",
+		"test rule",
+	)
+	require.NoError(t, err, "failed to add peer filtering rule")
+
+	_, err = manager.AddRouteFiltering(
+		[]netip.Prefix{netip.MustParsePrefix("192.168.2.0/24")},
+		netip.MustParsePrefix("10.1.0.0/24"),
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{443}},
+		fw.ActionAccept,
+	)
+	require.NoError(t, err, "failed to add route filtering rule")
+
+	pair := fw.RouterPair{
+		Source:      netip.MustParsePrefix("192.168.1.0/24"),
+		Destination: netip.MustParsePrefix("10.0.0.0/24"),
+		Masquerade:  true,
+	}
+	err = manager.AddNatRule(pair)
+	require.NoError(t, err, "failed to add NAT rule")
+
+	stdout, stderr = runIptablesSave(t)
+	verifyIptablesOutput(t, stdout, stderr)
+}

client/firewall/nftables/router_linux.go

@@ -2,7 +2,6 @@ package nftables
 
 import (
 	"bytes"
-	"context"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -10,6 +9,8 @@ import (
 	"net/netip"
 	"strings"
 
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/google/nftables"
 	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
@@ -20,11 +21,12 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl/id"
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
 	chainNameRoutingFw  = "netbird-rt-fwd"
-	chainNameRoutingNat = "netbird-rt-nat"
+	chainNameRoutingNat = "netbird-rt-postrouting"
 	chainNameForward    = "FORWARD"
 
 	userDataAcceptForwardRuleIif = "frwacceptiif"
@@ -38,8 +40,6 @@ var (
 )
 
 type router struct {
-	ctx         context.Context
-	stop        context.CancelFunc
 	conn        *nftables.Conn
 	workTable   *nftables.Table
 	filterTable *nftables.Table
@@ -52,12 +52,8 @@ type router struct {
 	legacyManagement bool
 }
 
-func newRouter(parentCtx context.Context, workTable *nftables.Table, wgIface iFaceMapper) (*router, error) {
-	ctx, cancel := context.WithCancel(parentCtx)
-
+func newRouter(workTable *nftables.Table, wgIface iFaceMapper) (*router, error) {
 	r := &router{
-		ctx:       ctx,
-		stop:      cancel,
 		conn:      &nftables.Conn{},
 		workTable: workTable,
 		chains:    make(map[string]*nftables.Chain),
@@ -76,20 +72,25 @@ func newRouter(parentCtx context.Context, workTable *nftables.Table, wgIface iFa
 		if errors.Is(err, errFilterTableNotFound) {
 			log.Warnf("table 'filter' not found for forward rules")
 		} else {
-			return nil, err
+			return nil, fmt.Errorf("load filter table: %w", err)
 		}
 	}
 
-	err = r.cleanUpDefaultForwardRules()
-	if err != nil {
+	return r, nil
+}
+
+func (r *router) init(workTable *nftables.Table) error {
+	r.workTable = workTable
+
+	if err := r.removeAcceptForwardRules(); err != nil {
 		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
 
-	err = r.createContainers()
-	if err != nil {
-		log.Errorf("failed to create containers for route: %s", err)
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
 	}
-	return r, err
+
+	return nil
 }
 
 // Reset cleans existing nftables default forward rules from the system
@@ -97,40 +98,7 @@ func (r *router) Reset() error {
 	// clear without deleting the ipsets, the nf table will be deleted by the caller
 	r.ipsetCounter.Clear()
 
-	return r.cleanUpDefaultForwardRules()
-}
-
-func (r *router) cleanUpDefaultForwardRules() error {
-	if r.filterTable == nil {
-		return nil
-	}
-
-	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
-	if err != nil {
-		return fmt.Errorf("list chains: %v", err)
-	}
-
-	for _, chain := range chains {
-		if chain.Table.Name != r.filterTable.Name || chain.Name != chainNameForward {
-			continue
-		}
-
-		rules, err := r.conn.GetRules(r.filterTable, chain)
-		if err != nil {
-			return fmt.Errorf("get rules: %v", err)
-		}
-
-		for _, rule := range rules {
-			if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
-				bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) {
-				if err := r.conn.DelRule(rule); err != nil {
-					return fmt.Errorf("delete rule: %v", err)
-				}
-			}
-		}
-	}
-
-	return r.conn.Flush()
+	return r.removeAcceptForwardRules()
 }
 
 func (r *router) loadFilterTable() (*nftables.Table, error) {
@@ -149,7 +117,6 @@ func (r *router) loadFilterTable() (*nftables.Table, error) {
 }
 
 func (r *router) createContainers() error {
-
 	r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{
 		Name:  chainNameRoutingFw,
 		Table: r.workTable,
@@ -157,25 +124,42 @@ func (r *router) createContainers() error {
 
 	insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw])
 
+	prio := *nftables.ChainPriorityNATSource - 1
 	r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
 		Name:     chainNameRoutingNat,
 		Table:    r.workTable,
 		Hooknum:  nftables.ChainHookPostrouting,
-		Priority: nftables.ChainPriorityNATSource - 1,
+		Priority: &prio,
 		Type:     nftables.ChainTypeNAT,
 	})
 
-	r.acceptForwardRules()
+	// Chain is created by acl manager
+	// TODO: move creation to a common place
+	r.chains[chainNamePrerouting] = &nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    r.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	}
 
-	err := r.refreshRulesMap()
-	if err != nil {
+	// Add the single NAT rule that matches on mark
+	if err := r.addPostroutingRules(); err != nil {
+		return fmt.Errorf("add single nat rule: %v", err)
+	}
+
+	if err := r.acceptForwardRules(); err != nil {
+		log.Errorf("failed to add accept rules for the forward chain: %s", err)
+	}
+
+	if err := r.refreshRulesMap(); err != nil {
 		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
 
-	err = r.conn.Flush()
-	if err != nil {
+	if err := r.conn.Flush(); err != nil {
 		return fmt.Errorf("nftables: unable to initialize table: %v", err)
 	}
+
 	return nil
 }
 
@@ -188,6 +172,7 @@ func (r *router) AddRouteFiltering(
 	dPort *firewall.Port,
 	action firewall.Action,
 ) (firewall.Rule, error) {
+
 	ruleKey := id.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action)
 	if _, ok := r.rules[string(ruleKey)]; ok {
 		return ruleKey, nil
@@ -248,9 +233,18 @@ func (r *router) AddRouteFiltering(
 		UserData: []byte(ruleKey),
 	}
 
-	r.rules[string(ruleKey)] = r.conn.AddRule(rule)
+	rule = r.conn.AddRule(rule)
 
-	return ruleKey, r.conn.Flush()
+	log.Tracef("Adding route rule %s", spew.Sdump(rule))
+	if err := r.conn.Flush(); err != nil {
+		return nil, fmt.Errorf(flushError, err)
+	}
+
+	r.rules[string(ruleKey)] = rule
+
+	log.Debugf("nftables: added route rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", sources, destination, proto, sPort, dPort, action)
+
+	return ruleKey, nil
 }
 
 func (r *router) getIpSetExprs(sources []netip.Prefix, exprs []expr.Any) ([]expr.Any, error) {
@@ -288,6 +282,10 @@ func (r *router) DeleteRouteRule(rule firewall.Rule) error {
 		return nil
 	}
 
+	if nftRule.Handle == 0 {
+		return fmt.Errorf("route rule %s has no handle", ruleKey)
+	}
+
 	setName := r.findSetNameInRule(nftRule)
 
 	if err := r.deleteNftRule(nftRule, ruleKey); err != nil {
@@ -439,44 +437,149 @@ func (r *router) addNatRule(pair firewall.RouterPair) error {
 	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
 	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
 
-	dir := expr.MetaKeyIIFNAME
+	op := expr.CmpOpEq
 	if pair.Inverse {
-		dir = expr.MetaKeyOIFNAME
+		op = expr.CmpOpNeq
 	}
 
-	intf := ifname(r.wgIface.Name())
 	exprs := []expr.Any{
+		// We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading.
+		// Masquerading will take care of the conntrack state, which means we won't need to mark established connections.
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: 1,
+		},
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+
+		// interface matching
 		&expr.Meta{
-			Key:      dir,
+			Key:      expr.MetaKeyIIFNAME,
 			Register: 1,
 		},
 		&expr.Cmp{
-			Op:       expr.CmpOpEq,
+			Op:       op,
 			Register: 1,
-			Data:     intf,
+			Data:     ifname(r.wgIface.Name()),
 		},
 	}
 
 	exprs = append(exprs, sourceExp...)
 	exprs = append(exprs, destExp...)
+
+	var markValue uint32 = nbnet.PreroutingFwmarkMasquerade
+	if pair.Inverse {
+		markValue = nbnet.PreroutingFwmarkMasqueradeReturn
+	}
+
 	exprs = append(exprs,
-		&expr.Counter{}, &expr.Masq{},
+		&expr.Immediate{
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(markValue),
+		},
+		&expr.Meta{
+			Key:            expr.MetaKeyMARK,
+			SourceRegister: true,
+			Register:       1,
+		},
 	)
 
-	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
+	ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
 
 	if _, exists := r.rules[ruleKey]; exists {
 		if err := r.removeNatRule(pair); err != nil {
-			return fmt.Errorf("remove routing rule: %w", err)
+			return fmt.Errorf("remove prerouting rule: %w", err)
 		}
 	}
 
 	r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{
 		Table:    r.workTable,
-		Chain:    r.chains[chainNameRoutingNat],
+		Chain:    r.chains[chainNamePrerouting],
 		Exprs:    exprs,
 		UserData: []byte(ruleKey),
 	})
+
+	return nil
+}
+
+// addPostroutingRules adds the masquerade rules
+func (r *router) addPostroutingRules() error {
+	// First masquerade rule for traffic coming in from WireGuard interface
+	exprs := []expr.Any{
+		// Match on the first fwmark
+		&expr.Meta{
+			Key:      expr.MetaKeyMARK,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade),
+		},
+
+		// We need to exclude the loopback interface as this changes the ebpf proxy port
+		&expr.Meta{
+			Key:      expr.MetaKeyOIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     ifname("lo"),
+		},
+		&expr.Counter{},
+		&expr.Masq{},
+	}
+
+	r.conn.AddRule(&nftables.Rule{
+		Table: r.workTable,
+		Chain: r.chains[chainNameRoutingNat],
+		Exprs: exprs,
+	})
+
+	// Second masquerade rule for traffic going out through WireGuard interface
+	exprs2 := []expr.Any{
+		// Match on the second fwmark
+		&expr.Meta{
+			Key:      expr.MetaKeyMARK,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn),
+		},
+
+		// Match WireGuard interface
+		&expr.Meta{
+			Key:      expr.MetaKeyOIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(r.wgIface.Name()),
+		},
+		&expr.Counter{},
+		&expr.Masq{},
+	}
+
+	r.conn.AddRule(&nftables.Rule{
+		Table: r.workTable,
+		Chain: r.chains[chainNameRoutingNat],
+		Exprs: exprs2,
+	})
+
 	return nil
 }
 
@@ -553,7 +656,10 @@ func (r *router) RemoveAllLegacyRouteRules() error {
 		}
 		if err := r.conn.DelRule(rule); err != nil {
 			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
 		}
+
 	}
 	return nberrors.FormatErrorOrNil(merr)
 }
@@ -562,19 +668,60 @@ func (r *router) RemoveAllLegacyRouteRules() error {
 // that our traffic is not dropped by existing rules there.
 // The existing FORWARD rules/policies decide outbound traffic towards our interface.
 // In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule.
-func (r *router) acceptForwardRules() {
+func (r *router) acceptForwardRules() error {
 	if r.filterTable == nil {
 		log.Debugf("table 'filter' not found for forward rules, skipping accept rules")
-		return
+		return nil
 	}
 
+	fw := "iptables"
+
+	defer func() {
+		log.Debugf("Used %s to add accept forward rules", fw)
+	}()
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		// filter table exists but iptables is not
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+
+		fw = "nftables"
+		return r.acceptForwardRulesNftables()
+	}
+
+	return r.acceptForwardRulesIptables(ipt)
+}
+
+func (r *router) acceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("add iptables rule: %v", err))
+		} else {
+			log.Debugf("added iptables rule: %v", rule)
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *router) getAcceptForwardRules() [][]string {
+	intf := r.wgIface.Name()
+	return [][]string{
+		{"-i", intf, "-j", "ACCEPT"},
+		{"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
+	}
+}
+
+func (r *router) acceptForwardRulesNftables() error {
 	intf := ifname(r.wgIface.Name())
 
 	// Rule for incoming interface (iif) with counter
 	iifRule := &nftables.Rule{
 		Table: r.filterTable,
 		Chain: &nftables.Chain{
-			Name:     "FORWARD",
+			Name:     chainNameForward,
 			Table:    r.filterTable,
 			Type:     nftables.ChainTypeFilter,
 			Hooknum:  nftables.ChainHookForward,
@@ -594,6 +741,15 @@ func (r *router) acceptForwardRules() {
 	}
 	r.conn.InsertRule(iifRule)
 
+	oifExprs := []expr.Any{
+		&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     intf,
+		},
+	}
+
 	// Rule for outgoing interface (oif) with counter
 	oifRule := &nftables.Rule{
 		Table: r.filterTable,
@@ -604,50 +760,86 @@ func (r *router) acceptForwardRules() {
 			Hooknum:  nftables.ChainHookForward,
 			Priority: nftables.ChainPriorityFilter,
 		},
-		Exprs: []expr.Any{
-			&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     intf,
-			},
-			&expr.Ct{
-				Key:      expr.CtKeySTATE,
-				Register: 2,
-			},
-			&expr.Bitwise{
-				SourceRegister: 2,
-				DestRegister:   2,
-				Len:            4,
-				Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
-				Xor:            binaryutil.NativeEndian.PutUint32(0),
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpNeq,
-				Register: 2,
-				Data:     []byte{0, 0, 0, 0},
-			},
-			&expr.Counter{},
-			&expr.Verdict{Kind: expr.VerdictAccept},
-		},
+		Exprs:    append(oifExprs, getEstablishedExprs(2)...),
 		UserData: []byte(userDataAcceptForwardRuleOif),
 	}
 
 	r.conn.InsertRule(oifRule)
+
+	return nil
 }
 
-// RemoveNatRule removes a nftables rule pair from nat chains
+func (r *router) removeAcceptForwardRules() error {
+	if r.filterTable == nil {
+		return nil
+	}
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+		return r.removeAcceptForwardRulesNftables()
+	}
+
+	return r.removeAcceptForwardRulesIptables(ipt)
+}
+
+func (r *router) removeAcceptForwardRulesNftables() error {
+	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return fmt.Errorf("list chains: %v", err)
+	}
+
+	for _, chain := range chains {
+		if chain.Table.Name != r.filterTable.Name || chain.Name != chainNameForward {
+			continue
+		}
+
+		rules, err := r.conn.GetRules(r.filterTable, chain)
+		if err != nil {
+			return fmt.Errorf("get rules: %v", err)
+		}
+
+		for _, rule := range rules {
+			if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
+				bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) {
+				if err := r.conn.DelRule(rule); err != nil {
+					return fmt.Errorf("delete rule: %v", err)
+				}
+			}
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("remove iptables rule: %v", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// RemoveNatRule removes the prerouting mark rule
 func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
 	if err := r.refreshRulesMap(); err != nil {
 		return fmt.Errorf(refreshRulesMapError, err)
 	}
 
 	if err := r.removeNatRule(pair); err != nil {
-		return fmt.Errorf("remove nat rule: %w", err)
+		return fmt.Errorf("remove prerouting rule: %w", err)
 	}
 
 	if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil {
-		return fmt.Errorf("remove inverse nat rule: %w", err)
+		return fmt.Errorf("remove inverse prerouting rule: %w", err)
 	}
 
 	if err := r.removeLegacyRouteRule(pair); err != nil {
@@ -658,25 +850,24 @@ func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
 		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
 	}
 
-	log.Debugf("nftables: removed rules for %s", pair.Destination)
+	log.Debugf("nftables: removed nat rules for %s", pair.Destination)
 	return nil
 }
 
-// removeNatRule adds a nftables rule to the removal queue and deletes it from the rules map
 func (r *router) removeNatRule(pair firewall.RouterPair) error {
-	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
+	ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
 
 	if rule, exists := r.rules[ruleKey]; exists {
 		err := r.conn.DelRule(rule)
 		if err != nil {
-			return fmt.Errorf("remove nat rule %s -> %s: %v", pair.Source, pair.Destination, err)
+			return fmt.Errorf("remove prerouting rule %s -> %s: %v", pair.Source, pair.Destination, err)
 		}
 
-		log.Debugf("nftables: removed nat rule %s -> %s", pair.Source, pair.Destination)
+		log.Debugf("nftables: removed prerouting rule %s -> %s", pair.Source, pair.Destination)
 
 		delete(r.rules, ruleKey)
 	} else {
-		log.Debugf("nftables: nat rule %s not found", ruleKey)
+		log.Debugf("nftables: prerouting rule %s not found", ruleKey)
 	}
 
 	return nil

client/firewall/nftables/router_linux_test.go

@@ -3,7 +3,6 @@
 package nftables
 
 import (
-	"context"
 	"encoding/binary"
 	"net/netip"
 	"os/exec"
@@ -11,6 +10,7 @@ import (
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -33,87 +33,87 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
 		t.Skip("nftables not supported on this OS")
 	}
 
-	table, err := createWorkTable()
-	require.NoError(t, err, "Failed to create work table")
-
-	defer deleteWorkTable()
-
 	for _, testCase := range test.InsertRuleTestCases {
 		t.Run(testCase.Name, func(t *testing.T) {
-			manager, err := newRouter(context.TODO(), table, ifaceMock)
-			require.NoError(t, err, "failed to create router")
+			// need fw manager to init both acl mgr and router for all chains to be present
+			manager, err := Create(ifaceMock)
+			t.Cleanup(func() {
+				require.NoError(t, manager.Reset(nil))
+			})
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 
 			nftablesTestingClient := &nftables.Conn{}
 
-			defer func(manager *router) {
-				require.NoError(t, manager.Reset(), "failed to reset rules")
-			}(manager)
-
-			require.NoError(t, err, "shouldn't return error")
-
-			err = manager.AddNatRule(testCase.InputPair)
+			rtr := manager.router
+			err = rtr.AddNatRule(testCase.InputPair)
 			require.NoError(t, err, "pair should be inserted")
 
-			defer func(manager *router, pair firewall.RouterPair) {
-				require.NoError(t, manager.RemoveNatRule(pair), "failed to remove rule")
-			}(manager, testCase.InputPair)
+			t.Cleanup(func() {
+				require.NoError(t, rtr.RemoveNatRule(testCase.InputPair), "failed to remove rule")
+			})
 
 			if testCase.InputPair.Masquerade {
-				sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
-				destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
-				testingExpression := append(sourceExp, destExp...) //nolint:gocritic
-				testingExpression = append(testingExpression,
-					&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+				// Build expected expressions for connection tracking
+				conntrackExprs := []expr.Any{
+					&expr.Ct{
+						Key:      expr.CtKeySTATE,
+						Register: 1,
+					},
+					&expr.Bitwise{
+						SourceRegister: 1,
+						DestRegister:   1,
+						Len:            4,
+						Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
+						Xor:            binaryutil.NativeEndian.PutUint32(0),
+					},
+					&expr.Cmp{
+						Op:       expr.CmpOpNeq,
+						Register: 1,
+						Data:     []byte{0, 0, 0, 0},
+					},
+				}
+
+				// Build interface matching expression
+				ifaceExprs := []expr.Any{
+					&expr.Meta{
+						Key:      expr.MetaKeyIIFNAME,
+						Register: 1,
+					},
 					&expr.Cmp{
 						Op:       expr.CmpOpEq,
 						Register: 1,
 						Data:     ifname(ifaceMock.Name()),
 					},
-				)
-
-				natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
-				found := 0
-				for _, chain := range manager.chains {
-					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-					for _, rule := range rules {
-						if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
-							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "nat rule elements should match")
-							found = 1
-						}
-					}
 				}
-				require.Equal(t, 1, found, "should find at least 1 rule to test")
-			}
 
-			if testCase.InputPair.Masquerade {
+				// Build CIDR matching expressions
 				sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
 				destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
-				testingExpression := append(sourceExp, destExp...) //nolint:gocritic
-				testingExpression = append(testingExpression,
-					&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
-					&expr.Cmp{
-						Op:       expr.CmpOpEq,
-						Register: 1,
-						Data:     ifname(ifaceMock.Name()),
-					},
-				)
 
-				inNatRuleKey := firewall.GenKey(firewall.NatFormat, firewall.GetInversePair(testCase.InputPair))
+				// Combine all expressions in the correct order
+				// nolint:gocritic
+				testingExpression := append(conntrackExprs, ifaceExprs...)
+				testingExpression = append(testingExpression, sourceExp...)
+				testingExpression = append(testingExpression, destExp...)
+
+				natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
 				found := 0
-				for _, chain := range manager.chains {
-					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-					for _, rule := range rules {
-						if len(rule.UserData) > 0 && string(rule.UserData) == inNatRuleKey {
-							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "income nat rule elements should match")
-							found = 1
+				for _, chain := range rtr.chains {
+					if chain.Name == chainNamePrerouting {
+						rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+						require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+						for _, rule := range rules {
+							if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+								// Compare expressions up to the mark setting expressions
+								require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "prerouting nat rule elements should match")
+								found = 1
+							}
 						}
 					}
 				}
-				require.Equal(t, 1, found, "should find at least 1 rule to test")
+				require.Equal(t, 1, found, "should find at least 1 rule in prerouting chain")
 			}
-
 		})
 	}
 }
@@ -123,67 +123,66 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) {
 		t.Skip("nftables not supported on this OS")
 	}
 
-	table, err := createWorkTable()
-	require.NoError(t, err, "Failed to create work table")
-
-	defer deleteWorkTable()
-
 	for _, testCase := range test.RemoveRuleTestCases {
 		t.Run(testCase.Name, func(t *testing.T) {
-			manager, err := newRouter(context.TODO(), table, ifaceMock)
-			require.NoError(t, err, "failed to create router")
-
-			nftablesTestingClient := &nftables.Conn{}
-
-			defer func(manager *router) {
-				require.NoError(t, manager.Reset(), "failed to reset rules")
-			}(manager)
-
-			sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
-			destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
-
-			natExp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) //nolint:gocritic
-			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
-
-			insertedNat := nftablesTestingClient.InsertRule(&nftables.Rule{
-				Table:    manager.workTable,
-				Chain:    manager.chains[chainNameRoutingNat],
-				Exprs:    natExp,
-				UserData: []byte(natRuleKey),
+			manager, err := Create(ifaceMock)
+			t.Cleanup(func() {
+				require.NoError(t, manager.Reset(nil))
 			})
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 
-			sourceExp = generateCIDRMatcherExpressions(true, firewall.GetInversePair(testCase.InputPair).Source)
-			destExp = generateCIDRMatcherExpressions(false, firewall.GetInversePair(testCase.InputPair).Destination)
+			rtr := manager.router
 
-			natExp = append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) //nolint:gocritic
-			inNatRuleKey := firewall.GenKey(firewall.NatFormat, firewall.GetInversePair(testCase.InputPair))
+			// First add the NAT rule using the router's method
+			err = rtr.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "should add NAT rule")
 
-			insertedInNat := nftablesTestingClient.InsertRule(&nftables.Rule{
-				Table:    manager.workTable,
-				Chain:    manager.chains[chainNameRoutingNat],
-				Exprs:    natExp,
-				UserData: []byte(inNatRuleKey),
-			})
-
-			err = nftablesTestingClient.Flush()
-			require.NoError(t, err, "shouldn't return error")
-
-			err = manager.Reset()
-			require.NoError(t, err, "shouldn't return error")
-
-			err = manager.RemoveNatRule(testCase.InputPair)
-			require.NoError(t, err, "shouldn't return error")
-
-			for _, chain := range manager.chains {
-				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-				for _, rule := range rules {
-					if len(rule.UserData) > 0 {
-						require.NotEqual(t, insertedNat.UserData, rule.UserData, "nat rule should not exist")
-						require.NotEqual(t, insertedInNat.UserData, rule.UserData, "income nat rule should not exist")
-					}
+			// Verify the rule was added
+			natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
+			found := false
+			rules, err := rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNamePrerouting])
+			require.NoError(t, err, "should list rules")
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+					found = true
+					break
 				}
 			}
+			require.True(t, found, "NAT rule should exist before removal")
+
+			// Now remove the rule
+			err = rtr.RemoveNatRule(testCase.InputPair)
+			require.NoError(t, err, "shouldn't return error when removing rule")
+
+			// Verify the rule was removed
+			found = false
+			rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNamePrerouting])
+			require.NoError(t, err, "should list rules after removal")
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+					found = true
+					break
+				}
+			}
+			require.False(t, found, "NAT rule should not exist after removal")
+
+			// Verify the static postrouting rules still exist
+			rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameRoutingNat])
+			require.NoError(t, err, "should list postrouting rules")
+			foundCounter := false
+			for _, rule := range rules {
+				for _, e := range rule.Exprs {
+					if _, ok := e.(*expr.Counter); ok {
+						foundCounter = true
+						break
+					}
+				}
+				if foundCounter {
+					break
+				}
+			}
+			require.True(t, foundCounter, "static postrouting rule should remain")
 		})
 	}
 }
@@ -198,8 +197,9 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 
 	defer deleteWorkTable()
 
-	r, err := newRouter(context.Background(), workTable, ifaceMock)
+	r, err := newRouter(workTable, ifaceMock)
 	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
 
 	defer func(r *router) {
 		require.NoError(t, r.Reset(), "Failed to reset rules")
@@ -314,6 +314,10 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 			ruleKey, err := r.AddRouteFiltering(tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.action)
 			require.NoError(t, err, "AddRouteFiltering failed")
 
+			t.Cleanup(func() {
+				require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule")
+			})
+
 			// Check if the rule is in the internal map
 			rule, ok := r.rules[ruleKey.GetRuleID()]
 			assert.True(t, ok, "Rule not found in internal map")
@@ -346,10 +350,6 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 
 			// Verify actual nftables rule content
 			verifyRule(t, nftRule, tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.direction, tt.action, tt.expectSet)
-
-			// Clean up
-			err = r.DeleteRouteRule(ruleKey)
-			require.NoError(t, err, "Failed to delete rule")
 		})
 	}
 }
@@ -364,8 +364,9 @@ func TestNftablesCreateIpSet(t *testing.T) {
 
 	defer deleteWorkTable()
 
-	r, err := newRouter(context.Background(), workTable, ifaceMock)
+	r, err := newRouter(workTable, ifaceMock)
 	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
 
 	defer func() {
 		require.NoError(t, r.Reset(), "Failed to reset router")

client/firewall/nftables/state_linux.go

@@ -0,0 +1,47 @@
+package nftables
+
+import (
+	"fmt"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "nftables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	nft, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create nftables manager: %w", err)
+	}
+
+	if err := nft.Reset(nil); err != nil {
+		return fmt.Errorf("reset nftables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/uspfilter/allow_netbird.go

@@ -2,8 +2,10 @@
 
 package uspfilter
 
+import "github.com/netbirdio/netbird/client/internal/statemanager"
+
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
@@ -11,7 +13,7 @@ func (m *Manager) Reset() error {
 	m.incomingRules = make(map[string]RuleSet)
 
 	if m.nativeFirewall != nil {
-		return m.nativeFirewall.Reset()
+		return m.nativeFirewall.Reset(stateManager)
 	}
 	return nil
 }

client/firewall/uspfilter/allow_netbird_windows.go

@@ -6,6 +6,8 @@ import (
 	"syscall"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 type action string
@@ -17,7 +19,7 @@ const (
 )
 
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(*statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 

client/firewall/uspfilter/uspfilter.go

@@ -14,6 +14,7 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const layerTypeAll = 0
@@ -97,6 +98,10 @@ func create(iface IFaceMapper) (*Manager, error) {
 	return m, nil
 }
 
+func (m *Manager) Init(*statemanager.Manager) error {
+	return nil
+}
+
 func (m *Manager) IsServerRouteSupported() bool {
 	if m.nativeFirewall == nil {
 		return false
@@ -190,7 +195,7 @@ func (m *Manager) AddPeerFiltering(
 	return []firewall.Rule{&r}, nil
 }
 
-func (m *Manager) AddRouteFiltering(sources [] netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action ) (firewall.Rule, error) {
+func (m *Manager) AddRouteFiltering(sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action) (firewall.Rule, error) {
 	if m.nativeFirewall == nil {
 		return nil, errRouteNotSupported
 	}
@@ -232,8 +237,11 @@ func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
 }
 
 // SetLegacyManagement doesn't need to be implemented for this manager
-func (m *Manager) SetLegacyManagement(_ bool) error {
-	return nil
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	if m.nativeFirewall == nil {
+		return nil
+	}
+	return m.nativeFirewall.SetLegacyManagement(isLegacy)
 }
 
 // Flush doesn't need to be implemented for this manager

client/firewall/uspfilter/uspfilter_test.go

@@ -259,7 +259,7 @@ func TestManagerReset(t *testing.T) {
 		return
 	}
 
-	err = m.Reset()
+	err = m.Reset(nil)
 	if err != nil {
 		t.Errorf("failed to reset Manager: %v", err)
 		return
@@ -330,7 +330,7 @@ func TestNotMatchByIP(t *testing.T) {
 		return
 	}
 
-	if err = m.Reset(); err != nil {
+	if err = m.Reset(nil); err != nil {
 		t.Errorf("failed to reset Manager: %v", err)
 		return
 	}
@@ -396,7 +396,7 @@ func TestUSPFilterCreatePerformance(t *testing.T) {
 			time.Sleep(time.Second)
 
 			defer func() {
-				if err := manager.Reset(); err != nil {
+				if err := manager.Reset(nil); err != nil {
 					t.Errorf("clear the manager state: %v", err)
 				}
 				time.Sleep(time.Second)

client/iface/bind/bind.go

@@ -1,142 +0,0 @@
-package bind
-
-import (
-	"fmt"
-	"net"
-	"runtime"
-	"sync"
-
-	"github.com/pion/stun/v2"
-	"github.com/pion/transport/v3"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/net/ipv4"
-	wgConn "golang.zx2c4.com/wireguard/conn"
-)
-
-type receiverCreator struct {
-	iceBind *ICEBind
-}
-
-func (rc receiverCreator) CreateIPv4ReceiverFn(msgPool *sync.Pool, pc *ipv4.PacketConn, conn *net.UDPConn) wgConn.ReceiveFunc {
-	return rc.iceBind.createIPv4ReceiverFn(msgPool, pc, conn)
-}
-
-type ICEBind struct {
-	*wgConn.StdNetBind
-
-	muUDPMux sync.Mutex
-
-	transportNet transport.Net
-	udpMux       *UniversalUDPMuxDefault
-
-	filterFn FilterFn
-}
-
-func NewICEBind(transportNet transport.Net, filterFn FilterFn) *ICEBind {
-	ib := &ICEBind{
-		transportNet: transportNet,
-		filterFn:     filterFn,
-	}
-
-	rc := receiverCreator{
-		ib,
-	}
-	ib.StdNetBind = wgConn.NewStdNetBindWithReceiverCreator(rc)
-	return ib
-}
-
-// GetICEMux returns the ICE UDPMux that was created and used by ICEBind
-func (s *ICEBind) GetICEMux() (*UniversalUDPMuxDefault, error) {
-	s.muUDPMux.Lock()
-	defer s.muUDPMux.Unlock()
-	if s.udpMux == nil {
-		return nil, fmt.Errorf("ICEBind has not been initialized yet")
-	}
-
-	return s.udpMux, nil
-}
-
-func (s *ICEBind) createIPv4ReceiverFn(ipv4MsgsPool *sync.Pool, pc *ipv4.PacketConn, conn *net.UDPConn) wgConn.ReceiveFunc {
-	s.muUDPMux.Lock()
-	defer s.muUDPMux.Unlock()
-
-	s.udpMux = NewUniversalUDPMuxDefault(
-		UniversalUDPMuxParams{
-			UDPConn:  conn,
-			Net:      s.transportNet,
-			FilterFn: s.filterFn,
-		},
-	)
-	return func(bufs [][]byte, sizes []int, eps []wgConn.Endpoint) (n int, err error) {
-		msgs := ipv4MsgsPool.Get().(*[]ipv4.Message)
-		defer ipv4MsgsPool.Put(msgs)
-		for i := range bufs {
-			(*msgs)[i].Buffers[0] = bufs[i]
-		}
-		var numMsgs int
-		if runtime.GOOS == "linux" {
-			numMsgs, err = pc.ReadBatch(*msgs, 0)
-			if err != nil {
-				return 0, err
-			}
-		} else {
-			msg := &(*msgs)[0]
-			msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
-			if err != nil {
-				return 0, err
-			}
-			numMsgs = 1
-		}
-		for i := 0; i < numMsgs; i++ {
-			msg := &(*msgs)[i]
-
-			// todo: handle err
-			ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr)
-			if ok {
-				sizes[i] = 0
-			} else {
-				sizes[i] = msg.N
-			}
-
-			addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
-			ep := &wgConn.StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
-			wgConn.GetSrcFromControl(msg.OOB[:msg.NN], ep)
-			eps[i] = ep
-		}
-		return numMsgs, nil
-	}
-}
-
-func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) {
-	for i := range buffers {
-		if !stun.IsMessage(buffers[i]) {
-			continue
-		}
-
-		msg, err := s.parseSTUNMessage(buffers[i][:n])
-		if err != nil {
-			buffers[i] = []byte{}
-			return true, err
-		}
-
-		muxErr := s.udpMux.HandleSTUNMessage(msg, addr)
-		if muxErr != nil {
-			log.Warnf("failed to handle STUN packet")
-		}
-
-		buffers[i] = []byte{}
-		return true, nil
-	}
-	return false, nil
-}
-
-func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) {
-	msg := &stun.Message{
-		Raw: raw,
-	}
-	if err := msg.Decode(); err != nil {
-		return nil, err
-	}
-
-	return msg, nil
-}

client/iface/bind/control_android.go

@@ -0,0 +1,12 @@
+package bind
+
+import (
+	wireguard "golang.zx2c4.com/wireguard/conn"
+
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func init() {
+	// ControlFns is not thread safe and should only be modified during init.
+	*wireguard.ControlFns = append(*wireguard.ControlFns, nbnet.ControlProtectSocket)
+}

client/iface/bind/endpoint.go

@@ -0,0 +1,5 @@
+package bind
+
+import wgConn "golang.zx2c4.com/wireguard/conn"
+
+type Endpoint = wgConn.StdNetEndpoint

client/iface/bind/ice_bind.go

@@ -0,0 +1,303 @@
+package bind
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/pion/stun/v2"
+	"github.com/pion/transport/v3"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+	wgConn "golang.zx2c4.com/wireguard/conn"
+)
+
+type RecvMessage struct {
+	Endpoint *Endpoint
+	Buffer   []byte
+}
+
+type receiverCreator struct {
+	iceBind *ICEBind
+}
+
+func (rc receiverCreator) CreateIPv4ReceiverFn(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool, msgPool *sync.Pool) wgConn.ReceiveFunc {
+	return rc.iceBind.createIPv4ReceiverFn(pc, conn, rxOffload, msgPool)
+}
+
+// ICEBind is a bind implementation with two main features:
+// 1. filter out STUN messages and handle them
+// 2. forward the received packets to the WireGuard interface from the relayed connection
+//
+// ICEBind.endpoints var is a map that stores the connection for each relayed peer. Fake address is just an IP address
+// without port, in the format of 127.1.x.x where x.x is the last two octets of the peer address. We try to avoid to
+// use the port because in the Send function the wgConn.Endpoint the port info is not exported.
+type ICEBind struct {
+	*wgConn.StdNetBind
+	RecvChan chan RecvMessage
+
+	transportNet transport.Net
+	filterFn     FilterFn
+	endpoints    map[netip.Addr]net.Conn
+	endpointsMu  sync.Mutex
+	// every time when Close() is called (i.e. BindUpdate()) we need to close exit from the receiveRelayed and create a
+	// new closed channel. With the closedChanMu we can safely close the channel and create a new one
+	closedChan   chan struct{}
+	closedChanMu sync.RWMutex // protect the closeChan recreation from reading from it.
+	closed       bool
+
+	muUDPMux sync.Mutex
+	udpMux   *UniversalUDPMuxDefault
+}
+
+func NewICEBind(transportNet transport.Net, filterFn FilterFn) *ICEBind {
+	b, _ := wgConn.NewStdNetBind().(*wgConn.StdNetBind)
+	ib := &ICEBind{
+		StdNetBind:   b,
+		RecvChan:     make(chan RecvMessage, 1),
+		transportNet: transportNet,
+		filterFn:     filterFn,
+		endpoints:    make(map[netip.Addr]net.Conn),
+		closedChan:   make(chan struct{}),
+		closed:       true,
+	}
+
+	rc := receiverCreator{
+		ib,
+	}
+	ib.StdNetBind = wgConn.NewStdNetBindWithReceiverCreator(rc)
+	return ib
+}
+
+func (s *ICEBind) Open(uport uint16) ([]wgConn.ReceiveFunc, uint16, error) {
+	s.closed = false
+	s.closedChanMu.Lock()
+	s.closedChan = make(chan struct{})
+	s.closedChanMu.Unlock()
+	fns, port, err := s.StdNetBind.Open(uport)
+	if err != nil {
+		return nil, 0, err
+	}
+	fns = append(fns, s.receiveRelayed)
+	return fns, port, nil
+}
+
+func (s *ICEBind) Close() error {
+	if s.closed {
+		return nil
+	}
+	s.closed = true
+
+	close(s.closedChan)
+
+	return s.StdNetBind.Close()
+}
+
+// GetICEMux returns the ICE UDPMux that was created and used by ICEBind
+func (s *ICEBind) GetICEMux() (*UniversalUDPMuxDefault, error) {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+	if s.udpMux == nil {
+		return nil, fmt.Errorf("ICEBind has not been initialized yet")
+	}
+
+	return s.udpMux, nil
+}
+
+func (b *ICEBind) SetEndpoint(peerAddress *net.UDPAddr, conn net.Conn) (*net.UDPAddr, error) {
+	fakeUDPAddr, err := fakeAddress(peerAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// force IPv4
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		return nil, fmt.Errorf("failed to convert IP to netip.Addr")
+	}
+
+	b.endpointsMu.Lock()
+	b.endpoints[fakeAddr] = conn
+	b.endpointsMu.Unlock()
+
+	return fakeUDPAddr, nil
+}
+
+func (b *ICEBind) RemoveEndpoint(fakeUDPAddr *net.UDPAddr) {
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		log.Warnf("failed to convert IP to netip.Addr")
+		return
+	}
+
+	b.endpointsMu.Lock()
+	defer b.endpointsMu.Unlock()
+	delete(b.endpoints, fakeAddr)
+}
+
+func (b *ICEBind) Send(bufs [][]byte, ep wgConn.Endpoint) error {
+	b.endpointsMu.Lock()
+	conn, ok := b.endpoints[ep.DstIP()]
+	b.endpointsMu.Unlock()
+	if !ok {
+		return b.StdNetBind.Send(bufs, ep)
+	}
+
+	for _, buf := range bufs {
+		if _, err := conn.Write(buf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *ICEBind) createIPv4ReceiverFn(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool, msgsPool *sync.Pool) wgConn.ReceiveFunc {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+
+	s.udpMux = NewUniversalUDPMuxDefault(
+		UniversalUDPMuxParams{
+			UDPConn:  conn,
+			Net:      s.transportNet,
+			FilterFn: s.filterFn,
+		},
+	)
+	return func(bufs [][]byte, sizes []int, eps []wgConn.Endpoint) (n int, err error) {
+		msgs := getMessages(msgsPool)
+		for i := range bufs {
+			(*msgs)[i].Buffers[0] = bufs[i]
+			(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
+		}
+		defer putMessages(msgs, msgsPool)
+		var numMsgs int
+		if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+			if rxOffload {
+				readAt := len(*msgs) - (wgConn.IdealBatchSize / wgConn.UdpSegmentMaxDatagrams)
+				//nolint
+				numMsgs, err = pc.ReadBatch((*msgs)[readAt:], 0)
+				if err != nil {
+					return 0, err
+				}
+				numMsgs, err = wgConn.SplitCoalescedMessages(*msgs, readAt, wgConn.GetGSOSize)
+				if err != nil {
+					return 0, err
+				}
+			} else {
+				numMsgs, err = pc.ReadBatch(*msgs, 0)
+				if err != nil {
+					return 0, err
+				}
+			}
+		} else {
+			msg := &(*msgs)[0]
+			msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
+			if err != nil {
+				return 0, err
+			}
+			numMsgs = 1
+		}
+		for i := 0; i < numMsgs; i++ {
+			msg := &(*msgs)[i]
+
+			// todo: handle err
+			ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr)
+			if ok {
+				continue
+			}
+			sizes[i] = msg.N
+			if sizes[i] == 0 {
+				continue
+			}
+			addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
+			ep := &wgConn.StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
+			wgConn.GetSrcFromControl(msg.OOB[:msg.NN], ep)
+			eps[i] = ep
+		}
+		return numMsgs, nil
+	}
+}
+
+func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) {
+	for i := range buffers {
+		if !stun.IsMessage(buffers[i]) {
+			continue
+		}
+
+		msg, err := s.parseSTUNMessage(buffers[i][:n])
+		if err != nil {
+			buffers[i] = []byte{}
+			return true, err
+		}
+
+		muxErr := s.udpMux.HandleSTUNMessage(msg, addr)
+		if muxErr != nil {
+			log.Warnf("failed to handle STUN packet")
+		}
+
+		buffers[i] = []byte{}
+		return true, nil
+	}
+	return false, nil
+}
+
+func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) {
+	msg := &stun.Message{
+		Raw: raw,
+	}
+	if err := msg.Decode(); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}
+
+// receiveRelayed is a receive function that is used to receive packets from the relayed connection and forward to the
+// WireGuard. Critical part is do not block if the Closed() has been called.
+func (c *ICEBind) receiveRelayed(buffs [][]byte, sizes []int, eps []wgConn.Endpoint) (int, error) {
+	c.closedChanMu.RLock()
+	defer c.closedChanMu.RUnlock()
+
+	select {
+	case <-c.closedChan:
+		return 0, net.ErrClosed
+	case msg, ok := <-c.RecvChan:
+		if !ok {
+			return 0, net.ErrClosed
+		}
+		copy(buffs[0], msg.Buffer)
+		sizes[0] = len(msg.Buffer)
+		eps[0] = wgConn.Endpoint(msg.Endpoint)
+		return 1, nil
+	}
+}
+
+// fakeAddress returns a fake address that is used to as an identifier for the peer.
+// The fake address is in the format of 127.1.x.x where x.x is the last two octets of the peer address.
+func fakeAddress(peerAddress *net.UDPAddr) (*net.UDPAddr, error) {
+	octets := strings.Split(peerAddress.IP.String(), ".")
+	if len(octets) != 4 {
+		return nil, fmt.Errorf("invalid IP format")
+	}
+
+	newAddr := &net.UDPAddr{
+		IP:   net.ParseIP(fmt.Sprintf("127.1.%s.%s", octets[2], octets[3])),
+		Port: peerAddress.Port,
+	}
+	return newAddr, nil
+}
+
+func getMessages(msgsPool *sync.Pool) *[]ipv6.Message {
+	return msgsPool.Get().(*[]ipv6.Message)
+}
+
+func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) {
+	for i := range *msgs {
+		(*msgs)[i].OOB = (*msgs)[i].OOB[:0]
+		(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
+	}
+	msgsPool.Put(msgs)
+}

client/iface/bind/udp_mux.go

@@ -162,12 +162,13 @@ func NewUDPMuxDefault(params UDPMuxParams) *UDPMuxDefault {
 		params.Logger.Warn("UDPMuxDefault should not listening on unspecified address, use NewMultiUDPMuxFromPort instead")
 		var networks []ice.NetworkType
 		switch {
-		case addr.IP.To4() != nil:
-			networks = []ice.NetworkType{ice.NetworkTypeUDP4}
 
 		case addr.IP.To16() != nil:
 			networks = []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}
 
+		case addr.IP.To4() != nil:
+			networks = []ice.NetworkType{ice.NetworkTypeUDP4}
+
 		default:
 			params.Logger.Errorf("LocalAddr expected IPV4 or IPV6, got %T", params.UDPConn.LocalAddr())
 		}

client/iface/device/device_android.go

@@ -5,7 +5,6 @@ package device
 import (
 	"strings"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/device"
@@ -31,13 +30,13 @@ type WGTunDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewTunDevice(address WGAddress, port int, key string, mtu int, transportNet transport.Net, tunAdapter TunAdapter, filterFn bind.FilterFn) *WGTunDevice {
+func NewTunDevice(address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind, tunAdapter TunAdapter) *WGTunDevice {
 	return &WGTunDevice{
 		address:    address,
 		port:       port,
 		key:        key,
 		mtu:        mtu,
-		iceBind:    bind.NewICEBind(transportNet, filterFn),
+		iceBind:    iceBind,
 		tunAdapter: tunAdapter,
 	}
 }

client/iface/device/device_darwin.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os/exec"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
@@ -29,14 +28,14 @@ type TunDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) *TunDevice {
+func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *TunDevice {
 	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 

client/iface/device/device_ios.go

@@ -6,7 +6,6 @@ package device
 import (
 	"os"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/device"
@@ -30,13 +29,13 @@ type TunDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewTunDevice(name string, address WGAddress, port int, key string, transportNet transport.Net, tunFd int, filterFn bind.FilterFn) *TunDevice {
+func NewTunDevice(name string, address WGAddress, port int, key string, iceBind *bind.ICEBind, tunFd int) *TunDevice {
 	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 		tunFd:   tunFd,
 	}
 }

client/iface/device/device_netstack.go

@@ -6,7 +6,6 @@ package device
 import (
 	"fmt"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 
@@ -31,7 +30,7 @@ type TunNetstackDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, mtu int, transportNet transport.Net, listenAddress string, filterFn bind.FilterFn) *TunNetstackDevice {
+func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, mtu int, iceBind *bind.ICEBind, listenAddress string) *TunNetstackDevice {
 	return &TunNetstackDevice{
 		name:          name,
 		address:       address,
@@ -39,7 +38,7 @@ func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, m
 		key:           key,
 		mtu:           mtu,
 		listenAddress: listenAddress,
-		iceBind:       bind.NewICEBind(transportNet, filterFn),
+		iceBind:       iceBind,
 	}
 }
 

client/iface/device/device_usp_unix.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
@@ -30,7 +29,7 @@ type USPDevice struct {
 	configurer     WGConfigurer
 }
 
-func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) *USPDevice {
+func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *USPDevice {
 	log.Infof("using userspace bind mode")
 
 	checkUser()
@@ -41,7 +40,8 @@ func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn)}
+		iceBind: iceBind,
+	}
 }
 
 func (t *USPDevice) Create() (WGConfigurer, error) {

client/iface/device/device_windows.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/netip"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/device"
@@ -32,14 +31,14 @@ type TunDevice struct {
 	configurer      WGConfigurer
 }
 
-func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) *TunDevice {
+func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *TunDevice {
 	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 

client/iface/iface.go

@@ -6,12 +6,16 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
+	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
+	"github.com/netbirdio/netbird/client/errors"
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 const (
@@ -22,14 +26,35 @@ const (
 
 type WGAddress = device.WGAddress
 
+type wgProxyFactory interface {
+	GetProxy() wgproxy.Proxy
+	Free() error
+}
+
+type WGIFaceOpts struct {
+	IFaceName    string
+	Address      string
+	WGPort       int
+	WGPrivKey    string
+	MTU          int
+	MobileArgs   *device.MobileIFaceArguments
+	TransportNet transport.Net
+	FilterFn     bind.FilterFn
+}
+
 // WGIface represents an interface instance
 type WGIface struct {
 	tun           WGTunDevice
 	userspaceBind bool
 	mu            sync.Mutex
 
-	configurer device.WGConfigurer
-	filter     device.PacketFilter
+	configurer     device.WGConfigurer
+	filter         device.PacketFilter
+	wgProxyFactory wgProxyFactory
+}
+
+func (w *WGIface) GetProxy() wgproxy.Proxy {
+	return w.wgProxyFactory.GetProxy()
 }
 
 // IsUserspaceBind indicates whether this interfaces is userspace with bind.ICEBind
@@ -124,22 +149,26 @@ func (w *WGIface) Close() error {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 
-	err := w.tun.Close()
-	if err != nil {
-		return fmt.Errorf("failed to close wireguard interface %s: %w", w.Name(), err)
+	var result *multierror.Error
+
+	if err := w.wgProxyFactory.Free(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to free WireGuard proxy: %w", err))
 	}
 
-	err = w.waitUntilRemoved()
-	if err != nil {
+	if err := w.tun.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close wireguard interface %s: %w", w.Name(), err))
+	}
+
+	if err := w.waitUntilRemoved(); err != nil {
 		log.Warnf("failed to remove WireGuard interface %s: %v", w.Name(), err)
-		err = w.Destroy()
-		if err != nil {
-			return fmt.Errorf("failed to remove WireGuard interface %s: %w", w.Name(), err)
+		if err := w.Destroy(); err != nil {
+			result = multierror.Append(result, fmt.Errorf("failed to remove WireGuard interface %s: %w", w.Name(), err))
+			return errors.FormatErrorOrNil(result)
 		}
 		log.Infof("interface %s successfully removed", w.Name())
 	}
 
-	return nil
+	return errors.FormatErrorOrNil(result)
 }
 
 // SetFilter sets packet filters for the userspace implementation

client/iface/iface_android.go

@@ -1,43 +0,0 @@
-package iface
-
-import (
-	"fmt"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{
-		tun:           device.NewTunDevice(wgAddress, wgPort, wgPrivKey, mtu, transportNet, args.TunAdapter, filterFn),
-		userspaceBind: true,
-	}
-	return wgIFace, nil
-}
-
-// CreateOnAndroid creates a new Wireguard interface, sets a given IP and brings it up.
-// Will reuse an existing one.
-func (w *WGIface) CreateOnAndroid(routes []string, dns string, searchDomains []string) error {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-
-	cfgr, err := w.tun.Create(routes, dns, searchDomains)
-	if err != nil {
-		return err
-	}
-	w.configurer = cfgr
-	return nil
-}
-
-// Create this function make sense on mobile only
-func (w *WGIface) Create() error {
-	return fmt.Errorf("this function has not implemented on this platform")
-}

client/iface/iface_create.go

@@ -2,6 +2,8 @@
 
 package iface
 
+import "fmt"
+
 // Create creates a new Wireguard interface, sets a given IP and brings it up.
 // Will reuse an existing one.
 // this function is different on Android
@@ -17,3 +19,8 @@ func (w *WGIface) Create() error {
 	w.configurer = cfgr
 	return nil
 }
+
+// CreateOnAndroid this function make sense on mobile only
+func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
+	return fmt.Errorf("this function has not implemented on non mobile")
+}

client/iface/iface_create_android.go

@@ -0,0 +1,24 @@
+package iface
+
+import (
+	"fmt"
+)
+
+// CreateOnAndroid creates a new Wireguard interface, sets a given IP and brings it up.
+// Will reuse an existing one.
+func (w *WGIface) CreateOnAndroid(routes []string, dns string, searchDomains []string) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	cfgr, err := w.tun.Create(routes, dns, searchDomains)
+	if err != nil {
+		return err
+	}
+	w.configurer = cfgr
+	return nil
+}
+
+// Create this function make sense on mobile only
+func (w *WGIface) Create() error {
+	return fmt.Errorf("this function has not implemented on this platform")
+}

client/iface/iface_create_darwin.go

@@ -7,39 +7,8 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-	"github.com/netbirdio/netbird/client/iface/netstack"
 )
 
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, _ *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{
-		userspaceBind: true,
-	}
-
-	if netstack.IsEnabled() {
-		wgIFace.tun = device.NewNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr(), filterFn)
-		return wgIFace, nil
-	}
-
-	wgIFace.tun = device.NewTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, filterFn)
-
-	return wgIFace, nil
-}
-
-// CreateOnAndroid this function make sense on mobile only
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("this function has not implemented on this platform")
-}
-
 // Create creates a new Wireguard interface, sets a given IP and brings it up.
 // Will reuse an existing one.
 // this function is different on Android
@@ -65,3 +34,8 @@ func (w *WGIface) Create() error {
 
 	return backoff.Retry(operation, backOff)
 }
+
+// CreateOnAndroid this function make sense on mobile only
+func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
+	return fmt.Errorf("this function has not implemented on this platform")
+}

client/iface/iface_guid_windows.go

@@ -0,0 +1,10 @@
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+// GetInterfaceGUIDString returns an interface GUID. This is useful on Windows only
+func (w *WGIface) GetInterfaceGUIDString() (string, error) {
+	return w.tun.(*device.TunDevice).GetInterfaceGUIDString()
+}

client/iface/iface_ios.go

@@ -1,31 +0,0 @@
-//go:build ios
-
-package iface
-
-import (
-	"fmt"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-	wgIFace := &WGIface{
-		tun:           device.NewTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, transportNet, args.TunFd, filterFn),
-		userspaceBind: true,
-	}
-	return wgIFace, nil
-}
-
-// CreateOnAndroid creates a new Wireguard interface, sets a given IP and brings it up.
-// Will reuse an existing one.
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("this function has not implemented on this platform")
-}

client/iface/iface_moc.go

@@ -1,4 +1,4 @@
-package internal
+package iface
 
 import (
 	"net"
@@ -9,6 +9,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type MockWGIface struct {
@@ -30,6 +31,7 @@ type MockWGIface struct {
 	GetDeviceFunc              func() *device.FilteredDevice
 	GetStatsFunc               func(peerKey string) (configurer.WGStats, error)
 	GetInterfaceGUIDStringFunc func() (string, error)
+	GetProxyFunc               func() wgproxy.Proxy
 }
 
 func (m *MockWGIface) GetInterfaceGUIDString() (string, error) {
@@ -103,3 +105,8 @@ func (m *MockWGIface) GetDevice() *device.FilteredDevice {
 func (m *MockWGIface) GetStats(peerKey string) (configurer.WGStats, error) {
 	return m.GetStatsFunc(peerKey)
 }
+
+func (m *MockWGIface) GetProxy() wgproxy.Proxy {
+	//TODO implement me
+	panic("implement me")
+}

client/iface/iface_new_android.go

@@ -0,0 +1,24 @@
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	wgIFace := &WGIface{
+		userspaceBind:  true,
+		tun:            device.NewTunDevice(wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, opts.MobileArgs.TunAdapter),
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+}

client/iface/iface_new_darwin.go

@@ -0,0 +1,34 @@
+//go:build !ios
+
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	var tun WGTunDevice
+	if netstack.IsEnabled() {
+		tun = device.NewNetstackDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, netstack.ListenAddr())
+	} else {
+		tun = device.NewTunDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind)
+	}
+
+	wgIFace := &WGIface{
+		userspaceBind:  true,
+		tun:            tun,
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+}

client/iface/iface_new_ios.go

@@ -0,0 +1,26 @@
+//go:build ios
+
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	wgIFace := &WGIface{
+		tun:            device.NewTunDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, iceBind, opts.MobileArgs.TunFd),
+		userspaceBind:  true,
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+}

client/iface/iface_new_unix.go

@@ -0,0 +1,45 @@
+//go:build (linux && !android) || freebsd
+
+package iface
+
+import (
+	"fmt"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	wgIFace := &WGIface{}
+
+	if netstack.IsEnabled() {
+		iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+		wgIFace.tun = device.NewNetstackDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, netstack.ListenAddr())
+		wgIFace.userspaceBind = true
+		wgIFace.wgProxyFactory = wgproxy.NewUSPFactory(iceBind)
+		return wgIFace, nil
+	}
+
+	if device.WireGuardModuleIsLoaded() {
+		wgIFace.tun = device.NewKernelDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, opts.TransportNet)
+		wgIFace.wgProxyFactory = wgproxy.NewKernelFactory(opts.WGPort)
+		return wgIFace, nil
+	}
+	if device.ModuleTunIsLoaded() {
+		iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+		wgIFace.tun = device.NewUSPDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind)
+		wgIFace.userspaceBind = true
+		wgIFace.wgProxyFactory = wgproxy.NewUSPFactory(iceBind)
+		return wgIFace, nil
+	}
+
+	return nil, fmt.Errorf("couldn't check or load tun module")
+}

client/iface/iface_new_windows.go

@@ -0,0 +1,32 @@
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+)
+
+// NewWGIFace Creates a new WireGuard interface instance
+func NewWGIFace(opts WGIFaceOpts) (*WGIface, error) {
+	wgAddress, err := device.ParseWGAddress(opts.Address)
+	if err != nil {
+		return nil, err
+	}
+	iceBind := bind.NewICEBind(opts.TransportNet, opts.FilterFn)
+
+	var tun WGTunDevice
+	if netstack.IsEnabled() {
+		tun = device.NewNetstackDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind, netstack.ListenAddr())
+	} else {
+		tun = device.NewTunDevice(opts.IFaceName, wgAddress, opts.WGPort, opts.WGPrivKey, opts.MTU, iceBind)
+	}
+
+	wgIFace := &WGIface{
+		userspaceBind:  true,
+		tun:            tun,
+		wgProxyFactory: wgproxy.NewUSPFactory(iceBind),
+	}
+	return wgIFace, nil
+
+}

client/iface/iface_test.go

@@ -45,7 +45,16 @@ func TestWGIface_UpdateAddr(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, addr, wgPort, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      addr,
+		WGPort:       wgPort,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -118,7 +127,16 @@ func Test_CreateInterface(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       33100,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +171,16 @@ func Test_Close(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       wgPort,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -189,7 +216,16 @@ func TestRecreation(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil, nil)
+			opts := WGIFaceOpts{
+				IFaceName:    ifaceName,
+				Address:      wgIP,
+				WGPort:       wgPort,
+				WGPrivKey:    key,
+				MTU:          DefaultMTU,
+				TransportNet: newNet,
+			}
+
+			iface, err := NewWGIFace(opts)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -252,7 +288,15 @@ func Test_ConfigureInterface(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	iface, err := NewWGIFace(ifaceName, wgIP, wgPort, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       wgPort,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -300,7 +344,16 @@ func Test_UpdatePeer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       33100,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -361,7 +414,16 @@ func Test_RemovePeer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	iface, err := NewWGIFace(ifaceName, wgIP, 33100, key, DefaultMTU, newNet, nil, nil)
+	opts := WGIFaceOpts{
+		IFaceName:    ifaceName,
+		Address:      wgIP,
+		WGPort:       33100,
+		WGPrivKey:    key,
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface, err := NewWGIFace(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -418,7 +480,15 @@ func Test_ConnectPeers(t *testing.T) {
 	guid := fmt.Sprintf("{%s}", uuid.New().String())
 	device.CustomWindowsGUIDString = strings.ToLower(guid)
 
-	iface1, err := NewWGIFace(peer1ifaceName, peer1wgIP, peer1wgPort, peer1Key.String(), DefaultMTU, newNet, nil, nil)
+	optsPeer1 := WGIFaceOpts{
+		IFaceName:    peer1ifaceName,
+		Address:      peer1wgIP,
+		WGPort:       peer1wgPort,
+		WGPrivKey:    peer1Key.String(),
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+	iface1, err := NewWGIFace(optsPeer1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -432,7 +502,12 @@ func Test_ConnectPeers(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", peer1wgPort))
+	localIP, err := getLocalIP()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP, peer1wgPort))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -444,7 +519,17 @@ func Test_ConnectPeers(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	iface2, err := NewWGIFace(peer2ifaceName, peer2wgIP, peer2wgPort, peer2Key.String(), DefaultMTU, newNet, nil, nil)
+
+	optsPeer2 := WGIFaceOpts{
+		IFaceName:    peer2ifaceName,
+		Address:      peer2wgIP,
+		WGPort:       peer2wgPort,
+		WGPrivKey:    peer2Key.String(),
+		MTU:          DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	iface2, err := NewWGIFace(optsPeer2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -458,7 +543,7 @@ func Test_ConnectPeers(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", peer2wgPort))
+	peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP, peer2wgPort))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -527,3 +612,28 @@ func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
 	}
 	return wgtypes.Peer{}, fmt.Errorf("peer not found")
 }
+
+func getLocalIP() (string, error) {
+	// Get all interfaces
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "", err
+	}
+
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok {
+			continue
+		}
+		if ipNet.IP.IsLoopback() {
+			continue
+		}
+
+		if ipNet.IP.To4() == nil {
+			continue
+		}
+		return ipNet.IP.String(), nil
+	}
+
+	return "", fmt.Errorf("no local IP found")
+}

client/iface/iface_unix.go

@@ -1,49 +0,0 @@
-//go:build (linux && !android) || freebsd
-
-package iface
-
-import (
-	"fmt"
-	"runtime"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-	"github.com/netbirdio/netbird/client/iface/netstack"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{}
-
-	// move the kernel/usp/netstack preference evaluation to upper layer
-	if netstack.IsEnabled() {
-		wgIFace.tun = device.NewNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr(), filterFn)
-		wgIFace.userspaceBind = true
-		return wgIFace, nil
-	}
-
-	if device.WireGuardModuleIsLoaded() {
-		wgIFace.tun = device.NewKernelDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet)
-		wgIFace.userspaceBind = false
-		return wgIFace, nil
-	}
-
-	if !device.ModuleTunIsLoaded() {
-		return nil, fmt.Errorf("couldn't check or load tun module")
-	}
-	wgIFace.tun = device.NewUSPDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, nil)
-	wgIFace.userspaceBind = true
-	return wgIFace, nil
-}
-
-// CreateOnAndroid this function make sense on mobile only
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("CreateOnAndroid function has not implemented on %s platform", runtime.GOOS)
-}

client/iface/iface_windows.go

@@ -1,41 +0,0 @@
-package iface
-
-import (
-	"fmt"
-
-	"github.com/pion/transport/v3"
-
-	"github.com/netbirdio/netbird/client/iface/bind"
-	"github.com/netbirdio/netbird/client/iface/device"
-	"github.com/netbirdio/netbird/client/iface/netstack"
-)
-
-// NewWGIFace Creates a new WireGuard interface instance
-func NewWGIFace(iFaceName string, address string, wgPort int, wgPrivKey string, mtu int, transportNet transport.Net, args *device.MobileIFaceArguments, filterFn bind.FilterFn) (*WGIface, error) {
-	wgAddress, err := device.ParseWGAddress(address)
-	if err != nil {
-		return nil, err
-	}
-
-	wgIFace := &WGIface{
-		userspaceBind: true,
-	}
-
-	if netstack.IsEnabled() {
-		wgIFace.tun = device.NewNetstackDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, netstack.ListenAddr(), filterFn)
-		return wgIFace, nil
-	}
-
-	wgIFace.tun = device.NewTunDevice(iFaceName, wgAddress, wgPort, wgPrivKey, mtu, transportNet, filterFn)
-	return wgIFace, nil
-}
-
-// CreateOnAndroid this function make sense on mobile only
-func (w *WGIface) CreateOnAndroid([]string, string, []string) error {
-	return fmt.Errorf("this function has not implemented on non mobile")
-}
-
-// GetInterfaceGUIDString returns an interface GUID. This is useful on Windows only
-func (w *WGIface) GetInterfaceGUIDString() (string, error) {
-	return w.tun.(*device.TunDevice).GetInterfaceGUIDString()
-}

client/iface/iwginterface.go

@@ -1,6 +1,6 @@
 //go:build !windows
 
-package internal
+package iface
 
 import (
 	"net"
@@ -11,6 +11,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type IWGIface interface {
@@ -22,6 +23,7 @@ type IWGIface interface {
 	ToInterface() *net.Interface
 	Up() (*bind.UniversalUDPMuxDefault, error)
 	UpdateAddr(newAddr string) error
+	GetProxy() wgproxy.Proxy
 	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
 	RemovePeer(peerKey string) error
 	AddAllowedIP(peerKey string, allowedIP string) error

client/iface/iwginterface_windows.go

@@ -1,4 +1,4 @@
-package internal
+package iface
 
 import (
 	"net"
@@ -9,6 +9,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
 )
 
 type IWGIface interface {
@@ -20,6 +21,7 @@ type IWGIface interface {
 	ToInterface() *net.Interface
 	Up() (*bind.UniversalUDPMuxDefault, error)
 	UpdateAddr(newAddr string) error
+	GetProxy() wgproxy.Proxy
 	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
 	RemovePeer(peerKey string) error
 	AddAllowedIP(peerKey string, allowedIP string) error

client/iface/wgproxy/bind/proxy.go

@@ -0,0 +1,141 @@
+package bind
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+)
+
+type ProxyBind struct {
+	Bind *bind.ICEBind
+
+	wgAddr     *net.UDPAddr
+	wgEndpoint *bind.Endpoint
+	remoteConn net.Conn
+	ctx        context.Context
+	cancel     context.CancelFunc
+	closeMu    sync.Mutex
+	closed     bool
+
+	pausedMu  sync.Mutex
+	paused    bool
+	isStarted bool
+}
+
+// AddTurnConn adds a new connection to the bind.
+// endpoint is the NetBird address of the remote peer. The SetEndpoint return with the address what will be used in the
+// WireGuard configuration.
+func (p *ProxyBind) AddTurnConn(ctx context.Context, nbAddr *net.UDPAddr, remoteConn net.Conn) error {
+	addr, err := p.Bind.SetEndpoint(nbAddr, remoteConn)
+	if err != nil {
+		return err
+	}
+
+	p.wgAddr = addr
+	p.wgEndpoint = addrToEndpoint(addr)
+	p.remoteConn = remoteConn
+	p.ctx, p.cancel = context.WithCancel(ctx)
+	return err
+
+}
+func (p *ProxyBind) EndpointAddr() *net.UDPAddr {
+	return p.wgAddr
+}
+
+func (p *ProxyBind) Work() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = false
+	p.pausedMu.Unlock()
+
+	// Start the proxy only once
+	if !p.isStarted {
+		p.isStarted = true
+		go p.proxyToLocal(p.ctx)
+	}
+}
+
+func (p *ProxyBind) Pause() {
+	if p.remoteConn == nil {
+		return
+	}
+
+	p.pausedMu.Lock()
+	p.paused = true
+	p.pausedMu.Unlock()
+}
+
+func (p *ProxyBind) CloseConn() error {
+	if p.cancel == nil {
+		return fmt.Errorf("proxy not started")
+	}
+	return p.close()
+}
+
+func (p *ProxyBind) close() error {
+	p.closeMu.Lock()
+	defer p.closeMu.Unlock()
+
+	if p.closed {
+		return nil
+	}
+	p.closed = true
+
+	p.cancel()
+
+	p.Bind.RemoveEndpoint(p.wgAddr)
+
+	if rErr := p.remoteConn.Close(); rErr != nil && !errors.Is(rErr, net.ErrClosed) {
+		return rErr
+	}
+	return nil
+}
+
+func (p *ProxyBind) proxyToLocal(ctx context.Context) {
+	defer func() {
+		if err := p.close(); err != nil {
+			log.Warnf("failed to close remote conn: %s", err)
+		}
+	}()
+
+	for {
+		buf := make([]byte, 1500)
+		n, err := p.remoteConn.Read(buf)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.RemoteAddr(), err)
+			return
+		}
+
+		p.pausedMu.Lock()
+		if p.paused {
+			p.pausedMu.Unlock()
+			continue
+		}
+
+		msg := bind.RecvMessage{
+			Endpoint: p.wgEndpoint,
+			Buffer:   buf[:n],
+		}
+		p.Bind.RecvChan <- msg
+		p.pausedMu.Unlock()
+	}
+}
+
+func addrToEndpoint(addr *net.UDPAddr) *bind.Endpoint {
+	ip, _ := netip.AddrFromSlice(addr.IP.To4())
+	addrPort := netip.AddrPortFrom(ip, uint16(addr.Port))
+	return &bind.Endpoint{AddrPort: addrPort}
+}

client/iface/wgproxy/ebpf/portlookup.go

@@ -5,9 +5,9 @@ import (
 	"net"
 )
 
-const (
+var (
 	portRangeStart = 3128
-	portRangeEnd   = 3228
+	portRangeEnd   = portRangeStart + 100
 )
 
 type portLookup struct {

client/iface/wgproxy/ebpf/portlookup_test.go

@@ -17,6 +17,9 @@ func Test_portLookup_searchFreePort(t *testing.T) {
 func Test_portLookup_on_allocated(t *testing.T) {
 	pl := portLookup{}
 
+	portRangeStart = 4128
+	portRangeEnd = portRangeStart + 100
+
 	allocatedPort, err := allocatePort(portRangeStart)
 	if err != nil {
 		t.Fatal(err)

client/iface/wgproxy/ebpf/proxy.go

@@ -119,7 +119,7 @@ func (p *WGEBPFProxy) Free() error {
 	p.ctxCancel()
 
 	var result *multierror.Error
-	if p.conn != nil { // p.conn will be nil if we have failed to listen
+	if p.conn != nil {
 		if err := p.conn.Close(); err != nil {
 			result = multierror.Append(result, err)
 		}

client/iface/wgproxy/ebpf/wrapper.go

@@ -28,7 +28,7 @@ type ProxyWrapper struct {
 	isStarted bool
 }
 
-func (p *ProxyWrapper) AddTurnConn(ctx context.Context, remoteConn net.Conn) error {
+func (p *ProxyWrapper) AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error {
 	addr, err := p.WgeBPFProxy.AddTurnConn(remoteConn)
 	if err != nil {
 		return fmt.Errorf("add turn conn: %w", err)
@@ -77,7 +77,7 @@ func (e *ProxyWrapper) CloseConn() error {
 
 	e.cancel()
 
-	if err := e.remoteConn.Close(); err != nil {
+	if err := e.remoteConn.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
 		return fmt.Errorf("failed to close remote conn: %w", err)
 	}
 	return nil

client/iface/wgproxy/factory_kernel.go

@@ -0,0 +1,49 @@
+//go:build linux && !android
+
+package wgproxy
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
+	udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
+)
+
+type KernelFactory struct {
+	wgPort int
+
+	ebpfProxy *ebpf.WGEBPFProxy
+}
+
+func NewKernelFactory(wgPort int) *KernelFactory {
+	f := &KernelFactory{
+		wgPort: wgPort,
+	}
+
+	ebpfProxy := ebpf.NewWGEBPFProxy(wgPort)
+	if err := ebpfProxy.Listen(); err != nil {
+		log.Infof("WireGuard Proxy Factory will produce UDP proxy")
+		log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err)
+		return f
+	}
+	log.Infof("WireGuard Proxy Factory will produce eBPF proxy")
+	f.ebpfProxy = ebpfProxy
+	return f
+}
+
+func (w *KernelFactory) GetProxy() Proxy {
+	if w.ebpfProxy == nil {
+		return udpProxy.NewWGUDPProxy(w.wgPort)
+	}
+
+	return &ebpf.ProxyWrapper{
+		WgeBPFProxy: w.ebpfProxy,
+	}
+}
+
+func (w *KernelFactory) Free() error {
+	if w.ebpfProxy == nil {
+		return nil
+	}
+	return w.ebpfProxy.Free()
+}

client/iface/wgproxy/factory_kernel_freebsd.go

@@ -0,0 +1,29 @@
+package wgproxy
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
+)
+
+// KernelFactory todo: check eBPF support on FreeBSD
+type KernelFactory struct {
+	wgPort int
+}
+
+func NewKernelFactory(wgPort int) *KernelFactory {
+	log.Infof("WireGuard Proxy Factory will produce UDP proxy")
+	f := &KernelFactory{
+		wgPort: wgPort,
+	}
+
+	return f
+}
+
+func (w *KernelFactory) GetProxy() Proxy {
+	return udpProxy.NewWGUDPProxy(w.wgPort)
+}
+
+func (w *KernelFactory) Free() error {
+	return nil
+}

client/iface/wgproxy/factory_usp.go

@@ -0,0 +1,30 @@
+package wgproxy
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+	proxyBind "github.com/netbirdio/netbird/client/iface/wgproxy/bind"
+)
+
+type USPFactory struct {
+	bind *bind.ICEBind
+}
+
+func NewUSPFactory(iceBind *bind.ICEBind) *USPFactory {
+	log.Infof("WireGuard Proxy Factory will produce bind proxy")
+	f := &USPFactory{
+		bind: iceBind,
+	}
+	return f
+}
+
+func (w *USPFactory) GetProxy() Proxy {
+	return &proxyBind.ProxyBind{
+		Bind: w.bind,
+	}
+}
+
+func (w *USPFactory) Free() error {
+	return nil
+}

client/iface/wgproxy/proxy.go

@@ -0,0 +1,15 @@
+package wgproxy
+
+import (
+	"context"
+	"net"
+)
+
+// Proxy is a transfer layer between the relayed connection and the WireGuard
+type Proxy interface {
+	AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error
+	EndpointAddr() *net.UDPAddr // EndpointAddr returns the address of the WireGuard peer endpoint
+	Work()                      // Work start or resume the proxy
+	Pause()                     // Pause to forward the packages from remote connection to WireGuard. The opposite way still works.
+	CloseConn() error
+}

client/iface/wgproxy/proxy_linux_test.go

@@ -0,0 +1,56 @@
+//go:build linux && !android
+
+package wgproxy
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
+)
+
+func TestProxyCloseByRemoteConnEBPF(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") != "true" {
+		t.Skip("Skipping test as it requires root privileges")
+	}
+	ctx := context.Background()
+
+	ebpfProxy := ebpf.NewWGEBPFProxy(51831)
+	if err := ebpfProxy.Listen(); err != nil {
+		t.Fatalf("failed to initialize ebpf proxy: %s", err)
+	}
+
+	defer func() {
+		if err := ebpfProxy.Free(); err != nil {
+			t.Errorf("failed to free ebpf proxy: %s", err)
+		}
+	}()
+
+	tests := []struct {
+		name  string
+		proxy Proxy
+	}{
+		{
+			name: "ebpf proxy",
+			proxy: &ebpf.ProxyWrapper{
+				WgeBPFProxy: ebpfProxy,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			relayedConn := newMockConn()
+			err := tt.proxy.AddTurnConn(ctx, nil, relayedConn)
+			if err != nil {
+				t.Errorf("error: %v", err)
+			}
+
+			_ = relayedConn.Close()
+			if err := tt.proxy.CloseConn(); err != nil {
+				t.Errorf("error: %v", err)
+			}
+		})
+	}
+}

client/iface/wgproxy/proxy_test.go

@@ -11,8 +11,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/netbirdio/netbird/client/internal/wgproxy/ebpf"
-	"github.com/netbirdio/netbird/client/internal/wgproxy/usp"
+	"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
+	udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -84,7 +84,7 @@ func TestProxyCloseByRemoteConn(t *testing.T) {
 	}{
 		{
 			name:  "userspace proxy",
-			proxy: usp.NewWGUserSpaceProxy(51830),
+			proxy: udpProxy.NewWGUDPProxy(51830),
 		},
 	}
 
@@ -114,7 +114,7 @@ func TestProxyCloseByRemoteConn(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			relayedConn := newMockConn()
-			err := tt.proxy.AddTurnConn(ctx, relayedConn)
+			err := tt.proxy.AddTurnConn(ctx, nil, relayedConn)
 			if err != nil {
 				t.Errorf("error: %v", err)
 			}

client/iface/wgproxy/udp/proxy.go

@@ -1,19 +1,21 @@
-package usp
+package udp
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"net"
 	"sync"
 
 	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/netbirdio/netbird/client/errors"
+	cerrors "github.com/netbirdio/netbird/client/errors"
 )
 
-// WGUserSpaceProxy proxies
-type WGUserSpaceProxy struct {
+// WGUDPProxy proxies
+type WGUDPProxy struct {
 	localWGListenPort int
 
 	remoteConn net.Conn
@@ -28,10 +30,10 @@ type WGUserSpaceProxy struct {
 	isStarted bool
 }
 
-// NewWGUserSpaceProxy instantiate a user space WireGuard proxy. This is not a thread safe implementation
-func NewWGUserSpaceProxy(wgPort int) *WGUserSpaceProxy {
+// NewWGUDPProxy instantiate a UDP based WireGuard proxy. This is not a thread safe implementation
+func NewWGUDPProxy(wgPort int) *WGUDPProxy {
 	log.Debugf("Initializing new user space proxy with port %d", wgPort)
-	p := &WGUserSpaceProxy{
+	p := &WGUDPProxy{
 		localWGListenPort: wgPort,
 	}
 	return p
@@ -42,7 +44,7 @@ func NewWGUserSpaceProxy(wgPort int) *WGUserSpaceProxy {
 // the connection is complete, an error is returned. Once successfully
 // connected, any expiration of the context will not affect the
 // connection.
-func (p *WGUserSpaceProxy) AddTurnConn(ctx context.Context, remoteConn net.Conn) error {
+func (p *WGUDPProxy) AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error {
 	dialer := net.Dialer{}
 	localConn, err := dialer.DialContext(ctx, "udp", fmt.Sprintf(":%d", p.localWGListenPort))
 	if err != nil {
@@ -57,7 +59,7 @@ func (p *WGUserSpaceProxy) AddTurnConn(ctx context.Context, remoteConn net.Conn)
 	return err
 }
 
-func (p *WGUserSpaceProxy) EndpointAddr() *net.UDPAddr {
+func (p *WGUDPProxy) EndpointAddr() *net.UDPAddr {
 	if p.localConn == nil {
 		return nil
 	}
@@ -66,7 +68,7 @@ func (p *WGUserSpaceProxy) EndpointAddr() *net.UDPAddr {
 }
 
 // Work starts the proxy or resumes it if it was paused
-func (p *WGUserSpaceProxy) Work() {
+func (p *WGUDPProxy) Work() {
 	if p.remoteConn == nil {
 		return
 	}
@@ -83,7 +85,7 @@ func (p *WGUserSpaceProxy) Work() {
 }
 
 // Pause pauses the proxy from receiving data from the remote peer
-func (p *WGUserSpaceProxy) Pause() {
+func (p *WGUDPProxy) Pause() {
 	if p.remoteConn == nil {
 		return
 	}
@@ -94,14 +96,14 @@ func (p *WGUserSpaceProxy) Pause() {
 }
 
 // CloseConn close the localConn
-func (p *WGUserSpaceProxy) CloseConn() error {
+func (p *WGUDPProxy) CloseConn() error {
 	if p.cancel == nil {
 		return fmt.Errorf("proxy not started")
 	}
 	return p.close()
 }
 
-func (p *WGUserSpaceProxy) close() error {
+func (p *WGUDPProxy) close() error {
 	p.closeMu.Lock()
 	defer p.closeMu.Unlock()
 
@@ -114,18 +116,18 @@ func (p *WGUserSpaceProxy) close() error {
 	p.cancel()
 
 	var result *multierror.Error
-	if err := p.remoteConn.Close(); err != nil {
+	if err := p.remoteConn.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
 		result = multierror.Append(result, fmt.Errorf("remote conn: %s", err))
 	}
 
 	if err := p.localConn.Close(); err != nil {
 		result = multierror.Append(result, fmt.Errorf("local conn: %s", err))
 	}
-	return errors.FormatErrorOrNil(result)
+	return cerrors.FormatErrorOrNil(result)
 }
 
 // proxyToRemote proxies from Wireguard to the RemoteKey
-func (p *WGUserSpaceProxy) proxyToRemote(ctx context.Context) {
+func (p *WGUDPProxy) proxyToRemote(ctx context.Context) {
 	defer func() {
 		if err := p.close(); err != nil {
 			log.Warnf("error in proxy to remote loop: %s", err)
@@ -157,21 +159,19 @@ func (p *WGUserSpaceProxy) proxyToRemote(ctx context.Context) {
 
 // proxyToLocal proxies from the Remote peer to local WireGuard
 // if the proxy is paused it will drain the remote conn and drop the packets
-func (p *WGUserSpaceProxy) proxyToLocal(ctx context.Context) {
+func (p *WGUDPProxy) proxyToLocal(ctx context.Context) {
 	defer func() {
 		if err := p.close(); err != nil {
-			log.Warnf("error in proxy to local loop: %s", err)
+			if !errors.Is(err, io.EOF) {
+				log.Warnf("error in proxy to local loop: %s", err)
+			}
 		}
 	}()
 
 	buf := make([]byte, 1500)
 	for {
-		n, err := p.remoteConn.Read(buf)
+		n, err := p.remoteConnRead(ctx, buf)
 		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.RemoteAddr(), err)
 			return
 		}
 
@@ -193,3 +193,15 @@ func (p *WGUserSpaceProxy) proxyToLocal(ctx context.Context) {
 		}
 	}
 }
+
+func (p *WGUDPProxy) remoteConnRead(ctx context.Context, buf []byte) (n int, err error) {
+	n, err = p.remoteConn.Read(buf)
+	if err != nil {
+		if ctx.Err() != nil {
+			return
+		}
+		log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.LocalAddr(), err)
+		return
+	}
+	return
+}

client/internal/acl/id/id.go

@@ -1,8 +1,11 @@
 package id
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net/netip"
+	"strconv"
 
 	"github.com/netbirdio/netbird/client/firewall/manager"
 )
@@ -21,5 +24,41 @@ func GenerateRouteRuleKey(
 	dPort *manager.Port,
 	action manager.Action,
 ) RuleID {
-	return RuleID(fmt.Sprintf("%s-%s-%s-%s-%s-%d", sources, destination, proto, sPort, dPort, action))
+	manager.SortPrefixes(sources)
+
+	h := sha256.New()
+
+	// Write all fields to the hasher, with delimiters
+	h.Write([]byte("sources:"))
+	for _, src := range sources {
+		h.Write([]byte(src.String()))
+		h.Write([]byte(","))
+	}
+
+	h.Write([]byte("destination:"))
+	h.Write([]byte(destination.String()))
+
+	h.Write([]byte("proto:"))
+	h.Write([]byte(proto))
+
+	h.Write([]byte("sPort:"))
+	if sPort != nil {
+		h.Write([]byte(sPort.String()))
+	} else {
+		h.Write([]byte("<nil>"))
+	}
+
+	h.Write([]byte("dPort:"))
+	if dPort != nil {
+		h.Write([]byte(dPort.String()))
+	} else {
+		h.Write([]byte("<nil>"))
+	}
+
+	h.Write([]byte("action:"))
+	h.Write([]byte(strconv.Itoa(int(action))))
+	hash := hex.EncodeToString(h.Sum(nil))
+
+	// prepend destination prefix to be able to identify the rule
+	return RuleID(fmt.Sprintf("%s-%s", destination.String(), hash[:16]))
 }

client/internal/acl/manager.go

@@ -3,6 +3,7 @@ package acl
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"net"
 	"net/netip"
@@ -10,14 +11,18 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl/id"
 	"github.com/netbirdio/netbird/client/ssh"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
 )
 
+var ErrSourceRangesEmpty = errors.New("sources range is empty")
+
 // Manager is a ACL rules manager
 type Manager interface {
 	ApplyFiltering(networkMap *mgmProto.NetworkMap)
@@ -167,31 +172,40 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) {
 }
 
 func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule) error {
-	var newRouteRules = make(map[id.RuleID]struct{})
+	newRouteRules := make(map[id.RuleID]struct{}, len(rules))
+	var merr *multierror.Error
+
+	// Apply new rules - firewall manager will return existing rule ID if already present
 	for _, rule := range rules {
 		id, err := d.applyRouteACL(rule)
 		if err != nil {
-			return fmt.Errorf("apply route ACL: %w", err)
+			if errors.Is(err, ErrSourceRangesEmpty) {
+				log.Debugf("skipping empty rule with destination %s: %v", rule.Destination, err)
+			} else {
+				merr = multierror.Append(merr, fmt.Errorf("add route rule: %w", err))
+			}
+			continue
 		}
 		newRouteRules[id] = struct{}{}
 	}
 
+	// Clean up old firewall rules
 	for id := range d.routeRules {
-		if _, ok := newRouteRules[id]; !ok {
+		if _, exists := newRouteRules[id]; !exists {
 			if err := d.firewall.DeleteRouteRule(id); err != nil {
-				log.Errorf("failed to delete route firewall rule: %v", err)
-				continue
+				merr = multierror.Append(merr, fmt.Errorf("delete route rule: %w", err))
 			}
-			delete(d.routeRules, id)
+			// implicitly deleted from the map
 		}
 	}
+
 	d.routeRules = newRouteRules
-	return nil
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule) (id.RuleID, error) {
 	if len(rule.SourceRanges) == 0 {
-		return "", fmt.Errorf("source ranges is empty")
+		return "", ErrSourceRangesEmpty
 	}
 
 	var sources []netip.Prefix

client/internal/acl/manager_test.go

@@ -1,7 +1,6 @@
 package acl
 
 import (
-	"context"
 	"net"
 	"testing"
 
@@ -52,13 +51,13 @@ func TestDefaultManager(t *testing.T) {
 	}).AnyTimes()
 
 	// we receive one rule from the management so for testing purposes ignore it
-	fw, err := firewall.NewFirewall(context.Background(), ifaceMock)
+	fw, err := firewall.NewFirewall(ifaceMock, nil)
 	if err != nil {
 		t.Errorf("create firewall: %v", err)
 		return
 	}
 	defer func(fw manager.Manager) {
-		_ = fw.Reset()
+		_ = fw.Reset(nil)
 	}(fw)
 	acl := NewDefaultManager(fw)
 
@@ -345,13 +344,13 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) {
 	}).AnyTimes()
 
 	// we receive one rule from the management so for testing purposes ignore it
-	fw, err := firewall.NewFirewall(context.Background(), ifaceMock)
+	fw, err := firewall.NewFirewall(ifaceMock, nil)
 	if err != nil {
 		t.Errorf("create firewall: %v", err)
 		return
 	}
 	defer func(fw manager.Manager) {
-		_ = fw.Reset()
+		_ = fw.Reset(nil)
 	}(fw)
 	acl := NewDefaultManager(fw)
 

client/internal/config.go

@@ -46,6 +46,7 @@ type ConfigInput struct {
 	ManagementURL       string
 	AdminURL            string
 	ConfigPath          string
+	StateFilePath       string
 	PreSharedKey        *string
 	ServerSSHAllowed    *bool
 	NATExternalIPs      []string
@@ -105,10 +106,10 @@ type Config struct {
 
 	// DNSRouteInterval is the interval in which the DNS routes are updated
 	DNSRouteInterval time.Duration
-	//Path to a certificate used for mTLS authentication
+	// Path to a certificate used for mTLS authentication
 	ClientCertPath string
 
-	//Path to corresponding private key of ClientCertPath
+	// Path to corresponding private key of ClientCertPath
 	ClientCertKeyPath string
 
 	ClientCertKeyPair *tls.Certificate `json:"-"`
@@ -116,7 +117,7 @@ type Config struct {
 
 // ReadConfig read config file and return with Config. If it is not exists create a new with default values
 func ReadConfig(configPath string) (*Config, error) {
-	if configFileIsExists(configPath) {
+	if fileExists(configPath) {
 		err := util.EnforcePermission(configPath)
 		if err != nil {
 			log.Errorf("failed to enforce permission on config dir: %v", err)
@@ -149,7 +150,7 @@ func ReadConfig(configPath string) (*Config, error) {
 
 // UpdateConfig update existing configuration according to input configuration and return with the configuration
 func UpdateConfig(input ConfigInput) (*Config, error) {
-	if !configFileIsExists(input.ConfigPath) {
+	if !fileExists(input.ConfigPath) {
 		return nil, status.Errorf(codes.NotFound, "config file doesn't exist")
 	}
 
@@ -158,13 +159,13 @@ func UpdateConfig(input ConfigInput) (*Config, error) {
 
 // UpdateOrCreateConfig reads existing config or generates a new one
 func UpdateOrCreateConfig(input ConfigInput) (*Config, error) {
-	if !configFileIsExists(input.ConfigPath) {
+	if !fileExists(input.ConfigPath) {
 		log.Infof("generating new config %s", input.ConfigPath)
 		cfg, err := createNewConfig(input)
 		if err != nil {
 			return nil, err
 		}
-		err = util.WriteJsonWithRestrictedPermission(input.ConfigPath, cfg)
+		err = util.WriteJsonWithRestrictedPermission(context.Background(), input.ConfigPath, cfg)
 		return cfg, err
 	}
 
@@ -185,7 +186,7 @@ func CreateInMemoryConfig(input ConfigInput) (*Config, error) {
 
 // WriteOutConfig write put the prepared config to the given path
 func WriteOutConfig(path string, config *Config) error {
-	return util.WriteJson(path, config)
+	return util.WriteJson(context.Background(), path, config)
 }
 
 // createNewConfig creates a new config generating a new Wireguard key and saving to file
@@ -215,7 +216,7 @@ func update(input ConfigInput) (*Config, error) {
 	}
 
 	if updated {
-		if err := util.WriteJson(input.ConfigPath, config); err != nil {
+		if err := util.WriteJson(context.Background(), input.ConfigPath, config); err != nil {
 			return nil, err
 		}
 	}
@@ -472,11 +473,19 @@ func isPreSharedKeyHidden(preSharedKey *string) bool {
 	return false
 }
 
-func configFileIsExists(path string) bool {
+func fileExists(path string) bool {
 	_, err := os.Stat(path)
 	return !os.IsNotExist(err)
 }
 
+func createFile(path string) error {
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	return file.Close()
+}
+
 // UpdateOldManagementURL checks whether client can switch to the new Management URL with port 443 and the management domain.
 // If it can switch, then it updates the config and returns a new one. Otherwise, it returns the provided config.
 // The check is performed only for the NetBird's managed version.

client/internal/connect.go

@@ -40,6 +40,8 @@ type ConnectClient struct {
 	statusRecorder *peer.Status
 	engine         *Engine
 	engineMutex    sync.Mutex
+
+	persistNetworkMap bool
 }
 
 func NewConnectClient(
@@ -62,10 +64,7 @@ func (c *ConnectClient) Run() error {
 }
 
 // RunWithProbes runs the client's main logic with probes attached
-func (c *ConnectClient) RunWithProbes(
-	probes *ProbeHolder,
-	runningChan chan error,
-) error {
+func (c *ConnectClient) RunWithProbes(probes *ProbeHolder, runningChan chan error) error {
 	return c.run(MobileDependency{}, probes, runningChan)
 }
 
@@ -92,6 +91,7 @@ func (c *ConnectClient) RunOniOS(
 	fileDescriptor int32,
 	networkChangeListener listener.NetworkChangeListener,
 	dnsManager dns.IosDnsManager,
+	stateFilePath string,
 ) error {
 	// Set GC percent to 5% to reduce memory usage as iOS only allows 50MB of memory for the extension.
 	debug.SetGCPercent(5)
@@ -100,15 +100,12 @@ func (c *ConnectClient) RunOniOS(
 		FileDescriptor:        fileDescriptor,
 		NetworkChangeListener: networkChangeListener,
 		DnsManager:            dnsManager,
+		StateFilePath:         stateFilePath,
 	}
 	return c.run(mobileDependency, nil, nil)
 }
 
-func (c *ConnectClient) run(
-	mobileDependency MobileDependency,
-	probes *ProbeHolder,
-	runningChan chan error,
-) error {
+func (c *ConnectClient) run(mobileDependency MobileDependency, probes *ProbeHolder, runningChan chan error) error {
 	defer func() {
 		if r := recover(); r != nil {
 			log.Panicf("Panic occurred: %v, stack trace: %s", r, string(debug.Stack()))
@@ -117,12 +114,6 @@ func (c *ConnectClient) run(
 
 	log.Infof("starting NetBird client version %s on %s/%s", version.NetbirdVersion(), runtime.GOOS, runtime.GOARCH)
 
-	// Check if client was not shut down in a clean way and restore DNS config if required.
-	// Otherwise, we might not be able to connect to the management server to retrieve new config.
-	if err := dns.CheckUncleanShutdown(c.config.WgIface); err != nil {
-		log.Errorf("checking unclean shutdown error: %s", err)
-	}
-
 	backOff := &backoff.ExponentialBackOff{
 		InitialInterval:     time.Second,
 		RandomizationFactor: 1,
@@ -170,7 +161,8 @@ func (c *ConnectClient) run(
 
 		engineCtx, cancel := context.WithCancel(c.ctx)
 		defer func() {
-			c.statusRecorder.MarkManagementDisconnected(state.err)
+			_, err := state.Status()
+			c.statusRecorder.MarkManagementDisconnected(err)
 			c.statusRecorder.CleanLocalPeerState()
 			cancel()
 		}()
@@ -220,7 +212,8 @@ func (c *ConnectClient) run(
 
 		c.statusRecorder.MarkSignalDisconnected(nil)
 		defer func() {
-			c.statusRecorder.MarkSignalDisconnected(state.err)
+			_, err := state.Status()
+			c.statusRecorder.MarkSignalDisconnected(err)
 		}()
 
 		// with the global Wiretrustee config in hand connect (just a connection, no stream yet) Signal
@@ -243,6 +236,7 @@ func (c *ConnectClient) run(
 
 		relayURLs, token := parseRelayInfo(loginResp)
 		relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String())
+		c.statusRecorder.SetRelayMgr(relayManager)
 		if len(relayURLs) > 0 {
 			if token != nil {
 				if err := relayManager.UpdateToken(token); err != nil {
@@ -253,9 +247,7 @@ func (c *ConnectClient) run(
 			log.Infof("connecting to the Relay service(s): %s", strings.Join(relayURLs, ", "))
 			if err = relayManager.Serve(); err != nil {
 				log.Error(err)
-				return wrapErr(err)
 			}
-			c.statusRecorder.SetRelayMgr(relayManager)
 		}
 
 		peerConfig := loginResp.GetPeerConfig()
@@ -270,7 +262,7 @@ func (c *ConnectClient) run(
 
 		c.engineMutex.Lock()
 		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, probes, checks)
-
+		c.engine.SetNetworkMapPersistence(c.persistNetworkMap)
 		c.engineMutex.Unlock()
 
 		if err := c.engine.Start(); err != nil {
@@ -348,6 +340,19 @@ func (c *ConnectClient) Engine() *Engine {
 	return e
 }
 
+// Status returns the current client status
+func (c *ConnectClient) Status() StatusType {
+	if c == nil {
+		return StatusIdle
+	}
+	status, err := CtxGetState(c.ctx).Status()
+	if err != nil {
+		return StatusIdle
+	}
+
+	return status
+}
+
 func (c *ConnectClient) Stop() error {
 	if c == nil {
 		return nil
@@ -358,7 +363,11 @@ func (c *ConnectClient) Stop() error {
 	if c.engine == nil {
 		return nil
 	}
-	return c.engine.Stop()
+	if err := c.engine.Stop(); err != nil {
+		return fmt.Errorf("stop engine: %w", err)
+	}
+
+	return nil
 }
 
 func (c *ConnectClient) isContextCancelled() bool {
@@ -370,6 +379,22 @@ func (c *ConnectClient) isContextCancelled() bool {
 	}
 }
 
+// SetNetworkMapPersistence enables or disables network map persistence.
+// When enabled, the last received network map will be stored and can be retrieved
+// through the Engine's getLatestNetworkMap method. When disabled, any stored
+// network map will be cleared. This functionality is primarily used for debugging
+// and should not be enabled during normal operation.
+func (c *ConnectClient) SetNetworkMapPersistence(enabled bool) {
+	c.engineMutex.Lock()
+	c.persistNetworkMap = enabled
+	c.engineMutex.Unlock()
+
+	engine := c.Engine()
+	if engine != nil {
+		engine.SetNetworkMapPersistence(enabled)
+	}
+}
+
 // createEngineConfig converts configuration received from Management Service to EngineConfig
 func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) {
 	nm := false

client/internal/dns/consts_freebsd.go

@@ -1,6 +1,5 @@
 package dns
 
 const (
-	fileUncleanShutdownResolvConfLocation  = "/var/db/netbird/resolv.conf"
-	fileUncleanShutdownManagerTypeLocation = "/var/db/netbird/manager"
+	fileUncleanShutdownResolvConfLocation = "/var/db/netbird/resolv.conf"
 )

client/internal/dns/consts_linux.go

@@ -3,6 +3,5 @@
 package dns
 
 const (
-	fileUncleanShutdownResolvConfLocation  = "/var/lib/netbird/resolv.conf"
-	fileUncleanShutdownManagerTypeLocation = "/var/lib/netbird/manager"
+	fileUncleanShutdownResolvConfLocation = "/var/lib/netbird/resolv.conf"
 )

client/internal/dns/file_repair_unix.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/fsnotify/fsnotify"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 var (
@@ -20,7 +22,7 @@ var (
 	}
 )
 
-type repairConfFn func([]string, string, *resolvConf) error
+type repairConfFn func([]string, string, *resolvConf, *statemanager.Manager) error
 
 type repair struct {
 	operationFile string
@@ -40,7 +42,7 @@ func newRepair(operationFile string, updateFn repairConfFn) *repair {
 	}
 }
 
-func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP string) {
+func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP string, stateManager *statemanager.Manager) {
 	if f.inotify != nil {
 		return
 	}
@@ -81,7 +83,7 @@ func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP strin
 				log.Errorf("failed to rm inotify watch for resolv.conf: %s", err)
 			}
 
-			err = f.updateFn(nbSearchDomains, nbNameserverIP, rConf)
+			err = f.updateFn(nbSearchDomains, nbNameserverIP, rConf, stateManager)
 			if err != nil {
 				log.Errorf("failed to repair resolv.conf: %v", err)
 			}

client/internal/dns/file_repair_unix_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -104,14 +105,14 @@ nameserver 8.8.8.8`,
 
 			var changed bool
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-			updateFn := func([]string, string, *resolvConf) error {
+			updateFn := func([]string, string, *resolvConf, *statemanager.Manager) error {
 				changed = true
 				cancel()
 				return nil
 			}
 
 			r := newRepair(operationFile, updateFn)
-			r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1")
+			r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1", nil)
 
 			err = os.WriteFile(operationFile, []byte(tt.touchedConfContent), 0755)
 			if err != nil {
@@ -151,14 +152,14 @@ searchdomain netbird.cloud something`
 
 	var changed bool
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	updateFn := func([]string, string, *resolvConf) error {
+	updateFn := func([]string, string, *resolvConf, *statemanager.Manager) error {
 		changed = true
 		cancel()
 		return nil
 	}
 
 	r := newRepair(tmpLink, updateFn)
-	r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1")
+	r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1", nil)
 
 	err = os.WriteFile(tmpLink, []byte(modifyContent), 0755)
 	if err != nil {

client/internal/dns/file_unix.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -36,7 +38,7 @@ type fileConfigurator struct {
 	nbNameserverIP string
 }
 
-func newFileConfigurator() (hostManager, error) {
+func newFileConfigurator() (*fileConfigurator, error) {
 	fc := &fileConfigurator{}
 	fc.repair = newRepair(defaultResolvConfPath, fc.updateConfig)
 	return fc, nil
@@ -46,7 +48,7 @@ func (f *fileConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	backupFileExist := f.isBackupFileExist()
 	if !config.RouteAll {
 		if backupFileExist {
@@ -76,15 +78,15 @@ func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig) error {
 
 	f.repair.stopWatchFileChanges()
 
-	err = f.updateConfig(nbSearchDomains, f.nbNameserverIP, resolvConf)
+	err = f.updateConfig(nbSearchDomains, f.nbNameserverIP, resolvConf, stateManager)
 	if err != nil {
 		return err
 	}
-	f.repair.watchFileChanges(nbSearchDomains, f.nbNameserverIP)
+	f.repair.watchFileChanges(nbSearchDomains, f.nbNameserverIP, stateManager)
 	return nil
 }
 
-func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP string, cfg *resolvConf) error {
+func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP string, cfg *resolvConf, stateManager *statemanager.Manager) error {
 	searchDomainList := mergeSearchDomains(nbSearchDomains, cfg.searchDomains)
 	nameServers := generateNsList(nbNameserverIP, cfg)
 
@@ -107,7 +109,7 @@ func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP
 	log.Infof("created a NetBird managed %s file with the DNS settings. Added %d search domains. Search list: %s", defaultResolvConfPath, len(searchDomainList), searchDomainList)
 
 	// create another backup for unclean shutdown detection right after overwriting the original resolv.conf
-	if err := createUncleanShutdownIndicator(fileDefaultResolvConfBackupLocation, fileManager, nbNameserverIP); err != nil {
+	if err := createUncleanShutdownIndicator(fileDefaultResolvConfBackupLocation, nbNameserverIP, stateManager); err != nil {
 		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
 	}
 
@@ -145,10 +147,6 @@ func (f *fileConfigurator) restore() error {
 		return fmt.Errorf("restoring %s from %s: %w", defaultResolvConfPath, fileDefaultResolvConfBackupLocation, err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return os.RemoveAll(fileDefaultResolvConfBackupLocation)
 }
 
@@ -176,7 +174,7 @@ func (f *fileConfigurator) restoreUncleanShutdownDNS(storedDNSAddress *netip.Add
 		return restoreResolvConfFile()
 	}
 
-	log.Info("restoring unclean shutdown: first current nameserver differs from saved nameserver pre-netbird: not restoring")
+	log.Infof("restoring unclean shutdown: first current nameserver differs from saved nameserver pre-netbird: %s (current) vs %s (stored): not restoring", currentDNSAddress, storedDNSAddress)
 	return nil
 }
 
@@ -192,10 +190,6 @@ func restoreResolvConfFile() error {
 		return fmt.Errorf("restoring %s from %s: %w", defaultResolvConfPath, fileUncleanShutdownResolvConfLocation, err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf file: %s", err)
-	}
-
 	return nil
 }
 

client/internal/dns/host.go

@@ -5,14 +5,14 @@ import (
 	"net/netip"
 	"strings"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
 type hostManager interface {
-	applyDNSConfig(config HostDNSConfig) error
+	applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error
 	restoreHostDNS() error
 	supportCustomPort() bool
-	restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error
 }
 
 type SystemDNSSettings struct {
@@ -35,15 +35,15 @@ type DomainConfig struct {
 }
 
 type mockHostConfigurator struct {
-	applyDNSConfigFunc            func(config HostDNSConfig) error
+	applyDNSConfigFunc            func(config HostDNSConfig, stateManager *statemanager.Manager) error
 	restoreHostDNSFunc            func() error
 	supportCustomPortFunc         func() bool
 	restoreUncleanShutdownDNSFunc func(*netip.Addr) error
 }
 
-func (m *mockHostConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (m *mockHostConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	if m.applyDNSConfigFunc != nil {
-		return m.applyDNSConfigFunc(config)
+		return m.applyDNSConfigFunc(config, stateManager)
 	}
 	return fmt.Errorf("method applyDNSSettings is not implemented")
 }
@@ -62,16 +62,9 @@ func (m *mockHostConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (m *mockHostConfigurator) restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error {
-	if m.restoreUncleanShutdownDNSFunc != nil {
-		return m.restoreUncleanShutdownDNSFunc(storedDNSAddress)
-	}
-	return fmt.Errorf("method restoreUncleanShutdownDNS is not implemented")
-}
-
 func newNoopHostMocker() hostManager {
 	return &mockHostConfigurator{
-		applyDNSConfigFunc:            func(config HostDNSConfig) error { return nil },
+		applyDNSConfigFunc:            func(config HostDNSConfig, stateManager *statemanager.Manager) error { return nil },
 		restoreHostDNSFunc:            func() error { return nil },
 		supportCustomPortFunc:         func() bool { return true },
 		restoreUncleanShutdownDNSFunc: func(*netip.Addr) error { return nil },

client/internal/dns/host_android.go

@@ -1,15 +1,17 @@
 package dns
 
-import "net/netip"
+import (
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
 
 type androidHostManager struct {
 }
 
-func newHostManager() (hostManager, error) {
+func newHostManager() (*androidHostManager, error) {
 	return &androidHostManager{}, nil
 }
 
-func (a androidHostManager) applyDNSConfig(config HostDNSConfig) error {
+func (a androidHostManager) applyDNSConfig(HostDNSConfig, *statemanager.Manager) error {
 	return nil
 }
 
@@ -20,7 +22,3 @@ func (a androidHostManager) restoreHostDNS() error {
 func (a androidHostManager) supportCustomPort() bool {
 	return false
 }
-
-func (a androidHostManager) restoreUncleanShutdownDNS(*netip.Addr) error {
-	return nil
-}

client/internal/dns/host_darwin.go

@@ -8,12 +8,13 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"net/netip"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -37,7 +38,7 @@ type systemConfigurator struct {
 	systemDNSSettings SystemDNSSettings
 }
 
-func newHostManager() (hostManager, error) {
+func newHostManager() (*systemConfigurator, error) {
 	return &systemConfigurator{
 		createdKeys: make(map[string]struct{}),
 	}, nil
@@ -47,12 +48,11 @@ func (s *systemConfigurator) supportCustomPort() bool {
 	return true
 }
 
-func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 
-	// create a file for unclean shutdown detection
-	if err := createUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to create unclean shutdown file: %s", err)
+	if err := stateManager.UpdateState(&ShutdownState{}); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	var (
@@ -123,10 +123,6 @@ func (s *systemConfigurator) restoreHostDNS() error {
 		}
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown file: %s", err)
-	}
-
 	return nil
 }
 
@@ -320,7 +316,7 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) {
 	return primaryService, router, nil
 }
 
-func (s *systemConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
+func (s *systemConfigurator) restoreUncleanShutdownDNS() error {
 	if err := s.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via scutil: %w", err)
 	}

client/internal/dns/host_ios.go

@@ -3,9 +3,10 @@ package dns
 import (
 	"encoding/json"
 	"fmt"
-	"net/netip"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 type iosHostManager struct {
@@ -13,13 +14,13 @@ type iosHostManager struct {
 	config     HostDNSConfig
 }
 
-func newHostManager(dnsManager IosDnsManager) (hostManager, error) {
+func newHostManager(dnsManager IosDnsManager) (*iosHostManager, error) {
 	return &iosHostManager{
 		dnsManager: dnsManager,
 	}, nil
 }
 
-func (a iosHostManager) applyDNSConfig(config HostDNSConfig) error {
+func (a iosHostManager) applyDNSConfig(config HostDNSConfig, _ *statemanager.Manager) error {
 	jsonData, err := json.Marshal(config)
 	if err != nil {
 		return fmt.Errorf("marshal: %w", err)
@@ -37,7 +38,3 @@ func (a iosHostManager) restoreHostDNS() error {
 func (a iosHostManager) supportCustomPort() bool {
 	return false
 }
-
-func (a iosHostManager) restoreUncleanShutdownDNS(*netip.Addr) error {
-	return nil
-}

client/internal/dns/host_unix.go

@@ -4,9 +4,9 @@ package dns
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
+	"net/netip"
 	"os"
 	"strings"
 
@@ -21,27 +21,8 @@ const (
 	resolvConfManager
 )
 
-var ErrUnknownOsManagerType = errors.New("unknown os manager type")
-
 type osManagerType int
 
-func newOsManagerType(osManager string) (osManagerType, error) {
-	switch osManager {
-	case "netbird":
-		return fileManager, nil
-	case "file":
-		return netbirdManager, nil
-	case "networkManager":
-		return networkManager, nil
-	case "systemd":
-		return systemdManager, nil
-	case "resolvconf":
-		return resolvConfManager, nil
-	default:
-		return 0, ErrUnknownOsManagerType
-	}
-}
-
 func (t osManagerType) String() string {
 	switch t {
 	case netbirdManager:
@@ -59,6 +40,11 @@ func (t osManagerType) String() string {
 	}
 }
 
+type restoreHostManager interface {
+	hostManager
+	restoreUncleanShutdownDNS(*netip.Addr) error
+}
+
 func newHostManager(wgInterface string) (hostManager, error) {
 	osManager, err := getOSDNSManagerType()
 	if err != nil {
@@ -69,7 +55,7 @@ func newHostManager(wgInterface string) (hostManager, error) {
 	return newHostManagerFromType(wgInterface, osManager)
 }
 
-func newHostManagerFromType(wgInterface string, osManager osManagerType) (hostManager, error) {
+func newHostManagerFromType(wgInterface string, osManager osManagerType) (restoreHostManager, error) {
 	switch osManager {
 	case networkManager:
 		return newNetworkManagerDbusConfigurator(wgInterface)

client/internal/dns/host_windows.go

@@ -3,11 +3,12 @@ package dns
 import (
 	"fmt"
 	"io"
-	"net/netip"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/registry"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -31,7 +32,7 @@ type registryConfigurator struct {
 	routingAll bool
 }
 
-func newHostManager(wgInterface WGIface) (hostManager, error) {
+func newHostManager(wgInterface WGIface) (*registryConfigurator, error) {
 	guid, err := wgInterface.GetInterfaceGUIDString()
 	if err != nil {
 		return nil, err
@@ -39,7 +40,7 @@ func newHostManager(wgInterface WGIface) (hostManager, error) {
 	return newHostManagerWithGuid(guid)
 }
 
-func newHostManagerWithGuid(guid string) (hostManager, error) {
+func newHostManagerWithGuid(guid string) (*registryConfigurator, error) {
 	return &registryConfigurator{
 		guid: guid,
 	}, nil
@@ -49,7 +50,7 @@ func (r *registryConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 	if config.RouteAll {
 		err = r.addDNSSetupForAll(config.ServerIP)
@@ -65,9 +66,8 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig) error {
 		log.Infof("removed %s as main DNS forwarder for this peer", config.ServerIP)
 	}
 
-	// create a file for unclean shutdown detection
-	if err := createUncleanShutdownIndicator(r.guid); err != nil {
-		log.Errorf("failed to create unclean shutdown file: %s", err)
+	if err := stateManager.UpdateState(&ShutdownState{Guid: r.guid}); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	var (
@@ -160,10 +160,6 @@ func (r *registryConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("remove interface registry key: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown file: %s", err)
-	}
-
 	return nil
 }
 
@@ -221,7 +217,7 @@ func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) {
 	return regKey, nil
 }
 
-func (r *registryConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
+func (r *registryConfigurator) restoreUncleanShutdownDNS() error {
 	if err := r.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via registry: %w", err)
 	}

client/internal/dns/network_manager_unix.go

@@ -16,6 +16,7 @@ import (
 	"github.com/miekg/dns"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbversion "github.com/netbirdio/netbird/version"
 )
 
@@ -53,6 +54,7 @@ var supportedNetworkManagerVersionConstraints = []string{
 type networkManagerDbusConfigurator struct {
 	dbusLinkObject dbus.ObjectPath
 	routingAll     bool
+	ifaceName      string
 }
 
 // the types below are based on dbus specification, each field is mapped to a dbus type
@@ -77,7 +79,7 @@ func (s networkManagerConnSettings) cleanDeprecatedSettings() {
 	}
 }
 
-func newNetworkManagerDbusConfigurator(wgInterface string) (hostManager, error) {
+func newNetworkManagerDbusConfigurator(wgInterface string) (*networkManagerDbusConfigurator, error) {
 	obj, closeConn, err := getDbusObject(networkManagerDest, networkManagerDbusObjectNode)
 	if err != nil {
 		return nil, fmt.Errorf("get nm dbus: %w", err)
@@ -93,6 +95,7 @@ func newNetworkManagerDbusConfigurator(wgInterface string) (hostManager, error)
 
 	return &networkManagerDbusConfigurator{
 		dbusLinkObject: dbus.ObjectPath(s),
+		ifaceName:      wgInterface,
 	}, nil
 }
 
@@ -100,7 +103,7 @@ func (n *networkManagerDbusConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	connSettings, configVersion, err := n.getAppliedConnectionSettings()
 	if err != nil {
 		return fmt.Errorf("retrieving the applied connection settings, error: %w", err)
@@ -151,10 +154,12 @@ func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig) er
 	connSettings[networkManagerDbusIPv4Key][networkManagerDbusDNSPriorityKey] = dbus.MakeVariant(priority)
 	connSettings[networkManagerDbusIPv4Key][networkManagerDbusDNSSearchKey] = dbus.MakeVariant(newDomainList)
 
-	// create a backup for unclean shutdown detection before adding domains, as these might end up in the resolv.conf file.
-	// The file content itself is not important for network-manager restoration
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, networkManager, dnsIP.String()); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: networkManager,
+		WgIface:     n.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	log.Infof("adding %d search domains and %d match domains. Search list: %s , Match list: %s", len(searchDomains), len(matchDomains), searchDomains, matchDomains)
@@ -171,10 +176,6 @@ func (n *networkManagerDbusConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("delete connection settings: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return nil
 }
 

client/internal/dns/resolvconf_unix.go

@@ -9,6 +9,8 @@ import (
 	"os/exec"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const resolvconfCommand = "resolvconf"
@@ -22,7 +24,7 @@ type resolvconf struct {
 }
 
 // supported "openresolv" only
-func newResolvConfConfigurator(wgInterface string) (hostManager, error) {
+func newResolvConfConfigurator(wgInterface string) (*resolvconf, error) {
 	resolvConfEntries, err := parseDefaultResolvConf()
 	if err != nil {
 		log.Errorf("could not read original search domains from %s: %s", defaultResolvConfPath, err)
@@ -40,7 +42,7 @@ func (r *resolvconf) supportCustomPort() bool {
 	return false
 }
 
-func (r *resolvconf) applyDNSConfig(config HostDNSConfig) error {
+func (r *resolvconf) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 	if !config.RouteAll {
 		err = r.restoreHostDNS()
@@ -60,9 +62,12 @@ func (r *resolvconf) applyDNSConfig(config HostDNSConfig) error {
 		append([]string{config.ServerIP}, r.originalNameServers...),
 		options)
 
-	// create a backup for unclean shutdown detection before the resolv.conf is changed
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, resolvConfManager, config.ServerIP); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: resolvConfManager,
+		WgIface:     r.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	err = r.applyConfig(buf)
@@ -79,11 +84,7 @@ func (r *resolvconf) restoreHostDNS() error {
 	cmd := exec.Command(resolvconfCommand, "-f", "-d", r.ifaceName)
 	_, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("removing resolvconf configuration for %s interface, error: %w", r.ifaceName, err)
-	}
-
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
+		return fmt.Errorf("removing resolvconf configuration for %s interface: %w", r.ifaceName, err)
 	}
 
 	return nil
@@ -95,7 +96,7 @@ func (r *resolvconf) applyConfig(content bytes.Buffer) error {
 	cmd.Stdin = &content
 	_, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("applying resolvconf configuration for %s interface, error: %w", r.ifaceName, err)
+		return fmt.Errorf("applying resolvconf configuration for %s interface: %w", r.ifaceName, err)
 	}
 	return nil
 }

client/internal/dns/server.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/netbirdio/netbird/client/internal/listener"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
@@ -63,6 +64,7 @@ type DefaultServer struct {
 	iosDnsManager        IosDnsManager
 
 	statusRecorder *peer.Status
+	stateManager   *statemanager.Manager
 }
 
 type handlerWithStop interface {
@@ -77,12 +79,7 @@ type muxUpdate struct {
 }
 
 // NewDefaultServer returns a new dns server
-func NewDefaultServer(
-	ctx context.Context,
-	wgInterface WGIface,
-	customAddress string,
-	statusRecorder *peer.Status,
-) (*DefaultServer, error) {
+func NewDefaultServer(ctx context.Context, wgInterface WGIface, customAddress string, statusRecorder *peer.Status, stateManager *statemanager.Manager) (*DefaultServer, error) {
 	var addrPort *netip.AddrPort
 	if customAddress != "" {
 		parsedAddrPort, err := netip.ParseAddrPort(customAddress)
@@ -99,7 +96,7 @@ func NewDefaultServer(
 		dnsService = newServiceViaListener(wgInterface, addrPort)
 	}
 
-	return newDefaultServer(ctx, wgInterface, dnsService, statusRecorder), nil
+	return newDefaultServer(ctx, wgInterface, dnsService, statusRecorder, stateManager), nil
 }
 
 // NewDefaultServerPermanentUpstream returns a new dns server. It optimized for mobile systems
@@ -112,7 +109,7 @@ func NewDefaultServerPermanentUpstream(
 	statusRecorder *peer.Status,
 ) *DefaultServer {
 	log.Debugf("host dns address list is: %v", hostsDnsList)
-	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil)
 	ds.hostsDNSHolder.set(hostsDnsList)
 	ds.permanent = true
 	ds.addHostRootZone()
@@ -130,12 +127,12 @@ func NewDefaultServerIos(
 	iosDnsManager IosDnsManager,
 	statusRecorder *peer.Status,
 ) *DefaultServer {
-	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil)
 	ds.iosDnsManager = iosDnsManager
 	return ds
 }
 
-func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService service, statusRecorder *peer.Status) *DefaultServer {
+func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService service, statusRecorder *peer.Status, stateManager *statemanager.Manager) *DefaultServer {
 	ctx, stop := context.WithCancel(ctx)
 	defaultServer := &DefaultServer{
 		ctx:       ctx,
@@ -147,6 +144,7 @@ func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService servi
 		},
 		wgInterface:    wgInterface,
 		statusRecorder: statusRecorder,
+		stateManager:   stateManager,
 		hostsDNSHolder: newHostsDNSHolder(),
 	}
 
@@ -169,6 +167,7 @@ func (s *DefaultServer) Initialize() (err error) {
 		}
 	}
 
+	s.stateManager.RegisterState(&ShutdownState{})
 	s.hostManager, err = s.initialize()
 	if err != nil {
 		return fmt.Errorf("initialize: %w", err)
@@ -191,9 +190,10 @@ func (s *DefaultServer) Stop() {
 	s.ctxCancel()
 
 	if s.hostManager != nil {
-		err := s.hostManager.restoreHostDNS()
-		if err != nil {
-			log.Error(err)
+		if err := s.hostManager.restoreHostDNS(); err != nil {
+			log.Error("failed to restore host DNS settings: ", err)
+		} else if err := s.stateManager.DeleteState(&ShutdownState{}); err != nil {
+			log.Errorf("failed to delete shutdown dns state: %v", err)
 		}
 	}
 
@@ -318,10 +318,17 @@ func (s *DefaultServer) applyConfiguration(update nbdns.Config) error {
 		hostUpdate.RouteAll = false
 	}
 
-	if err = s.hostManager.applyDNSConfig(hostUpdate); err != nil {
+	if err = s.hostManager.applyDNSConfig(hostUpdate, s.stateManager); err != nil {
 		log.Error(err)
 	}
 
+	go func() {
+		// persist dns state right away
+		if err := s.stateManager.PersistState(s.ctx); err != nil {
+			log.Errorf("Failed to persist dns state: %v", err)
+		}
+	}()
+
 	if s.searchDomainNotifier != nil {
 		s.searchDomainNotifier.onNewSearchDomains(s.SearchDomains())
 	}
@@ -367,6 +374,8 @@ func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.Nam
 			continue
 		}
 
+		log.Debugf("received a nameserver group with %#v nameservers for \"%s\" domains", nsGroup.NameServers, nsGroup.Domains)
+
 		handler, err := newUpstreamResolver(
 			s.ctx,
 			s.wgInterface.Name(),
@@ -437,9 +446,11 @@ func (s *DefaultServer) updateMux(muxUpdates []muxUpdate) {
 	var isContainRootUpdate bool
 
 	for _, update := range muxUpdates {
+		log.Debugf("registering a new handler for domain %s", update.domain)
 		s.service.RegisterMux(update.domain, update.handler)
 		muxUpdateMap[update.domain] = update.handler
 		if existingHandler, ok := s.dnsMuxMap[update.domain]; ok {
+			log.Debugf("stopping the existing handler for domain %s", update.domain)
 			existingHandler.stop()
 		}
 
@@ -455,6 +466,7 @@ func (s *DefaultServer) updateMux(muxUpdates []muxUpdate) {
 				s.addHostRootZone()
 				existingHandler.stop()
 			} else {
+				log.Debugf("stopping the existing handler for domain %s", key)
 				existingHandler.stop()
 				s.service.DeregisterMux(key)
 			}
@@ -521,10 +533,16 @@ func (s *DefaultServer) upstreamCallbacks(
 			}
 		}
 
-		if err := s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+		if err := s.hostManager.applyDNSConfig(s.currentConfig, s.stateManager); err != nil {
 			l.Errorf("Failed to apply nameserver deactivation on the host: %v", err)
 		}
 
+		go func() {
+			if err := s.stateManager.PersistState(s.ctx); err != nil {
+				l.Errorf("Failed to persist dns state: %v", err)
+			}
+		}()
+
 		if runtime.GOOS == "android" && nsGroup.Primary && len(s.hostsDNSHolder.get()) > 0 {
 			s.addHostRootZone()
 		}
@@ -551,7 +569,7 @@ func (s *DefaultServer) upstreamCallbacks(
 			s.currentConfig.RouteAll = true
 			s.service.RegisterMux(nbdns.RootZone, handler)
 		}
-		if err := s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+		if err := s.hostManager.applyDNSConfig(s.currentConfig, s.stateManager); err != nil {
 			l.WithError(err).Error("reactivate temporary disabled nameserver group, DNS update apply")
 		}
 

client/internal/dns/server_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/device"
 	pfmock "github.com/netbirdio/netbird/client/iface/mocks"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/formatter"
@@ -267,7 +268,17 @@ func TestUpdateDNSServer(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+
+			opts := iface.WGIFaceOpts{
+				IFaceName:    fmt.Sprintf("utun230%d", n),
+				Address:      fmt.Sprintf("100.66.100.%d/32", n+1),
+				WGPort:       33100,
+				WGPrivKey:    privKey.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+
+			wgIface, err := iface.NewWGIFace(opts)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -281,7 +292,7 @@ func TestUpdateDNSServer(t *testing.T) {
 					t.Log(err)
 				}
 			}()
-			dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{})
+			dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{}, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -345,7 +356,15 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 	}
 
 	privKey, _ := wgtypes.GeneratePrivateKey()
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+	opts := iface.WGIFaceOpts{
+		IFaceName:    "utun2301",
+		Address:      "100.66.100.1/32",
+		WGPort:       33100,
+		WGPrivKey:    privKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+	wgIface, err := iface.NewWGIFace(opts)
 	if err != nil {
 		t.Errorf("build interface wireguard: %v", err)
 		return
@@ -382,7 +401,7 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 		return
 	}
 
-	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{})
+	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{}, nil)
 	if err != nil {
 		t.Errorf("create DNS server: %v", err)
 		return
@@ -477,7 +496,7 @@ func TestDNSServerStartStop(t *testing.T) {
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			dnsServer, err := NewDefaultServer(context.Background(), &mocWGIface{}, testCase.addrPort, &peer.Status{})
+			dnsServer, err := NewDefaultServer(context.Background(), &mocWGIface{}, testCase.addrPort, &peer.Status{}, nil)
 			if err != nil {
 				t.Fatalf("%v", err)
 			}
@@ -536,6 +555,7 @@ func TestDNSServerStartStop(t *testing.T) {
 func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	hostManager := &mockHostConfigurator{}
 	server := DefaultServer{
+		ctx:     context.Background(),
 		service: NewServiceViaMemory(&mocWGIface{}),
 		localResolver: &localResolver{
 			registeredMap: make(registrationMap),
@@ -552,7 +572,7 @@ func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	}
 
 	var domainsUpdate string
-	hostManager.applyDNSConfigFunc = func(config HostDNSConfig) error {
+	hostManager.applyDNSConfigFunc = func(config HostDNSConfig, statemanager *statemanager.Manager) error {
 		domains := []string{}
 		for _, item := range config.Domains {
 			if item.Disabled {
@@ -762,7 +782,7 @@ func TestDNSPermanent_matchOnly(t *testing.T) {
 						Port:   53,
 					},
 				},
-				Domains: []string{"customdomain.com"},
+				Domains: []string{"google.com"},
 				Primary: false,
 			},
 		},
@@ -784,7 +804,7 @@ func TestDNSPermanent_matchOnly(t *testing.T) {
 	if ips[0] != zoneRecords[0].RData {
 		t.Fatalf("invalid zone record: %v", err)
 	}
-	_, err = resolver.LookupHost(context.Background(), "customdomain.com")
+	_, err = resolver.LookupHost(context.Background(), "google.com")
 	if err != nil {
 		t.Errorf("failed to resolve: %s", err)
 	}
@@ -803,7 +823,17 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) {
 	}
 
 	privKey, _ := wgtypes.GeneratePrivateKey()
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+
+	opts := iface.WGIFaceOpts{
+		IFaceName:    "utun2301",
+		Address:      "100.66.100.2/24",
+		WGPort:       33100,
+		WGPrivKey:    privKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	wgIface, err := iface.NewWGIFace(opts)
 	if err != nil {
 		t.Fatalf("build interface wireguard: %v", err)
 		return nil, err