upgrade zcalusic/sysinfo@v1.1.3 (add serial for ARM arch) (#2954)

Signed-off-by: Edouard Vanbelle <edouard.vanbelle@shadow.tech>

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.go]
+indent_style = tab

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [netbirdio]

.github/ISSUE_TEMPLATE/bug-issue-report.md

@@ -31,9 +31,14 @@ Please specify whether you use NetBird Cloud or self-host NetBird's control plan
 
 `netbird version`
 
-**NetBird status -d output:**
+**NetBird status -dA output:**
 
-If applicable, add the `netbird status -d' command output.
+If applicable, add the `netbird status -dA' command output.
+
+**Do you face any (non-mobile) client issues?**
+
+Please provide the file created by `netbird debug for 1m -AS`.
+We advise reviewing the anonymized files for any remaining PII.
 
 **Screenshots**
 

.github/workflows/golang-test-darwin.yml

@@ -18,14 +18,14 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: macos-go-${{ hashFiles('**/go.sum') }}
@@ -42,4 +42,4 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...
+        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...

.github/workflows/golang-test-freebsd.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - name: Test in FreeBSD
@@ -21,19 +21,26 @@ jobs:
         uses: vmactions/freebsd-vm@v1
         with:
           usesh: true
+          copyback: false
+          release: "14.1"
           prepare: |
-            pkg install -y curl
-            pkg install -y git
+            pkg install -y go
 
+          # -x - to print all executed commands
+          # -e - to faile on first error
           run: |
-            set -x
-            curl -o go.tar.gz https://go.dev/dl/go1.21.11.freebsd-amd64.tar.gz -L
-            tar zxf go.tar.gz
-            mv go /usr/local/go
-            ln -s /usr/local/go/bin/go /usr/local/bin/go
-            go mod tidy
-            go test -timeout 5m -p 1 ./iface/...
-            go test -timeout 5m -p 1 ./client/... 
-            cd client
-            go build .
-            cd ..
+            set -e -x
+            time go build -o netbird client/main.go
+            # check all component except management, since we do not support management server on freebsd
+            time go test -timeout 1m -failfast ./base62/...
+            # NOTE: without -p1 `client/internal/dns` will fail becasue of `listen udp4 :33100: bind: address already in use`
+            time go test -timeout 8m -failfast -p 1 ./client/...
+            time go test -timeout 1m -failfast ./dns/...
+            time go test -timeout 1m -failfast ./encryption/...
+            time go test -timeout 1m -failfast ./formatter/...
+            time go test -timeout 1m -failfast ./client/iface/...
+            time go test -timeout 1m -failfast ./route/...
+            time go test -timeout 1m -failfast ./sharedsock/...
+            time go test -timeout 1m -failfast ./signal/...
+            time go test -timeout 1m -failfast ./util/...
+            time go test -timeout 1m -failfast ./version/...

.github/workflows/golang-test-linux.yml

@@ -13,19 +13,20 @@ concurrency:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
         arch: [ '386','amd64' ]
         store: [ 'sqlite', 'postgres']
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -33,7 +34,7 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
@@ -49,18 +50,24 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
 
-  test_client_on_docker:
-    runs-on: ubuntu-20.04
+  benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ '386','amd64' ]
+        store: [ 'sqlite', 'postgres' ]
+    runs-on: ubuntu-22.04
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
+
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -68,7 +75,42 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: matrix.arch == '386'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
+
+      - name: Install modules
+        run: go mod tidy
+
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
+      - name: Test
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
+
+  test_client_on_docker:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Checkout code
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
@@ -79,9 +121,6 @@ jobs:
       - name: check git status
         run: git --no-pager diff --exit-code
 
-      - name: Generate Iface Test bin
-        run: CGO_ENABLED=0 go test -c -o iface-testing.bin ./iface/
-
       - name: Generate Shared Sock Test bin
         run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock
 
@@ -98,7 +137,7 @@ jobs:
         run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal
 
       - name: Generate Peer Test bin
-        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/...
+        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/
 
       - run: chmod +x *testing.bin
 
@@ -106,7 +145,7 @@ jobs:
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/sharedsock --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/sharedsock-testing.bin -test.timeout 5m -test.parallel 1
 
       - name: Run Iface tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/iface --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/iface-testing.bin -test.timeout 5m -test.parallel 1
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/netbird -v /tmp/cache:/tmp/cache -v /tmp/modcache:/tmp/modcache -w /netbird -e GOCACHE=/tmp/cache -e GOMODCACHE=/tmp/modcache -e CGO_ENABLED=0 golang:1.23-alpine go test  -test.timeout 5m -test.parallel 1 ./client/iface/...
 
       - name: Run RouteManager tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/routemanager-testing.bin  -test.timeout 5m -test.parallel 1
@@ -124,4 +163,4 @@ jobs:
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="sqlite" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
 
       - name: Run Peer tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer  --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/peer-testing.bin  -test.timeout 5m -test.parallel 1
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer  --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/peer-testing.bin  -test.timeout 5m -test.parallel 1

.github/workflows/golang-test-windows.yml

@@ -17,13 +17,13 @@ jobs:
     runs-on: windows-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         id: go
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
       - name: Download wintun
         uses: carlosperate/download-file-action@v2

.github/workflows/golangci-lint.yml

@@ -15,11 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          ignore_words_list: erro,clienta,hastable,
+          ignore_words_list: erro,clienta,hastable,iif,groupd
           skip: go.mod,go.sum
           only_warn: 1
   golangci:
@@ -32,15 +32,15 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check for duplicate constants
         if: matrix.os == 'ubuntu-latest'
         run: |
           ! awk '/const \(/,/)/{print $0}' management/server/activity/codes.go | grep -o '= [0-9]*' | sort | uniq -d | grep .
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
           cache: false
       - name: Install dependencies
         if: matrix.os == 'ubuntu-latest'
@@ -49,4 +49,4 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           version: latest
-          args: --timeout=12m
+          args: --timeout=12m

.github/workflows/install-script-test.yml

@@ -13,6 +13,7 @@ concurrency:
 jobs:
   test-install-script:
     strategy:
+      fail-fast: false
       max-parallel: 2
       matrix:
         os: [ubuntu-latest, macos-latest]
@@ -21,7 +22,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: run install script
         env:

.github/workflows/mobile-build-validation.yml

@@ -15,23 +15,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
       - name: Setup Android SDK
         uses: android-actions/setup-android@v3
         with:
           cmdline-tools-version: 8512546
       - name: Setup Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: "11"
           distribution: "adopt"
       - name: NDK Cache
         id: ndk-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /usr/local/lib/android/sdk/ndk
           key: ndk-cache-23.1.7779620
@@ -50,11 +50,11 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
       - name: install gomobile
         run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20240404231514-09dbf07665ed
       - name: gomobile init
@@ -62,4 +62,4 @@ jobs:
       - name: build iOS netbird lib
         run: PATH=$PATH:$(go env GOPATH) gomobile bind -target=ios -bundleid=io.netbird.framework -ldflags="-X github.com/netbirdio/netbird/version.version=buildtest" -o ./NetBirdSDK.xcframework ./client/ios/NetBirdSDK
         env:
-          CGO_ENABLED: 0
+          CGO_ENABLED: 0

.github/workflows/release.yml

@@ -3,15 +3,16 @@ name: Release
 on:
   push:
     tags:
-      - 'v*'
+      - "v*"
     branches:
       - main
   pull_request:
 
-
 env:
-  SIGN_PIPE_VER: "v0.0.11"
-  GORELEASER_VER: "v1.14.1"
+  SIGN_PIPE_VER: "v0.0.17"
+  GORELEASER_VER: "v2.3.2"
+  PRODUCT_NAME: "NetBird"
+  COPYRIGHT: "Wiretrustee UG (haftungsbeschreankt)"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
@@ -19,26 +20,30 @@ concurrency:
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       flags: ""
     steps:
+      - name: Parse semver string
+        id: semver_parser
+        uses: booxmedialtd/ws-action-parse-semver@v1
+        with:
+          input_string: ${{ (startsWith(github.ref, 'refs/tags/v') && github.ref) || 'refs/tags/v0.0.0' }}
+          version_extractor_regex: '\/v(.*)$'
+
       - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
         run: echo "flags=--snapshot" >> $GITHUB_ENV
-      -
-        name: Checkout
-        uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
-      -
-        name: Set up Go
-        uses: actions/setup-go@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.23"
           cache: false
-      -
-        name: Cache Go modules
-        uses: actions/cache@v3
+      - name: Cache Go modules
+        uses: actions/cache@v4
         with:
           path: |
             ~/go/pkg/mod
@@ -46,73 +51,57 @@ jobs:
           key: ${{ runner.os }}-go-releaser-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-releaser-
-      -
-        name: Install modules
+      - name: Install modules
         run: go mod tidy
-      -
-        name: check git status
+      - name: check git status
         run: git --no-pager diff --exit-code
-      -
-        name: Set up QEMU
+      - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      -
-        name: Login to Docker hub
+      - name: Login to Docker hub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:
-          username: netbirdio
+          username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Install OS build dependencies
         run: sudo apt update && sudo apt install -y -q gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
 
-      - name: Install rsrc
-        run: go install github.com/akavel/rsrc@v0.10.2
-      - name: Generate windows rsrc amd64
-        run: rsrc -arch amd64 -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_amd64.syso
-      - name: Generate windows rsrc arm64
-        run: rsrc -arch arm64 -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_arm64.syso
-      - name: Generate windows rsrc arm
-        run: rsrc -arch arm -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_arm.syso
-      - name: Generate windows rsrc 386
-        run: rsrc -arch 386 -ico client/ui/netbird.ico -manifest client/manifest.xml -o client/resources_windows_386.syso
-      -
-        name: Run GoReleaser
+      - name: Install goversioninfo
+        run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e
+      - name: Generate windows syso amd64
+        run: goversioninfo  -icon client/ui/netbird.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_amd64.syso
+      - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
-          args: release --rm-dist ${{ env.flags }}
+          args: release --clean ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
           UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
           UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
-      -
-        name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v3
+      - name: upload non tags for debug purposes
+        uses: actions/upload-artifact@v4
         with:
           name: release
           path: dist/
           retention-days: 3
-      -
-        name: upload linux packages
-        uses: actions/upload-artifact@v3
+      - name: upload linux packages
+        uses: actions/upload-artifact@v4
         with:
           name: linux-packages
           path: dist/netbird_linux**
           retention-days: 3
-      -
-        name: upload windows packages
-        uses: actions/upload-artifact@v3
+      - name: upload windows packages
+        uses: actions/upload-artifact@v4
         with:
           name: windows-packages
           path: dist/netbird_windows**
           retention-days: 3
-      -
-        name: upload macos packages
-        uses: actions/upload-artifact@v3
+      - name: upload macos packages
+        uses: actions/upload-artifact@v4
         with:
           name: macos-packages
           path: dist/netbird_darwin**
@@ -121,22 +110,29 @@ jobs:
   release_ui:
     runs-on: ubuntu-latest
     steps:
+      - name: Parse semver string
+        id: semver_parser
+        uses: booxmedialtd/ws-action-parse-semver@v1
+        with:
+          input_string: ${{ (startsWith(github.ref, 'refs/tags/v') && github.ref) || 'refs/tags/v0.0.0' }}
+          version_extractor_regex: '\/v(.*)$'
+
       - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
         run: echo "flags=--snapshot" >> $GITHUB_ENV
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.23"
           cache: false
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
-          path: | 
+          path: |
             ~/go/pkg/mod
             ~/.cache/go-build
           key: ${{ runner.os }}-ui-go-releaser-${{ hashFiles('**/go.sum') }}
@@ -151,22 +147,23 @@ jobs:
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libappindicator3-dev gir1.2-appindicator3-0.1 libxxf86vm-dev gcc-mingw-w64-x86-64
-      - name: Install rsrc
-        run: go install github.com/akavel/rsrc@v0.10.2
-      - name: Generate windows rsrc
-        run: rsrc -arch amd64 -ico client/ui/netbird.ico -manifest client/ui/manifest.xml -o client/ui/resources_windows_amd64.syso
+      - name: Install goversioninfo
+        run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e
+      - name: Generate windows syso amd64
+        run: goversioninfo -64 -icon client/ui/netbird.ico -manifest client/ui/manifest.xml -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_amd64.syso
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
-          args: release --config .goreleaser_ui.yaml --rm-dist ${{ env.flags }}
+          args: release --config .goreleaser_ui.yaml --clean ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
           UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
           UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
       - name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: release-ui
           path: dist/
@@ -177,20 +174,17 @@ jobs:
     steps:
       - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
         run: echo "flags=--snapshot" >> $GITHUB_ENV
-      -
-        name: Checkout
-        uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
-      -
-        name: Set up Go
-        uses: actions/setup-go@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.23"
           cache: false
-      -
-        name: Cache Go modules
-        uses: actions/cache@v3
+      - name: Cache Go modules
+        uses: actions/cache@v4
         with:
           path: |
             ~/go/pkg/mod
@@ -198,53 +192,35 @@ jobs:
           key: ${{ runner.os }}-ui-go-releaser-darwin-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-ui-go-releaser-darwin-
-      -
-        name: Install modules
+      - name: Install modules
         run: go mod tidy
-      - 
-        name: check git status
+      - name: check git status
         run: git --no-pager diff --exit-code
-      -
-        name: Run GoReleaser
+      - name: Run GoReleaser
         id: goreleaser
         uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
-          args: release --config .goreleaser_ui_darwin.yaml --rm-dist ${{ env.flags }}
+          args: release --config .goreleaser_ui_darwin.yaml --clean ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v3
+      - name: upload non tags for debug purposes
+        uses: actions/upload-artifact@v4
         with:
           name: release-ui-darwin
           path: dist/
           retention-days: 3
 
-  trigger_windows_signer:
+  trigger_signer:
     runs-on: ubuntu-latest
-    needs: [release,release_ui]
+    needs: [release, release_ui, release_ui_darwin]
     if: startsWith(github.ref, 'refs/tags/')
     steps:
-      - name: Trigger Windows binaries sign pipeline
+      - name: Trigger binaries sign pipelines
         uses: benc-uk/workflow-dispatch@v1
         with:
-          workflow: Sign windows bin and installer
+          workflow: Sign bin and installer
           repo: netbirdio/sign-pipelines
           ref: ${{ env.SIGN_PIPE_VER }}
           token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'
-
-  trigger_darwin_signer:
-    runs-on: ubuntu-latest
-    needs: [release,release_ui_darwin]
-    if: startsWith(github.ref, 'refs/tags/')
-    steps:
-      - name: Trigger Darwin App binaries sign pipeline
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: Sign darwin ui app with dispatch
-          repo: netbirdio/sign-pipelines
-          ref: ${{ env.SIGN_PIPE_VER }}
-          token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'
+          inputs: '{ "tag": "${{ github.ref }}", "skipRelease": false }'

.github/workflows/test-infrastructure-files.yml

@@ -18,7 +18,31 @@ concurrency:
 jobs:
   test-docker-compose:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        store: [ 'sqlite', 'postgres' ]
+    services:
+      postgres:
+        image: ${{ (matrix.store == 'postgres') && 'postgres' || '' }}
+        env:
+          POSTGRES_USER: netbird
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: netbird
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+        ports:
+          - 5432:5432
     steps:
+      - name: Set Database Connection String
+        run: |
+          if [ "${{ matrix.store }}" == "postgres" ]; then
+            echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN=host=$(hostname -I | awk '{print $1}') user=netbird password=postgres dbname=netbird port=5432" >> $GITHUB_ENV
+          else
+            echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN==" >> $GITHUB_ENV
+          fi
+
       - name: Install jq
         run: sudo apt-get install -y jq
 
@@ -26,12 +50,12 @@ jobs:
         run: sudo apt-get install -y curl
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21.x"
+          go-version: "1.23.x"
 
       - name: Cache Go modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -39,7 +63,7 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: cp setup.env
         run: cp infrastructure_files/tests/setup.env infrastructure_files/
@@ -58,7 +82,8 @@ jobs:
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
           CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
-          CI_NETBIRD_STORE_CONFIG_ENGINE: "sqlite"
+          CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
+          NETBIRD_STORE_ENGINE_POSTGRES_DSN: ${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}
           CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
 
       - name: check values
@@ -85,7 +110,8 @@ jobs:
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
           CI_NETBIRD_SIGNAL_PORT: 12345
-          CI_NETBIRD_STORE_CONFIG_ENGINE: "sqlite"
+          CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
+          NETBIRD_STORE_ENGINE_POSTGRES_DSN: '${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}$'
           CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
           CI_NETBIRD_TURN_EXTERNAL_IP: "1.2.3.4"
 
@@ -123,6 +149,14 @@ jobs:
           grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
           grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000"
           grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP
+          grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN"
+          # check relay values
+          grep "NB_EXPOSED_ADDRESS=$CI_NETBIRD_DOMAIN:33445" docker-compose.yml
+          grep "NB_LISTEN_ADDRESS=:33445" docker-compose.yml
+          grep '33445:33445' docker-compose.yml
+          grep -A 10 'relay:' docker-compose.yml | egrep 'NB_AUTH_SECRET=.+$'
+          grep -A 7 Relay management.json | grep "rel://$CI_NETBIRD_DOMAIN:33445"
+          grep -A 7 Relay management.json | egrep '"Secret": ".+"'
 
       - name: Install modules
         run: go mod tidy
@@ -148,26 +182,35 @@ jobs:
         run: |
           docker build -t netbirdio/signal:latest .
 
+      - name: Build relay binary
+        working-directory: relay
+        run: CGO_ENABLED=0 go build -o netbird-relay main.go
+
+      - name: Build relay docker image
+        working-directory: relay
+        run: |
+          docker build -t netbirdio/relay:latest .
+
       - name: run docker compose up
         working-directory: infrastructure_files/artifacts
         run: |
-          docker-compose up -d
+          docker compose up -d
           sleep 5
-          docker-compose ps
-          docker-compose logs --tail=20
+          docker compose ps
+          docker compose logs --tail=20
 
       - name: test running containers
         run: |
           count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running)
-          test $count -eq 4
+          test $count -eq 5 || docker compose logs
         working-directory: infrastructure_files/artifacts
 
       - name: test geolocation databases
         working-directory: infrastructure_files/artifacts
         run: |
           sleep 30
-          docker compose exec management ls -l /var/lib/netbird/ | grep -i GeoLite2-City.mmdb
-          docker compose exec management ls -l /var/lib/netbird/ | grep -i geonames.db
+          docker compose exec management ls -l /var/lib/netbird/ | grep -i GeoLite2-City_[0-9]*.mmdb
+          docker compose exec management ls -l /var/lib/netbird/ | grep -i geonames_[0-9]*.db
 
   test-getting-started-script:
     runs-on: ubuntu-latest
@@ -176,7 +219,7 @@ jobs:
         run: sudo apt-get install -y jq
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: run script with Zitadel PostgreSQL
         run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh
@@ -191,7 +234,7 @@ jobs:
         run: test -f management.json
 
       - name: test turnserver.conf file gen postgres
-        run: | 
+        run: |
           set -x
           test -f turnserver.conf
           grep external-ip turnserver.conf
@@ -202,12 +245,15 @@ jobs:
       - name: test dashboard.env file gen postgres
         run: test -f dashboard.env
 
+      - name: test relay.env file gen postgres
+        run: test -f relay.env
+
       - name: test zdb.env file gen postgres
         run: test -f zdb.env
 
       - name: Postgres run cleanup
         run: |
-          docker-compose down --volumes --rmi all
+          docker compose down --volumes --rmi all
           rm -rf docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json zdb.env
 
       - name: run script with Zitadel CockroachDB
@@ -226,7 +272,7 @@ jobs:
         run: test -f management.json
 
       - name: test turnserver.conf file gen CockroachDB
-        run: | 
+        run: |
           set -x
           test -f turnserver.conf
           grep external-ip turnserver.conf
@@ -237,20 +283,5 @@ jobs:
       - name: test dashboard.env file gen CockroachDB
         run: test -f dashboard.env
 
-  test-download-geolite2-script:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install jq
-        run: sudo apt-get update && sudo apt-get install -y unzip sqlite3
-
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: test script
-        run: bash -x infrastructure_files/download-geolite2.sh
-
-      - name: test mmdb file exists
-        run: test -f GeoLite2-City.mmdb
-
-      - name: test geonames file exists
-        run: test -f geonames.db
+      - name: test relay.env file gen CockroachDB
+        run: test -f relay.env

.gitignore

@@ -29,4 +29,3 @@ infrastructure_files/setup.env
 infrastructure_files/setup-*.env
 .vscode
 .DS_Store
-GeoLite2-City*

.goreleaser.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 project_name: netbird
 builds:
   - id: netbird
@@ -22,7 +24,7 @@ builds:
         goarch: 386
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
     tags:
       - load_wgnt_from_rsrc
 
@@ -42,19 +44,19 @@ builds:
       - softfloat
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
     tags:
       - load_wgnt_from_rsrc
 
   - id: netbird-mgmt
     dir: management
     env:
-    - CGO_ENABLED=1
-    - >-
-      {{- if eq .Runtime.Goos "linux" }}
-        {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }}
-        {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }}
-      {{- end }}
+      - CGO_ENABLED=1
+      - >-
+        {{- if eq .Runtime.Goos "linux" }}
+          {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }}
+          {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }}
+        {{- end }}
     binary: netbird-mgmt
     goos:
       - linux
@@ -64,7 +66,7 @@ builds:
       - arm
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: netbird-signal
     dir: signal
@@ -78,7 +80,24 @@ builds:
       - arm
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
+
+  - id: netbird-relay
+    dir: relay
+    env: [CGO_ENABLED=0]
+    binary: netbird-relay
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - arm
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: "{{ .CommitTimestamp }}"
+
+universal_binaries:
+  - id: netbird
 
 archives:
   - builds:
@@ -86,7 +105,6 @@ archives:
       - netbird-static
 
 nfpms:
-
   - maintainer: Netbird <dev@netbird.io>
     description: Netbird client.
     homepage: https://netbird.io/
@@ -161,6 +179,52 @@ dockers:
       - "--label=org.opencontainers.image.revision={{.FullCommit}}"
       - "--label=org.opencontainers.image.version={{.Version}}"
       - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-amd64
+    ids:
+      - netbird-relay
+    goarch: amd64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+    ids:
+      - netbird-relay
+    goarch: arm64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm
+    ids:
+      - netbird-relay
+    goarch: arm
+    goarm: 6
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
   - image_templates:
       - netbirdio/signal:{{ .Version }}-amd64
     ids:
@@ -313,6 +377,18 @@ docker_manifests:
       - netbirdio/netbird:{{ .Version }}-arm
       - netbirdio/netbird:{{ .Version }}-amd64
 
+  - name_template: netbirdio/relay:{{ .Version }}
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
+  - name_template: netbirdio/relay:latest
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
   - name_template: netbirdio/signal:{{ .Version }}
     image_templates:
       - netbirdio/signal:{{ .Version }}-arm64v8
@@ -344,10 +420,9 @@ docker_manifests:
       - netbirdio/management:{{ .Version }}-debug-amd64
 
 brews:
-  -
-    ids:
+  - ids:
       - default
-    tap:
+    repository:
       owner: netbirdio
       name: homebrew-tap
       token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}"
@@ -364,7 +439,7 @@ brews:
 uploads:
   - name: debian
     ids:
-    - netbird-deb
+      - netbird-deb
     mode: archive
     target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package=
     username: dev@wiretrustee.com
@@ -386,4 +461,4 @@ checksum:
 release:
   extra_files:
     - glob: ./infrastructure_files/getting-started-with-zitadel.sh
-    - glob: ./release_files/install.sh
+    - glob: ./release_files/install.sh

.goreleaser_ui.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 project_name: netbird-ui
 builds:
   - id: netbird-ui
@@ -11,9 +13,7 @@ builds:
       - amd64
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    tags:
-      - legacy_appindicator
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: netbird-ui-windows
     dir: client/ui
@@ -28,7 +28,7 @@ builds:
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
       - -H windowsgui
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
 
 archives:
   - id: linux-arch
@@ -41,7 +41,6 @@ archives:
       - netbird-ui-windows
 
 nfpms:
-
   - maintainer: Netbird <dev@netbird.io>
     description: Netbird client UI.
     homepage: https://netbird.io/
@@ -79,7 +78,7 @@ nfpms:
 uploads:
   - name: debian
     ids:
-    - netbird-ui-deb
+      - netbird-ui-deb
     mode: archive
     target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package=
     username: dev@wiretrustee.com

.goreleaser_ui_darwin.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 project_name: netbird-ui
 builds:
   - id: netbird-ui-darwin
@@ -17,10 +19,13 @@ builds:
       - softfloat
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
     tags:
       - load_wgnt_from_rsrc
 
+universal_binaries:
+  - id: netbird-ui-darwin
+
 archives:
   - builds:
       - netbird-ui-darwin
@@ -28,4 +33,4 @@ archives:
 checksum:
   name_template: "{{ .ProjectName }}_darwin_checksums.txt"
 changelog:
-  skip: true
+  disable: true

CONTRIBUTING.md

@@ -96,7 +96,7 @@ They can be executed from the repository root before every push or PR:
 
 **Goreleaser**
 ```shell
-goreleaser --snapshot --rm-dist
+goreleaser build --snapshot --clean
 ```
 **golangci-lint**
 ```shell

README.md

@@ -10,13 +10,19 @@
   <img width="234" src="docs/media/logo-full.png"/>
 </p>
   <p>
+   <a href="https://img.shields.io/badge/license-BSD--3-blue)">
+       <img src="https://sonarcloud.io/api/project_badges/measure?project=netbirdio_netbird&metric=alert_status" />
+     </a> 
      <a href="https://github.com/netbirdio/netbird/blob/main/LICENSE">
        <img src="https://img.shields.io/badge/license-BSD--3-blue" />
      </a> 
-   <a href="https://www.codacy.com/gh/netbirdio/netbird/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=netbirdio/netbird&amp;utm_campaign=Badge_Grade"><img src="https://app.codacy.com/project/badge/Grade/e3013d046aec44cdb7462c8673b00976"/></a>
     <br>
-    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">
+    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2utg2ncdz-W7LEB6toRBLE1Jca37dYpg">
         <img src="https://img.shields.io/badge/slack-@netbird-red.svg?logo=slack"/>
+     </a>  
+     <br>
+    <a href="https://gurubase.io/g/netbird">
+        <img src="https://img.shields.io/badge/Gurubase-Ask%20NetBird%20Guru-006BFF"/>
      </a>    
   </p>
 </div>
@@ -28,7 +34,7 @@
   <br/>
   See <a href="https://netbird.io/docs/">Documentation</a>
   <br/>
-   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">Slack channel</a>
+   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2utg2ncdz-W7LEB6toRBLE1Jca37dYpg">Slack channel</a>
   <br/>
  
 </strong>
@@ -47,6 +53,8 @@
 
 ![netbird_2](https://github.com/netbirdio/netbird/assets/700848/46bc3b73-508d-4a0e-bb9a-f465d68646ab)
 
+### NetBird on Lawrence Systems (Video)
+[![Watch the video](https://img.youtube.com/vi/Kwrff6h0rEw/0.jpg)](https://www.youtube.com/watch?v=Kwrff6h0rEw)
 
 ### Key features
 
@@ -60,6 +68,7 @@
 |                                                                                                                              |                                                                                                          | <ul><li> - \[x] [Quantum-resistance with Rosenpass](https://netbird.io/knowledge-hub/the-first-quantum-resistant-mesh-vpn) </ul></li> |                                                                                                                                          | <ul><li> - \[x] OpenWRT </ul></li>                                                      |
 |                                                                                                                              |                                                                                                          | <ui><li> - \[x] [Periodic re-authentication](https://docs.netbird.io/how-to/enforce-periodic-user-authentication)</ul></li>           |                                                                                                                                          | <ul><li> - \[x] [Serverless](https://docs.netbird.io/how-to/netbird-on-faas) </ul></li> |
 |                                                                                                                              |                                                                                                          |                                                                                                                                       |                                                                                                                                          | <ul><li> - \[x] Docker </ul></li>                                                       |
+
 ### Quickstart with NetBird Cloud
 
 - Download and install NetBird at [https://app.netbird.io/install](https://app.netbird.io/install)

client/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 RUN apk add --no-cache ca-certificates iptables ip6tables
 ENV NB_FOREGROUND_MODE=true
 ENTRYPOINT [ "/usr/local/bin/netbird","up"]

client/android/client.go

@@ -8,6 +8,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/iface/device"
 	"github.com/netbirdio/netbird/client/internal"
 	"github.com/netbirdio/netbird/client/internal/dns"
 	"github.com/netbirdio/netbird/client/internal/listener"
@@ -15,7 +16,6 @@ import (
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/formatter"
-	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/util/net"
 )
 
@@ -26,7 +26,7 @@ type ConnectionListener interface {
 
 // TunAdapter export internal TunAdapter for mobile
 type TunAdapter interface {
-	iface.TunAdapter
+	device.TunAdapter
 }
 
 // IFaceDiscover export internal IFaceDiscover for mobile
@@ -51,7 +51,7 @@ func init() {
 // Client struct manage the life circle of background service
 type Client struct {
 	cfgFile               string
-	tunAdapter            iface.TunAdapter
+	tunAdapter            device.TunAdapter
 	iFaceDiscover         IFaceDiscover
 	recorder              *peer.Status
 	ctxCancel             context.CancelFunc

client/android/login.go

@@ -84,7 +84,7 @@ func (a *Auth) SaveConfigIfSSOSupported(listener SSOListener) {
 func (a *Auth) saveConfigIfSSOSupported() (bool, error) {
 	supportsSSO := true
 	err := a.withBackOff(a.ctx, func() (err error) {
-		_, err = internal.GetPKCEAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
+		_, err = internal.GetPKCEAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL, nil)
 		if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) {
 			_, err = internal.GetDeviceAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
 			s, ok := gstatus.FromError(err)

client/anonymize/anonymize.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 )
 
+const anonTLD = ".domain"
+
 type Anonymizer struct {
 	ipAnonymizer     map[netip.Addr]netip.Addr
 	domainAnonymizer map[string]string
@@ -83,29 +85,39 @@ func (a *Anonymizer) AnonymizeIPString(ip string) string {
 }
 
 func (a *Anonymizer) AnonymizeDomain(domain string) string {
-	if strings.HasSuffix(domain, "netbird.io") ||
-		strings.HasSuffix(domain, "netbird.selfhosted") ||
-		strings.HasSuffix(domain, "netbird.cloud") ||
-		strings.HasSuffix(domain, "netbird.stage") ||
-		strings.HasSuffix(domain, ".domain") {
+	baseDomain := domain
+	hasDot := strings.HasSuffix(domain, ".")
+	if hasDot {
+		baseDomain = domain[:len(domain)-1]
+	}
+
+	if strings.HasSuffix(baseDomain, "netbird.io") ||
+		strings.HasSuffix(baseDomain, "netbird.selfhosted") ||
+		strings.HasSuffix(baseDomain, "netbird.cloud") ||
+		strings.HasSuffix(baseDomain, "netbird.stage") ||
+		strings.HasSuffix(baseDomain, anonTLD) {
 		return domain
 	}
 
-	parts := strings.Split(domain, ".")
+	parts := strings.Split(baseDomain, ".")
 	if len(parts) < 2 {
 		return domain
 	}
 
-	baseDomain := parts[len(parts)-2] + "." + parts[len(parts)-1]
+	baseForLookup := parts[len(parts)-2] + "." + parts[len(parts)-1]
 
-	anonymized, ok := a.domainAnonymizer[baseDomain]
+	anonymized, ok := a.domainAnonymizer[baseForLookup]
 	if !ok {
-		anonymizedBase := "anon-" + generateRandomString(5) + ".domain"
-		a.domainAnonymizer[baseDomain] = anonymizedBase
+		anonymizedBase := "anon-" + generateRandomString(5) + anonTLD
+		a.domainAnonymizer[baseForLookup] = anonymizedBase
 		anonymized = anonymizedBase
 	}
 
-	return strings.Replace(domain, baseDomain, anonymized, 1)
+	result := strings.Replace(baseDomain, baseForLookup, anonymized, 1)
+	if hasDot {
+		result += "."
+	}
+	return result
 }
 
 func (a *Anonymizer) AnonymizeURI(uri string) string {
@@ -152,9 +164,9 @@ func (a *Anonymizer) AnonymizeString(str string) string {
 	return str
 }
 
-// AnonymizeSchemeURI finds and anonymizes URIs with stun, stuns, turn, and turns schemes.
+// AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes.
 func (a *Anonymizer) AnonymizeSchemeURI(text string) string {
-	re := regexp.MustCompile(`(?i)\b(stuns?:|turns?:|https?://)\S+\b`)
+	re := regexp.MustCompile(`(?i)\b(wss?://|rels?://|stuns?:|turns?:|https?://)\S+\b`)
 
 	return re.ReplaceAllStringFunc(text, a.AnonymizeURI)
 }
@@ -168,16 +180,31 @@ func (a *Anonymizer) AnonymizeDNSLogLine(logEntry string) string {
 		parts := strings.Split(match, `"`)
 		if len(parts) >= 2 {
 			domain := parts[1]
-			if strings.HasSuffix(domain, ".domain") {
+			if strings.HasSuffix(domain, anonTLD) {
 				return match
 			}
-			randomDomain := generateRandomString(10) + ".domain"
+			randomDomain := generateRandomString(10) + anonTLD
 			return strings.Replace(match, domain, randomDomain, 1)
 		}
 		return match
 	})
 }
 
+// AnonymizeRoute anonymizes a route string by replacing IP addresses with anonymized versions and
+// domain names with random strings.
+func (a *Anonymizer) AnonymizeRoute(route string) string {
+	prefix, err := netip.ParsePrefix(route)
+	if err == nil {
+		ip := a.AnonymizeIPString(prefix.Addr().String())
+		return fmt.Sprintf("%s/%d", ip, prefix.Bits())
+	}
+	domains := strings.Split(route, ", ")
+	for i, domain := range domains {
+		domains[i] = a.AnonymizeDomain(domain)
+	}
+	return strings.Join(domains, ", ")
+}
+
 func isWellKnown(addr netip.Addr) bool {
 	wellKnown := []string{
 		"8.8.8.8", "8.8.4.4", // Google DNS IPv4
@@ -186,6 +213,8 @@ func isWellKnown(addr netip.Addr) bool {
 		"2606:4700:4700::1111", "2606:4700:4700::1001", // Cloudflare DNS IPv6
 		"9.9.9.9", "149.112.112.112", // Quad9 DNS IPv4
 		"2620:fe::fe", "2620:fe::9", // Quad9 DNS IPv6
+
+		"128.0.0.0", "8000::", // 2nd split subnet for default routes
 	}
 
 	if slices.Contains(wellKnown, addr.String()) {

client/anonymize/anonymize_test.go

@@ -67,18 +67,36 @@ func TestAnonymizeDomain(t *testing.T) {
 			`^anon-[a-zA-Z0-9]+\.domain$`,
 			true,
 		},
+		{
+			"Domain with Trailing Dot",
+			"example.com.",
+			`^anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
 		{
 			"Subdomain",
 			"sub.example.com",
 			`^sub\.anon-[a-zA-Z0-9]+\.domain$`,
 			true,
 		},
+		{
+			"Subdomain with Trailing Dot",
+			"sub.example.com.",
+			`^sub\.anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
 		{
 			"Protected Domain",
 			"netbird.io",
 			`^netbird\.io$`,
 			false,
 		},
+		{
+			"Protected Domain with Trailing Dot",
+			"netbird.io.",
+			`^netbird\.io.$`,
+			false,
+		},
 	}
 
 	for _, tc := range tests {
@@ -140,8 +158,16 @@ func TestAnonymizeSchemeURI(t *testing.T) {
 		expect string
 	}{
 		{"STUN URI in text", "Connection made via stun:example.com", `Connection made via stun:anon-[a-zA-Z0-9]+\.domain`},
+		{"STUNS URI in message", "Secure connection to stuns:example.com:443", `Secure connection to stuns:anon-[a-zA-Z0-9]+\.domain:443`},
 		{"TURN URI in log", "Failed attempt turn:some.example.com:3478?transport=tcp: retrying", `Failed attempt turn:some.anon-[a-zA-Z0-9]+\.domain:3478\?transport=tcp: retrying`},
+		{"TURNS URI in message", "Secure connection to turns:example.com:5349", `Secure connection to turns:anon-[a-zA-Z0-9]+\.domain:5349`},
+		{"HTTP URI in text", "Visit http://example.com for more", `Visit http://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"HTTPS URI in CAPS", "Visit HTTPS://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
 		{"HTTPS URI in message", "Visit https://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"WS URI in log", "Connection established to ws://example.com:8080", `Connection established to ws://anon-[a-zA-Z0-9]+\.domain:8080`},
+		{"WSS URI in message", "Secure connection to wss://example.com", `Secure connection to wss://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rel URI in text", "Relaying to rel://example.com", `Relaying to rel://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rels URI in message", "Relaying to rels://example.com", `Relaying to rels://anon-[a-zA-Z0-9]+\.domain`},
 	}
 
 	for _, tc := range tests {

client/cmd/debug.go

@@ -3,8 +3,10 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc/status"
 
@@ -13,6 +15,8 @@ import (
 	"github.com/netbirdio/netbird/client/server"
 )
 
+const errCloseConnection = "Failed to close connection: %v"
+
 var debugCmd = &cobra.Command{
 	Use:   "debug",
 	Short: "Debugging commands",
@@ -58,17 +62,31 @@ var forCmd = &cobra.Command{
 	RunE:    runForDuration,
 }
 
+var persistenceCmd = &cobra.Command{
+	Use:     "persistence [on|off]",
+	Short:   "Set network map memory persistence",
+	Long:    `Configure whether the latest network map should persist in memory. When enabled, the last known network map will be kept in memory.`,
+	Example: "  netbird debug persistence on",
+	Args:    cobra.ExactArgs(1),
+	RunE:    setNetworkMapPersistence,
+}
+
 func debugBundle(cmd *cobra.Command, _ []string) error {
 	conn, err := getClient(cmd)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
 
 	client := proto.NewDaemonServiceClient(conn)
 	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
-		Anonymize: anonymizeFlag,
-		Status:    getStatusOutput(cmd),
+		Anonymize:  anonymizeFlag,
+		Status:     getStatusOutput(cmd),
+		SystemInfo: debugSystemInfoFlag,
 	})
 	if err != nil {
 		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
@@ -84,7 +102,11 @@ func setLogLevel(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
 
 	client := proto.NewDaemonServiceClient(conn)
 	level := server.ParseLogLevel(args[0])
@@ -113,7 +135,11 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
 
 	client := proto.NewDaemonServiceClient(conn)
 
@@ -122,17 +148,20 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to get status: %v", status.Convert(err).Message())
 	}
 
-	restoreUp := stat.Status == string(internal.StatusConnected) || stat.Status == string(internal.StatusConnecting)
+	stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting)
 
 	initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{})
 	if err != nil {
 		return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message())
 	}
 
-	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
-		return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+	if stateWasDown {
+		if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
+			return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
+		}
+		cmd.Println("Netbird up")
+		time.Sleep(time.Second * 10)
 	}
-	cmd.Println("Netbird down")
 
 	initialLevelTrace := initialLogLevel.GetLevel() >= proto.LogLevel_TRACE
 	if !initialLevelTrace {
@@ -145,8 +174,20 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		cmd.Println("Log level set to trace.")
 	}
 
+	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+		return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+	}
+	cmd.Println("Netbird down")
+
 	time.Sleep(1 * time.Second)
 
+	// Enable network map persistence before bringing the service up
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: true,
+	}); err != nil {
+		return fmt.Errorf("failed to enable network map persistence: %v", status.Convert(err).Message())
+	}
+
 	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
 		return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
 	}
@@ -162,21 +203,32 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	}
 	cmd.Println("\nDuration completed")
 
+	cmd.Println("Creating debug bundle...")
+
 	headerPreDown := fmt.Sprintf("----- Netbird pre-down - Timestamp: %s - Duration: %s", time.Now().Format(time.RFC3339), duration)
 	statusOutput = fmt.Sprintf("%s\n%s\n%s", statusOutput, headerPreDown, getStatusOutput(cmd))
 
-	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
-		return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
+		Anonymize:  anonymizeFlag,
+		Status:     statusOutput,
+		SystemInfo: debugSystemInfoFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
 	}
-	cmd.Println("Netbird down")
 
-	time.Sleep(1 * time.Second)
+	// Disable network map persistence after creating the debug bundle
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: false,
+	}); err != nil {
+		return fmt.Errorf("failed to disable network map persistence: %v", status.Convert(err).Message())
+	}
 
-	if restoreUp {
-		if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
-			return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
+	if stateWasDown {
+		if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+			return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
 		}
-		cmd.Println("Netbird up")
+		cmd.Println("Netbird down")
 	}
 
 	if !initialLevelTrace {
@@ -186,21 +238,39 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		cmd.Println("Log level restored to", initialLogLevel.GetLevel())
 	}
 
-	cmd.Println("Creating debug bundle...")
-
-	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
-		Anonymize: anonymizeFlag,
-		Status:    statusOutput,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
-	}
-
 	cmd.Println(resp.GetPath())
 
 	return nil
 }
 
+func setNetworkMapPersistence(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	persistence := strings.ToLower(args[0])
+	if persistence != "on" && persistence != "off" {
+		return fmt.Errorf("invalid persistence value: %s. Use 'on' or 'off'", args[0])
+	}
+
+	client := proto.NewDaemonServiceClient(conn)
+	_, err = client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: persistence == "on",
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set network map persistence: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("Network map persistence set to: %s\n", persistence)
+	return nil
+}
+
 func getStatusOutput(cmd *cobra.Command) string {
 	var statusOutputString string
 	statusResp, err := getStatus(cmd.Context())

client/cmd/down.go

@@ -26,7 +26,7 @@ var downCmd = &cobra.Command{
 			return err
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*7)
 		defer cancel()
 
 		conn, err := DialClientGRPCServer(ctx, daemonAddr)
@@ -42,6 +42,8 @@ var downCmd = &cobra.Command{
 			log.Errorf("call service down method: %v", err)
 			return err
 		}
+
+		cmd.Println("Disconnected")
 		return nil
 	},
 }

client/cmd/login.go

@@ -39,6 +39,11 @@ var loginCmd = &cobra.Command{
 			ctx = context.WithValue(ctx, system.DeviceNameCtxKey, hostName)
 		}
 
+		providedSetupKey, err := getSetupKey()
+		if err != nil {
+			return err
+		}
+
 		// workaround to run without service
 		if logFile == "console" {
 			err = handleRebrand(cmd)
@@ -62,7 +67,7 @@ var loginCmd = &cobra.Command{
 
 			config, _ = internal.UpdateOldManagementURL(ctx, config, configPath)
 
-			err = foregroundLogin(ctx, cmd, config, setupKey)
+			err = foregroundLogin(ctx, cmd, config, providedSetupKey)
 			if err != nil {
 				return fmt.Errorf("foreground login failed: %v", err)
 			}
@@ -81,7 +86,7 @@ var loginCmd = &cobra.Command{
 		client := proto.NewDaemonServiceClient(conn)
 
 		loginRequest := proto.LoginRequest{
-			SetupKey:             setupKey,
+			SetupKey:             providedSetupKey,
 			ManagementUrl:        managementURL,
 			IsLinuxDesktopClient: isLinuxRunningDesktop(),
 			Hostname:             hostName,

client/cmd/login_test.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/internal"
-	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/util"
 )
 

client/cmd/pprof.go

@@ -0,0 +1,33 @@
+//go:build pprof
+// +build pprof
+
+package cmd
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	addr := pprofAddr()
+	go pprof(addr)
+}
+
+func pprofAddr() string {
+	listenAddr := os.Getenv("NB_PPROF_ADDR")
+	if listenAddr == "" {
+		return "localhost:6969"
+	}
+
+	return listenAddr
+}
+
+func pprof(listenAddr string) {
+	log.Infof("listening pprof on: %s\n", listenAddr)
+	if err := http.ListenAndServe(listenAddr, nil); err != nil {
+		log.Fatalf("Failed to start pprof: %v", err)
+	}
+}

client/cmd/root.go

@@ -37,6 +37,7 @@ const (
 	serverSSHAllowedFlag    = "allow-server-ssh"
 	extraIFaceBlackListFlag = "extra-iface-blacklist"
 	dnsRouteIntervalFlag    = "dns-router-interval"
+	systemInfoFlag          = "system-info"
 )
 
 var (
@@ -55,6 +56,7 @@ var (
 	managementURL           string
 	adminURL                string
 	setupKey                string
+	setupKeyPath            string
 	hostName                string
 	preSharedKey            string
 	natExternalIPs          []string
@@ -69,6 +71,7 @@ var (
 	autoConnectDisabled     bool
 	extraIFaceBlackList     []string
 	anonymizeFlag           bool
+	debugSystemInfoFlag     bool
 	dnsRouteInterval        time.Duration
 
 	rootCmd = &cobra.Command{
@@ -91,12 +94,15 @@ func init() {
 	oldDefaultConfigPathDir = "/etc/wiretrustee/"
 	oldDefaultLogFileDir = "/var/log/wiretrustee/"
 
-	if runtime.GOOS == "windows" {
+	switch runtime.GOOS {
+	case "windows":
 		defaultConfigPathDir = os.Getenv("PROGRAMDATA") + "\\Netbird\\"
 		defaultLogFileDir = os.Getenv("PROGRAMDATA") + "\\Netbird\\"
 
 		oldDefaultConfigPathDir = os.Getenv("PROGRAMDATA") + "\\Wiretrustee\\"
 		oldDefaultLogFileDir = os.Getenv("PROGRAMDATA") + "\\Wiretrustee\\"
+	case "freebsd":
+		defaultConfigPathDir = "/var/db/netbird/"
 	}
 
 	defaultConfigPath = defaultConfigPathDir + "config.json"
@@ -121,8 +127,10 @@ func init() {
 	rootCmd.PersistentFlags().StringVarP(&serviceName, "service", "s", defaultServiceName, "Netbird system service name")
 	rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "Netbird config file location")
 	rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "sets Netbird log level")
-	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the log will be output to stdout")
+	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the log will be output to stdout. If syslog is specified the log will be sent to syslog daemon.")
 	rootCmd.PersistentFlags().StringVarP(&setupKey, "setup-key", "k", "", "Setup key obtained from the Management Service Dashboard (used to register peer)")
+	rootCmd.PersistentFlags().StringVar(&setupKeyPath, "setup-key-file", "", "The path to a setup key obtained from the Management Service Dashboard (used to register peer) This is ignored if the setup-key flag is provided.")
+	rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file")
 	rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets Wireguard PreSharedKey property. If set, then only peers that have the same key can communicate.")
 	rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device")
 	rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output")
@@ -147,6 +155,7 @@ func init() {
 	debugCmd.AddCommand(logCmd)
 	logCmd.AddCommand(logLevelCmd)
 	debugCmd.AddCommand(forCmd)
+	debugCmd.AddCommand(persistenceCmd)
 
 	upCmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
 		`Sets external IPs maps between local addresses and interfaces.`+
@@ -165,6 +174,8 @@ func init() {
 	upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.")
 	upCmd.PersistentFlags().BoolVar(&serverSSHAllowed, serverSSHAllowedFlag, false, "Allow SSH server on peer. If enabled, the SSH server will be permitted")
 	upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.")
+
+	debugCmd.PersistentFlags().BoolVarP(&debugSystemInfoFlag, systemInfoFlag, "S", false, "Adds system information to the debug bundle")
 }
 
 // SetupCloseHandler handles SIGTERM signal and exits with success
@@ -246,6 +257,21 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{
 	Clock:               backoff.SystemClock,
 }
 
+func getSetupKey() (string, error) {
+	if setupKeyPath != "" && setupKey == "" {
+		return getSetupKeyFromFile(setupKeyPath)
+	}
+	return setupKey, nil
+}
+
+func getSetupKeyFromFile(setupKeyPath string) (string, error) {
+	data, err := os.ReadFile(setupKeyPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read setup key file: %v", err)
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
 func handleRebrand(cmd *cobra.Command) error {
 	var err error
 	if logFile == defaultLogFile {

client/cmd/root_test.go

@@ -4,6 +4,10 @@ import (
 	"fmt"
 	"io"
 	"testing"
+
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/client/iface"
 )
 
 func TestInitCommands(t *testing.T) {
@@ -34,3 +38,44 @@ func TestInitCommands(t *testing.T) {
 		})
 	}
 }
+
+func TestSetFlagsFromEnvVars(t *testing.T) {
+	var cmd = &cobra.Command{
+		Use:          "netbird",
+		Long:         "test",
+		SilenceUsage: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			SetFlagsFromEnvVars(cmd)
+		},
+	}
+
+	cmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
+		`comma separated list of external IPs to map to the Wireguard interface`)
+	cmd.PersistentFlags().StringVar(&interfaceName, interfaceNameFlag, iface.WgInterfaceDefault, "Wireguard interface name")
+	cmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "Enable Rosenpass feature Rosenpass.")
+	cmd.PersistentFlags().Uint16Var(&wireguardPort, wireguardPortFlag, iface.DefaultWgPort, "Wireguard interface listening port")
+
+	t.Setenv("NB_EXTERNAL_IP_MAP", "abc,dec")
+	t.Setenv("NB_INTERFACE_NAME", "test-name")
+	t.Setenv("NB_ENABLE_ROSENPASS", "true")
+	t.Setenv("NB_WIREGUARD_PORT", "10000")
+	err := cmd.Execute()
+	if err != nil {
+		t.Fatalf("expected no error while running netbird command, got %v", err)
+	}
+	if len(natExternalIPs) != 2 {
+		t.Errorf("expected 2 external ips, got %d", len(natExternalIPs))
+	}
+	if natExternalIPs[0] != "abc" || natExternalIPs[1] != "dec" {
+		t.Errorf("expected abc,dec, got %s,%s", natExternalIPs[0], natExternalIPs[1])
+	}
+	if interfaceName != "test-name" {
+		t.Errorf("expected test-name, got %s", interfaceName)
+	}
+	if !rosenpassEnabled {
+		t.Errorf("expected rosenpassEnabled to be true, got false")
+	}
+	if wireguardPort != 10000 {
+		t.Errorf("expected wireguardPort to be 10000, got %d", wireguardPort)
+	}
+}

client/cmd/service.go

@@ -2,18 +2,23 @@ package cmd
 
 import (
 	"context"
+	"sync"
+
 	"github.com/kardianos/service"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
 
 	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/server"
 )
 
 type program struct {
-	ctx    context.Context
-	cancel context.CancelFunc
-	serv   *grpc.Server
+	ctx              context.Context
+	cancel           context.CancelFunc
+	serv             *grpc.Server
+	serverInstance   *server.Server
+	serverInstanceMu sync.Mutex
 }
 
 func newProgram(ctx context.Context, cancel context.CancelFunc) *program {

client/cmd/service_controller.go

@@ -61,6 +61,10 @@ func (p *program) Start(svc service.Service) error {
 		}
 		proto.RegisterDaemonServiceServer(p.serv, serverInstance)
 
+		p.serverInstanceMu.Lock()
+		p.serverInstance = serverInstance
+		p.serverInstanceMu.Unlock()
+
 		log.Printf("started daemon server: %v", split[1])
 		if err := p.serv.Serve(listen); err != nil {
 			log.Errorf("failed to serve daemon requests: %v", err)
@@ -70,6 +74,16 @@ func (p *program) Start(svc service.Service) error {
 }
 
 func (p *program) Stop(srv service.Service) error {
+	p.serverInstanceMu.Lock()
+	if p.serverInstance != nil {
+		in := new(proto.DownRequest)
+		_, err := p.serverInstance.Down(p.ctx, in)
+		if err != nil {
+			log.Errorf("failed to stop daemon: %v", err)
+		}
+	}
+	p.serverInstanceMu.Unlock()
+
 	p.cancel()
 
 	if p.serv != nil {

client/cmd/service_installer.go

@@ -31,6 +31,8 @@ var installCmd = &cobra.Command{
 			configPath,
 			"--log-level",
 			logLevel,
+			"--daemon-addr",
+			daemonAddr,
 		}
 
 		if managementURL != "" {

client/cmd/state.go

@@ -0,0 +1,181 @@
+package cmd
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/proto"
+)
+
+var (
+	allFlag bool
+)
+
+var stateCmd = &cobra.Command{
+	Use:   "state",
+	Short: "Manage daemon state",
+	Long:  "Provides commands for managing and inspecting the Netbird daemon state.",
+}
+
+var stateListCmd = &cobra.Command{
+	Use:     "list",
+	Aliases: []string{"ls"},
+	Short:   "List all stored states",
+	Long:    "Lists all registered states with their status and basic information.",
+	Example: "  netbird state list",
+	RunE:    stateList,
+}
+
+var stateCleanCmd = &cobra.Command{
+	Use:   "clean [state-name]",
+	Short: "Clean stored states",
+	Long: `Clean specific state or all states. The daemon must not be running.
+This will perform cleanup operations and remove the state.`,
+	Example: `  netbird state clean dns_state
+  netbird state clean --all`,
+	RunE: stateClean,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Check mutual exclusivity between --all flag and state-name argument
+		if allFlag && len(args) > 0 {
+			return fmt.Errorf("cannot specify both --all flag and state name")
+		}
+		if !allFlag && len(args) != 1 {
+			return fmt.Errorf("requires a state name argument or --all flag")
+		}
+		return nil
+	},
+}
+
+var stateDeleteCmd = &cobra.Command{
+	Use:   "delete [state-name]",
+	Short: "Delete stored states",
+	Long: `Delete specific state or all states from storage. The daemon must not be running.
+This will remove the state without performing any cleanup operations.`,
+	Example: `  netbird state delete dns_state
+  netbird state delete --all`,
+	RunE: stateDelete,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Check mutual exclusivity between --all flag and state-name argument
+		if allFlag && len(args) > 0 {
+			return fmt.Errorf("cannot specify both --all flag and state name")
+		}
+		if !allFlag && len(args) != 1 {
+			return fmt.Errorf("requires a state name argument or --all flag")
+		}
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(stateCmd)
+	stateCmd.AddCommand(stateListCmd, stateCleanCmd, stateDeleteCmd)
+
+	stateCleanCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Clean all states")
+	stateDeleteCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Delete all states")
+}
+
+func stateList(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.ListStates(cmd.Context(), &proto.ListStatesRequest{})
+	if err != nil {
+		return fmt.Errorf("failed to list states: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("\nStored states:\n\n")
+	for _, state := range resp.States {
+		cmd.Printf("- %s\n", state.Name)
+	}
+
+	return nil
+}
+
+func stateClean(cmd *cobra.Command, args []string) error {
+	var stateName string
+	if !allFlag {
+		stateName = args[0]
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.CleanState(cmd.Context(), &proto.CleanStateRequest{
+		StateName: stateName,
+		All:       allFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to clean state: %v", status.Convert(err).Message())
+	}
+
+	if resp.CleanedStates == 0 {
+		cmd.Println("No states were cleaned")
+		return nil
+	}
+
+	if allFlag {
+		cmd.Printf("Successfully cleaned %d states\n", resp.CleanedStates)
+	} else {
+		cmd.Printf("Successfully cleaned state %q\n", stateName)
+	}
+
+	return nil
+}
+
+func stateDelete(cmd *cobra.Command, args []string) error {
+	var stateName string
+	if !allFlag {
+		stateName = args[0]
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.DeleteState(cmd.Context(), &proto.DeleteStateRequest{
+		StateName: stateName,
+		All:       allFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete state: %v", status.Convert(err).Message())
+	}
+
+	if resp.DeletedStates == 0 {
+		cmd.Println("No states were deleted")
+		return nil
+	}
+
+	if allFlag {
+		cmd.Printf("Successfully deleted %d states\n", resp.DeletedStates)
+	} else {
+		cmd.Printf("Successfully deleted state %q\n", stateName)
+	}
+
+	return nil
+}

client/cmd/status.go

@@ -31,9 +31,9 @@ type peerStateDetailOutput struct {
 	Status                 string           `json:"status" yaml:"status"`
 	LastStatusUpdate       time.Time        `json:"lastStatusUpdate" yaml:"lastStatusUpdate"`
 	ConnType               string           `json:"connectionType" yaml:"connectionType"`
-	Direct                 bool             `json:"direct" yaml:"direct"`
 	IceCandidateType       iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"`
 	IceCandidateEndpoint   iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"`
+	RelayAddress           string           `json:"relayAddress" yaml:"relayAddress"`
 	LastWireguardHandshake time.Time        `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"`
 	TransferReceived       int64            `json:"transferReceived" yaml:"transferReceived"`
 	TransferSent           int64            `json:"transferSent" yaml:"transferSent"`
@@ -335,16 +335,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput {
 
 func mapPeers(peers []*proto.PeerState) peersStateOutput {
 	var peersStateDetail []peerStateDetailOutput
-	localICE := ""
-	remoteICE := ""
-	localICEEndpoint := ""
-	remoteICEEndpoint := ""
-	connType := ""
 	peersConnected := 0
-	lastHandshake := time.Time{}
-	transferReceived := int64(0)
-	transferSent := int64(0)
 	for _, pbPeerState := range peers {
+		localICE := ""
+		remoteICE := ""
+		localICEEndpoint := ""
+		remoteICEEndpoint := ""
+		relayServerAddress := ""
+		connType := ""
+		lastHandshake := time.Time{}
+		transferReceived := int64(0)
+		transferSent := int64(0)
+
 		isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String()
 		if skipDetailByFilters(pbPeerState, isPeerConnected) {
 			continue
@@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			if pbPeerState.Relayed {
 				connType = "Relayed"
 			}
+			relayServerAddress = pbPeerState.GetRelayAddress()
 			lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local()
 			transferReceived = pbPeerState.GetBytesRx()
 			transferSent = pbPeerState.GetBytesTx()
@@ -372,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			Status:           pbPeerState.GetConnStatus(),
 			LastStatusUpdate: timeLocal,
 			ConnType:         connType,
-			Direct:           pbPeerState.GetDirect(),
 			IceCandidateType: iceCandidateType{
 				Local:  localICE,
 				Remote: remoteICE,
@@ -381,6 +383,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 				Local:  localICEEndpoint,
 				Remote: remoteICEEndpoint,
 			},
+			RelayAddress:           relayServerAddress,
 			FQDN:                   pbPeerState.GetFqdn(),
 			LastWireguardHandshake: lastHandshake,
 			TransferReceived:       transferReceived,
@@ -641,9 +644,9 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 				"  Status: %s\n"+
 				"  -- detail --\n"+
 				"  Connection type: %s\n"+
-				"  Direct: %t\n"+
 				"  ICE candidate (Local/Remote): %s/%s\n"+
 				"  ICE candidate endpoints (Local/Remote): %s/%s\n"+
+				"  Relay server address: %s\n"+
 				"  Last connection update: %s\n"+
 				"  Last WireGuard handshake: %s\n"+
 				"  Transfer status (received/sent) %s/%s\n"+
@@ -655,11 +658,11 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 			peerState.PubKey,
 			peerState.Status,
 			peerState.ConnType,
-			peerState.Direct,
 			localICE,
 			remoteICE,
 			localICEEndpoint,
 			remoteICEEndpoint,
+			peerState.RelayAddress,
 			timeAgo(peerState.LastStatusUpdate),
 			timeAgo(peerState.LastWireguardHandshake),
 			toIEC(peerState.TransferReceived),
@@ -677,7 +680,7 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 func skipDetailByFilters(peerState *proto.PeerState, isConnected bool) bool {
 	statusEval := false
 	ipEval := false
-	nameEval := false
+	nameEval := true
 
 	if statusFilter != "" {
 		lowerStatusFilter := strings.ToLower(statusFilter)
@@ -697,11 +700,13 @@ func skipDetailByFilters(peerState *proto.PeerState, isConnected bool) bool {
 
 	if len(prefixNamesFilter) > 0 {
 		for prefixNameFilter := range prefixNamesFilterMap {
-			if !strings.HasPrefix(peerState.Fqdn, prefixNameFilter) {
-				nameEval = true
+			if strings.HasPrefix(peerState.Fqdn, prefixNameFilter) {
+				nameEval = false
 				break
 			}
 		}
+	} else {
+		nameEval = false
 	}
 
 	return statusEval || ipEval || nameEval
@@ -802,12 +807,15 @@ func anonymizePeerDetail(a *anonymize.Anonymizer, peer *peerStateDetailOutput) {
 	if remoteIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Remote); err == nil {
 		peer.IceCandidateEndpoint.Remote = fmt.Sprintf("%s:%s", a.AnonymizeIPString(remoteIP), port)
 	}
+
+	peer.RelayAddress = a.AnonymizeURI(peer.RelayAddress)
+
 	for i, route := range peer.Routes {
 		peer.Routes[i] = a.AnonymizeIPString(route)
 	}
 
 	for i, route := range peer.Routes {
-		peer.Routes[i] = anonymizeRoute(a, route)
+		peer.Routes[i] = a.AnonymizeRoute(route)
 	}
 }
 
@@ -843,21 +851,8 @@ func anonymizeOverview(a *anonymize.Anonymizer, overview *statusOutputOverview)
 	}
 
 	for i, route := range overview.Routes {
-		overview.Routes[i] = anonymizeRoute(a, route)
+		overview.Routes[i] = a.AnonymizeRoute(route)
 	}
 
 	overview.FQDN = a.AnonymizeDomain(overview.FQDN)
 }
-
-func anonymizeRoute(a *anonymize.Anonymizer, route string) string {
-	prefix, err := netip.ParsePrefix(route)
-	if err == nil {
-		ip := a.AnonymizeIPString(prefix.Addr().String())
-		return fmt.Sprintf("%s/%d", ip, prefix.Bits())
-	}
-	domains := strings.Split(route, ", ")
-	for i, domain := range domains {
-		domains[i] = a.AnonymizeDomain(domain)
-	}
-	return strings.Join(domains, ", ")
-}

client/cmd/status_test.go

@@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)),
 				Relayed:                    false,
-				Direct:                     true,
 				LocalIceCandidateType:      "",
 				RemoteIceCandidateType:     "",
 				LocalIceCandidateEndpoint:  "",
@@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)),
 				Relayed:                    true,
-				Direct:                     false,
 				LocalIceCandidateType:      "relay",
 				RemoteIceCandidateType:     "prflx",
 				LocalIceCandidateEndpoint:  "10.0.0.1:10001",
@@ -137,7 +135,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC),
 				ConnType:         "P2P",
-				Direct:           true,
 				IceCandidateType: iceCandidateType{
 					Local:  "",
 					Remote: "",
@@ -161,7 +158,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC),
 				ConnType:         "Relayed",
-				Direct:           false,
 				IceCandidateType: iceCandidateType{
 					Local:  "relay",
 					Remote: "prflx",
@@ -283,7 +279,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2001-01-01T01:01:01Z",
                 "connectionType": "P2P",
-                "direct": true,
                 "iceCandidateType": {
                   "local": "",
                   "remote": ""
@@ -292,6 +287,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "",
                   "remote": ""
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2001-01-01T01:01:02Z",
                 "transferReceived": 200,
                 "transferSent": 100,
@@ -308,7 +304,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2002-02-02T02:02:02Z",
                 "connectionType": "Relayed",
-                "direct": false,
                 "iceCandidateType": {
                   "local": "relay",
                   "remote": "prflx"
@@ -317,6 +312,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "10.0.0.1:10001",
                   "remote": "10.0.10.1:10002"
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2002-02-02T02:02:03Z",
                 "transferReceived": 2000,
                 "transferSent": 1000,
@@ -408,13 +404,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2001-01-01T01:01:01Z
           connectionType: P2P
-          direct: true
           iceCandidateType:
             local: ""
             remote: ""
           iceCandidateEndpoint:
             local: ""
             remote: ""
+          relayAddress: ""
           lastWireguardHandshake: 2001-01-01T01:01:02Z
           transferReceived: 200
           transferSent: 100
@@ -428,13 +424,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2002-02-02T02:02:02Z
           connectionType: Relayed
-          direct: false
           iceCandidateType:
             local: relay
             remote: prflx
           iceCandidateEndpoint:
             local: 10.0.0.1:10001
             remote: 10.0.10.1:10002
+          relayAddress: ""
           lastWireguardHandshake: 2002-02-02T02:02:03Z
           transferReceived: 2000
           transferSent: 1000
@@ -505,9 +501,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: P2P
-  Direct: true
   ICE candidate (Local/Remote): -/-
   ICE candidate endpoints (Local/Remote): -/-
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 200 B/100 B
@@ -521,9 +517,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: Relayed
-  Direct: false
   ICE candidate (Local/Remote): relay/prflx
   ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 2.0 KiB/1000 B

client/cmd/testutil_test.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"net"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -11,6 +10,7 @@ import (
 	"go.opentelemetry.io/otel"
 
 	"github.com/netbirdio/netbird/management/server/activity"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 
 	"github.com/netbirdio/netbird/util"
 
@@ -33,18 +33,12 @@ func startTestingServices(t *testing.T) string {
 	if err != nil {
 		t.Fatal(err)
 	}
-	testDir := t.TempDir()
-	config.Datadir = testDir
-	err = util.CopyFileContents("../testdata/store.json", filepath.Join(testDir, "store.json"))
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	_, signalLis := startSignal(t)
 	signalAddr := signalLis.Addr().String()
 	config.Signal.URI = signalAddr
 
-	_, mgmLis := startManagement(t, config)
+	_, mgmLis := startManagement(t, config, "../testdata/store.sql")
 	mgmAddr := mgmLis.Addr().String()
 	return mgmAddr
 }
@@ -56,7 +50,7 @@ func startSignal(t *testing.T) (*grpc.Server, net.Listener) {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	srv, err := sig.NewServer(otel.Meter(""))
+	srv, err := sig.NewServer(context.Background(), otel.Meter(""))
 	require.NoError(t, err)
 
 	sigProto.RegisterSignalExchangeServer(s, srv)
@@ -69,14 +63,15 @@ func startSignal(t *testing.T) (*grpc.Server, net.Listener) {
 	return s, lis
 }
 
-func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Listener) {
+func startManagement(t *testing.T, config *mgmt.Config, testFile string) (*grpc.Server, net.Listener) {
 	t.Helper()
+
 	lis, err := net.Listen("tcp", ":0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, cleanUp, err := mgmt.NewTestStoreFromJson(context.Background(), config.Datadir)
+	store, cleanUp, err := mgmt.NewTestStoreFromSQL(context.Background(), testFile, t.TempDir())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -88,12 +83,17 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 		return nil, nil
 	}
 	iv, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
-	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv)
+
+	metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
+	require.NoError(t, err)
+
+	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv, metrics)
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+
+	secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/cmd/up.go

@@ -15,11 +15,11 @@ import (
 	gstatus "google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/durationpb"
 
+	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/internal"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/proto"
 	"github.com/netbirdio/netbird/client/system"
-	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -147,6 +147,11 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 		ic.DNSRouteInterval = &dnsRouteInterval
 	}
 
+	providedSetupKey, err := getSetupKey()
+	if err != nil {
+		return err
+	}
+
 	config, err := internal.UpdateOrCreateConfig(ic)
 	if err != nil {
 		return fmt.Errorf("get config file: %v", err)
@@ -154,7 +159,7 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 
 	config, _ = internal.UpdateOldManagementURL(ctx, config, configPath)
 
-	err = foregroundLogin(ctx, cmd, config, setupKey)
+	err = foregroundLogin(ctx, cmd, config, providedSetupKey)
 	if err != nil {
 		return fmt.Errorf("foreground login failed: %v", err)
 	}
@@ -163,7 +168,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 	ctx, cancel = context.WithCancel(ctx)
 	SetupCloseHandler(ctx, cancel)
 
-	connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String()))
+	r := peer.NewRecorder(config.ManagementURL.String())
+	r.GetFullStatus()
+
+	connectClient := internal.NewConnectClient(ctx, config, r)
 	return connectClient.Run()
 }
 
@@ -199,8 +207,13 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error {
 		return nil
 	}
 
+	providedSetupKey, err := getSetupKey()
+	if err != nil {
+		return err
+	}
+
 	loginRequest := proto.LoginRequest{
-		SetupKey:             setupKey,
+		SetupKey:             providedSetupKey,
 		ManagementUrl:        managementURL,
 		AdminURL:             adminURL,
 		NatExternalIPs:       natExternalIPs,

client/cmd/up_daemon_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"os"
 	"testing"
 	"time"
 
@@ -40,6 +41,36 @@ func TestUpDaemon(t *testing.T) {
 		return
 	}
 
+	// Test the setup-key-file flag.
+	tempFile, err := os.CreateTemp("", "setup-key")
+	if err != nil {
+		t.Errorf("could not create temp file, got error %v", err)
+		return
+	}
+	defer os.Remove(tempFile.Name())
+	if _, err := tempFile.Write([]byte("A2C8E62B-38F5-4553-B31E-DD66C696CEBB")); err != nil {
+		t.Errorf("could not write to temp file, got error %v", err)
+		return
+	}
+	if err := tempFile.Close(); err != nil {
+		t.Errorf("unable to close file, got error %v", err)
+	}
+	rootCmd.SetArgs([]string{
+		"login",
+		"--daemon-addr", "tcp://" + cliAddr,
+		"--setup-key-file", tempFile.Name(),
+		"--log-file", "",
+	})
+	if err := rootCmd.Execute(); err != nil {
+		t.Errorf("expected no error while running up command, got %v", err)
+		return
+	}
+	time.Sleep(time.Second * 3)
+	if status, err := state.Status(); err != nil && status != internal.StatusIdle {
+		t.Errorf("wrong status after login: %s, %v", internal.StatusIdle, err)
+		return
+	}
+
 	rootCmd.SetArgs([]string{
 		"up",
 		"--daemon-addr", "tcp://" + cliAddr,

client/errors/errors.go

@@ -8,8 +8,8 @@ import (
 )
 
 func formatError(es []error) string {
-	if len(es) == 0 {
-		return fmt.Sprintf("0 error occurred:\n\t* %s", es[0])
+	if len(es) == 1 {
+		return fmt.Sprintf("1 error occurred:\n\t* %s", es[0])
 	}
 
 	points := make([]string, len(es))

client/firewall/create.go

@@ -3,7 +3,6 @@
 package firewall
 
 import (
-	"context"
 	"fmt"
 	"runtime"
 
@@ -11,10 +10,11 @@ import (
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 // NewFirewall creates a firewall manager instance
-func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, _ *statemanager.Manager) (firewall.Manager, error) {
 	if !iface.IsUserspaceBind() {
 		return nil, fmt.Errorf("not implemented for this OS: %s", runtime.GOOS)
 	}

client/firewall/create_linux.go

@@ -3,7 +3,7 @@
 package firewall
 
 import (
-	"context"
+	"errors"
 	"fmt"
 	"os"
 
@@ -15,6 +15,7 @@ import (
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	nbnftables "github.com/netbirdio/netbird/client/firewall/nftables"
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -32,54 +33,65 @@ const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK"
 // FWType is the type for the firewall type
 type FWType int
 
-func NewFirewall(context context.Context, iface IFaceMapper) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
 	// on the linux system we try to user nftables or iptables
 	// in any case, because we need to allow netbird interface traffic
 	// so we use AllowNetbird traffic from these firewall managers
 	// for the userspace packet filtering firewall
-	var fm firewall.Manager
-	var errFw error
+	fm, err := createNativeFirewall(iface, stateManager)
 
+	if !iface.IsUserspaceBind() {
+		return fm, err
+	}
+
+	if err != nil {
+		log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err)
+	}
+	return createUserspaceFirewall(iface, fm)
+}
+
+func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+	fm, err := createFW(iface)
+	if err != nil {
+		return nil, fmt.Errorf("create firewall: %s", err)
+	}
+
+	if err = fm.Init(stateManager); err != nil {
+		return nil, fmt.Errorf("init firewall: %s", err)
+	}
+
+	return fm, nil
+}
+
+func createFW(iface IFaceMapper) (firewall.Manager, error) {
 	switch check() {
 	case IPTABLES:
 		log.Info("creating an iptables firewall manager")
-		fm, errFw = nbiptables.Create(context, iface)
-		if errFw != nil {
-			log.Errorf("failed to create iptables manager: %s", errFw)
-		}
+		return nbiptables.Create(iface)
 	case NFTABLES:
 		log.Info("creating an nftables firewall manager")
-		fm, errFw = nbnftables.Create(context, iface)
-		if errFw != nil {
-			log.Errorf("failed to create nftables manager: %s", errFw)
-		}
+		return nbnftables.Create(iface)
 	default:
-		errFw = fmt.Errorf("no firewall manager found")
 		log.Info("no firewall manager found, trying to use userspace packet filtering firewall")
+		return nil, errors.New("no firewall manager found")
+	}
+}
+
+func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager) (firewall.Manager, error) {
+	var errUsp error
+	if fm != nil {
+		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
+	} else {
+		fm, errUsp = uspfilter.Create(iface)
 	}
 
-	if iface.IsUserspaceBind() {
-		var errUsp error
-		if errFw == nil {
-			fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
-		} else {
-			fm, errUsp = uspfilter.Create(iface)
-		}
-		if errUsp != nil {
-			log.Debugf("failed to create userspace filtering firewall: %s", errUsp)
-			return nil, errUsp
-		}
-
-		if err := fm.AllowNetbird(); err != nil {
-			log.Errorf("failed to allow netbird interface traffic: %v", err)
-		}
-		return fm, nil
+	if errUsp != nil {
+		return nil, fmt.Errorf("create userspace firewall: %s", errUsp)
 	}
 
-	if errFw != nil {
-		return nil, errFw
+	if err := fm.AllowNetbird(); err != nil {
+		log.Errorf("failed to allow netbird interface traffic: %v", err)
 	}
-
 	return fm, nil
 }
 

client/firewall/iface.go

@@ -1,11 +1,13 @@
 package firewall
 
-import "github.com/netbirdio/netbird/iface"
+import (
+	"github.com/netbirdio/netbird/client/iface/device"
+)
 
 // IFaceMapper defines subset methods of interface required for manager
 type IFaceMapper interface {
 	Name() string
-	Address() iface.WGAddress
+	Address() device.WGAddress
 	IsUserspaceBind() bool
-	SetFilter(iface.PacketFilter) error
+	SetFilter(device.PacketFilter) error
 }

client/firewall/iptables/acl_linux.go

@@ -11,6 +11,8 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
@@ -19,49 +21,65 @@ const (
 	// rules chains contains the effective ACL rules
 	chainNameInputRules  = "NETBIRD-ACL-INPUT"
 	chainNameOutputRules = "NETBIRD-ACL-OUTPUT"
-
-	postRoutingMark = "0x000007e4"
 )
 
-type aclManager struct {
-	iptablesClient      *iptables.IPTables
-	wgIface             iFaceMapper
-	routeingFwChainName string
+type aclEntries map[string][][]string
 
-	entries    map[string][][]string
-	ipsetStore *ipsetStore
+type entry struct {
+	spec     []string
+	position int
 }
 
-func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routeingFwChainName string) (*aclManager, error) {
+type aclManager struct {
+	iptablesClient     *iptables.IPTables
+	wgIface            iFaceMapper
+	routingFwChainName string
+
+	entries         aclEntries
+	optionalEntries map[string][]entry
+	ipsetStore      *ipsetStore
+
+	stateManager *statemanager.Manager
+}
+
+func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routingFwChainName string) (*aclManager, error) {
 	m := &aclManager{
-		iptablesClient:      iptablesClient,
-		wgIface:             wgIface,
-		routeingFwChainName: routeingFwChainName,
+		iptablesClient:     iptablesClient,
+		wgIface:            wgIface,
+		routingFwChainName: routingFwChainName,
 
-		entries:    make(map[string][][]string),
-		ipsetStore: newIpsetStore(),
+		entries:         make(map[string][][]string),
+		optionalEntries: make(map[string][]entry),
+		ipsetStore:      newIpsetStore(),
 	}
 
-	err := ipset.Init()
-	if err != nil {
-		return nil, fmt.Errorf("failed to init ipset: %w", err)
+	if err := ipset.Init(); err != nil {
+		return nil, fmt.Errorf("init ipset: %w", err)
 	}
 
-	m.seedInitialEntries()
-
-	err = m.cleanChains()
-	if err != nil {
-		return nil, err
-	}
-
-	err = m.createDefaultChains()
-	if err != nil {
-		return nil, err
-	}
 	return m, nil
 }
 
-func (m *aclManager) AddFiltering(
+func (m *aclManager) init(stateManager *statemanager.Manager) error {
+	m.stateManager = stateManager
+
+	m.seedInitialEntries()
+	m.seedInitialOptionalEntries()
+
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
+	}
+
+	if err := m.createDefaultChains(); err != nil {
+		return fmt.Errorf("create default chains: %w", err)
+	}
+
+	m.updateState()
+
+	return nil
+}
+
+func (m *aclManager) AddPeerFiltering(
 	ip net.IP,
 	protocol firewall.Protocol,
 	sPort *firewall.Port,
@@ -127,7 +145,7 @@ func (m *aclManager) AddFiltering(
 		return nil, fmt.Errorf("rule already exists")
 	}
 
-	if err := m.iptablesClient.Insert("filter", chain, 1, specs...); err != nil {
+	if err := m.iptablesClient.Append("filter", chain, specs...); err != nil {
 		return nil, err
 	}
 
@@ -139,28 +157,18 @@ func (m *aclManager) AddFiltering(
 		chain:     chain,
 	}
 
-	if !shouldAddToPrerouting(protocol, dPort, direction) {
-		return []firewall.Rule{rule}, nil
-	}
+	m.updateState()
 
-	rulePrerouting, err := m.addPreroutingFilter(ipsetName, string(protocol), dPortVal, ip)
-	if err != nil {
-		return []firewall.Rule{rule}, err
-	}
-	return []firewall.Rule{rule, rulePrerouting}, nil
+	return []firewall.Rule{rule}, nil
 }
 
-// DeleteRule from the firewall by rule definition
-func (m *aclManager) DeleteRule(rule firewall.Rule) error {
+// DeletePeerRule from the firewall by rule definition
+func (m *aclManager) DeletePeerRule(rule firewall.Rule) error {
 	r, ok := rule.(*Rule)
 	if !ok {
 		return fmt.Errorf("invalid rule type")
 	}
 
-	if r.chain == "PREROUTING" {
-		goto DELETERULE
-	}
-
 	if ipsetList, ok := m.ipsetStore.ipset(r.ipsetName); ok {
 		// delete IP from ruleset IPs list and ipset
 		if _, ok := ipsetList.ips[r.ip]; ok {
@@ -185,60 +193,23 @@ func (m *aclManager) DeleteRule(rule firewall.Rule) error {
 		}
 	}
 
-DELETERULE:
-	var table string
-	if r.chain == "PREROUTING" {
-		table = "mangle"
-	} else {
-		table = "filter"
+	if err := m.iptablesClient.Delete(tableName, r.chain, r.specs...); err != nil {
+		return fmt.Errorf("failed to delete rule: %s, %v: %w", r.chain, r.specs, err)
 	}
-	err := m.iptablesClient.Delete(table, r.chain, r.specs...)
-	if err != nil {
-		log.Debugf("failed to delete rule, %s, %v: %s", r.chain, r.specs, err)
-	}
-	return err
+
+	m.updateState()
+
+	return nil
 }
 
 func (m *aclManager) Reset() error {
-	return m.cleanChains()
-}
-
-func (m *aclManager) addPreroutingFilter(ipsetName string, protocol string, port string, ip net.IP) (*Rule, error) {
-	var src []string
-	if ipsetName != "" {
-		src = []string{"-m", "set", "--set", ipsetName, "src"}
-	} else {
-		src = []string{"-s", ip.String()}
-	}
-	specs := []string{
-		"-d", m.wgIface.Address().IP.String(),
-		"-p", protocol,
-		"--dport", port,
-		"-j", "MARK", "--set-mark", postRoutingMark,
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
 	}
 
-	specs = append(src, specs...)
+	m.updateState()
 
-	ok, err := m.iptablesClient.Exists("mangle", "PREROUTING", specs...)
-	if err != nil {
-		return nil, fmt.Errorf("failed to check rule: %w", err)
-	}
-	if ok {
-		return nil, fmt.Errorf("rule already exists")
-	}
-
-	if err := m.iptablesClient.Insert("mangle", "PREROUTING", 1, specs...); err != nil {
-		return nil, err
-	}
-
-	rule := &Rule{
-		ruleID:    uuid.New().String(),
-		specs:     specs,
-		ipsetName: ipsetName,
-		ip:        ip.String(),
-		chain:     "PREROUTING",
-	}
-	return rule, nil
+	return nil
 }
 
 // todo write less destructive cleanup mechanism
@@ -293,8 +264,7 @@ func (m *aclManager) cleanChains() error {
 
 	ok, err = m.iptablesClient.ChainExists("mangle", "PREROUTING")
 	if err != nil {
-		log.Debugf("failed to list chains: %s", err)
-		return err
+		return fmt.Errorf("list chains: %w", err)
 	}
 	if ok {
 		for _, rule := range m.entries["PREROUTING"] {
@@ -303,11 +273,6 @@ func (m *aclManager) cleanChains() error {
 				log.Errorf("failed to delete rule: %v, %s", rule, err)
 			}
 		}
-		err = m.iptablesClient.ClearChain("mangle", "PREROUTING")
-		if err != nil {
-			log.Debugf("failed to clear %s chain: %s", "PREROUTING", err)
-			return err
-		}
 	}
 
 	for _, ipsetName := range m.ipsetStore.ipsetNames() {
@@ -338,64 +303,98 @@ func (m *aclManager) createDefaultChains() error {
 
 	for chainName, rules := range m.entries {
 		for _, rule := range rules {
-			if chainName == "FORWARD" {
-				// position 2 because we add it after router's, jump rule
-				if err := m.iptablesClient.InsertUnique(tableName, "FORWARD", 2, rule...); err != nil {
-					log.Debugf("failed to create input chain jump rule: %s", err)
-					return err
-				}
-			} else {
-				if err := m.iptablesClient.AppendUnique(tableName, chainName, rule...); err != nil {
-					log.Debugf("failed to create input chain jump rule: %s", err)
-					return err
-				}
+			if err := m.iptablesClient.InsertUnique(tableName, chainName, 1, rule...); err != nil {
+				log.Debugf("failed to create input chain jump rule: %s", err)
+				return err
 			}
 		}
 	}
 
+	for chainName, entries := range m.optionalEntries {
+		for _, entry := range entries {
+			if err := m.iptablesClient.InsertUnique(tableName, chainName, entry.position, entry.spec...); err != nil {
+				log.Errorf("failed to insert optional entry %v: %v", entry.spec, err)
+				continue
+			}
+			m.entries[chainName] = append(m.entries[chainName], entry.spec)
+		}
+	}
+	clear(m.optionalEntries)
+
 	return nil
 }
 
+// seedInitialEntries adds default rules to the entries map, rules are inserted on pos 1, hence the order is reversed.
+// We want to make sure our traffic is not dropped by existing rules.
+
+// The existing FORWARD rules/policies decide outbound traffic towards our interface.
+// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule.
+
+// The OUTPUT chain gets an extra rule to allow traffic to any set up routes, the return traffic is handled by the INPUT related/established rule.
 func (m *aclManager) seedInitialEntries() {
-	m.appendToEntries("INPUT",
-		[]string{"-i", m.wgIface.Name(), "!", "-s", m.wgIface.Address().String(), "-d", m.wgIface.Address().String(), "-j", "ACCEPT"})
 
-	m.appendToEntries("INPUT",
-		[]string{"-i", m.wgIface.Name(), "-s", m.wgIface.Address().String(), "!", "-d", m.wgIface.Address().String(), "-j", "ACCEPT"})
-
-	m.appendToEntries("INPUT",
-		[]string{"-i", m.wgIface.Name(), "-s", m.wgIface.Address().String(), "-d", m.wgIface.Address().String(), "-j", chainNameInputRules})
+	established := getConntrackEstablished()
 
 	m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", "DROP"})
-
-	m.appendToEntries("OUTPUT",
-		[]string{"-o", m.wgIface.Name(), "!", "-s", m.wgIface.Address().String(), "-d", m.wgIface.Address().String(), "-j", "ACCEPT"})
-
-	m.appendToEntries("OUTPUT",
-		[]string{"-o", m.wgIface.Name(), "-s", m.wgIface.Address().String(), "!", "-d", m.wgIface.Address().String(), "-j", "ACCEPT"})
-
-	m.appendToEntries("OUTPUT",
-		[]string{"-o", m.wgIface.Name(), "-s", m.wgIface.Address().String(), "-d", m.wgIface.Address().String(), "-j", chainNameOutputRules})
+	m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", chainNameInputRules})
+	m.appendToEntries("INPUT", append([]string{"-i", m.wgIface.Name()}, established...))
 
 	m.appendToEntries("OUTPUT", []string{"-o", m.wgIface.Name(), "-j", "DROP"})
+	m.appendToEntries("OUTPUT", []string{"-o", m.wgIface.Name(), "-j", chainNameOutputRules})
+	m.appendToEntries("OUTPUT", []string{"-o", m.wgIface.Name(), "!", "-d", m.wgIface.Address().String(), "-j", "ACCEPT"})
+	m.appendToEntries("OUTPUT", append([]string{"-o", m.wgIface.Name()}, established...))
 
 	m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", "DROP"})
-	m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", chainNameInputRules})
-	m.appendToEntries("FORWARD",
-		[]string{"-o", m.wgIface.Name(), "-m", "mark", "--mark", postRoutingMark, "-j", "ACCEPT"})
-	m.appendToEntries("FORWARD",
-		[]string{"-i", m.wgIface.Name(), "-m", "mark", "--mark", postRoutingMark, "-j", "ACCEPT"})
-	m.appendToEntries("FORWARD", []string{"-o", m.wgIface.Name(), "-j", m.routeingFwChainName})
-	m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", m.routeingFwChainName})
+	m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", m.routingFwChainName})
+	m.appendToEntries("FORWARD", append([]string{"-o", m.wgIface.Name()}, established...))
+}
 
-	m.appendToEntries("PREROUTING",
-		[]string{"-t", "mangle", "-i", m.wgIface.Name(), "!", "-s", m.wgIface.Address().String(), "-d", m.wgIface.Address().IP.String(), "-m", "mark", "--mark", postRoutingMark})
+func (m *aclManager) seedInitialOptionalEntries() {
+	m.optionalEntries["FORWARD"] = []entry{
+		{
+			spec:     []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", chainNameInputRules},
+			position: 2,
+		},
+	}
+
+	m.optionalEntries["PREROUTING"] = []entry{
+		{
+			spec:     []string{"-t", "mangle", "-i", m.wgIface.Name(), "-m", "addrtype", "--dst-type", "LOCAL", "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected)},
+			position: 1,
+		},
+	}
 }
 
 func (m *aclManager) appendToEntries(chainName string, spec []string) {
 	m.entries[chainName] = append(m.entries[chainName], spec)
 }
 
+func (m *aclManager) updateState() {
+	if m.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := m.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.ACLEntries = m.entries
+	currentState.ACLIPsetStore = m.ipsetStore
+
+	if err := m.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
 // filterRuleSpecs returns the specs of a filtering rule
 func filterRuleSpecs(
 	ip net.IP, protocol string, sPort, dPort string, direction firewall.RuleDirection, action firewall.Action, ipsetName string,
@@ -456,18 +455,3 @@ func transformIPsetName(ipsetName string, sPort, dPort string) string {
 		return ipsetName
 	}
 }
-
-func shouldAddToPrerouting(proto firewall.Protocol, dPort *firewall.Port, direction firewall.RuleDirection) bool {
-	if proto == "all" {
-		return false
-	}
-
-	if direction != firewall.RuleDirectionIN {
-		return false
-	}
-
-	if dPort == nil {
-		return false
-	}
-	return true
-}

client/firewall/iptables/manager_linux.go

@@ -4,13 +4,17 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/netip"
 	"sync"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 // Manager of iptables firewall
@@ -21,7 +25,7 @@ type Manager struct {
 
 	ipv4Client *iptables.IPTables
 	aclMgr     *aclManager
-	router     *routerManager
+	router     *router
 }
 
 // iFaceMapper defines subset methods of interface required for manager
@@ -32,10 +36,10 @@ type iFaceMapper interface {
 }
 
 // Create iptables firewall manager
-func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	if err != nil {
-		return nil, fmt.Errorf("iptables is not installed in the system or not supported")
+		return nil, fmt.Errorf("init iptables: %w", err)
 	}
 
 	m := &Manager{
@@ -43,24 +47,55 @@ func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
 		ipv4Client: iptablesClient,
 	}
 
-	m.router, err = newRouterManager(context, iptablesClient)
+	m.router, err = newRouter(iptablesClient, wgIface)
 	if err != nil {
-		log.Debugf("failed to initialize route related chains: %s", err)
-		return nil, err
+		return nil, fmt.Errorf("create router: %w", err)
 	}
-	m.aclMgr, err = newAclManager(iptablesClient, wgIface, m.router.RouteingFwChainName())
+
+	m.aclMgr, err = newAclManager(iptablesClient, wgIface, chainRTFWD)
 	if err != nil {
-		log.Debugf("failed to initialize ACL manager: %s", err)
-		return nil, err
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
-// AddFiltering rule to the firewall
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	state := &ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}
+	stateManager.RegisterState(state)
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	if err := m.router.init(stateManager); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclMgr.init(stateManager); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	// persist early to ensure cleanup of chains
+	go func() {
+		if err := stateManager.PersistState(context.Background()); err != nil {
+			log.Errorf("failed to persist state: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+// AddPeerFiltering adds a rule to the firewall
 //
 // Comment will be ignored because some system this feature is not supported
-func (m *Manager) AddFiltering(
+func (m *Manager) AddPeerFiltering(
 	ip net.IP,
 	protocol firewall.Protocol,
 	sPort *firewall.Port,
@@ -73,50 +108,86 @@ func (m *Manager) AddFiltering(
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.aclMgr.AddFiltering(ip, protocol, sPort, dPort, direction, action, ipsetName)
+	return m.aclMgr.AddPeerFiltering(ip, protocol, sPort, dPort, direction, action, ipsetName)
 }
 
-// DeleteRule from the firewall by rule definition
-func (m *Manager) DeleteRule(rule firewall.Rule) error {
+func (m *Manager) AddRouteFiltering(
+	sources []netip.Prefix,
+	destination netip.Prefix,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	action firewall.Action,
+) (firewall.Rule, error) {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.aclMgr.DeleteRule(rule)
+	if !destination.Addr().Is4() {
+		return nil, fmt.Errorf("unsupported IP version: %s", destination.Addr().String())
+	}
+
+	return m.router.AddRouteFiltering(sources, destination, proto, sPort, dPort, action)
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.aclMgr.DeletePeerRule(rule)
+}
+
+func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.DeleteRouteRule(rule)
 }
 
 func (m *Manager) IsServerRouteSupported() bool {
 	return true
 }
 
-func (m *Manager) InsertRoutingRules(pair firewall.RouterPair) error {
+func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.router.InsertRoutingRules(pair)
+	return m.router.AddNatRule(pair)
 }
 
-func (m *Manager) RemoveRoutingRules(pair firewall.RouterPair) error {
+func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.router.RemoveRoutingRules(pair)
+	return m.router.RemoveNatRule(pair)
+}
+
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	return firewall.SetLegacyManagement(m.router, isLegacy)
 }
 
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	errAcl := m.aclMgr.Reset()
-	if errAcl != nil {
-		log.Errorf("failed to clean up ACL rules from firewall: %s", errAcl)
+	var merr *multierror.Error
+
+	if err := m.aclMgr.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset acl manager: %w", err))
 	}
-	errMgr := m.router.Reset()
-	if errMgr != nil {
-		log.Errorf("failed to clean up router rules from firewall: %s", errMgr)
-		return errMgr
+	if err := m.router.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset router: %w", err))
 	}
-	return errAcl
+
+	// attempt to delete state only if all other operations succeeded
+	if merr == nil {
+		if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("delete state: %w", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 // AllowNetbird allows netbird interface traffic
@@ -125,7 +196,7 @@ func (m *Manager) AllowNetbird() error {
 		return nil
 	}
 
-	_, err := m.AddFiltering(
+	_, err := m.AddPeerFiltering(
 		net.ParseIP("0.0.0.0"),
 		"all",
 		nil,
@@ -138,7 +209,7 @@ func (m *Manager) AllowNetbird() error {
 	if err != nil {
 		return fmt.Errorf("failed to allow netbird interface traffic: %w", err)
 	}
-	_, err = m.AddFiltering(
+	_, err = m.AddPeerFiltering(
 		net.ParseIP("0.0.0.0"),
 		"all",
 		nil,
@@ -153,3 +224,7 @@ func (m *Manager) AllowNetbird() error {
 
 // Flush doesn't need to be implemented for this manager
 func (m *Manager) Flush() error { return nil }
+
+func getConntrackEstablished() []string {
+	return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}
+}

client/firewall/iptables/manager_linux_test.go

@@ -1,7 +1,6 @@
 package iptables
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"testing"
@@ -11,9 +10,24 @@ import (
 	"github.com/stretchr/testify/require"
 
 	fw "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/client/iface"
 )
 
+var ifaceMock = &iFaceMock{
+	NameFunc: func() string {
+		return "lo"
+	},
+	AddressFunc: func() iface.WGAddress {
+		return iface.WGAddress{
+			IP: net.ParseIP("10.20.0.1"),
+			Network: &net.IPNet{
+				IP:   net.ParseIP("10.20.0.0"),
+				Mask: net.IPv4Mask(255, 255, 255, 0),
+			},
+		}
+	},
+}
+
 // iFaceMapper defines subset methods of interface required for manager
 type iFaceMock struct {
 	NameFunc    func() string
@@ -40,29 +54,15 @@ func TestIptablesManager(t *testing.T) {
 	ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err)
 
-	mock := &iFaceMock{
-		NameFunc: func() string {
-			return "lo"
-		},
-		AddressFunc: func() iface.WGAddress {
-			return iface.WGAddress{
-				IP: net.ParseIP("10.20.0.1"),
-				Network: &net.IPNet{
-					IP:   net.ParseIP("10.20.0.0"),
-					Mask: net.IPv4Mask(255, 255, 255, 0),
-				},
-			}
-		},
-	}
-
 	// just check on the local interface
-	manager, err := Create(context.Background(), mock)
+	manager, err := Create(ifaceMock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 
 	time.Sleep(time.Second)
 
 	defer func() {
-		err := manager.Reset()
+		err := manager.Reset(nil)
 		require.NoError(t, err, "clear the manager state")
 
 		time.Sleep(time.Second)
@@ -72,7 +72,7 @@ func TestIptablesManager(t *testing.T) {
 	t.Run("add first rule", func(t *testing.T) {
 		ip := net.ParseIP("10.20.0.2")
 		port := &fw.Port{Values: []int{8080}}
-		rule1, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+		rule1, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
 		require.NoError(t, err, "failed to add rule")
 
 		for _, r := range rule1 {
@@ -87,7 +87,7 @@ func TestIptablesManager(t *testing.T) {
 		port := &fw.Port{
 			Values: []int{8043: 8046},
 		}
-		rule2, err = manager.AddFiltering(
+		rule2, err = manager.AddPeerFiltering(
 			ip, "tcp", port, nil, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTPS traffic from ports range")
 		require.NoError(t, err, "failed to add rule")
 
@@ -99,7 +99,7 @@ func TestIptablesManager(t *testing.T) {
 
 	t.Run("delete first rule", func(t *testing.T) {
 		for _, r := range rule1 {
-			err := manager.DeleteRule(r)
+			err := manager.DeletePeerRule(r)
 			require.NoError(t, err, "failed to delete rule")
 
 			checkRuleSpecs(t, ipv4Client, chainNameOutputRules, false, r.(*Rule).specs...)
@@ -108,7 +108,7 @@ func TestIptablesManager(t *testing.T) {
 
 	t.Run("delete second rule", func(t *testing.T) {
 		for _, r := range rule2 {
-			err := manager.DeleteRule(r)
+			err := manager.DeletePeerRule(r)
 			require.NoError(t, err, "failed to delete rule")
 		}
 
@@ -119,10 +119,10 @@ func TestIptablesManager(t *testing.T) {
 		// add second rule
 		ip := net.ParseIP("10.20.0.3")
 		port := &fw.Port{Values: []int{5353}}
-		_, err = manager.AddFiltering(ip, "udp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept Fake DNS traffic")
+		_, err = manager.AddPeerFiltering(ip, "udp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept Fake DNS traffic")
 		require.NoError(t, err, "failed to add rule")
 
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 
 		ok, err := ipv4Client.ChainExists("filter", chainNameInputRules)
@@ -154,13 +154,14 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	}
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), mock)
+	manager, err := Create(mock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 
 	time.Sleep(time.Second)
 
 	defer func() {
-		err := manager.Reset()
+		err := manager.Reset(nil)
 		require.NoError(t, err, "clear the manager state")
 
 		time.Sleep(time.Second)
@@ -170,7 +171,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	t.Run("add first rule with set", func(t *testing.T) {
 		ip := net.ParseIP("10.20.0.2")
 		port := &fw.Port{Values: []int{8080}}
-		rule1, err = manager.AddFiltering(
+		rule1, err = manager.AddPeerFiltering(
 			ip, "tcp", nil, port, fw.RuleDirectionOUT,
 			fw.ActionAccept, "default", "accept HTTP traffic",
 		)
@@ -189,7 +190,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 		port := &fw.Port{
 			Values: []int{443},
 		}
-		rule2, err = manager.AddFiltering(
+		rule2, err = manager.AddPeerFiltering(
 			ip, "tcp", port, nil, fw.RuleDirectionIN, fw.ActionAccept,
 			"default", "accept HTTPS traffic from ports range",
 		)
@@ -202,7 +203,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 
 	t.Run("delete first rule", func(t *testing.T) {
 		for _, r := range rule1 {
-			err := manager.DeleteRule(r)
+			err := manager.DeletePeerRule(r)
 			require.NoError(t, err, "failed to delete rule")
 
 			require.NotContains(t, manager.aclMgr.ipsetStore.ipsets, r.(*Rule).ruleID, "rule must be removed form the ruleset index")
@@ -211,7 +212,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 
 	t.Run("delete second rule", func(t *testing.T) {
 		for _, r := range rule2 {
-			err := manager.DeleteRule(r)
+			err := manager.DeletePeerRule(r)
 			require.NoError(t, err, "failed to delete rule")
 
 			require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty")
@@ -219,7 +220,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	})
 
 	t.Run("reset check", func(t *testing.T) {
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 	})
 }
@@ -251,12 +252,13 @@ func TestIptablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(context.Background(), mock)
+			manager, err := Create(mock)
 			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second)
 
 			defer func() {
-				err := manager.Reset()
+				err := manager.Reset(nil)
 				require.NoError(t, err, "clear the manager state")
 
 				time.Sleep(time.Second)
@@ -269,9 +271,9 @@ func TestIptablesCreatePerformance(t *testing.T) {
 			for i := 0; i < testMax; i++ {
 				port := &fw.Port{Values: []int{1000 + i}}
 				if i%2 == 0 {
-					_, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
 				} else {
-					_, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
 				}
 
 				require.NoError(t, err, "failed to add rule")

client/firewall/iptables/router_linux.go

@@ -3,370 +3,597 @@
 package iptables
 
 import (
-	"context"
 	"fmt"
+	"net/netip"
+	"strconv"
 	"strings"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/hashicorp/go-multierror"
+	"github.com/nadoo/ipset"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-)
-
-const (
-	Ipv4Forwarding = "netbird-rt-forwarding"
-	ipv4Nat        = "netbird-rt-nat"
+	"github.com/netbirdio/netbird/client/internal/acl/id"
+	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 // constants needed to manage and create iptable rules
 const (
 	tableFilter             = "filter"
 	tableNat                = "nat"
-	chainFORWARD            = "FORWARD"
+	tableMangle             = "mangle"
 	chainPOSTROUTING        = "POSTROUTING"
+	chainPREROUTING         = "PREROUTING"
 	chainRTNAT              = "NETBIRD-RT-NAT"
 	chainRTFWD              = "NETBIRD-RT-FWD"
+	chainRTPRE              = "NETBIRD-RT-PRE"
 	routingFinalForwardJump = "ACCEPT"
 	routingFinalNatJump     = "MASQUERADE"
+
+	jumpPre  = "jump-pre"
+	jumpNat  = "jump-nat"
+	matchSet = "--match-set"
 )
 
-type routerManager struct {
-	ctx            context.Context
-	stop           context.CancelFunc
-	iptablesClient *iptables.IPTables
-	rules          map[string][]string
+type routeFilteringRuleParams struct {
+	Sources     []netip.Prefix
+	Destination netip.Prefix
+	Proto       firewall.Protocol
+	SPort       *firewall.Port
+	DPort       *firewall.Port
+	Direction   firewall.RuleDirection
+	Action      firewall.Action
+	SetName     string
 }
 
-func newRouterManager(parentCtx context.Context, iptablesClient *iptables.IPTables) (*routerManager, error) {
-	ctx, cancel := context.WithCancel(parentCtx)
-	m := &routerManager{
-		ctx:            ctx,
-		stop:           cancel,
+type routeRules map[string][]string
+
+type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}]
+
+type router struct {
+	iptablesClient   *iptables.IPTables
+	rules            routeRules
+	ipsetCounter     *ipsetCounter
+	wgIface          iFaceMapper
+	legacyManagement bool
+
+	stateManager *statemanager.Manager
+}
+
+func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
+	r := &router{
 		iptablesClient: iptablesClient,
 		rules:          make(map[string][]string),
+		wgIface:        wgIface,
 	}
 
-	err := m.cleanUpDefaultForwardRules()
-	if err != nil {
-		log.Errorf("failed to cleanup routing rules: %s", err)
-		return nil, err
+	r.ipsetCounter = refcounter.New(
+		func(name string, sources []netip.Prefix) (struct{}, error) {
+			return struct{}{}, r.createIpSet(name, sources)
+		},
+		func(name string, _ struct{}) error {
+			return r.deleteIpSet(name)
+		},
+	)
+
+	if err := ipset.Init(); err != nil {
+		return nil, fmt.Errorf("init ipset: %w", err)
 	}
-	err = m.createContainers()
-	if err != nil {
-		log.Errorf("failed to create containers for route: %s", err)
-	}
-	return m, err
+
+	return r, nil
 }
 
-// InsertRoutingRules inserts an iptables rule pair to the forwarding chain and if enabled, to the nat chain
-func (i *routerManager) InsertRoutingRules(pair firewall.RouterPair) error {
-	err := i.insertRoutingRule(firewall.ForwardingFormat, tableFilter, chainRTFWD, routingFinalForwardJump, pair)
-	if err != nil {
-		return err
+func (r *router) init(stateManager *statemanager.Manager) error {
+	r.stateManager = stateManager
+
+	if err := r.cleanUpDefaultForwardRules(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
 	}
 
-	err = i.insertRoutingRule(firewall.InForwardingFormat, tableFilter, chainRTFWD, routingFinalForwardJump, firewall.GetInPair(pair))
-	if err != nil {
-		return err
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+func (r *router) AddRouteFiltering(
+	sources []netip.Prefix,
+	destination netip.Prefix,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	action firewall.Action,
+) (firewall.Rule, error) {
+	ruleKey := id.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action)
+	if _, ok := r.rules[string(ruleKey)]; ok {
+		return ruleKey, nil
+	}
+
+	var setName string
+	if len(sources) > 1 {
+		setName = firewall.GenerateSetName(sources)
+		if _, err := r.ipsetCounter.Increment(setName, sources); err != nil {
+			return nil, fmt.Errorf("create or get ipset: %w", err)
+		}
+	}
+
+	params := routeFilteringRuleParams{
+		Sources:     sources,
+		Destination: destination,
+		Proto:       proto,
+		SPort:       sPort,
+		DPort:       dPort,
+		Action:      action,
+		SetName:     setName,
+	}
+
+	rule := genRouteFilteringRuleSpec(params)
+	if err := r.iptablesClient.Append(tableFilter, chainRTFWD, rule...); err != nil {
+		return nil, fmt.Errorf("add route rule: %v", err)
+	}
+
+	r.rules[string(ruleKey)] = rule
+
+	r.updateState()
+
+	return ruleKey, nil
+}
+
+func (r *router) DeleteRouteRule(rule firewall.Rule) error {
+	ruleKey := rule.GetRuleID()
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		setName := r.findSetNameInRule(rule)
+
+		if err := r.iptablesClient.Delete(tableFilter, chainRTFWD, rule...); err != nil {
+			return fmt.Errorf("delete route rule: %v", err)
+		}
+		delete(r.rules, ruleKey)
+
+		if setName != "" {
+			if _, err := r.ipsetCounter.Decrement(setName); err != nil {
+				return fmt.Errorf("failed to remove ipset: %w", err)
+			}
+		}
+	} else {
+		log.Debugf("route rule %s not found", ruleKey)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+func (r *router) findSetNameInRule(rule []string) string {
+	for i, arg := range rule {
+		if arg == "-m" && i+3 < len(rule) && rule[i+1] == "set" && rule[i+2] == matchSet {
+			return rule[i+3]
+		}
+	}
+	return ""
+}
+
+func (r *router) createIpSet(setName string, sources []netip.Prefix) error {
+	if err := ipset.Create(setName, ipset.OptTimeout(0)); err != nil {
+		return fmt.Errorf("create set %s: %w", setName, err)
+	}
+
+	for _, prefix := range sources {
+		if err := ipset.AddPrefix(setName, prefix); err != nil {
+			return fmt.Errorf("add element to set %s: %w", setName, err)
+		}
+	}
+
+	return nil
+}
+
+func (r *router) deleteIpSet(setName string) error {
+	if err := ipset.Destroy(setName); err != nil {
+		return fmt.Errorf("destroy set %s: %w", setName, err)
+	}
+	return nil
+}
+
+// AddNatRule inserts an iptables rule pair into the nat chain
+func (r *router) AddNatRule(pair firewall.RouterPair) error {
+	if r.legacyManagement {
+		log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination)
+		if err := r.addLegacyRouteRule(pair); err != nil {
+			return fmt.Errorf("add legacy routing rule: %w", err)
+		}
 	}
 
 	if !pair.Masquerade {
 		return nil
 	}
 
-	err = i.addNATRule(firewall.NatFormat, tableNat, chainRTNAT, routingFinalNatJump, pair)
-	if err != nil {
-		return err
+	if err := r.addNatRule(pair); err != nil {
+		return fmt.Errorf("add nat rule: %w", err)
 	}
 
-	err = i.addNATRule(firewall.InNatFormat, tableNat, chainRTNAT, routingFinalNatJump, firewall.GetInPair(pair))
-	if err != nil {
-		return err
+	if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil {
+		return fmt.Errorf("add inverse nat rule: %w", err)
 	}
 
+	r.updateState()
+
 	return nil
 }
 
-// insertRoutingRule inserts an iptables rule
-func (i *routerManager) insertRoutingRule(keyFormat, table, chain, jump string, pair firewall.RouterPair) error {
-	var err error
+// RemoveNatRule removes an iptables rule pair from forwarding and nat chains
+func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
+	if err := r.removeNatRule(pair); err != nil {
+		return fmt.Errorf("remove nat rule: %w", err)
+	}
 
-	ruleKey := firewall.GenKey(keyFormat, pair.ID)
-	rule := genRuleSpec(jump, pair.Source, pair.Destination)
-	existingRule, found := i.rules[ruleKey]
-	if found {
-		err = i.iptablesClient.DeleteIfExists(table, chain, existingRule...)
-		if err != nil {
-			return fmt.Errorf("error while removing existing %s rule for %s: %v", getIptablesRuleType(table), pair.Destination, err)
+	if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil {
+		return fmt.Errorf("remove inverse nat rule: %w", err)
+	}
+
+	if err := r.removeLegacyRouteRule(pair); err != nil {
+		return fmt.Errorf("remove legacy routing rule: %w", err)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls
+func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if err := r.removeLegacyRouteRule(pair); err != nil {
+		return err
+	}
+
+	rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", routingFinalForwardJump}
+	if err := r.iptablesClient.Append(tableFilter, chainRTFWD, rule...); err != nil {
+		return fmt.Errorf("add legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err)
+	}
+
+	r.rules[ruleKey] = rule
+
+	return nil
+}
+
+func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWD, rule...); err != nil {
+			return fmt.Errorf("remove legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err)
 		}
-		delete(i.rules, ruleKey)
-	}
-
-	err = i.iptablesClient.Insert(table, chain, 1, rule...)
-	if err != nil {
-		return fmt.Errorf("error while adding new %s rule for %s: %v", getIptablesRuleType(table), pair.Destination, err)
-	}
-
-	i.rules[ruleKey] = rule
-
-	return nil
-}
-
-// RemoveRoutingRules removes an iptables rule pair from forwarding and nat chains
-func (i *routerManager) RemoveRoutingRules(pair firewall.RouterPair) error {
-	err := i.removeRoutingRule(firewall.ForwardingFormat, tableFilter, chainRTFWD, pair)
-	if err != nil {
-		return err
-	}
-
-	err = i.removeRoutingRule(firewall.InForwardingFormat, tableFilter, chainRTFWD, firewall.GetInPair(pair))
-	if err != nil {
-		return err
-	}
-
-	if !pair.Masquerade {
-		return nil
-	}
-
-	err = i.removeRoutingRule(firewall.NatFormat, tableNat, chainRTNAT, pair)
-	if err != nil {
-		return err
-	}
-
-	err = i.removeRoutingRule(firewall.InNatFormat, tableNat, chainRTNAT, firewall.GetInPair(pair))
-	if err != nil {
-		return err
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("legacy forwarding rule %s not found", ruleKey)
 	}
 
 	return nil
 }
 
-func (i *routerManager) removeRoutingRule(keyFormat, table, chain string, pair firewall.RouterPair) error {
-	var err error
+// GetLegacyManagement returns the current legacy management mode
+func (r *router) GetLegacyManagement() bool {
+	return r.legacyManagement
+}
 
-	ruleKey := firewall.GenKey(keyFormat, pair.ID)
-	existingRule, found := i.rules[ruleKey]
-	if found {
-		err = i.iptablesClient.DeleteIfExists(table, chain, existingRule...)
-		if err != nil {
-			return fmt.Errorf("error while removing existing %s rule for %s: %v", getIptablesRuleType(table), pair.Destination, err)
+// SetLegacyManagement sets the route manager to use legacy management mode
+func (r *router) SetLegacyManagement(isLegacy bool) {
+	r.legacyManagement = isLegacy
+}
+
+// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls
+func (r *router) RemoveAllLegacyRouteRules() error {
+	var merr *multierror.Error
+	for k, rule := range r.rules {
+		if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) {
+			continue
+		}
+		if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWD, rule...); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
 		}
 	}
-	delete(i.rules, ruleKey)
 
-	return nil
+	r.updateState()
+
+	return nberrors.FormatErrorOrNil(merr)
 }
 
-func (i *routerManager) RouteingFwChainName() string {
-	return chainRTFWD
-}
-
-func (i *routerManager) Reset() error {
-	err := i.cleanUpDefaultForwardRules()
-	if err != nil {
-		return err
+func (r *router) Reset() error {
+	var merr *multierror.Error
+	if err := r.cleanUpDefaultForwardRules(); err != nil {
+		merr = multierror.Append(merr, err)
 	}
-	i.rules = make(map[string][]string)
-	return nil
+	r.rules = make(map[string][]string)
+
+	if err := r.ipsetCounter.Flush(); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+
+	r.updateState()
+
+	return nberrors.FormatErrorOrNil(merr)
 }
 
-func (i *routerManager) cleanUpDefaultForwardRules() error {
-	err := i.cleanJumpRules()
-	if err != nil {
-		return err
+func (r *router) cleanUpDefaultForwardRules() error {
+	if err := r.cleanJumpRules(); err != nil {
+		return fmt.Errorf("clean jump rules: %w", err)
 	}
 
 	log.Debug("flushing routing related tables")
-	ok, err := i.iptablesClient.ChainExists(tableFilter, chainRTFWD)
-	if err != nil {
-		log.Errorf("failed check chain %s,error: %v", chainRTFWD, err)
-		return err
-	} else if ok {
-		err = i.iptablesClient.ClearAndDeleteChain(tableFilter, chainRTFWD)
+	for _, chainInfo := range []struct {
+		chain string
+		table string
+	}{
+		{chainRTFWD, tableFilter},
+		{chainRTNAT, tableNat},
+		{chainRTPRE, tableMangle},
+	} {
+		ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain)
 		if err != nil {
-			log.Errorf("failed cleaning chain %s,error: %v", chainRTFWD, err)
-			return err
+			return fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
+		} else if ok {
+			if err = r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil {
+				return fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
+			}
 		}
 	}
 
-	ok, err = i.iptablesClient.ChainExists(tableNat, chainRTNAT)
+	return nil
+}
+
+func (r *router) createContainers() error {
+	for _, chainInfo := range []struct {
+		chain string
+		table string
+	}{
+		{chainRTFWD, tableFilter},
+		{chainRTPRE, tableMangle},
+		{chainRTNAT, tableNat},
+	} {
+		if err := r.createAndSetupChain(chainInfo.chain); err != nil {
+			return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
+		}
+	}
+
+	if err := r.insertEstablishedRule(chainRTFWD); err != nil {
+		return fmt.Errorf("insert established rule: %w", err)
+	}
+
+	if err := r.addPostroutingRules(); err != nil {
+		return fmt.Errorf("add static nat rules: %w", err)
+	}
+
+	if err := r.addJumpRules(); err != nil {
+		return fmt.Errorf("add jump rules: %w", err)
+	}
+
+	return nil
+}
+
+func (r *router) addPostroutingRules() error {
+	// First rule for outbound masquerade
+	rule1 := []string{
+		"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+		"!", "-o", "lo",
+		"-j", routingFinalNatJump,
+	}
+	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil {
+		return fmt.Errorf("add outbound masquerade rule: %v", err)
+	}
+	r.rules["static-nat-outbound"] = rule1
+
+	// Second rule for return traffic masquerade
+	rule2 := []string{
+		"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+		"-o", r.wgIface.Name(),
+		"-j", routingFinalNatJump,
+	}
+	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil {
+		return fmt.Errorf("add return masquerade rule: %v", err)
+	}
+	r.rules["static-nat-return"] = rule2
+
+	return nil
+}
+
+func (r *router) createAndSetupChain(chain string) error {
+	table := r.getTableForChain(chain)
+
+	if err := r.iptablesClient.NewChain(table, chain); err != nil {
+		return fmt.Errorf("failed creating chain %s, error: %v", chain, err)
+	}
+
+	return nil
+}
+
+func (r *router) getTableForChain(chain string) string {
+	switch chain {
+	case chainRTNAT:
+		return tableNat
+	case chainRTPRE:
+		return tableMangle
+	default:
+		return tableFilter
+	}
+}
+
+func (r *router) insertEstablishedRule(chain string) error {
+	establishedRule := getConntrackEstablished()
+
+	err := r.iptablesClient.Insert(tableFilter, chain, 1, establishedRule...)
 	if err != nil {
-		log.Errorf("failed check chain %s,error: %v", chainRTNAT, err)
-		return err
-	} else if ok {
-		err = i.iptablesClient.ClearAndDeleteChain(tableNat, chainRTNAT)
-		if err != nil {
-			log.Errorf("failed cleaning chain %s,error: %v", chainRTNAT, err)
-			return err
+		return fmt.Errorf("failed to insert established rule: %v", err)
+	}
+
+	ruleKey := "established-" + chain
+	r.rules[ruleKey] = establishedRule
+
+	return nil
+}
+
+func (r *router) addJumpRules() error {
+	// Jump to NAT chain
+	natRule := []string{"-j", chainRTNAT}
+	if err := r.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, natRule...); err != nil {
+		return fmt.Errorf("add nat jump rule: %v", err)
+	}
+	r.rules[jumpNat] = natRule
+
+	// Jump to prerouting chain
+	preRule := []string{"-j", chainRTPRE}
+	if err := r.iptablesClient.Insert(tableMangle, chainPREROUTING, 1, preRule...); err != nil {
+		return fmt.Errorf("add prerouting jump rule: %v", err)
+	}
+	r.rules[jumpPre] = preRule
+
+	return nil
+}
+
+func (r *router) cleanJumpRules() error {
+	for _, ruleKey := range []string{jumpNat, jumpPre} {
+		if rule, exists := r.rules[ruleKey]; exists {
+			table := tableNat
+			chain := chainPOSTROUTING
+			if ruleKey == jumpPre {
+				table = tableMangle
+				chain = chainPREROUTING
+			}
+
+			if err := r.iptablesClient.DeleteIfExists(table, chain, rule...); err != nil {
+				return fmt.Errorf("delete rule from chain %s in table %s, err: %v", chain, table, err)
+			}
+			delete(r.rules, ruleKey)
 		}
 	}
 	return nil
 }
 
-func (i *routerManager) createContainers() error {
-	if i.rules[Ipv4Forwarding] != nil {
+func (r *router) addNatRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil {
+			return fmt.Errorf("error while removing existing marking rule for %s: %v", pair.Destination, err)
+		}
+		delete(r.rules, ruleKey)
+	}
+
+	markValue := nbnet.PreroutingFwmarkMasquerade
+	if pair.Inverse {
+		markValue = nbnet.PreroutingFwmarkMasqueradeReturn
+	}
+
+	rule := []string{"-i", r.wgIface.Name()}
+	if pair.Inverse {
+		rule = []string{"!", "-i", r.wgIface.Name()}
+	}
+
+	rule = append(rule,
+		"-m", "conntrack",
+		"--ctstate", "NEW",
+		"-s", pair.Source.String(),
+		"-d", pair.Destination.String(),
+		"-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue),
+	)
+
+	if err := r.iptablesClient.Append(tableMangle, chainRTPRE, rule...); err != nil {
+		return fmt.Errorf("error while adding marking rule for %s: %v", pair.Destination, err)
+	}
+
+	r.rules[ruleKey] = rule
+	return nil
+}
+
+func (r *router) removeNatRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil {
+			return fmt.Errorf("error while removing marking rule for %s: %v", pair.Destination, err)
+		}
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("marking rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+func (r *router) updateState() {
+	if r.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := r.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.RouteRules = r.rules
+	currentState.RouteIPsetCounter = r.ipsetCounter
+
+	if err := r.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
+func genRouteFilteringRuleSpec(params routeFilteringRuleParams) []string {
+	var rule []string
+
+	if params.SetName != "" {
+		rule = append(rule, "-m", "set", matchSet, params.SetName, "src")
+	} else if len(params.Sources) > 0 {
+		source := params.Sources[0]
+		rule = append(rule, "-s", source.String())
+	}
+
+	rule = append(rule, "-d", params.Destination.String())
+
+	if params.Proto != firewall.ProtocolALL {
+		rule = append(rule, "-p", strings.ToLower(string(params.Proto)))
+		rule = append(rule, applyPort("--sport", params.SPort)...)
+		rule = append(rule, applyPort("--dport", params.DPort)...)
+	}
+
+	rule = append(rule, "-j", actionToStr(params.Action))
+
+	return rule
+}
+
+func applyPort(flag string, port *firewall.Port) []string {
+	if port == nil {
 		return nil
 	}
 
-	errMSGFormat := "failed creating chain %s,error: %v"
-	err := i.createChain(tableFilter, chainRTFWD)
-	if err != nil {
-		return fmt.Errorf(errMSGFormat, chainRTFWD, err)
+	if port.IsRange && len(port.Values) == 2 {
+		return []string{flag, fmt.Sprintf("%d:%d", port.Values[0], port.Values[1])}
 	}
 
-	err = i.createChain(tableNat, chainRTNAT)
-	if err != nil {
-		return fmt.Errorf(errMSGFormat, chainRTNAT, err)
+	if len(port.Values) > 1 {
+		portList := make([]string, len(port.Values))
+		for i, p := range port.Values {
+			portList[i] = strconv.Itoa(p)
+		}
+		return []string{"-m", "multiport", flag, strings.Join(portList, ",")}
 	}
 
-	err = i.addJumpRules()
-	if err != nil {
-		return fmt.Errorf("error while creating jump rules: %v", err)
-	}
-
-	return nil
-}
-
-// addJumpRules create jump rules to send packets to NetBird chains
-func (i *routerManager) addJumpRules() error {
-	rule := []string{"-j", chainRTFWD}
-	err := i.iptablesClient.Insert(tableFilter, chainFORWARD, 1, rule...)
-	if err != nil {
-		return err
-	}
-	i.rules[Ipv4Forwarding] = rule
-
-	rule = []string{"-j", chainRTNAT}
-	err = i.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, rule...)
-	if err != nil {
-		return err
-	}
-	i.rules[ipv4Nat] = rule
-
-	return nil
-}
-
-// cleanJumpRules cleans jump rules that was sending packets to NetBird chains
-func (i *routerManager) cleanJumpRules() error {
-	var err error
-	errMSGFormat := "failed cleaning rule from chain %s,err: %v"
-	rule, found := i.rules[Ipv4Forwarding]
-	if found {
-		err = i.iptablesClient.DeleteIfExists(tableFilter, chainFORWARD, rule...)
-		if err != nil {
-			return fmt.Errorf(errMSGFormat, chainFORWARD, err)
-		}
-	}
-	rule, found = i.rules[ipv4Nat]
-	if found {
-		err = i.iptablesClient.DeleteIfExists(tableNat, chainPOSTROUTING, rule...)
-		if err != nil {
-			return fmt.Errorf(errMSGFormat, chainPOSTROUTING, err)
-		}
-	}
-
-	rules, err := i.iptablesClient.List("nat", "POSTROUTING")
-	if err != nil {
-		return fmt.Errorf("failed to list rules: %s", err)
-	}
-
-	for _, ruleString := range rules {
-		if !strings.Contains(ruleString, "NETBIRD") {
-			continue
-		}
-		rule := strings.Fields(ruleString)
-		err := i.iptablesClient.DeleteIfExists("nat", "POSTROUTING", rule[2:]...)
-		if err != nil {
-			return fmt.Errorf("failed to delete postrouting jump rule: %s", err)
-		}
-	}
-
-	rules, err = i.iptablesClient.List(tableFilter, "FORWARD")
-	if err != nil {
-		return fmt.Errorf("failed to list rules in FORWARD chain: %s", err)
-	}
-
-	for _, ruleString := range rules {
-		if !strings.Contains(ruleString, "NETBIRD") {
-			continue
-		}
-		rule := strings.Fields(ruleString)
-		err := i.iptablesClient.DeleteIfExists(tableFilter, "FORWARD", rule[2:]...)
-		if err != nil {
-			return fmt.Errorf("failed to delete FORWARD jump rule: %s", err)
-		}
-	}
-	return nil
-}
-
-func (i *routerManager) createChain(table, newChain string) error {
-	chains, err := i.iptablesClient.ListChains(table)
-	if err != nil {
-		return fmt.Errorf("couldn't get %s table chains, error: %v", table, err)
-	}
-
-	shouldCreateChain := true
-	for _, chain := range chains {
-		if chain == newChain {
-			shouldCreateChain = false
-		}
-	}
-
-	if shouldCreateChain {
-		err = i.iptablesClient.NewChain(table, newChain)
-		if err != nil {
-			return fmt.Errorf("couldn't create chain %s in %s table, error: %v", newChain, table, err)
-		}
-
-		// Add the loopback return rule to the NAT chain
-		loopbackRule := []string{"-o", "lo", "-j", "RETURN"}
-		err = i.iptablesClient.Insert(table, newChain, 1, loopbackRule...)
-		if err != nil {
-			return fmt.Errorf("failed to add loopback return rule to %s: %v", chainRTNAT, err)
-		}
-
-		err = i.iptablesClient.Append(table, newChain, "-j", "RETURN")
-		if err != nil {
-			return fmt.Errorf("couldn't create chain %s default rule, error: %v", newChain, err)
-		}
-
-	}
-	return nil
-}
-
-// addNATRule appends an iptables rule pair to the nat chain
-func (i *routerManager) addNATRule(keyFormat, table, chain, jump string, pair firewall.RouterPair) error {
-	ruleKey := firewall.GenKey(keyFormat, pair.ID)
-	rule := genRuleSpec(jump, pair.Source, pair.Destination)
-	existingRule, found := i.rules[ruleKey]
-	if found {
-		err := i.iptablesClient.DeleteIfExists(table, chain, existingRule...)
-		if err != nil {
-			return fmt.Errorf("error while removing existing NAT rule for %s: %v", pair.Destination, err)
-		}
-		delete(i.rules, ruleKey)
-	}
-
-	// inserting after loopback ignore rule
-	err := i.iptablesClient.Insert(table, chain, 2, rule...)
-	if err != nil {
-		return fmt.Errorf("error while appending new NAT rule for %s: %v", pair.Destination, err)
-	}
-
-	i.rules[ruleKey] = rule
-
-	return nil
-}
-
-// genRuleSpec generates rule specification
-func genRuleSpec(jump, source, destination string) []string {
-	return []string{"-s", source, "-d", destination, "-j", jump}
-}
-
-func getIptablesRuleType(table string) string {
-	ruleType := "forwarding"
-	if table == tableNat {
-		ruleType = "nat"
-	}
-	return ruleType
+	return []string{flag, strconv.Itoa(port.Values[0])}
 }

client/firewall/iptables/router_linux_test.go

@@ -3,16 +3,18 @@
 package iptables
 
 import (
-	"context"
+	"fmt"
+	"net/netip"
 	"os/exec"
 	"testing"
 
 	"github.com/coreos/go-iptables/iptables"
-	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/firewall/test"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 func isIptablesSupported() bool {
@@ -28,45 +30,45 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "failed to init iptables client")
 
-	manager, err := newRouterManager(context.TODO(), iptablesClient)
+	manager, err := newRouter(iptablesClient, ifaceMock)
 	require.NoError(t, err, "should return a valid iptables manager")
+	require.NoError(t, manager.init(nil))
 
 	defer func() {
-		_ = manager.Reset()
+		assert.NoError(t, manager.Reset(), "shouldn't return error")
 	}()
 
-	require.Len(t, manager.rules, 2, "should have created rules map")
+	// Now 5 rules:
+	// 1. established rule in forward chain
+	// 2. jump rule to NAT chain
+	// 3. jump rule to PRE chain
+	// 4. static outbound masquerade rule
+	// 5. static return masquerade rule
+	require.Len(t, manager.rules, 5, "should have created rules map")
 
-	exists, err := manager.iptablesClient.Exists(tableFilter, chainFORWARD, manager.rules[Ipv4Forwarding]...)
-	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableFilter, chainFORWARD)
-	require.True(t, exists, "forwarding rule should exist")
-
-	exists, err = manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, manager.rules[ipv4Nat]...)
+	exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, "-j", chainRTNAT)
 	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPOSTROUTING)
-	require.True(t, exists, "postrouting rule should exist")
+	require.True(t, exists, "postrouting jump rule should exist")
+
+	exists, err = manager.iptablesClient.Exists(tableMangle, chainPREROUTING, "-j", chainRTPRE)
+	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPREROUTING)
+	require.True(t, exists, "prerouting jump rule should exist")
 
 	pair := firewall.RouterPair{
 		ID:          "abc",
-		Source:      "100.100.100.1/32",
-		Destination: "100.100.100.0/24",
+		Source:      netip.MustParsePrefix("100.100.100.1/32"),
+		Destination: netip.MustParsePrefix("100.100.100.0/24"),
 		Masquerade:  true,
 	}
-	forward4Rule := genRuleSpec(routingFinalForwardJump, pair.Source, pair.Destination)
 
-	err = manager.iptablesClient.Insert(tableFilter, chainRTFWD, 1, forward4Rule...)
-	require.NoError(t, err, "inserting rule should not return error")
-
-	nat4Rule := genRuleSpec(routingFinalNatJump, pair.Source, pair.Destination)
-
-	err = manager.iptablesClient.Insert(tableNat, chainRTNAT, 1, nat4Rule...)
-	require.NoError(t, err, "inserting rule should not return error")
+	err = manager.AddNatRule(pair)
+	require.NoError(t, err, "adding NAT rule should not return error")
 
 	err = manager.Reset()
 	require.NoError(t, err, "shouldn't return error")
 }
 
-func TestIptablesManager_InsertRoutingRules(t *testing.T) {
-
+func TestIptablesManager_AddNatRule(t *testing.T) {
 	if !isIptablesSupported() {
 		t.SkipNow()
 	}
@@ -76,78 +78,71 @@ func TestIptablesManager_InsertRoutingRules(t *testing.T) {
 			iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 			require.NoError(t, err, "failed to init iptables client")
 
-			manager, err := newRouterManager(context.TODO(), iptablesClient)
+			manager, err := newRouter(iptablesClient, ifaceMock)
 			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
 
 			defer func() {
-				err := manager.Reset()
-				if err != nil {
-					log.Errorf("failed to reset iptables manager: %s", err)
-				}
+				assert.NoError(t, manager.Reset(), "shouldn't return error")
 			}()
 
-			err = manager.InsertRoutingRules(testCase.InputPair)
-			require.NoError(t, err, "forwarding pair should be inserted")
+			err = manager.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "marking rule should be inserted")
 
-			forwardRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID)
-			forwardRule := genRuleSpec(routingFinalForwardJump, testCase.InputPair.Source, testCase.InputPair.Destination)
-
-			exists, err := iptablesClient.Exists(tableFilter, chainRTFWD, forwardRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableFilter, chainRTFWD)
-			require.True(t, exists, "forwarding rule should exist")
-
-			foundRule, found := manager.rules[forwardRuleKey]
-			require.True(t, found, "forwarding rule should exist in the manager map")
-			require.Equal(t, forwardRule[:4], foundRule[:4], "stored forwarding rule should match")
-
-			inForwardRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID)
-			inForwardRule := genRuleSpec(routingFinalForwardJump, firewall.GetInPair(testCase.InputPair).Source, firewall.GetInPair(testCase.InputPair).Destination)
-
-			exists, err = iptablesClient.Exists(tableFilter, chainRTFWD, inForwardRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableFilter, chainRTFWD)
-			require.True(t, exists, "income forwarding rule should exist")
-
-			foundRule, found = manager.rules[inForwardRuleKey]
-			require.True(t, found, "income forwarding rule should exist in the manager map")
-			require.Equal(t, inForwardRule[:4], foundRule[:4], "stored income forwarding rule should match")
-
-			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID)
-			natRule := genRuleSpec(routingFinalNatJump, testCase.InputPair.Source, testCase.InputPair.Destination)
-
-			exists, err = iptablesClient.Exists(tableNat, chainRTNAT, natRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
-			if testCase.InputPair.Masquerade {
-				require.True(t, exists, "nat rule should be created")
-				foundNatRule, foundNat := manager.rules[natRuleKey]
-				require.True(t, foundNat, "nat rule should exist in the map")
-				require.Equal(t, natRule[:4], foundNatRule[:4], "stored nat rule should match")
-			} else {
-				require.False(t, exists, "nat rule should not be created")
-				_, foundNat := manager.rules[natRuleKey]
-				require.False(t, foundNat, "nat rule should not exist in the map")
+			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
+			markingRule := []string{
+				"-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", testCase.InputPair.Source.String(),
+				"-d", testCase.InputPair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
 			}
 
-			inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID)
-			inNatRule := genRuleSpec(routingFinalNatJump, firewall.GetInPair(testCase.InputPair).Source, firewall.GetInPair(testCase.InputPair).Destination)
-
-			exists, err = iptablesClient.Exists(tableNat, chainRTNAT, inNatRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
+			exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
 			if testCase.InputPair.Masquerade {
-				require.True(t, exists, "income nat rule should be created")
-				foundNatRule, foundNat := manager.rules[inNatRuleKey]
-				require.True(t, foundNat, "income nat rule should exist in the map")
-				require.Equal(t, inNatRule[:4], foundNatRule[:4], "stored income nat rule should match")
+				require.True(t, exists, "marking rule should be created")
+				foundRule, found := manager.rules[natRuleKey]
+				require.True(t, found, "marking rule should exist in the map")
+				require.Equal(t, markingRule, foundRule, "stored marking rule should match")
 			} else {
-				require.False(t, exists, "nat rule should not be created")
-				_, foundNat := manager.rules[inNatRuleKey]
-				require.False(t, foundNat, "income nat rule should not exist in the map")
+				require.False(t, exists, "marking rule should not be created")
+				_, found := manager.rules[natRuleKey]
+				require.False(t, found, "marking rule should not exist in the map")
+			}
+
+			// Check inverse rule
+			inversePair := firewall.GetInversePair(testCase.InputPair)
+			inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair)
+			inverseMarkingRule := []string{
+				"!", "-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", inversePair.Source.String(),
+				"-d", inversePair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+			}
+
+			exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			if testCase.InputPair.Masquerade {
+				require.True(t, exists, "inverse marking rule should be created")
+				foundRule, found := manager.rules[inverseRuleKey]
+				require.True(t, found, "inverse marking rule should exist in the map")
+				require.Equal(t, inverseMarkingRule, foundRule, "stored inverse marking rule should match")
+			} else {
+				require.False(t, exists, "inverse marking rule should not be created")
+				_, found := manager.rules[inverseRuleKey]
+				require.False(t, found, "inverse marking rule should not exist in the map")
 			}
 		})
 	}
 }
 
-func TestIptablesManager_RemoveRoutingRules(t *testing.T) {
-
+func TestIptablesManager_RemoveNatRule(t *testing.T) {
 	if !isIptablesSupported() {
 		t.SkipNow()
 	}
@@ -156,72 +151,226 @@ func TestIptablesManager_RemoveRoutingRules(t *testing.T) {
 		t.Run(testCase.Name, func(t *testing.T) {
 			iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 
-			manager, err := newRouterManager(context.TODO(), iptablesClient)
+			manager, err := newRouter(iptablesClient, ifaceMock)
 			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
 			defer func() {
-				_ = manager.Reset()
+				assert.NoError(t, manager.Reset(), "shouldn't return error")
 			}()
 
+			err = manager.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "should add NAT rule without error")
+
+			err = manager.RemoveNatRule(testCase.InputPair)
 			require.NoError(t, err, "shouldn't return error")
 
-			forwardRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID)
-			forwardRule := genRuleSpec(routingFinalForwardJump, testCase.InputPair.Source, testCase.InputPair.Destination)
+			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
+			markingRule := []string{
+				"-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", testCase.InputPair.Source.String(),
+				"-d", testCase.InputPair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+			}
 
-			err = iptablesClient.Insert(tableFilter, chainRTFWD, 1, forwardRule...)
-			require.NoError(t, err, "inserting rule should not return error")
+			exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			require.False(t, exists, "marking rule should not exist")
 
-			inForwardRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID)
-			inForwardRule := genRuleSpec(routingFinalForwardJump, firewall.GetInPair(testCase.InputPair).Source, firewall.GetInPair(testCase.InputPair).Destination)
+			_, found := manager.rules[natRuleKey]
+			require.False(t, found, "marking rule should not exist in the manager map")
 
-			err = iptablesClient.Insert(tableFilter, chainRTFWD, 1, inForwardRule...)
-			require.NoError(t, err, "inserting rule should not return error")
+			// Check inverse rule removal
+			inversePair := firewall.GetInversePair(testCase.InputPair)
+			inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair)
+			inverseMarkingRule := []string{
+				"!", "-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", inversePair.Source.String(),
+				"-d", inversePair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+			}
 
-			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID)
-			natRule := genRuleSpec(routingFinalNatJump, testCase.InputPair.Source, testCase.InputPair.Destination)
-
-			err = iptablesClient.Insert(tableNat, chainRTNAT, 1, natRule...)
-			require.NoError(t, err, "inserting rule should not return error")
-
-			inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID)
-			inNatRule := genRuleSpec(routingFinalNatJump, firewall.GetInPair(testCase.InputPair).Source, firewall.GetInPair(testCase.InputPair).Destination)
-
-			err = iptablesClient.Insert(tableNat, chainRTNAT, 1, inNatRule...)
-			require.NoError(t, err, "inserting rule should not return error")
-
-			err = manager.Reset()
-			require.NoError(t, err, "shouldn't return error")
-
-			err = manager.RemoveRoutingRules(testCase.InputPair)
-			require.NoError(t, err, "shouldn't return error")
-
-			exists, err := iptablesClient.Exists(tableFilter, chainRTFWD, forwardRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableFilter, chainRTFWD)
-			require.False(t, exists, "forwarding rule should not exist")
-
-			_, found := manager.rules[forwardRuleKey]
-			require.False(t, found, "forwarding rule should exist in the manager map")
-
-			exists, err = iptablesClient.Exists(tableFilter, chainRTFWD, inForwardRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableFilter, chainRTFWD)
-			require.False(t, exists, "income forwarding rule should not exist")
-
-			_, found = manager.rules[inForwardRuleKey]
-			require.False(t, found, "income forwarding rule should exist in the manager map")
-
-			exists, err = iptablesClient.Exists(tableNat, chainRTNAT, natRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
-			require.False(t, exists, "nat rule should not exist")
-
-			_, found = manager.rules[natRuleKey]
-			require.False(t, found, "nat rule should exist in the manager map")
-
-			exists, err = iptablesClient.Exists(tableNat, chainRTNAT, inNatRule...)
-			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainRTNAT)
-			require.False(t, exists, "income nat rule should not exist")
-
-			_, found = manager.rules[inNatRuleKey]
-			require.False(t, found, "income nat rule should exist in the manager map")
+			exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			require.False(t, exists, "inverse marking rule should not exist")
 
+			_, found = manager.rules[inverseRuleKey]
+			require.False(t, found, "inverse marking rule should not exist in the map")
+		})
+	}
+}
+
+func TestRouter_AddRouteFiltering(t *testing.T) {
+	if !isIptablesSupported() {
+		t.Skip("iptables not supported on this system")
+	}
+
+	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	require.NoError(t, err, "Failed to create iptables client")
+
+	r, err := newRouter(iptablesClient, ifaceMock)
+	require.NoError(t, err, "Failed to create router manager")
+	require.NoError(t, r.init(nil))
+
+	defer func() {
+		err := r.Reset()
+		require.NoError(t, err, "Failed to reset router")
+	}()
+
+	tests := []struct {
+		name        string
+		sources     []netip.Prefix
+		destination netip.Prefix
+		proto       firewall.Protocol
+		sPort       *firewall.Port
+		dPort       *firewall.Port
+		direction   firewall.RuleDirection
+		action      firewall.Action
+		expectSet   bool
+	}{
+		{
+			name:        "Basic TCP rule with single source",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{80}},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name: "UDP rule with multiple sources",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("172.16.0.0/16"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       &firewall.Port{Values: []int{1024, 2048}, IsRange: true},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionDrop,
+			expectSet:   true,
+		},
+		{
+			name:        "All protocols rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")},
+			destination: netip.MustParsePrefix("0.0.0.0/0"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "ICMP rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolICMP,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with multiple source ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("172.16.0.0/12")},
+			destination: netip.MustParsePrefix("192.168.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{80, 443, 8080}},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "UDP rule with single IP and port range",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.1/32")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{5000, 5100}, IsRange: true},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with source and destination ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")},
+			destination: netip.MustParsePrefix("172.16.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{1024, 65535}, IsRange: true},
+			dPort:       &firewall.Port{Values: []int{22}},
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "Drop all incoming traffic",
+			sources:     []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")},
+			destination: netip.MustParsePrefix("192.168.0.0/24"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ruleKey, err := r.AddRouteFiltering(tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.action)
+			require.NoError(t, err, "AddRouteFiltering failed")
+
+			// Check if the rule is in the internal map
+			rule, ok := r.rules[ruleKey.GetRuleID()]
+			assert.True(t, ok, "Rule not found in internal map")
+
+			// Log the internal rule
+			t.Logf("Internal rule: %v", rule)
+
+			// Check if the rule exists in iptables
+			exists, err := iptablesClient.Exists(tableFilter, chainRTFWD, rule...)
+			assert.NoError(t, err, "Failed to check rule existence")
+			assert.True(t, exists, "Rule not found in iptables")
+
+			// Verify rule content
+			params := routeFilteringRuleParams{
+				Sources:     tt.sources,
+				Destination: tt.destination,
+				Proto:       tt.proto,
+				SPort:       tt.sPort,
+				DPort:       tt.dPort,
+				Action:      tt.action,
+				SetName:     "",
+			}
+
+			expectedRule := genRouteFilteringRuleSpec(params)
+
+			if tt.expectSet {
+				setName := firewall.GenerateSetName(tt.sources)
+				params.SetName = setName
+				expectedRule = genRouteFilteringRuleSpec(params)
+
+				// Check if the set was created
+				_, exists := r.ipsetCounter.Get(setName)
+				assert.True(t, exists, "IPSet not created")
+			}
+
+			assert.Equal(t, expectedRule, rule, "Rule content mismatch")
+
+			// Clean up
+			err = r.DeleteRouteRule(ruleKey)
+			require.NoError(t, err, "Failed to delete rule")
 		})
 	}
 }

client/firewall/iptables/rulestore_linux.go

@@ -1,14 +1,16 @@
 package iptables
 
+import "encoding/json"
+
 type ipList struct {
 	ips map[string]struct{}
 }
 
-func newIpList(ip string) ipList {
+func newIpList(ip string) *ipList {
 	ips := make(map[string]struct{})
 	ips[ip] = struct{}{}
 
-	return ipList{
+	return &ipList{
 		ips: ips,
 	}
 }
@@ -17,27 +19,52 @@ func (s *ipList) addIP(ip string) {
 	s.ips[ip] = struct{}{}
 }
 
+// MarshalJSON implements json.Marshaler
+func (s *ipList) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{
+		IPs: s.ips,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipList) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ips = temp.IPs
+
+	if temp.IPs == nil {
+		temp.IPs = make(map[string]struct{})
+	}
+
+	return nil
+}
+
 type ipsetStore struct {
-	ipsets map[string]ipList // ipsetName -> ruleset
+	ipsets map[string]*ipList
 }
 
 func newIpsetStore() *ipsetStore {
 	return &ipsetStore{
-		ipsets: make(map[string]ipList),
+		ipsets: make(map[string]*ipList),
 	}
 }
 
-func (s *ipsetStore) ipset(ipsetName string) (ipList, bool) {
+func (s *ipsetStore) ipset(ipsetName string) (*ipList, bool) {
 	r, ok := s.ipsets[ipsetName]
 	return r, ok
 }
 
-func (s *ipsetStore) addIpList(ipsetName string, list ipList) {
+func (s *ipsetStore) addIpList(ipsetName string, list *ipList) {
 	s.ipsets[ipsetName] = list
 }
 
 func (s *ipsetStore) deleteIpset(ipsetName string) {
-	s.ipsets[ipsetName] = ipList{}
 	delete(s.ipsets, ipsetName)
 }
 
@@ -48,3 +75,29 @@ func (s *ipsetStore) ipsetNames() []string {
 	}
 	return names
 }
+
+// MarshalJSON implements json.Marshaler
+func (s *ipsetStore) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{
+		IPSets: s.ipsets,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipsetStore) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ipsets = temp.IPSets
+
+	if temp.IPSets == nil {
+		temp.IPSets = make(map[string]*ipList)
+	}
+
+	return nil
+}

client/firewall/iptables/state_linux.go

@@ -0,0 +1,70 @@
+package iptables
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	sync.Mutex
+
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+
+	RouteRules        routeRules    `json:"route_rules,omitempty"`
+	RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"`
+
+	ACLEntries    aclEntries  `json:"acl_entries,omitempty"`
+	ACLIPsetStore *ipsetStore `json:"acl_ipset_store,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "iptables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	ipt, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create iptables manager: %w", err)
+	}
+
+	if s.RouteRules != nil {
+		ipt.router.rules = s.RouteRules
+	}
+	if s.RouteIPsetCounter != nil {
+		ipt.router.ipsetCounter.LoadData(s.RouteIPsetCounter)
+	}
+
+	if s.ACLEntries != nil {
+		ipt.aclMgr.entries = s.ACLEntries
+	}
+	if s.ACLIPsetStore != nil {
+		ipt.aclMgr.ipsetStore = s.ACLIPsetStore
+	}
+
+	if err := ipt.Reset(nil); err != nil {
+		return fmt.Errorf("reset iptables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/manager/firewall.go

@@ -1,15 +1,24 @@
 package manager
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net"
+	"net/netip"
+	"sort"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
-	NatFormat          = "netbird-nat-%s"
-	ForwardingFormat   = "netbird-fwd-%s"
-	InNatFormat        = "netbird-nat-in-%s"
-	InForwardingFormat = "netbird-fwd-in-%s"
+	ForwardingFormatPrefix = "netbird-fwd-"
+	ForwardingFormat       = "netbird-fwd-%s-%t"
+	PreroutingFormat       = "netbird-prerouting-%s-%t"
+	NatFormat              = "netbird-nat-%s-%t"
 )
 
 // Rule abstraction should be implemented by each firewall manager
@@ -46,14 +55,16 @@ const (
 // It declares methods which handle actions required by the
 // Netbird client for ACL and routing functionality
 type Manager interface {
+	Init(stateManager *statemanager.Manager) error
+
 	// AllowNetbird allows netbird interface traffic
 	AllowNetbird() error
 
-	// AddFiltering rule to the firewall
+	// AddPeerFiltering adds a rule to the firewall
 	//
 	// If comment argument is empty firewall manager should set
 	// rule ID as comment for the rule
-	AddFiltering(
+	AddPeerFiltering(
 		ip net.IP,
 		proto Protocol,
 		sPort *Port,
@@ -64,25 +75,116 @@ type Manager interface {
 		comment string,
 	) ([]Rule, error)
 
-	// DeleteRule from the firewall by rule definition
-	DeleteRule(rule Rule) error
+	// DeletePeerRule from the firewall by rule definition
+	DeletePeerRule(rule Rule) error
 
 	// IsServerRouteSupported returns true if the firewall supports server side routing operations
 	IsServerRouteSupported() bool
 
-	// InsertRoutingRules inserts a routing firewall rule
-	InsertRoutingRules(pair RouterPair) error
+	AddRouteFiltering(source []netip.Prefix, destination netip.Prefix, proto Protocol, sPort *Port, dPort *Port, action Action) (Rule, error)
 
-	// RemoveRoutingRules removes a routing firewall rule
-	RemoveRoutingRules(pair RouterPair) error
+	// DeleteRouteRule deletes a routing rule
+	DeleteRouteRule(rule Rule) error
+
+	// AddNatRule inserts a routing NAT rule
+	AddNatRule(pair RouterPair) error
+
+	// RemoveNatRule removes a routing NAT rule
+	RemoveNatRule(pair RouterPair) error
+
+	// SetLegacyManagement sets the legacy management mode
+	SetLegacyManagement(legacy bool) error
 
 	// Reset firewall to the default state
-	Reset() error
+	Reset(stateManager *statemanager.Manager) error
 
 	// Flush the changes to firewall controller
 	Flush() error
 }
 
-func GenKey(format string, input string) string {
-	return fmt.Sprintf(format, input)
+func GenKey(format string, pair RouterPair) string {
+	return fmt.Sprintf(format, pair.ID, pair.Inverse)
+}
+
+// LegacyManager defines the interface for legacy management operations
+type LegacyManager interface {
+	RemoveAllLegacyRouteRules() error
+	GetLegacyManagement() bool
+	SetLegacyManagement(bool)
+}
+
+// SetLegacyManagement sets the route manager to use legacy management
+func SetLegacyManagement(router LegacyManager, isLegacy bool) error {
+	oldLegacy := router.GetLegacyManagement()
+
+	if oldLegacy != isLegacy {
+		router.SetLegacyManagement(isLegacy)
+		log.Debugf("Set legacy management to %v", isLegacy)
+	}
+
+	// client reconnected to a newer mgmt, we need to clean up the legacy rules
+	if !isLegacy && oldLegacy {
+		if err := router.RemoveAllLegacyRouteRules(); err != nil {
+			return fmt.Errorf("remove legacy routing rules: %v", err)
+		}
+
+		log.Debugf("Legacy routing rules removed")
+	}
+
+	return nil
+}
+
+// GenerateSetName generates a unique name for an ipset based on the given sources.
+func GenerateSetName(sources []netip.Prefix) string {
+	// sort for consistent naming
+	SortPrefixes(sources)
+
+	var sourcesStr strings.Builder
+	for _, src := range sources {
+		sourcesStr.WriteString(src.String())
+	}
+
+	hash := sha256.Sum256([]byte(sourcesStr.String()))
+	shortHash := hex.EncodeToString(hash[:])[:8]
+
+	return fmt.Sprintf("nb-%s", shortHash)
+}
+
+// MergeIPRanges merges overlapping IP ranges and returns a slice of non-overlapping netip.Prefix
+func MergeIPRanges(prefixes []netip.Prefix) []netip.Prefix {
+	if len(prefixes) == 0 {
+		return prefixes
+	}
+
+	merged := []netip.Prefix{prefixes[0]}
+	for _, prefix := range prefixes[1:] {
+		last := merged[len(merged)-1]
+		if last.Contains(prefix.Addr()) {
+			// If the current prefix is contained within the last merged prefix, skip it
+			continue
+		}
+		if prefix.Contains(last.Addr()) {
+			// If the current prefix contains the last merged prefix, replace it
+			merged[len(merged)-1] = prefix
+		} else {
+			// Otherwise, add the current prefix to the merged list
+			merged = append(merged, prefix)
+		}
+	}
+
+	return merged
+}
+
+// SortPrefixes sorts the given slice of netip.Prefix in place.
+// It sorts first by IP address, then by prefix length (most specific to least specific).
+func SortPrefixes(prefixes []netip.Prefix) {
+	sort.Slice(prefixes, func(i, j int) bool {
+		addrCmp := prefixes[i].Addr().Compare(prefixes[j].Addr())
+		if addrCmp != 0 {
+			return addrCmp < 0
+		}
+
+		// If IP addresses are the same, compare prefix lengths (longer prefixes first)
+		return prefixes[i].Bits() > prefixes[j].Bits()
+	})
 }

client/firewall/manager/firewall_test.go

@@ -0,0 +1,192 @@
+package manager_test
+
+import (
+	"net/netip"
+	"reflect"
+	"regexp"
+	"testing"
+
+	"github.com/netbirdio/netbird/client/firewall/manager"
+)
+
+func TestGenerateSetName(t *testing.T) {
+	t.Run("Different orders result in same hash", func(t *testing.T) {
+		prefixes1 := []netip.Prefix{
+			netip.MustParsePrefix("192.168.1.0/24"),
+			netip.MustParsePrefix("10.0.0.0/8"),
+		}
+		prefixes2 := []netip.Prefix{
+			netip.MustParsePrefix("10.0.0.0/8"),
+			netip.MustParsePrefix("192.168.1.0/24"),
+		}
+
+		result1 := manager.GenerateSetName(prefixes1)
+		result2 := manager.GenerateSetName(prefixes2)
+
+		if result1 != result2 {
+			t.Errorf("Different orders produced different hashes: %s != %s", result1, result2)
+		}
+	})
+
+	t.Run("Result format is correct", func(t *testing.T) {
+		prefixes := []netip.Prefix{
+			netip.MustParsePrefix("192.168.1.0/24"),
+			netip.MustParsePrefix("10.0.0.0/8"),
+		}
+
+		result := manager.GenerateSetName(prefixes)
+
+		matched, err := regexp.MatchString(`^nb-[0-9a-f]{8}$`, result)
+		if err != nil {
+			t.Fatalf("Error matching regex: %v", err)
+		}
+		if !matched {
+			t.Errorf("Result format is incorrect: %s", result)
+		}
+	})
+
+	t.Run("Empty input produces consistent result", func(t *testing.T) {
+		result1 := manager.GenerateSetName([]netip.Prefix{})
+		result2 := manager.GenerateSetName([]netip.Prefix{})
+
+		if result1 != result2 {
+			t.Errorf("Empty input produced inconsistent results: %s != %s", result1, result2)
+		}
+	})
+
+	t.Run("IPv4 and IPv6 mixing", func(t *testing.T) {
+		prefixes1 := []netip.Prefix{
+			netip.MustParsePrefix("192.168.1.0/24"),
+			netip.MustParsePrefix("2001:db8::/32"),
+		}
+		prefixes2 := []netip.Prefix{
+			netip.MustParsePrefix("2001:db8::/32"),
+			netip.MustParsePrefix("192.168.1.0/24"),
+		}
+
+		result1 := manager.GenerateSetName(prefixes1)
+		result2 := manager.GenerateSetName(prefixes2)
+
+		if result1 != result2 {
+			t.Errorf("Different orders of IPv4 and IPv6 produced different hashes: %s != %s", result1, result2)
+		}
+	})
+}
+
+func TestMergeIPRanges(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []netip.Prefix
+		expected []netip.Prefix
+	}{
+		{
+			name:     "Empty input",
+			input:    []netip.Prefix{},
+			expected: []netip.Prefix{},
+		},
+		{
+			name: "Single range",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+		},
+		{
+			name: "Two non-overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("10.0.0.0/8"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("10.0.0.0/8"),
+			},
+		},
+		{
+			name: "One range containing another",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+		{
+			name: "One range containing another (different order)",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+		{
+			name: "Overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.1.128/25"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+		},
+		{
+			name: "Overlapping ranges (different order)",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.128/25"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+		},
+		{
+			name: "Multiple overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.2.0/24"),
+				netip.MustParsePrefix("192.168.1.128/25"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+		{
+			name: "Partially overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/23"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.2.0/25"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/23"),
+				netip.MustParsePrefix("192.168.2.0/25"),
+			},
+		},
+		{
+			name: "IPv6 ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("2001:db8::/32"),
+				netip.MustParsePrefix("2001:db8:1::/48"),
+				netip.MustParsePrefix("2001:db8:2::/48"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("2001:db8::/32"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := manager.MergeIPRanges(tt.input)
+			if !reflect.DeepEqual(result, tt.expected) {
+				t.Errorf("MergeIPRanges() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}

client/firewall/manager/routerpair.go

@@ -1,18 +1,26 @@
 package manager
 
+import (
+	"net/netip"
+
+	"github.com/netbirdio/netbird/route"
+)
+
 type RouterPair struct {
-	ID          string
-	Source      string
-	Destination string
+	ID          route.ID
+	Source      netip.Prefix
+	Destination netip.Prefix
 	Masquerade  bool
+	Inverse     bool
 }
 
-func GetInPair(pair RouterPair) RouterPair {
+func GetInversePair(pair RouterPair) RouterPair {
 	return RouterPair{
 		ID: pair.ID,
 		// invert Source/Destination
 		Source:      pair.Destination,
 		Destination: pair.Source,
 		Masquerade:  pair.Masquerade,
+		Inverse:     true,
 	}
 }

client/firewall/nftables/acl_linux.go

@@ -11,12 +11,13 @@ import (
 	"time"
 
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/iface"
+	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
 const (
@@ -29,72 +30,63 @@ const (
 	chainNameInputFilter   = "netbird-acl-input-filter"
 	chainNameOutputFilter  = "netbird-acl-output-filter"
 	chainNameForwardFilter = "netbird-acl-forward-filter"
+	chainNamePrerouting    = "netbird-rt-prerouting"
 
 	allowNetbirdInputRuleID = "allow Netbird incoming traffic"
 )
 
+const flushError = "flush: %w"
+
 var (
-	anyIP           = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-	postroutingMark = []byte{0xe4, 0x7, 0x0, 0x00}
+	anyIP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
 )
 
 type AclManager struct {
-	rConn               *nftables.Conn
-	sConn               *nftables.Conn
-	wgIface             iFaceMapper
-	routeingFwChainName string
+	rConn              *nftables.Conn
+	sConn              *nftables.Conn
+	wgIface            iFaceMapper
+	routingFwChainName string
 
 	workTable        *nftables.Table
 	chainInputRules  *nftables.Chain
 	chainOutputRules *nftables.Chain
-	chainFwFilter    *nftables.Chain
-	chainPrerouting  *nftables.Chain
 
 	ipsetStore *ipsetStore
 	rules      map[string]*Rule
 }
 
-// iFaceMapper defines subset methods of interface required for manager
-type iFaceMapper interface {
-	Name() string
-	Address() iface.WGAddress
-	IsUserspaceBind() bool
-}
-
-func newAclManager(table *nftables.Table, wgIface iFaceMapper, routeingFwChainName string) (*AclManager, error) {
+func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) {
 	// sConn is used for creating sets and adding/removing elements from them
 	// it's differ then rConn (which does create new conn for each flush operation)
-	// and is permanent. Using same connection for booth type of operations
+	// and is permanent. Using same connection for both type of operations
 	// overloads netlink with high amount of rules ( > 10000)
 	sConn, err := nftables.New(nftables.AsLasting())
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create nf conn: %w", err)
 	}
 
-	m := &AclManager{
-		rConn:               &nftables.Conn{},
-		sConn:               sConn,
-		wgIface:             wgIface,
-		workTable:           table,
-		routeingFwChainName: routeingFwChainName,
+	return &AclManager{
+		rConn:              &nftables.Conn{},
+		sConn:              sConn,
+		wgIface:            wgIface,
+		workTable:          table,
+		routingFwChainName: routingFwChainName,
 
 		ipsetStore: newIpsetStore(),
 		rules:      make(map[string]*Rule),
-	}
-
-	err = m.createDefaultChains()
-	if err != nil {
-		return nil, err
-	}
-
-	return m, nil
+	}, nil
 }
 
-// AddFiltering rule to the firewall
+func (m *AclManager) init(workTable *nftables.Table) error {
+	m.workTable = workTable
+	return m.createDefaultChains()
+}
+
+// AddPeerFiltering rule to the firewall
 //
 // If comment argument is empty firewall manager should set
 // rule ID as comment for the rule
-func (m *AclManager) AddFiltering(
+func (m *AclManager) AddPeerFiltering(
 	ip net.IP,
 	proto firewall.Protocol,
 	sPort *firewall.Port,
@@ -120,20 +112,11 @@ func (m *AclManager) AddFiltering(
 	}
 
 	newRules = append(newRules, ioRule)
-	if !shouldAddToPrerouting(proto, dPort, direction) {
-		return newRules, nil
-	}
-
-	preroutingRule, err := m.addPreroutingFiltering(ipset, proto, dPort, ip)
-	if err != nil {
-		return newRules, err
-	}
-	newRules = append(newRules, preroutingRule)
 	return newRules, nil
 }
 
-// DeleteRule from the firewall by rule definition
-func (m *AclManager) DeleteRule(rule firewall.Rule) error {
+// DeletePeerRule from the firewall by rule definition
+func (m *AclManager) DeletePeerRule(rule firewall.Rule) error {
 	r, ok := rule.(*Rule)
 	if !ok {
 		return fmt.Errorf("invalid rule type")
@@ -199,8 +182,7 @@ func (m *AclManager) DeleteRule(rule firewall.Rule) error {
 	return nil
 }
 
-// createDefaultAllowRules In case if the USP firewall manager can use the native firewall manager we must to create allow rules for
-// input and output chains
+// createDefaultAllowRules creates default allow rules for the input and output chains
 func (m *AclManager) createDefaultAllowRules() error {
 	expIn := []expr.Any{
 		&expr.Payload{
@@ -214,13 +196,13 @@ func (m *AclManager) createDefaultAllowRules() error {
 			SourceRegister: 1,
 			DestRegister:   1,
 			Len:            4,
-			Mask:           []byte{0x00, 0x00, 0x00, 0x00},
-			Xor:            zeroXor,
+			Mask:           []byte{0, 0, 0, 0},
+			Xor:            []byte{0, 0, 0, 0},
 		},
 		// net address
 		&expr.Cmp{
 			Register: 1,
-			Data:     []byte{0x00, 0x00, 0x00, 0x00},
+			Data:     []byte{0, 0, 0, 0},
 		},
 		&expr.Verdict{
 			Kind: expr.VerdictAccept,
@@ -246,13 +228,13 @@ func (m *AclManager) createDefaultAllowRules() error {
 			SourceRegister: 1,
 			DestRegister:   1,
 			Len:            4,
-			Mask:           []byte{0x00, 0x00, 0x00, 0x00},
-			Xor:            zeroXor,
+			Mask:           []byte{0, 0, 0, 0},
+			Xor:            []byte{0, 0, 0, 0},
 		},
 		// net address
 		&expr.Cmp{
 			Register: 1,
-			Data:     []byte{0x00, 0x00, 0x00, 0x00},
+			Data:     []byte{0, 0, 0, 0},
 		},
 		&expr.Verdict{
 			Kind: expr.VerdictAccept,
@@ -266,10 +248,8 @@ func (m *AclManager) createDefaultAllowRules() error {
 		Exprs:    expOut,
 	})
 
-	err := m.rConn.Flush()
-	if err != nil {
-		log.Debugf("failed to create default allow rules: %s", err)
-		return err
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
 	}
 	return nil
 }
@@ -290,15 +270,11 @@ func (m *AclManager) Flush() error {
 		log.Errorf("failed to refresh rule handles IPv4 output chain: %v", err)
 	}
 
-	if err := m.refreshRuleHandles(m.chainPrerouting); err != nil {
-		log.Errorf("failed to refresh rule handles IPv4 prerouting chain: %v", err)
-	}
-
 	return nil
 }
 
 func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, ipset *nftables.Set, comment string) (*Rule, error) {
-	ruleId := generateRuleId(ip, sPort, dPort, direction, action, ipset)
+	ruleId := generatePeerRuleId(ip, sPort, dPort, direction, action, ipset)
 	if r, ok := m.rules[ruleId]; ok {
 		return &Rule{
 			r.nftRule,
@@ -308,18 +284,7 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f
 		}, nil
 	}
 
-	ifaceKey := expr.MetaKeyIIFNAME
-	if direction == firewall.RuleDirectionOUT {
-		ifaceKey = expr.MetaKeyOIFNAME
-	}
-	expressions := []expr.Any{
-		&expr.Meta{Key: ifaceKey, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-	}
+	var expressions []expr.Any
 
 	if proto != firewall.ProtocolALL {
 		expressions = append(expressions, &expr.Payload{
@@ -329,21 +294,15 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f
 			Len:          uint32(1),
 		})
 
-		var protoData []byte
-		switch proto {
-		case firewall.ProtocolTCP:
-			protoData = []byte{unix.IPPROTO_TCP}
-		case firewall.ProtocolUDP:
-			protoData = []byte{unix.IPPROTO_UDP}
-		case firewall.ProtocolICMP:
-			protoData = []byte{unix.IPPROTO_ICMP}
-		default:
-			return nil, fmt.Errorf("unsupported protocol: %s", proto)
+		protoData, err := protoToInt(proto)
+		if err != nil {
+			return nil, fmt.Errorf("convert protocol to number: %v", err)
 		}
+
 		expressions = append(expressions, &expr.Cmp{
 			Register: 1,
 			Op:       expr.CmpOpEq,
-			Data:     protoData,
+			Data:     []byte{protoData},
 		})
 	}
 
@@ -432,10 +391,9 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f
 	} else {
 		chain = m.chainOutputRules
 	}
-	nftRule := m.rConn.InsertRule(&nftables.Rule{
+	nftRule := m.rConn.AddRule(&nftables.Rule{
 		Table:    m.workTable,
 		Chain:    chain,
-		Position: 0,
 		Exprs:    expressions,
 		UserData: userData,
 	})
@@ -453,139 +411,13 @@ func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *f
 	return rule, nil
 }
 
-func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall.Protocol, port *firewall.Port, ip net.IP) (*Rule, error) {
-	var protoData []byte
-	switch proto {
-	case firewall.ProtocolTCP:
-		protoData = []byte{unix.IPPROTO_TCP}
-	case firewall.ProtocolUDP:
-		protoData = []byte{unix.IPPROTO_UDP}
-	case firewall.ProtocolICMP:
-		protoData = []byte{unix.IPPROTO_ICMP}
-	default:
-		return nil, fmt.Errorf("unsupported protocol: %s", proto)
-	}
-
-	ruleId := generateRuleIdForMangle(ipset, ip, proto, port)
-	if r, ok := m.rules[ruleId]; ok {
-		return &Rule{
-			r.nftRule,
-			r.nftSet,
-			r.ruleID,
-			ip,
-		}, nil
-	}
-
-	var ipExpression expr.Any
-	// add individual IP for match if no ipset defined
-	rawIP := ip.To4()
-	if ipset == nil {
-		ipExpression = &expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     rawIP,
-		}
-	} else {
-		ipExpression = &expr.Lookup{
-			SourceRegister: 1,
-			SetName:        ipset.Name,
-			SetID:          ipset.ID,
-		}
-	}
-
-	expressions := []expr.Any{
-		&expr.Payload{
-			DestRegister: 1,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       12,
-			Len:          4,
-		},
-		ipExpression,
-		&expr.Payload{
-			DestRegister: 1,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       16,
-			Len:          4,
-		},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     m.wgIface.Address().IP.To4(),
-		},
-		&expr.Payload{
-			DestRegister: 1,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       uint32(9),
-			Len:          uint32(1),
-		},
-		&expr.Cmp{
-			Register: 1,
-			Op:       expr.CmpOpEq,
-			Data:     protoData,
-		},
-	}
-
-	if port != nil {
-		expressions = append(expressions,
-			&expr.Payload{
-				DestRegister: 1,
-				Base:         expr.PayloadBaseTransportHeader,
-				Offset:       2,
-				Len:          2,
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     encodePort(*port),
-			},
-		)
-	}
-
-	expressions = append(expressions,
-		&expr.Immediate{
-			Register: 1,
-			Data:     postroutingMark,
-		},
-		&expr.Meta{
-			Key:            expr.MetaKeyMARK,
-			SourceRegister: true,
-			Register:       1,
-		},
-	)
-
-	nftRule := m.rConn.InsertRule(&nftables.Rule{
-		Table:    m.workTable,
-		Chain:    m.chainPrerouting,
-		Position: 0,
-		Exprs:    expressions,
-		UserData: []byte(ruleId),
-	})
-
-	if err := m.rConn.Flush(); err != nil {
-		return nil, fmt.Errorf("flush insert rule: %v", err)
-	}
-
-	rule := &Rule{
-		nftRule: nftRule,
-		nftSet:  ipset,
-		ruleID:  ruleId,
-		ip:      ip,
-	}
-
-	m.rules[ruleId] = rule
-	if ipset != nil {
-		m.ipsetStore.AddReferenceToIpset(ipset.Name)
-	}
-	return rule, nil
-}
-
 func (m *AclManager) createDefaultChains() (err error) {
 	// chainNameInputRules
 	chain := m.createChain(chainNameInputRules)
 	err = m.rConn.Flush()
 	if err != nil {
 		log.Debugf("failed to create chain (%s): %s", chain.Name, err)
-		return err
+		return fmt.Errorf(flushError, err)
 	}
 	m.chainInputRules = chain
 
@@ -601,9 +433,6 @@ func (m *AclManager) createDefaultChains() (err error) {
 	// netbird-acl-input-filter
 	// type filter hook input priority filter; policy accept;
 	chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput)
-	//netbird-acl-input-filter iifname "wt0" ip saddr 100.72.0.0/16 ip daddr != 100.72.0.0/16 accept
-	m.addRouteAllowRule(chain, expr.MetaKeyIIFNAME)
-	m.addFwdAllow(chain, expr.MetaKeyIIFNAME)
 	m.addJumpRule(chain, m.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules
 	m.addDropExpressions(chain, expr.MetaKeyIIFNAME)
 	err = m.rConn.Flush()
@@ -615,7 +444,6 @@ func (m *AclManager) createDefaultChains() (err error) {
 	// netbird-acl-output-filter
 	// type filter hook output priority filter; policy accept;
 	chain = m.createFilterChainWithHook(chainNameOutputFilter, nftables.ChainHookOutput)
-	m.addRouteAllowRule(chain, expr.MetaKeyOIFNAME)
 	m.addFwdAllow(chain, expr.MetaKeyOIFNAME)
 	m.addJumpRule(chain, m.chainOutputRules.Name, expr.MetaKeyOIFNAME) // to netbird-acl-output-rules
 	m.addDropExpressions(chain, expr.MetaKeyOIFNAME)
@@ -626,29 +454,106 @@ func (m *AclManager) createDefaultChains() (err error) {
 	}
 
 	// netbird-acl-forward-filter
-	m.chainFwFilter = m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
-	m.addJumpRulesToRtForward() // to
-	m.addMarkAccept()
-	m.addJumpRuleToInputChain() // to netbird-acl-input-rules
-	m.addDropExpressions(m.chainFwFilter, expr.MetaKeyIIFNAME)
+	chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
+	m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd
+	m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME)
+
 	err = m.rConn.Flush()
 	if err != nil {
 		log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err)
-		return err
+		return fmt.Errorf(flushError, err)
 	}
 
-	// netbird-acl-output-filter
-	// type filter hook output priority filter; policy accept;
-	m.chainPrerouting = m.createPreroutingMangle()
-	err = m.rConn.Flush()
-	if err != nil {
-		log.Debugf("failed to create chain (%s): %s", m.chainPrerouting.Name, err)
-		return err
+	if err := m.allowRedirectedTraffic(chainFwFilter); err != nil {
+		log.Errorf("failed to allow redirected traffic: %s", err)
 	}
+
 	return nil
 }
 
-func (m *AclManager) addJumpRulesToRtForward() {
+// Makes redirected traffic originally destined for the host itself (now subject to the forward filter)
+// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the
+// netbird peer IP.
+func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error {
+	preroutingChain := m.rConn.AddChain(&nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    m.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	})
+
+	m.addPreroutingRule(preroutingChain)
+
+	m.addFwmarkToForward(chainFwFilter)
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (m *AclManager) addPreroutingRule(preroutingChain *nftables.Chain) {
+	m.rConn.AddRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: preroutingChain,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyIIFNAME,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     ifname(m.wgIface.Name()),
+			},
+			&expr.Fib{
+				Register:       1,
+				ResultADDRTYPE: true,
+				FlagDADDR:      true,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL),
+			},
+			&expr.Immediate{
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
+			},
+			&expr.Meta{
+				Key:            expr.MetaKeyMARK,
+				Register:       1,
+				SourceRegister: true,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) {
+	m.rConn.InsertRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: chainFwFilter,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyMARK,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
+			},
+			&expr.Verdict{
+				Kind:  expr.VerdictJump,
+				Chain: m.chainInputRules.Name,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) {
 	expressions := []expr.Any{
 		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
 		&expr.Cmp{
@@ -658,68 +563,15 @@ func (m *AclManager) addJumpRulesToRtForward() {
 		},
 		&expr.Verdict{
 			Kind:  expr.VerdictJump,
-			Chain: m.routeingFwChainName,
+			Chain: m.routingFwChainName,
 		},
 	}
 
 	_ = m.rConn.AddRule(&nftables.Rule{
 		Table: m.workTable,
-		Chain: m.chainFwFilter,
+		Chain: chainFwFilter,
 		Exprs: expressions,
 	})
-
-	expressions = []expr.Any{
-		&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-		&expr.Verdict{
-			Kind:  expr.VerdictJump,
-			Chain: m.routeingFwChainName,
-		},
-	}
-
-	_ = m.rConn.AddRule(&nftables.Rule{
-		Table: m.workTable,
-		Chain: m.chainFwFilter,
-		Exprs: expressions,
-	})
-}
-
-func (m *AclManager) addMarkAccept() {
-	// oifname "wt0" meta mark 0x000007e4 accept
-	// iifname "wt0" meta mark 0x000007e4 accept
-	ifaces := []expr.MetaKey{expr.MetaKeyIIFNAME, expr.MetaKeyOIFNAME}
-	for _, iface := range ifaces {
-		expressions := []expr.Any{
-			&expr.Meta{Key: iface, Register: 1},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     ifname(m.wgIface.Name()),
-			},
-			&expr.Meta{
-				Key:      expr.MetaKeyMARK,
-				Register: 1,
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     postroutingMark,
-			},
-			&expr.Verdict{
-				Kind: expr.VerdictAccept,
-			},
-		}
-
-		_ = m.rConn.AddRule(&nftables.Rule{
-			Table: m.workTable,
-			Chain: m.chainFwFilter,
-			Exprs: expressions,
-		})
-	}
 }
 
 func (m *AclManager) createChain(name string) *nftables.Chain {
@@ -729,10 +581,13 @@ func (m *AclManager) createChain(name string) *nftables.Chain {
 	}
 
 	chain = m.rConn.AddChain(chain)
+
+	insertReturnTrafficRule(m.rConn, m.workTable, chain)
+
 	return chain
 }
 
-func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.ChainHook) *nftables.Chain {
+func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain {
 	polAccept := nftables.ChainPolicyAccept
 	chain := &nftables.Chain{
 		Name:     name,
@@ -746,74 +601,6 @@ func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.Cha
 	return m.rConn.AddChain(chain)
 }
 
-func (m *AclManager) createPreroutingMangle() *nftables.Chain {
-	polAccept := nftables.ChainPolicyAccept
-	chain := &nftables.Chain{
-		Name:     "netbird-acl-prerouting-filter",
-		Table:    m.workTable,
-		Hooknum:  nftables.ChainHookPrerouting,
-		Priority: nftables.ChainPriorityMangle,
-		Type:     nftables.ChainTypeFilter,
-		Policy:   &polAccept,
-	}
-
-	chain = m.rConn.AddChain(chain)
-
-	ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4())
-	expressions := []expr.Any{
-		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-		&expr.Payload{
-			DestRegister: 2,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       12,
-			Len:          4,
-		},
-		&expr.Bitwise{
-			SourceRegister: 2,
-			DestRegister:   2,
-			Len:            4,
-			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-			Mask:           m.wgIface.Address().Network.Mask,
-		},
-		&expr.Cmp{
-			Op:       expr.CmpOpNeq,
-			Register: 2,
-			Data:     ip.Unmap().AsSlice(),
-		},
-		&expr.Payload{
-			DestRegister: 1,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       16,
-			Len:          4,
-		},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     m.wgIface.Address().IP.To4(),
-		},
-		&expr.Immediate{
-			Register: 1,
-			Data:     postroutingMark,
-		},
-		&expr.Meta{
-			Key:            expr.MetaKeyMARK,
-			SourceRegister: true,
-			Register:       1,
-		},
-	}
-	_ = m.rConn.AddRule(&nftables.Rule{
-		Table: m.workTable,
-		Chain: chain,
-		Exprs: expressions,
-	})
-	return chain
-}
-
 func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any {
 	expressions := []expr.Any{
 		&expr.Meta{Key: ifaceKey, Register: 1},
@@ -832,101 +619,9 @@ func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.Met
 	return nil
 }
 
-func (m *AclManager) addJumpRuleToInputChain() {
-	expressions := []expr.Any{
-		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-		&expr.Verdict{
-			Kind:  expr.VerdictJump,
-			Chain: m.chainInputRules.Name,
-		},
-	}
-
-	_ = m.rConn.AddRule(&nftables.Rule{
-		Table: m.workTable,
-		Chain: m.chainFwFilter,
-		Exprs: expressions,
-	})
-}
-
-func (m *AclManager) addRouteAllowRule(chain *nftables.Chain, netIfName expr.MetaKey) {
-	ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4())
-	var srcOp, dstOp expr.CmpOp
-	if netIfName == expr.MetaKeyIIFNAME {
-		srcOp = expr.CmpOpEq
-		dstOp = expr.CmpOpNeq
-	} else {
-		srcOp = expr.CmpOpNeq
-		dstOp = expr.CmpOpEq
-	}
-	expressions := []expr.Any{
-		&expr.Meta{Key: netIfName, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-		&expr.Payload{
-			DestRegister: 2,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       12,
-			Len:          4,
-		},
-		&expr.Bitwise{
-			SourceRegister: 2,
-			DestRegister:   2,
-			Len:            4,
-			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-			Mask:           m.wgIface.Address().Network.Mask,
-		},
-		&expr.Cmp{
-			Op:       srcOp,
-			Register: 2,
-			Data:     ip.Unmap().AsSlice(),
-		},
-		&expr.Payload{
-			DestRegister: 2,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       16,
-			Len:          4,
-		},
-		&expr.Bitwise{
-			SourceRegister: 2,
-			DestRegister:   2,
-			Len:            4,
-			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-			Mask:           m.wgIface.Address().Network.Mask,
-		},
-		&expr.Cmp{
-			Op:       dstOp,
-			Register: 2,
-			Data:     ip.Unmap().AsSlice(),
-		},
-		&expr.Verdict{
-			Kind: expr.VerdictAccept,
-		},
-	}
-	_ = m.rConn.AddRule(&nftables.Rule{
-		Table: chain.Table,
-		Chain: chain,
-		Exprs: expressions,
-	})
-}
-
 func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) {
 	ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4())
-	var srcOp, dstOp expr.CmpOp
-	if iifname == expr.MetaKeyIIFNAME {
-		srcOp = expr.CmpOpNeq
-		dstOp = expr.CmpOpEq
-	} else {
-		srcOp = expr.CmpOpEq
-		dstOp = expr.CmpOpNeq
-	}
+	dstOp := expr.CmpOpNeq
 	expressions := []expr.Any{
 		&expr.Meta{Key: iifname, Register: 1},
 		&expr.Cmp{
@@ -934,24 +629,6 @@ func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) {
 			Register: 1,
 			Data:     ifname(m.wgIface.Name()),
 		},
-		&expr.Payload{
-			DestRegister: 2,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       12,
-			Len:          4,
-		},
-		&expr.Bitwise{
-			SourceRegister: 2,
-			DestRegister:   2,
-			Len:            4,
-			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-			Mask:           m.wgIface.Address().Network.Mask,
-		},
-		&expr.Cmp{
-			Op:       srcOp,
-			Register: 2,
-			Data:     ip.Unmap().AsSlice(),
-		},
 		&expr.Payload{
 			DestRegister: 2,
 			Base:         expr.PayloadBaseNetworkHeader,
@@ -982,7 +659,6 @@ func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) {
 }
 
 func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) {
-	ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4())
 	expressions := []expr.Any{
 		&expr.Meta{Key: ifaceKey, Register: 1},
 		&expr.Cmp{
@@ -990,47 +666,12 @@ func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr
 			Register: 1,
 			Data:     ifname(m.wgIface.Name()),
 		},
-		&expr.Payload{
-			DestRegister: 2,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       12,
-			Len:          4,
-		},
-		&expr.Bitwise{
-			SourceRegister: 2,
-			DestRegister:   2,
-			Len:            4,
-			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-			Mask:           m.wgIface.Address().Network.Mask,
-		},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 2,
-			Data:     ip.Unmap().AsSlice(),
-		},
-		&expr.Payload{
-			DestRegister: 2,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       16,
-			Len:          4,
-		},
-		&expr.Bitwise{
-			SourceRegister: 2,
-			DestRegister:   2,
-			Len:            4,
-			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-			Mask:           m.wgIface.Address().Network.Mask,
-		},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 2,
-			Data:     ip.Unmap().AsSlice(),
-		},
 		&expr.Verdict{
 			Kind:  expr.VerdictJump,
 			Chain: to,
 		},
 	}
+
 	_ = m.rConn.AddRule(&nftables.Rule{
 		Table: chain.Table,
 		Chain: chain,
@@ -1132,7 +773,7 @@ func (m *AclManager) refreshRuleHandles(chain *nftables.Chain) error {
 	return nil
 }
 
-func generateRuleId(
+func generatePeerRuleId(
 	ip net.IP,
 	sPort *firewall.Port,
 	dPort *firewall.Port,
@@ -1155,33 +796,6 @@ func generateRuleId(
 	}
 	return "set:" + ipset.Name + rulesetID
 }
-func generateRuleIdForMangle(ipset *nftables.Set, ip net.IP, proto firewall.Protocol, port *firewall.Port) string {
-	// case of icmp port is empty
-	var p string
-	if port != nil {
-		p = port.String()
-	}
-	if ipset != nil {
-		return fmt.Sprintf("p:set:%s:%s:%v", ipset.Name, proto, p)
-	} else {
-		return fmt.Sprintf("p:ip:%s:%s:%v", ip.String(), proto, p)
-	}
-}
-
-func shouldAddToPrerouting(proto firewall.Protocol, dPort *firewall.Port, direction firewall.RuleDirection) bool {
-	if proto == "all" {
-		return false
-	}
-
-	if direction != firewall.RuleDirectionIN {
-		return false
-	}
-
-	if dPort == nil && proto != firewall.ProtocolICMP {
-		return false
-	}
-	return true
-}
 
 func encodePort(port firewall.Port) []byte {
 	bs := make([]byte, 2)
@@ -1191,6 +805,19 @@ func encodePort(port firewall.Port) []byte {
 
 func ifname(n string) []byte {
 	b := make([]byte, 16)
-	copy(b, []byte(n+"\x00"))
+	copy(b, n+"\x00")
 	return b
 }
+
+func protoToInt(protocol firewall.Protocol) (uint8, error) {
+	switch protocol {
+	case firewall.ProtocolTCP:
+		return unix.IPPROTO_TCP, nil
+	case firewall.ProtocolUDP:
+		return unix.IPPROTO_UDP, nil
+	case firewall.ProtocolICMP:
+		return unix.IPPROTO_ICMP, nil
+	}
+
+	return 0, fmt.Errorf("unsupported protocol: %s", protocol)
+}

client/firewall/nftables/manager_linux.go

@@ -5,20 +5,34 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/netip"
 	"sync"
 
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
-	// tableName is the name of the table that is used for filtering by the Netbird client
-	tableName = "netbird"
+	// tableNameNetbird is the name of the table that is used for filtering by the Netbird client
+	tableNameNetbird = "netbird"
+
+	tableNameFilter = "filter"
+	chainNameInput  = "INPUT"
 )
 
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMapper interface {
+	Name() string
+	Address() iface.WGAddress
+	IsUserspaceBind() bool
+}
+
 // Manager of iptables firewall
 type Manager struct {
 	mutex   sync.Mutex
@@ -30,35 +44,75 @@ type Manager struct {
 }
 
 // Create nftables firewall manager
-func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
 	m := &Manager{
 		rConn:   &nftables.Conn{},
 		wgIface: wgIface,
 	}
 
-	workTable, err := m.createWorkTable()
+	workTable := &nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4}
+
+	var err error
+	m.router, err = newRouter(workTable, wgIface)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create router: %w", err)
 	}
 
-	m.router, err = newRouter(context, workTable)
+	m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw)
 	if err != nil {
-		return nil, err
-	}
-
-	m.aclManager, err = newAclManager(workTable, wgIface, m.router.RouteingFwChainName())
-	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
-// AddFiltering rule to the firewall
+// Init nftables firewall manager
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	workTable, err := m.createWorkTable()
+	if err != nil {
+		return fmt.Errorf("create work table: %w", err)
+	}
+
+	if err := m.router.init(workTable); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclManager.init(workTable); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	stateManager.RegisterState(&ShutdownState{})
+
+	// We only need to record minimal interface state for potential recreation.
+	// Unlike iptables, which requires tracking individual rules, nftables maintains
+	// a known state (our netbird table plus a few static rules). This allows for easy
+	// cleanup using Reset() without needing to store specific rules.
+	if err := stateManager.UpdateState(&ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	// persist early
+	go func() {
+		if err := stateManager.PersistState(context.Background()); err != nil {
+			log.Errorf("failed to persist state: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+// AddPeerFiltering rule to the firewall
 //
 // If comment argument is empty firewall manager should set
 // rule ID as comment for the rule
-func (m *Manager) AddFiltering(
+func (m *Manager) AddPeerFiltering(
 	ip net.IP,
 	proto firewall.Protocol,
 	sPort *firewall.Port,
@@ -76,33 +130,52 @@ func (m *Manager) AddFiltering(
 		return nil, fmt.Errorf("unsupported IP version: %s", ip.String())
 	}
 
-	return m.aclManager.AddFiltering(ip, proto, sPort, dPort, direction, action, ipsetName, comment)
+	return m.aclManager.AddPeerFiltering(ip, proto, sPort, dPort, direction, action, ipsetName, comment)
 }
 
-// DeleteRule from the firewall by rule definition
-func (m *Manager) DeleteRule(rule firewall.Rule) error {
+func (m *Manager) AddRouteFiltering(sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action) (firewall.Rule, error) {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.aclManager.DeleteRule(rule)
+	if !destination.Addr().Is4() {
+		return nil, fmt.Errorf("unsupported IP version: %s", destination.Addr().String())
+	}
+
+	return m.router.AddRouteFiltering(sources, destination, proto, sPort, dPort, action)
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.aclManager.DeletePeerRule(rule)
+}
+
+// DeleteRouteRule deletes a routing rule
+func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.DeleteRouteRule(rule)
 }
 
 func (m *Manager) IsServerRouteSupported() bool {
 	return true
 }
 
-func (m *Manager) InsertRoutingRules(pair firewall.RouterPair) error {
+func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.router.AddRoutingRules(pair)
+	return m.router.AddNatRule(pair)
 }
 
-func (m *Manager) RemoveRoutingRules(pair firewall.RouterPair) error {
+func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	return m.router.RemoveRoutingRules(pair)
+	return m.router.RemoveNatRule(pair)
 }
 
 // AllowNetbird allows netbird interface traffic
@@ -126,7 +199,7 @@ func (m *Manager) AllowNetbird() error {
 
 	var chain *nftables.Chain
 	for _, c := range chains {
-		if c.Table.Name == "filter" && c.Name == "INPUT" {
+		if c.Table.Name == tableNameFilter && c.Name == chainNameInput {
 			chain = c
 			break
 		}
@@ -157,47 +230,86 @@ func (m *Manager) AllowNetbird() error {
 	return nil
 }
 
+// SetLegacyManagement sets the route manager to use legacy management
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	return firewall.SetLegacyManagement(m.router, isLegacy)
+}
+
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	chains, err := m.rConn.ListChains()
-	if err != nil {
-		return fmt.Errorf("list of chains: %w", err)
+	if err := m.resetNetbirdInputRules(); err != nil {
+		return fmt.Errorf("reset netbird input rules: %v", err)
 	}
 
+	if err := m.router.Reset(); err != nil {
+		return fmt.Errorf("reset router: %v", err)
+	}
+
+	if err := m.cleanupNetbirdTables(); err != nil {
+		return fmt.Errorf("cleanup netbird tables: %v", err)
+	}
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+		return fmt.Errorf("delete state: %v", err)
+	}
+
+	return nil
+}
+
+func (m *Manager) resetNetbirdInputRules() error {
+	chains, err := m.rConn.ListChains()
+	if err != nil {
+		return fmt.Errorf("list chains: %w", err)
+	}
+
+	m.deleteNetbirdInputRules(chains)
+
+	return nil
+}
+
+func (m *Manager) deleteNetbirdInputRules(chains []*nftables.Chain) {
 	for _, c := range chains {
-		// delete Netbird allow input traffic rule if it exists
-		if c.Table.Name == "filter" && c.Name == "INPUT" {
+		if c.Table.Name == tableNameFilter && c.Name == chainNameInput {
 			rules, err := m.rConn.GetRules(c.Table, c)
 			if err != nil {
 				log.Errorf("get rules for chain %q: %v", c.Name, err)
 				continue
 			}
-			for _, r := range rules {
-				if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
-					if err := m.rConn.DelRule(r); err != nil {
-						log.Errorf("delete rule: %v", err)
-					}
-				}
+
+			m.deleteMatchingRules(rules)
+		}
+	}
+}
+
+func (m *Manager) deleteMatchingRules(rules []*nftables.Rule) {
+	for _, r := range rules {
+		if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
+			if err := m.rConn.DelRule(r); err != nil {
+				log.Errorf("delete rule: %v", err)
 			}
 		}
 	}
+}
 
-	m.router.ResetForwardRules()
-
+func (m *Manager) cleanupNetbirdTables() error {
 	tables, err := m.rConn.ListTables()
 	if err != nil {
-		return fmt.Errorf("list of tables: %w", err)
+		return fmt.Errorf("list tables: %w", err)
 	}
+
 	for _, t := range tables {
-		if t.Name == tableName {
+		if t.Name == tableNameNetbird {
 			m.rConn.DelTable(t)
 		}
 	}
-
-	return m.rConn.Flush()
+	return nil
 }
 
 // Flush rule/chain/set operations from the buffer
@@ -218,12 +330,12 @@ func (m *Manager) createWorkTable() (*nftables.Table, error) {
 	}
 
 	for _, t := range tables {
-		if t.Name == tableName {
+		if t.Name == tableNameNetbird {
 			m.rConn.DelTable(t)
 		}
 	}
 
-	table := m.rConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4})
+	table := m.rConn.AddTable(&nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4})
 	err = m.rConn.Flush()
 	return table, err
 }
@@ -251,7 +363,7 @@ func (m *Manager) applyAllowNetbirdRules(chain *nftables.Chain) {
 func (m *Manager) detectAllowNetbirdRule(existedRules []*nftables.Rule) *nftables.Rule {
 	ifName := ifname(m.wgIface.Name())
 	for _, rule := range existedRules {
-		if rule.Table.Name == "filter" && rule.Chain.Name == "INPUT" {
+		if rule.Table.Name == tableNameFilter && rule.Chain.Name == chainNameInput {
 			if len(rule.Exprs) < 4 {
 				if e, ok := rule.Exprs[0].(*expr.Meta); !ok || e.Key != expr.MetaKeyIIFNAME {
 					continue
@@ -265,3 +377,38 @@ func (m *Manager) detectAllowNetbirdRule(existedRules []*nftables.Rule) *nftable
 	}
 	return nil
 }
+
+func insertReturnTrafficRule(conn *nftables.Conn, table *nftables.Table, chain *nftables.Chain) {
+	rule := &nftables.Rule{
+		Table: table,
+		Chain: chain,
+		Exprs: getEstablishedExprs(1),
+	}
+
+	conn.InsertRule(rule)
+}
+
+func getEstablishedExprs(register uint32) []expr.Any {
+	return []expr.Any{
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: register,
+		},
+		&expr.Bitwise{
+			SourceRegister: register,
+			DestRegister:   register,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: register,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+}

client/firewall/nftables/manager_linux_test.go

@@ -1,22 +1,39 @@
 package nftables
 
 import (
-	"context"
+	"bytes"
 	"fmt"
 	"net"
 	"net/netip"
+	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/unix"
 
 	fw "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/client/iface"
 )
 
+var ifaceMock = &iFaceMock{
+	NameFunc: func() string {
+		return "lo"
+	},
+	AddressFunc: func() iface.WGAddress {
+		return iface.WGAddress{
+			IP: net.ParseIP("100.96.0.1"),
+			Network: &net.IPNet{
+				IP:   net.ParseIP("100.96.0.0"),
+				Mask: net.IPv4Mask(255, 255, 255, 0),
+			},
+		}
+	},
+}
+
 // iFaceMapper defines subset methods of interface required for manager
 type iFaceMock struct {
 	NameFunc    func() string
@@ -40,28 +57,15 @@ func (i *iFaceMock) Address() iface.WGAddress {
 func (i *iFaceMock) IsUserspaceBind() bool { return false }
 
 func TestNftablesManager(t *testing.T) {
-	mock := &iFaceMock{
-		NameFunc: func() string {
-			return "lo"
-		},
-		AddressFunc: func() iface.WGAddress {
-			return iface.WGAddress{
-				IP: net.ParseIP("100.96.0.1"),
-				Network: &net.IPNet{
-					IP:   net.ParseIP("100.96.0.0"),
-					Mask: net.IPv4Mask(255, 255, 255, 0),
-				},
-			}
-		},
-	}
 
 	// just check on the local interface
-	manager, err := Create(context.Background(), mock)
+	manager, err := Create(ifaceMock)
 	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
 	time.Sleep(time.Second * 3)
 
 	defer func() {
-		err = manager.Reset()
+		err = manager.Reset(nil)
 		require.NoError(t, err, "failed to reset")
 		time.Sleep(time.Second)
 	}()
@@ -70,7 +74,7 @@ func TestNftablesManager(t *testing.T) {
 
 	testClient := &nftables.Conn{}
 
-	rule, err := manager.AddFiltering(
+	rule, err := manager.AddPeerFiltering(
 		ip,
 		fw.ProtocolTCP,
 		nil,
@@ -88,17 +92,35 @@ func TestNftablesManager(t *testing.T) {
 	rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
 	require.NoError(t, err, "failed to get rules")
 
-	require.Len(t, rules, 1, "expected 1 rules")
+	require.Len(t, rules, 2, "expected 2 rules")
+
+	expectedExprs1 := []expr.Any{
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: 1,
+		},
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+	require.ElementsMatch(t, rules[0].Exprs, expectedExprs1, "expected the same expressions")
 
 	ipToAdd, _ := netip.AddrFromSlice(ip)
 	add := ipToAdd.Unmap()
-	expectedExprs := []expr.Any{
-		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname("lo"),
-		},
+	expectedExprs2 := []expr.Any{
 		&expr.Payload{
 			DestRegister: 1,
 			Base:         expr.PayloadBaseNetworkHeader,
@@ -134,10 +156,10 @@ func TestNftablesManager(t *testing.T) {
 		},
 		&expr.Verdict{Kind: expr.VerdictDrop},
 	}
-	require.ElementsMatch(t, rules[0].Exprs, expectedExprs, "expected the same expressions")
+	require.ElementsMatch(t, rules[1].Exprs, expectedExprs2, "expected the same expressions")
 
 	for _, r := range rule {
-		err = manager.DeleteRule(r)
+		err = manager.DeletePeerRule(r)
 		require.NoError(t, err, "failed to delete rule")
 	}
 
@@ -146,9 +168,10 @@ func TestNftablesManager(t *testing.T) {
 
 	rules, err = testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
 	require.NoError(t, err, "failed to get rules")
-	require.Len(t, rules, 0, "expected 0 rules after deletion")
+	// established rule remains
+	require.Len(t, rules, 1, "expected 1 rules after deletion")
 
-	err = manager.Reset()
+	err = manager.Reset(nil)
 	require.NoError(t, err, "failed to reset")
 }
 
@@ -171,12 +194,13 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(context.Background(), mock)
+			manager, err := Create(mock)
 			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second * 3)
 
 			defer func() {
-				if err := manager.Reset(); err != nil {
+				if err := manager.Reset(nil); err != nil {
 					t.Errorf("clear the manager state: %v", err)
 				}
 				time.Sleep(time.Second)
@@ -187,9 +211,9 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 			for i := 0; i < testMax; i++ {
 				port := &fw.Port{Values: []int{1000 + i}}
 				if i%2 == 0 {
-					_, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
 				} else {
-					_, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
 				}
 				require.NoError(t, err, "failed to add rule")
 
@@ -203,3 +227,105 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 		})
 	}
 }
+
+func runIptablesSave(t *testing.T) (string, string) {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("iptables-save")
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	require.NoError(t, err, "iptables-save failed to run")
+
+	return stdout.String(), stderr.String()
+}
+
+func verifyIptablesOutput(t *testing.T, stdout, stderr string) {
+	t.Helper()
+	// Check for any incompatibility warnings
+	require.NotContains(t,
+		stderr,
+		"incompatible",
+		"iptables-save produced compatibility warning. Full stderr: %s",
+		stderr,
+	)
+
+	// Verify standard tables are present
+	expectedTables := []string{
+		"*filter",
+		"*nat",
+		"*mangle",
+	}
+
+	for _, table := range expectedTables {
+		require.Contains(t,
+			stdout,
+			table,
+			"iptables-save output missing expected table: %s\nFull stdout: %s",
+			table,
+			stdout,
+		)
+	}
+}
+
+func TestNftablesManagerCompatibilityWithIptables(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
+
+	if _, err := exec.LookPath("iptables-save"); err != nil {
+		t.Skipf("iptables-save not available on this system: %v", err)
+	}
+
+	// First ensure iptables-nft tables exist by running iptables-save
+	stdout, stderr := runIptablesSave(t)
+	verifyIptablesOutput(t, stdout, stderr)
+
+	manager, err := Create(ifaceMock)
+	require.NoError(t, err, "failed to create manager")
+	require.NoError(t, manager.Init(nil))
+
+	t.Cleanup(func() {
+		err := manager.Reset(nil)
+		require.NoError(t, err, "failed to reset manager state")
+
+		// Verify iptables output after reset
+		stdout, stderr := runIptablesSave(t)
+		verifyIptablesOutput(t, stdout, stderr)
+	})
+
+	ip := net.ParseIP("100.96.0.1")
+	_, err = manager.AddPeerFiltering(
+		ip,
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{80}},
+		fw.RuleDirectionIN,
+		fw.ActionAccept,
+		"",
+		"test rule",
+	)
+	require.NoError(t, err, "failed to add peer filtering rule")
+
+	_, err = manager.AddRouteFiltering(
+		[]netip.Prefix{netip.MustParsePrefix("192.168.2.0/24")},
+		netip.MustParsePrefix("10.1.0.0/24"),
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{443}},
+		fw.ActionAccept,
+	)
+	require.NoError(t, err, "failed to add route filtering rule")
+
+	pair := fw.RouterPair{
+		Source:      netip.MustParsePrefix("192.168.1.0/24"),
+		Destination: netip.MustParsePrefix("10.0.0.0/24"),
+		Masquerade:  true,
+	}
+	err = manager.AddNatRule(pair)
+	require.NoError(t, err, "failed to add NAT rule")
+
+	stdout, stderr = runIptablesSave(t)
+	verifyIptablesOutput(t, stdout, stderr)
+}

client/firewall/nftables/route_linux.go

@@ -1,431 +0,0 @@
-package nftables
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"net"
-	"net/netip"
-
-	"github.com/google/nftables"
-	"github.com/google/nftables/binaryutil"
-	"github.com/google/nftables/expr"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/netbirdio/netbird/client/firewall/manager"
-)
-
-const (
-	chainNameRouteingFw = "netbird-rt-fwd"
-	chainNameRoutingNat = "netbird-rt-nat"
-
-	userDataAcceptForwardRuleSrc = "frwacceptsrc"
-	userDataAcceptForwardRuleDst = "frwacceptdst"
-
-	loopbackInterface = "lo\x00"
-)
-
-// some presets for building nftable rules
-var (
-	zeroXor = binaryutil.NativeEndian.PutUint32(0)
-
-	exprCounterAccept = []expr.Any{
-		&expr.Counter{},
-		&expr.Verdict{
-			Kind: expr.VerdictAccept,
-		},
-	}
-
-	errFilterTableNotFound = fmt.Errorf("nftables: 'filter' table not found")
-)
-
-type router struct {
-	ctx         context.Context
-	stop        context.CancelFunc
-	conn        *nftables.Conn
-	workTable   *nftables.Table
-	filterTable *nftables.Table
-	chains      map[string]*nftables.Chain
-	// rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules
-	rules                    map[string]*nftables.Rule
-	isDefaultFwdRulesEnabled bool
-}
-
-func newRouter(parentCtx context.Context, workTable *nftables.Table) (*router, error) {
-	ctx, cancel := context.WithCancel(parentCtx)
-
-	r := &router{
-		ctx:       ctx,
-		stop:      cancel,
-		conn:      &nftables.Conn{},
-		workTable: workTable,
-		chains:    make(map[string]*nftables.Chain),
-		rules:     make(map[string]*nftables.Rule),
-	}
-
-	var err error
-	r.filterTable, err = r.loadFilterTable()
-	if err != nil {
-		if errors.Is(err, errFilterTableNotFound) {
-			log.Warnf("table 'filter' not found for forward rules")
-		} else {
-			return nil, err
-		}
-	}
-
-	err = r.cleanUpDefaultForwardRules()
-	if err != nil {
-		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
-	}
-
-	err = r.createContainers()
-	if err != nil {
-		log.Errorf("failed to create containers for route: %s", err)
-	}
-	return r, err
-}
-
-func (r *router) RouteingFwChainName() string {
-	return chainNameRouteingFw
-}
-
-// ResetForwardRules cleans existing nftables default forward rules from the system
-func (r *router) ResetForwardRules() {
-	err := r.cleanUpDefaultForwardRules()
-	if err != nil {
-		log.Errorf("failed to reset forward rules: %s", err)
-	}
-}
-
-func (r *router) loadFilterTable() (*nftables.Table, error) {
-	tables, err := r.conn.ListTablesOfFamily(nftables.TableFamilyIPv4)
-	if err != nil {
-		return nil, fmt.Errorf("nftables: unable to list tables: %v", err)
-	}
-
-	for _, table := range tables {
-		if table.Name == "filter" {
-			return table, nil
-		}
-	}
-
-	return nil, errFilterTableNotFound
-}
-
-func (r *router) createContainers() error {
-
-	r.chains[chainNameRouteingFw] = r.conn.AddChain(&nftables.Chain{
-		Name:  chainNameRouteingFw,
-		Table: r.workTable,
-	})
-
-	r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
-		Name:     chainNameRoutingNat,
-		Table:    r.workTable,
-		Hooknum:  nftables.ChainHookPostrouting,
-		Priority: nftables.ChainPriorityNATSource - 1,
-		Type:     nftables.ChainTypeNAT,
-	})
-
-	// Add RETURN rule for loopback interface
-	loRule := &nftables.Rule{
-		Table: r.workTable,
-		Chain: r.chains[chainNameRoutingNat],
-		Exprs: []expr.Any{
-			&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     []byte(loopbackInterface),
-			},
-			&expr.Verdict{Kind: expr.VerdictReturn},
-		},
-	}
-	r.conn.InsertRule(loRule)
-
-	err := r.refreshRulesMap()
-	if err != nil {
-		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
-	}
-
-	err = r.conn.Flush()
-	if err != nil {
-		return fmt.Errorf("nftables: unable to initialize table: %v", err)
-	}
-	return nil
-}
-
-// AddRoutingRules appends a nftable rule pair to the forwarding chain and if enabled, to the nat chain
-func (r *router) AddRoutingRules(pair manager.RouterPair) error {
-	err := r.refreshRulesMap()
-	if err != nil {
-		return err
-	}
-
-	err = r.addRoutingRule(manager.ForwardingFormat, chainNameRouteingFw, pair, false)
-	if err != nil {
-		return err
-	}
-	err = r.addRoutingRule(manager.InForwardingFormat, chainNameRouteingFw, manager.GetInPair(pair), false)
-	if err != nil {
-		return err
-	}
-
-	if pair.Masquerade {
-		err = r.addRoutingRule(manager.NatFormat, chainNameRoutingNat, pair, true)
-		if err != nil {
-			return err
-		}
-		err = r.addRoutingRule(manager.InNatFormat, chainNameRoutingNat, manager.GetInPair(pair), true)
-		if err != nil {
-			return err
-		}
-	}
-
-	if r.filterTable != nil && !r.isDefaultFwdRulesEnabled {
-		log.Debugf("add default accept forward rule")
-		r.acceptForwardRule(pair.Source)
-	}
-
-	err = r.conn.Flush()
-	if err != nil {
-		return fmt.Errorf("nftables: unable to insert rules for %s: %v", pair.Destination, err)
-	}
-	return nil
-}
-
-// addRoutingRule inserts a nftable rule to the conn client flush queue
-func (r *router) addRoutingRule(format, chainName string, pair manager.RouterPair, isNat bool) error {
-	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
-	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
-
-	var expression []expr.Any
-	if isNat {
-		expression = append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) // nolint:gocritic
-	} else {
-		expression = append(sourceExp, append(destExp, exprCounterAccept...)...) // nolint:gocritic
-	}
-
-	ruleKey := manager.GenKey(format, pair.ID)
-
-	_, exists := r.rules[ruleKey]
-	if exists {
-		err := r.removeRoutingRule(format, pair)
-		if err != nil {
-			return err
-		}
-	}
-
-	r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{
-		Table:    r.workTable,
-		Chain:    r.chains[chainName],
-		Exprs:    expression,
-		UserData: []byte(ruleKey),
-	})
-	return nil
-}
-
-func (r *router) acceptForwardRule(sourceNetwork string) {
-	src := generateCIDRMatcherExpressions(true, sourceNetwork)
-	dst := generateCIDRMatcherExpressions(false, "0.0.0.0/0")
-
-	var exprs []expr.Any
-	exprs = append(src, append(dst, &expr.Verdict{ // nolint:gocritic
-		Kind: expr.VerdictAccept,
-	})...)
-
-	rule := &nftables.Rule{
-		Table: r.filterTable,
-		Chain: &nftables.Chain{
-			Name:     "FORWARD",
-			Table:    r.filterTable,
-			Type:     nftables.ChainTypeFilter,
-			Hooknum:  nftables.ChainHookForward,
-			Priority: nftables.ChainPriorityFilter,
-		},
-		Exprs:    exprs,
-		UserData: []byte(userDataAcceptForwardRuleSrc),
-	}
-
-	r.conn.AddRule(rule)
-
-	src = generateCIDRMatcherExpressions(true, "0.0.0.0/0")
-	dst = generateCIDRMatcherExpressions(false, sourceNetwork)
-
-	exprs = append(src, append(dst, &expr.Verdict{ //nolint:gocritic
-		Kind: expr.VerdictAccept,
-	})...)
-
-	rule = &nftables.Rule{
-		Table: r.filterTable,
-		Chain: &nftables.Chain{
-			Name:     "FORWARD",
-			Table:    r.filterTable,
-			Type:     nftables.ChainTypeFilter,
-			Hooknum:  nftables.ChainHookForward,
-			Priority: nftables.ChainPriorityFilter,
-		},
-		Exprs:    exprs,
-		UserData: []byte(userDataAcceptForwardRuleDst),
-	}
-	r.conn.AddRule(rule)
-	r.isDefaultFwdRulesEnabled = true
-}
-
-// RemoveRoutingRules removes a nftable rule pair from forwarding and nat chains
-func (r *router) RemoveRoutingRules(pair manager.RouterPair) error {
-	err := r.refreshRulesMap()
-	if err != nil {
-		return err
-	}
-
-	err = r.removeRoutingRule(manager.ForwardingFormat, pair)
-	if err != nil {
-		return err
-	}
-
-	err = r.removeRoutingRule(manager.InForwardingFormat, manager.GetInPair(pair))
-	if err != nil {
-		return err
-	}
-
-	err = r.removeRoutingRule(manager.NatFormat, pair)
-	if err != nil {
-		return err
-	}
-
-	err = r.removeRoutingRule(manager.InNatFormat, manager.GetInPair(pair))
-	if err != nil {
-		return err
-	}
-
-	if len(r.rules) == 0 {
-		err := r.cleanUpDefaultForwardRules()
-		if err != nil {
-			log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
-		}
-	}
-
-	err = r.conn.Flush()
-	if err != nil {
-		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
-	}
-	log.Debugf("nftables: removed rules for %s", pair.Destination)
-	return nil
-}
-
-// removeRoutingRule add a nftable rule to the removal queue and delete from rules map
-func (r *router) removeRoutingRule(format string, pair manager.RouterPair) error {
-	ruleKey := manager.GenKey(format, pair.ID)
-
-	rule, found := r.rules[ruleKey]
-	if found {
-		ruleType := "forwarding"
-		if rule.Chain.Type == nftables.ChainTypeNAT {
-			ruleType = "nat"
-		}
-
-		err := r.conn.DelRule(rule)
-		if err != nil {
-			return fmt.Errorf("nftables: unable to remove %s rule for %s: %v", ruleType, pair.Destination, err)
-		}
-
-		log.Debugf("nftables: removing %s rule for %s", ruleType, pair.Destination)
-
-		delete(r.rules, ruleKey)
-	}
-	return nil
-}
-
-// refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid
-// duplicates and to get missing attributes that we don't have when adding new rules
-func (r *router) refreshRulesMap() error {
-	for _, chain := range r.chains {
-		rules, err := r.conn.GetRules(chain.Table, chain)
-		if err != nil {
-			return fmt.Errorf("nftables: unable to list rules: %v", err)
-		}
-		for _, rule := range rules {
-			if len(rule.UserData) > 0 {
-				r.rules[string(rule.UserData)] = rule
-			}
-		}
-	}
-	return nil
-}
-
-func (r *router) cleanUpDefaultForwardRules() error {
-	if r.filterTable == nil {
-		r.isDefaultFwdRulesEnabled = false
-		return nil
-	}
-
-	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
-	if err != nil {
-		return err
-	}
-
-	var rules []*nftables.Rule
-	for _, chain := range chains {
-		if chain.Table.Name != r.filterTable.Name {
-			continue
-		}
-		if chain.Name != "FORWARD" {
-			continue
-		}
-
-		rules, err = r.conn.GetRules(r.filterTable, chain)
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, rule := range rules {
-		if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleSrc)) || bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleDst)) {
-			err := r.conn.DelRule(rule)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	r.isDefaultFwdRulesEnabled = false
-	return r.conn.Flush()
-}
-
-// generateCIDRMatcherExpressions generates nftables expressions that matches a CIDR
-func generateCIDRMatcherExpressions(source bool, cidr string) []expr.Any {
-	ip, network, _ := net.ParseCIDR(cidr)
-	ipToAdd, _ := netip.AddrFromSlice(ip)
-	add := ipToAdd.Unmap()
-
-	var offSet uint32
-	if source {
-		offSet = 12 // src offset
-	} else {
-		offSet = 16 // dst offset
-	}
-
-	return []expr.Any{
-		// fetch src add
-		&expr.Payload{
-			DestRegister: 1,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       offSet,
-			Len:          4,
-		},
-		// net mask
-		&expr.Bitwise{
-			DestRegister:   1,
-			SourceRegister: 1,
-			Len:            4,
-			Mask:           network.Mask,
-			Xor:            zeroXor,
-		},
-		// net address
-		&expr.Cmp{
-			Register: 1,
-			Data:     add.AsSlice(),
-		},
-	}
-}

client/firewall/nftables/router_linux.go

@@ -0,0 +1,989 @@
+package nftables
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/acl/id"
+	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+const (
+	chainNameRoutingFw  = "netbird-rt-fwd"
+	chainNameRoutingNat = "netbird-rt-postrouting"
+	chainNameForward    = "FORWARD"
+
+	userDataAcceptForwardRuleIif = "frwacceptiif"
+	userDataAcceptForwardRuleOif = "frwacceptoif"
+)
+
+const refreshRulesMapError = "refresh rules map: %w"
+
+var (
+	errFilterTableNotFound = fmt.Errorf("nftables: 'filter' table not found")
+)
+
+type router struct {
+	conn        *nftables.Conn
+	workTable   *nftables.Table
+	filterTable *nftables.Table
+	chains      map[string]*nftables.Chain
+	// rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules
+	rules        map[string]*nftables.Rule
+	ipsetCounter *refcounter.Counter[string, []netip.Prefix, *nftables.Set]
+
+	wgIface          iFaceMapper
+	legacyManagement bool
+}
+
+func newRouter(workTable *nftables.Table, wgIface iFaceMapper) (*router, error) {
+	r := &router{
+		conn:      &nftables.Conn{},
+		workTable: workTable,
+		chains:    make(map[string]*nftables.Chain),
+		rules:     make(map[string]*nftables.Rule),
+		wgIface:   wgIface,
+	}
+
+	r.ipsetCounter = refcounter.New(
+		r.createIpSet,
+		r.deleteIpSet,
+	)
+
+	var err error
+	r.filterTable, err = r.loadFilterTable()
+	if err != nil {
+		if errors.Is(err, errFilterTableNotFound) {
+			log.Warnf("table 'filter' not found for forward rules")
+		} else {
+			return nil, fmt.Errorf("load filter table: %w", err)
+		}
+	}
+
+	return r, nil
+}
+
+func (r *router) init(workTable *nftables.Table) error {
+	r.workTable = workTable
+
+	if err := r.removeAcceptForwardRules(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
+	}
+
+	return nil
+}
+
+// Reset cleans existing nftables default forward rules from the system
+func (r *router) Reset() error {
+	// clear without deleting the ipsets, the nf table will be deleted by the caller
+	r.ipsetCounter.Clear()
+
+	return r.removeAcceptForwardRules()
+}
+
+func (r *router) loadFilterTable() (*nftables.Table, error) {
+	tables, err := r.conn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return nil, fmt.Errorf("nftables: unable to list tables: %v", err)
+	}
+
+	for _, table := range tables {
+		if table.Name == "filter" {
+			return table, nil
+		}
+	}
+
+	return nil, errFilterTableNotFound
+}
+
+func (r *router) createContainers() error {
+	r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{
+		Name:  chainNameRoutingFw,
+		Table: r.workTable,
+	})
+
+	insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw])
+
+	prio := *nftables.ChainPriorityNATSource - 1
+	r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
+		Name:     chainNameRoutingNat,
+		Table:    r.workTable,
+		Hooknum:  nftables.ChainHookPostrouting,
+		Priority: &prio,
+		Type:     nftables.ChainTypeNAT,
+	})
+
+	// Chain is created by acl manager
+	// TODO: move creation to a common place
+	r.chains[chainNamePrerouting] = &nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    r.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	}
+
+	// Add the single NAT rule that matches on mark
+	if err := r.addPostroutingRules(); err != nil {
+		return fmt.Errorf("add single nat rule: %v", err)
+	}
+
+	if err := r.acceptForwardRules(); err != nil {
+		log.Errorf("failed to add accept rules for the forward chain: %s", err)
+	}
+
+	if err := r.refreshRulesMap(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf("nftables: unable to initialize table: %v", err)
+	}
+
+	return nil
+}
+
+// AddRouteFiltering appends a nftables rule to the routing chain
+func (r *router) AddRouteFiltering(
+	sources []netip.Prefix,
+	destination netip.Prefix,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	action firewall.Action,
+) (firewall.Rule, error) {
+
+	ruleKey := id.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action)
+	if _, ok := r.rules[string(ruleKey)]; ok {
+		return ruleKey, nil
+	}
+
+	chain := r.chains[chainNameRoutingFw]
+	var exprs []expr.Any
+
+	switch {
+	case len(sources) == 1 && sources[0].Bits() == 0:
+		// If it's 0.0.0.0/0, we don't need to add any source matching
+	case len(sources) == 1:
+		// If there's only one source, we can use it directly
+		exprs = append(exprs, generateCIDRMatcherExpressions(true, sources[0])...)
+	default:
+		// If there are multiple sources, create or get an ipset
+		var err error
+		exprs, err = r.getIpSetExprs(sources, exprs)
+		if err != nil {
+			return nil, fmt.Errorf("get ipset expressions: %w", err)
+		}
+	}
+
+	// Handle destination
+	exprs = append(exprs, generateCIDRMatcherExpressions(false, destination)...)
+
+	// Handle protocol
+	if proto != firewall.ProtocolALL {
+		protoNum, err := protoToInt(proto)
+		if err != nil {
+			return nil, fmt.Errorf("convert protocol to number: %w", err)
+		}
+		exprs = append(exprs, &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1})
+		exprs = append(exprs, &expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     []byte{protoNum},
+		})
+
+		exprs = append(exprs, applyPort(sPort, true)...)
+		exprs = append(exprs, applyPort(dPort, false)...)
+	}
+
+	exprs = append(exprs, &expr.Counter{})
+
+	var verdict expr.VerdictKind
+	if action == firewall.ActionAccept {
+		verdict = expr.VerdictAccept
+	} else {
+		verdict = expr.VerdictDrop
+	}
+	exprs = append(exprs, &expr.Verdict{Kind: verdict})
+
+	rule := &nftables.Rule{
+		Table:    r.workTable,
+		Chain:    chain,
+		Exprs:    exprs,
+		UserData: []byte(ruleKey),
+	}
+
+	rule = r.conn.AddRule(rule)
+
+	log.Tracef("Adding route rule %s", spew.Sdump(rule))
+	if err := r.conn.Flush(); err != nil {
+		return nil, fmt.Errorf(flushError, err)
+	}
+
+	r.rules[string(ruleKey)] = rule
+
+	log.Debugf("nftables: added route rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", sources, destination, proto, sPort, dPort, action)
+
+	return ruleKey, nil
+}
+
+func (r *router) getIpSetExprs(sources []netip.Prefix, exprs []expr.Any) ([]expr.Any, error) {
+	setName := firewall.GenerateSetName(sources)
+	ref, err := r.ipsetCounter.Increment(setName, sources)
+	if err != nil {
+		return nil, fmt.Errorf("create or get ipset for sources: %w", err)
+	}
+
+	exprs = append(exprs,
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       12,
+			Len:          4,
+		},
+		&expr.Lookup{
+			SourceRegister: 1,
+			SetName:        ref.Out.Name,
+			SetID:          ref.Out.ID,
+		},
+	)
+	return exprs, nil
+}
+
+func (r *router) DeleteRouteRule(rule firewall.Rule) error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	ruleKey := rule.GetRuleID()
+	nftRule, exists := r.rules[ruleKey]
+	if !exists {
+		log.Debugf("route rule %s not found", ruleKey)
+		return nil
+	}
+
+	if nftRule.Handle == 0 {
+		return fmt.Errorf("route rule %s has no handle", ruleKey)
+	}
+
+	setName := r.findSetNameInRule(nftRule)
+
+	if err := r.deleteNftRule(nftRule, ruleKey); err != nil {
+		return fmt.Errorf("delete: %w", err)
+	}
+
+	if setName != "" {
+		if _, err := r.ipsetCounter.Decrement(setName); err != nil {
+			return fmt.Errorf("decrement ipset reference: %w", err)
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (r *router) createIpSet(setName string, sources []netip.Prefix) (*nftables.Set, error) {
+	// overlapping prefixes will result in an error, so we need to merge them
+	sources = firewall.MergeIPRanges(sources)
+
+	set := &nftables.Set{
+		Name:  setName,
+		Table: r.workTable,
+		// required for prefixes
+		Interval: true,
+		KeyType:  nftables.TypeIPAddr,
+	}
+
+	var elements []nftables.SetElement
+	for _, prefix := range sources {
+		// TODO: Implement IPv6 support
+		if prefix.Addr().Is6() {
+			log.Printf("Skipping IPv6 prefix %s: IPv6 support not yet implemented", prefix)
+			continue
+		}
+
+		// nftables needs half-open intervals [firstIP, lastIP) for prefixes
+		// e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc
+		firstIP := prefix.Addr()
+		lastIP := calculateLastIP(prefix).Next()
+
+		elements = append(elements,
+			// the nft tool also adds a line like this, see https://github.com/google/nftables/issues/247
+			// nftables.SetElement{Key: []byte{0, 0, 0, 0}, IntervalEnd: true},
+			nftables.SetElement{Key: firstIP.AsSlice()},
+			nftables.SetElement{Key: lastIP.AsSlice(), IntervalEnd: true},
+		)
+	}
+
+	if err := r.conn.AddSet(set, elements); err != nil {
+		return nil, fmt.Errorf("error adding elements to set %s: %w", setName, err)
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return nil, fmt.Errorf("flush error: %w", err)
+	}
+
+	log.Printf("Created new ipset: %s with %d elements", setName, len(elements)/2)
+
+	return set, nil
+}
+
+// calculateLastIP determines the last IP in a given prefix.
+func calculateLastIP(prefix netip.Prefix) netip.Addr {
+	hostMask := ^uint32(0) >> prefix.Masked().Bits()
+	lastIP := uint32FromNetipAddr(prefix.Addr()) | hostMask
+
+	return netip.AddrFrom4(uint32ToBytes(lastIP))
+}
+
+// Utility function to convert netip.Addr to uint32.
+func uint32FromNetipAddr(addr netip.Addr) uint32 {
+	b := addr.As4()
+	return binary.BigEndian.Uint32(b[:])
+}
+
+// Utility function to convert uint32 to a netip-compatible byte slice.
+func uint32ToBytes(ip uint32) [4]byte {
+	var b [4]byte
+	binary.BigEndian.PutUint32(b[:], ip)
+	return b
+}
+
+func (r *router) deleteIpSet(setName string, set *nftables.Set) error {
+	r.conn.DelSet(set)
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	log.Debugf("Deleted unused ipset %s", setName)
+	return nil
+}
+
+func (r *router) findSetNameInRule(rule *nftables.Rule) string {
+	for _, e := range rule.Exprs {
+		if lookup, ok := e.(*expr.Lookup); ok {
+			return lookup.SetName
+		}
+	}
+	return ""
+}
+
+func (r *router) deleteNftRule(rule *nftables.Rule, ruleKey string) error {
+	if err := r.conn.DelRule(rule); err != nil {
+		return fmt.Errorf("delete rule %s: %w", ruleKey, err)
+	}
+	delete(r.rules, ruleKey)
+
+	log.Debugf("removed route rule %s", ruleKey)
+
+	return nil
+}
+
+// AddNatRule appends a nftables rule pair to the nat chain
+func (r *router) AddNatRule(pair firewall.RouterPair) error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	if r.legacyManagement {
+		log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination)
+		if err := r.addLegacyRouteRule(pair); err != nil {
+			return fmt.Errorf("add legacy routing rule: %w", err)
+		}
+	}
+
+	if pair.Masquerade {
+		if err := r.addNatRule(pair); err != nil {
+			return fmt.Errorf("add nat rule: %w", err)
+		}
+
+		if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil {
+			return fmt.Errorf("add inverse nat rule: %w", err)
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf("nftables: insert rules for %s: %v", pair.Destination, err)
+	}
+
+	return nil
+}
+
+// addNatRule inserts a nftables rule to the conn client flush queue
+func (r *router) addNatRule(pair firewall.RouterPair) error {
+	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
+	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
+
+	op := expr.CmpOpEq
+	if pair.Inverse {
+		op = expr.CmpOpNeq
+	}
+
+	exprs := []expr.Any{
+		// We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading.
+		// Masquerading will take care of the conntrack state, which means we won't need to mark established connections.
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: 1,
+		},
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+
+		// interface matching
+		&expr.Meta{
+			Key:      expr.MetaKeyIIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       op,
+			Register: 1,
+			Data:     ifname(r.wgIface.Name()),
+		},
+	}
+
+	exprs = append(exprs, sourceExp...)
+	exprs = append(exprs, destExp...)
+
+	var markValue uint32 = nbnet.PreroutingFwmarkMasquerade
+	if pair.Inverse {
+		markValue = nbnet.PreroutingFwmarkMasqueradeReturn
+	}
+
+	exprs = append(exprs,
+		&expr.Immediate{
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(markValue),
+		},
+		&expr.Meta{
+			Key:            expr.MetaKeyMARK,
+			SourceRegister: true,
+			Register:       1,
+		},
+	)
+
+	ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
+
+	if _, exists := r.rules[ruleKey]; exists {
+		if err := r.removeNatRule(pair); err != nil {
+			return fmt.Errorf("remove prerouting rule: %w", err)
+		}
+	}
+
+	r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{
+		Table:    r.workTable,
+		Chain:    r.chains[chainNamePrerouting],
+		Exprs:    exprs,
+		UserData: []byte(ruleKey),
+	})
+
+	return nil
+}
+
+// addPostroutingRules adds the masquerade rules
+func (r *router) addPostroutingRules() error {
+	// First masquerade rule for traffic coming in from WireGuard interface
+	exprs := []expr.Any{
+		// Match on the first fwmark
+		&expr.Meta{
+			Key:      expr.MetaKeyMARK,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade),
+		},
+
+		// We need to exclude the loopback interface as this changes the ebpf proxy port
+		&expr.Meta{
+			Key:      expr.MetaKeyOIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     ifname("lo"),
+		},
+		&expr.Counter{},
+		&expr.Masq{},
+	}
+
+	r.conn.AddRule(&nftables.Rule{
+		Table: r.workTable,
+		Chain: r.chains[chainNameRoutingNat],
+		Exprs: exprs,
+	})
+
+	// Second masquerade rule for traffic going out through WireGuard interface
+	exprs2 := []expr.Any{
+		// Match on the second fwmark
+		&expr.Meta{
+			Key:      expr.MetaKeyMARK,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn),
+		},
+
+		// Match WireGuard interface
+		&expr.Meta{
+			Key:      expr.MetaKeyOIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(r.wgIface.Name()),
+		},
+		&expr.Counter{},
+		&expr.Masq{},
+	}
+
+	r.conn.AddRule(&nftables.Rule{
+		Table: r.workTable,
+		Chain: r.chains[chainNameRoutingNat],
+		Exprs: exprs2,
+	})
+
+	return nil
+}
+
+// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls
+func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error {
+	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
+	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
+
+	exprs := []expr.Any{
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+
+	expression := append(sourceExp, append(destExp, exprs...)...) // nolint:gocritic
+
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if _, exists := r.rules[ruleKey]; exists {
+		if err := r.removeLegacyRouteRule(pair); err != nil {
+			return fmt.Errorf("remove legacy routing rule: %w", err)
+		}
+	}
+
+	r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{
+		Table:    r.workTable,
+		Chain:    r.chains[chainNameRoutingFw],
+		Exprs:    expression,
+		UserData: []byte(ruleKey),
+	})
+	return nil
+}
+
+// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls
+func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.conn.DelRule(rule); err != nil {
+			return fmt.Errorf("remove legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err)
+		}
+
+		log.Debugf("nftables: removed legacy forwarding rule %s -> %s", pair.Source, pair.Destination)
+
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("nftables: legacy forwarding rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+// GetLegacyManagement returns the route manager's legacy management mode
+func (r *router) GetLegacyManagement() bool {
+	return r.legacyManagement
+}
+
+// SetLegacyManagement sets the route manager to use legacy management mode
+func (r *router) SetLegacyManagement(isLegacy bool) {
+	r.legacyManagement = isLegacy
+}
+
+// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls
+func (r *router) RemoveAllLegacyRouteRules() error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	var merr *multierror.Error
+	for k, rule := range r.rules {
+		if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) {
+			continue
+		}
+		if err := r.conn.DelRule(rule); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
+		}
+
+	}
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// acceptForwardRules adds iif/oif rules in the filter table/forward chain to make sure
+// that our traffic is not dropped by existing rules there.
+// The existing FORWARD rules/policies decide outbound traffic towards our interface.
+// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule.
+func (r *router) acceptForwardRules() error {
+	if r.filterTable == nil {
+		log.Debugf("table 'filter' not found for forward rules, skipping accept rules")
+		return nil
+	}
+
+	fw := "iptables"
+
+	defer func() {
+		log.Debugf("Used %s to add accept forward rules", fw)
+	}()
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		// filter table exists but iptables is not
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+
+		fw = "nftables"
+		return r.acceptForwardRulesNftables()
+	}
+
+	return r.acceptForwardRulesIptables(ipt)
+}
+
+func (r *router) acceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("add iptables rule: %v", err))
+		} else {
+			log.Debugf("added iptables rule: %v", rule)
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *router) getAcceptForwardRules() [][]string {
+	intf := r.wgIface.Name()
+	return [][]string{
+		{"-i", intf, "-j", "ACCEPT"},
+		{"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
+	}
+}
+
+func (r *router) acceptForwardRulesNftables() error {
+	intf := ifname(r.wgIface.Name())
+
+	// Rule for incoming interface (iif) with counter
+	iifRule := &nftables.Rule{
+		Table: r.filterTable,
+		Chain: &nftables.Chain{
+			Name:     chainNameForward,
+			Table:    r.filterTable,
+			Type:     nftables.ChainTypeFilter,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityFilter,
+		},
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     intf,
+			},
+			&expr.Counter{},
+			&expr.Verdict{Kind: expr.VerdictAccept},
+		},
+		UserData: []byte(userDataAcceptForwardRuleIif),
+	}
+	r.conn.InsertRule(iifRule)
+
+	oifExprs := []expr.Any{
+		&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     intf,
+		},
+	}
+
+	// Rule for outgoing interface (oif) with counter
+	oifRule := &nftables.Rule{
+		Table: r.filterTable,
+		Chain: &nftables.Chain{
+			Name:     "FORWARD",
+			Table:    r.filterTable,
+			Type:     nftables.ChainTypeFilter,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityFilter,
+		},
+		Exprs:    append(oifExprs, getEstablishedExprs(2)...),
+		UserData: []byte(userDataAcceptForwardRuleOif),
+	}
+
+	r.conn.InsertRule(oifRule)
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRules() error {
+	if r.filterTable == nil {
+		return nil
+	}
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+		return r.removeAcceptForwardRulesNftables()
+	}
+
+	return r.removeAcceptForwardRulesIptables(ipt)
+}
+
+func (r *router) removeAcceptForwardRulesNftables() error {
+	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return fmt.Errorf("list chains: %v", err)
+	}
+
+	for _, chain := range chains {
+		if chain.Table.Name != r.filterTable.Name || chain.Name != chainNameForward {
+			continue
+		}
+
+		rules, err := r.conn.GetRules(r.filterTable, chain)
+		if err != nil {
+			return fmt.Errorf("get rules: %v", err)
+		}
+
+		for _, rule := range rules {
+			if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
+				bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) {
+				if err := r.conn.DelRule(rule); err != nil {
+					return fmt.Errorf("delete rule: %v", err)
+				}
+			}
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("remove iptables rule: %v", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// RemoveNatRule removes the prerouting mark rule
+func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	if err := r.removeNatRule(pair); err != nil {
+		return fmt.Errorf("remove prerouting rule: %w", err)
+	}
+
+	if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil {
+		return fmt.Errorf("remove inverse prerouting rule: %w", err)
+	}
+
+	if err := r.removeLegacyRouteRule(pair); err != nil {
+		return fmt.Errorf("remove legacy routing rule: %w", err)
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
+	}
+
+	log.Debugf("nftables: removed nat rules for %s", pair.Destination)
+	return nil
+}
+
+func (r *router) removeNatRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		err := r.conn.DelRule(rule)
+		if err != nil {
+			return fmt.Errorf("remove prerouting rule %s -> %s: %v", pair.Source, pair.Destination, err)
+		}
+
+		log.Debugf("nftables: removed prerouting rule %s -> %s", pair.Source, pair.Destination)
+
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("nftables: prerouting rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+// refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid
+// duplicates and to get missing attributes that we don't have when adding new rules
+func (r *router) refreshRulesMap() error {
+	for _, chain := range r.chains {
+		rules, err := r.conn.GetRules(chain.Table, chain)
+		if err != nil {
+			return fmt.Errorf("nftables: unable to list rules: %v", err)
+		}
+		for _, rule := range rules {
+			if len(rule.UserData) > 0 {
+				r.rules[string(rule.UserData)] = rule
+			}
+		}
+	}
+	return nil
+}
+
+// generateCIDRMatcherExpressions generates nftables expressions that matches a CIDR
+func generateCIDRMatcherExpressions(source bool, prefix netip.Prefix) []expr.Any {
+	var offset uint32
+	if source {
+		offset = 12 // src offset
+	} else {
+		offset = 16 // dst offset
+	}
+
+	ones := prefix.Bits()
+	// 0.0.0.0/0 doesn't need extra expressions
+	if ones == 0 {
+		return nil
+	}
+
+	mask := net.CIDRMask(ones, 32)
+
+	return []expr.Any{
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       offset,
+			Len:          4,
+		},
+		// netmask
+		&expr.Bitwise{
+			DestRegister:   1,
+			SourceRegister: 1,
+			Len:            4,
+			Mask:           mask,
+			Xor:            []byte{0, 0, 0, 0},
+		},
+		// net address
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     prefix.Masked().Addr().AsSlice(),
+		},
+	}
+}
+
+func applyPort(port *firewall.Port, isSource bool) []expr.Any {
+	if port == nil {
+		return nil
+	}
+
+	var exprs []expr.Any
+
+	offset := uint32(2) // Default offset for destination port
+	if isSource {
+		offset = 0 // Offset for source port
+	}
+
+	exprs = append(exprs, &expr.Payload{
+		DestRegister: 1,
+		Base:         expr.PayloadBaseTransportHeader,
+		Offset:       offset,
+		Len:          2,
+	})
+
+	if port.IsRange && len(port.Values) == 2 {
+		// Handle port range
+		exprs = append(exprs,
+			&expr.Cmp{
+				Op:       expr.CmpOpGte,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(uint16(port.Values[0])),
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpLte,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(uint16(port.Values[1])),
+			},
+		)
+	} else {
+		// Handle single port or multiple ports
+		for i, p := range port.Values {
+			if i > 0 {
+				// Add a bitwise OR operation between port checks
+				exprs = append(exprs, &expr.Bitwise{
+					SourceRegister: 1,
+					DestRegister:   1,
+					Len:            4,
+					Mask:           []byte{0x00, 0x00, 0xff, 0xff},
+					Xor:            []byte{0x00, 0x00, 0x00, 0x00},
+				})
+			}
+			exprs = append(exprs, &expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(uint16(p)),
+			})
+		}
+	}
+
+	return exprs
+}

client/firewall/nftables/router_linux_test.go

@@ -3,12 +3,16 @@
 package nftables
 
 import (
-	"context"
+	"encoding/binary"
+	"net/netip"
+	"os/exec"
 	"testing"
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/google/nftables/expr"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
@@ -24,195 +28,627 @@ const (
 	NFTABLES
 )
 
-func TestNftablesManager_InsertRoutingRules(t *testing.T) {
+func TestNftablesManager_AddNatRule(t *testing.T) {
 	if check() != NFTABLES {
 		t.Skip("nftables not supported on this OS")
 	}
 
-	table, err := createWorkTable()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer deleteWorkTable()
-
 	for _, testCase := range test.InsertRuleTestCases {
 		t.Run(testCase.Name, func(t *testing.T) {
-			manager, err := newRouter(context.TODO(), table)
-			require.NoError(t, err, "failed to create router")
+			// need fw manager to init both acl mgr and router for all chains to be present
+			manager, err := Create(ifaceMock)
+			t.Cleanup(func() {
+				require.NoError(t, manager.Reset(nil))
+			})
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
 
 			nftablesTestingClient := &nftables.Conn{}
 
-			defer manager.ResetForwardRules()
+			rtr := manager.router
+			err = rtr.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "pair should be inserted")
 
-			require.NoError(t, err, "shouldn't return error")
-
-			err = manager.AddRoutingRules(testCase.InputPair)
-			defer func() {
-				_ = manager.RemoveRoutingRules(testCase.InputPair)
-			}()
-			require.NoError(t, err, "forwarding pair should be inserted")
-
-			sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
-			destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
-			testingExpression := append(sourceExp, destExp...) //nolint:gocritic
-			fwdRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID)
-
-			found := 0
-			for _, chain := range manager.chains {
-				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-				for _, rule := range rules {
-					if len(rule.UserData) > 0 && string(rule.UserData) == fwdRuleKey {
-						require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "forwarding rule elements should match")
-						found = 1
-					}
-				}
-			}
-
-			require.Equal(t, 1, found, "should find at least 1 rule to test")
+			t.Cleanup(func() {
+				require.NoError(t, rtr.RemoveNatRule(testCase.InputPair), "failed to remove rule")
+			})
 
 			if testCase.InputPair.Masquerade {
-				natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID)
+				// Build expected expressions for connection tracking
+				conntrackExprs := []expr.Any{
+					&expr.Ct{
+						Key:      expr.CtKeySTATE,
+						Register: 1,
+					},
+					&expr.Bitwise{
+						SourceRegister: 1,
+						DestRegister:   1,
+						Len:            4,
+						Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
+						Xor:            binaryutil.NativeEndian.PutUint32(0),
+					},
+					&expr.Cmp{
+						Op:       expr.CmpOpNeq,
+						Register: 1,
+						Data:     []byte{0, 0, 0, 0},
+					},
+				}
+
+				// Build interface matching expression
+				ifaceExprs := []expr.Any{
+					&expr.Meta{
+						Key:      expr.MetaKeyIIFNAME,
+						Register: 1,
+					},
+					&expr.Cmp{
+						Op:       expr.CmpOpEq,
+						Register: 1,
+						Data:     ifname(ifaceMock.Name()),
+					},
+				}
+
+				// Build CIDR matching expressions
+				sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
+				destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
+
+				// Combine all expressions in the correct order
+				// nolint:gocritic
+				testingExpression := append(conntrackExprs, ifaceExprs...)
+				testingExpression = append(testingExpression, sourceExp...)
+				testingExpression = append(testingExpression, destExp...)
+
+				natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
 				found := 0
-				for _, chain := range manager.chains {
-					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-					for _, rule := range rules {
-						if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
-							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "nat rule elements should match")
-							found = 1
+				for _, chain := range rtr.chains {
+					if chain.Name == chainNamePrerouting {
+						rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+						require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+						for _, rule := range rules {
+							if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+								// Compare expressions up to the mark setting expressions
+								require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "prerouting nat rule elements should match")
+								found = 1
+							}
 						}
 					}
 				}
-				require.Equal(t, 1, found, "should find at least 1 rule to test")
-			}
-
-			sourceExp = generateCIDRMatcherExpressions(true, firewall.GetInPair(testCase.InputPair).Source)
-			destExp = generateCIDRMatcherExpressions(false, firewall.GetInPair(testCase.InputPair).Destination)
-			testingExpression = append(sourceExp, destExp...) //nolint:gocritic
-			inFwdRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID)
-
-			found = 0
-			for _, chain := range manager.chains {
-				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-				for _, rule := range rules {
-					if len(rule.UserData) > 0 && string(rule.UserData) == inFwdRuleKey {
-						require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "income forwarding rule elements should match")
-						found = 1
-					}
-				}
-			}
-
-			require.Equal(t, 1, found, "should find at least 1 rule to test")
-
-			if testCase.InputPair.Masquerade {
-				inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID)
-				found := 0
-				for _, chain := range manager.chains {
-					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-					for _, rule := range rules {
-						if len(rule.UserData) > 0 && string(rule.UserData) == inNatRuleKey {
-							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "income nat rule elements should match")
-							found = 1
-						}
-					}
-				}
-				require.Equal(t, 1, found, "should find at least 1 rule to test")
+				require.Equal(t, 1, found, "should find at least 1 rule in prerouting chain")
 			}
 		})
 	}
 }
 
-func TestNftablesManager_RemoveRoutingRules(t *testing.T) {
+func TestNftablesManager_RemoveNatRule(t *testing.T) {
 	if check() != NFTABLES {
 		t.Skip("nftables not supported on this OS")
 	}
 
-	table, err := createWorkTable()
-	if err != nil {
-		t.Fatal(err)
+	for _, testCase := range test.RemoveRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			manager, err := Create(ifaceMock)
+			t.Cleanup(func() {
+				require.NoError(t, manager.Reset(nil))
+			})
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
+
+			rtr := manager.router
+
+			// First add the NAT rule using the router's method
+			err = rtr.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "should add NAT rule")
+
+			// Verify the rule was added
+			natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
+			found := false
+			rules, err := rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNamePrerouting])
+			require.NoError(t, err, "should list rules")
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+					found = true
+					break
+				}
+			}
+			require.True(t, found, "NAT rule should exist before removal")
+
+			// Now remove the rule
+			err = rtr.RemoveNatRule(testCase.InputPair)
+			require.NoError(t, err, "shouldn't return error when removing rule")
+
+			// Verify the rule was removed
+			found = false
+			rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNamePrerouting])
+			require.NoError(t, err, "should list rules after removal")
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+					found = true
+					break
+				}
+			}
+			require.False(t, found, "NAT rule should not exist after removal")
+
+			// Verify the static postrouting rules still exist
+			rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameRoutingNat])
+			require.NoError(t, err, "should list postrouting rules")
+			foundCounter := false
+			for _, rule := range rules {
+				for _, e := range rule.Exprs {
+					if _, ok := e.(*expr.Counter); ok {
+						foundCounter = true
+						break
+					}
+				}
+				if foundCounter {
+					break
+				}
+			}
+			require.True(t, foundCounter, "static postrouting rule should remain")
+		})
 	}
+}
+
+func TestRouter_AddRouteFiltering(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
+
+	workTable, err := createWorkTable()
+	require.NoError(t, err, "Failed to create work table")
 
 	defer deleteWorkTable()
 
-	for _, testCase := range test.RemoveRuleTestCases {
-		t.Run(testCase.Name, func(t *testing.T) {
-			manager, err := newRouter(context.TODO(), table)
-			require.NoError(t, err, "failed to create router")
+	r, err := newRouter(workTable, ifaceMock)
+	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
 
-			nftablesTestingClient := &nftables.Conn{}
+	defer func(r *router) {
+		require.NoError(t, r.Reset(), "Failed to reset rules")
+	}(r)
 
-			defer manager.ResetForwardRules()
+	tests := []struct {
+		name        string
+		sources     []netip.Prefix
+		destination netip.Prefix
+		proto       firewall.Protocol
+		sPort       *firewall.Port
+		dPort       *firewall.Port
+		direction   firewall.RuleDirection
+		action      firewall.Action
+		expectSet   bool
+	}{
+		{
+			name:        "Basic TCP rule with single source",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{80}},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name: "UDP rule with multiple sources",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("172.16.0.0/16"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       &firewall.Port{Values: []int{1024, 2048}, IsRange: true},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionDrop,
+			expectSet:   true,
+		},
+		{
+			name:        "All protocols rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")},
+			destination: netip.MustParsePrefix("0.0.0.0/0"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "ICMP rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolICMP,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with multiple source ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("172.16.0.0/12")},
+			destination: netip.MustParsePrefix("192.168.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{80, 443, 8080}},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "UDP rule with single IP and port range",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.1/32")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{5000, 5100}, IsRange: true},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with source and destination ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")},
+			destination: netip.MustParsePrefix("172.16.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{1024, 65535}, IsRange: true},
+			dPort:       &firewall.Port{Values: []int{22}},
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "Drop all incoming traffic",
+			sources:     []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")},
+			destination: netip.MustParsePrefix("192.168.0.0/24"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+	}
 
-			sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
-			destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ruleKey, err := r.AddRouteFiltering(tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.action)
+			require.NoError(t, err, "AddRouteFiltering failed")
 
-			forwardExp := append(sourceExp, append(destExp, exprCounterAccept...)...) //nolint:gocritic
-			forwardRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID)
-			insertedForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
-				Table:    manager.workTable,
-				Chain:    manager.chains[chainNameRouteingFw],
-				Exprs:    forwardExp,
-				UserData: []byte(forwardRuleKey),
+			t.Cleanup(func() {
+				require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule")
 			})
 
-			natExp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) //nolint:gocritic
-			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID)
+			// Check if the rule is in the internal map
+			rule, ok := r.rules[ruleKey.GetRuleID()]
+			assert.True(t, ok, "Rule not found in internal map")
 
-			insertedNat := nftablesTestingClient.InsertRule(&nftables.Rule{
-				Table:    manager.workTable,
-				Chain:    manager.chains[chainNameRoutingNat],
-				Exprs:    natExp,
-				UserData: []byte(natRuleKey),
-			})
+			t.Log("Internal rule expressions:")
+			for i, expr := range rule.Exprs {
+				t.Logf("  [%d] %T: %+v", i, expr, expr)
+			}
 
-			sourceExp = generateCIDRMatcherExpressions(true, firewall.GetInPair(testCase.InputPair).Source)
-			destExp = generateCIDRMatcherExpressions(false, firewall.GetInPair(testCase.InputPair).Destination)
+			// Verify internal rule content
+			verifyRule(t, rule, tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.direction, tt.action, tt.expectSet)
 
-			forwardExp = append(sourceExp, append(destExp, exprCounterAccept...)...) //nolint:gocritic
-			inForwardRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID)
-			insertedInForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
-				Table:    manager.workTable,
-				Chain:    manager.chains[chainNameRouteingFw],
-				Exprs:    forwardExp,
-				UserData: []byte(inForwardRuleKey),
-			})
+			// Check if the rule exists in nftables and verify its content
+			rules, err := r.conn.GetRules(r.workTable, r.chains[chainNameRoutingFw])
+			require.NoError(t, err, "Failed to get rules from nftables")
 
-			natExp = append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) //nolint:gocritic
-			inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID)
+			var nftRule *nftables.Rule
+			for _, rule := range rules {
+				if string(rule.UserData) == ruleKey.GetRuleID() {
+					nftRule = rule
+					break
+				}
+			}
 
-			insertedInNat := nftablesTestingClient.InsertRule(&nftables.Rule{
-				Table:    manager.workTable,
-				Chain:    manager.chains[chainNameRoutingNat],
-				Exprs:    natExp,
-				UserData: []byte(inNatRuleKey),
-			})
+			require.NotNil(t, nftRule, "Rule not found in nftables")
+			t.Log("Actual nftables rule expressions:")
+			for i, expr := range nftRule.Exprs {
+				t.Logf("  [%d] %T: %+v", i, expr, expr)
+			}
 
-			err = nftablesTestingClient.Flush()
-			require.NoError(t, err, "shouldn't return error")
+			// Verify actual nftables rule content
+			verifyRule(t, nftRule, tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.direction, tt.action, tt.expectSet)
+		})
+	}
+}
 
-			manager.ResetForwardRules()
+func TestNftablesCreateIpSet(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
 
-			err = manager.RemoveRoutingRules(testCase.InputPair)
-			require.NoError(t, err, "shouldn't return error")
+	workTable, err := createWorkTable()
+	require.NoError(t, err, "Failed to create work table")
 
-			for _, chain := range manager.chains {
-				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
-				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
-				for _, rule := range rules {
-					if len(rule.UserData) > 0 {
-						require.NotEqual(t, insertedForwarding.UserData, rule.UserData, "forwarding rule should not exist")
-						require.NotEqual(t, insertedNat.UserData, rule.UserData, "nat rule should not exist")
-						require.NotEqual(t, insertedInForwarding.UserData, rule.UserData, "income forwarding rule should not exist")
-						require.NotEqual(t, insertedInNat.UserData, rule.UserData, "income nat rule should not exist")
+	defer deleteWorkTable()
+
+	r, err := newRouter(workTable, ifaceMock)
+	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
+
+	defer func() {
+		require.NoError(t, r.Reset(), "Failed to reset router")
+	}()
+
+	tests := []struct {
+		name     string
+		sources  []netip.Prefix
+		expected []netip.Prefix
+	}{
+		{
+			name:    "Single IP",
+			sources: []netip.Prefix{netip.MustParsePrefix("192.168.1.1/32")},
+		},
+		{
+			name: "Multiple IPs",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.1/32"),
+				netip.MustParsePrefix("10.0.0.1/32"),
+				netip.MustParsePrefix("172.16.0.1/32"),
+			},
+		},
+		{
+			name:    "Single Subnet",
+			sources: []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")},
+		},
+		{
+			name: "Multiple Subnets with Various Prefix Lengths",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("10.0.0.0/8"),
+				netip.MustParsePrefix("172.16.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("203.0.113.0/26"),
+			},
+		},
+		{
+			name: "Mix of Single IPs and Subnets in Different Positions",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.1/32"),
+				netip.MustParsePrefix("10.0.0.0/16"),
+				netip.MustParsePrefix("172.16.0.1/32"),
+				netip.MustParsePrefix("203.0.113.0/24"),
+			},
+		},
+		{
+			name: "Overlapping IPs/Subnets",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("10.0.0.0/8"),
+				netip.MustParsePrefix("10.0.0.0/16"),
+				netip.MustParsePrefix("10.0.0.1/32"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.1.1/32"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("10.0.0.0/8"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+	}
+
+	// Add this helper function inside TestNftablesCreateIpSet
+	printNftSets := func() {
+		cmd := exec.Command("nft", "list", "sets")
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Logf("Failed to run 'nft list sets': %v", err)
+		} else {
+			t.Logf("Current nft sets:\n%s", output)
+		}
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setName := firewall.GenerateSetName(tt.sources)
+			set, err := r.createIpSet(setName, tt.sources)
+			if err != nil {
+				t.Logf("Failed to create IP set: %v", err)
+				printNftSets()
+				require.NoError(t, err, "Failed to create IP set")
+			}
+			require.NotNil(t, set, "Created set is nil")
+
+			// Verify set properties
+			assert.Equal(t, setName, set.Name, "Set name mismatch")
+			assert.Equal(t, r.workTable, set.Table, "Set table mismatch")
+			assert.True(t, set.Interval, "Set interval property should be true")
+			assert.Equal(t, nftables.TypeIPAddr, set.KeyType, "Set key type mismatch")
+
+			// Fetch the created set from nftables
+			fetchedSet, err := r.conn.GetSetByName(r.workTable, setName)
+			require.NoError(t, err, "Failed to fetch created set")
+			require.NotNil(t, fetchedSet, "Fetched set is nil")
+
+			// Verify set elements
+			elements, err := r.conn.GetSetElements(fetchedSet)
+			require.NoError(t, err, "Failed to get set elements")
+
+			// Count the number of unique prefixes (excluding interval end markers)
+			uniquePrefixes := make(map[string]bool)
+			for _, elem := range elements {
+				if !elem.IntervalEnd {
+					ip := netip.AddrFrom4(*(*[4]byte)(elem.Key))
+					uniquePrefixes[ip.String()] = true
+				}
+			}
+
+			// Check against expected merged prefixes
+			expectedCount := len(tt.expected)
+			if expectedCount == 0 {
+				expectedCount = len(tt.sources)
+			}
+			assert.Equal(t, expectedCount, len(uniquePrefixes), "Number of unique prefixes in set doesn't match expected")
+
+			// Verify each expected prefix is in the set
+			for _, expected := range tt.expected {
+				found := false
+				for _, elem := range elements {
+					if !elem.IntervalEnd {
+						ip := netip.AddrFrom4(*(*[4]byte)(elem.Key))
+						if expected.Contains(ip) {
+							found = true
+							break
+						}
+					}
+				}
+				assert.True(t, found, "Expected prefix %s not found in set", expected)
+			}
+
+			r.conn.DelSet(set)
+			if err := r.conn.Flush(); err != nil {
+				t.Logf("Failed to delete set: %v", err)
+				printNftSets()
+			}
+			require.NoError(t, err, "Failed to delete set")
+		})
+	}
+}
+
+func verifyRule(t *testing.T, rule *nftables.Rule, sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, expectSet bool) {
+	t.Helper()
+
+	assert.NotNil(t, rule, "Rule should not be nil")
+
+	// Verify sources and destination
+	if expectSet {
+		assert.True(t, containsSetLookup(rule.Exprs), "Rule should contain set lookup for multiple sources")
+	} else if len(sources) == 1 && sources[0].Bits() != 0 {
+		if direction == firewall.RuleDirectionIN {
+			assert.True(t, containsCIDRMatcher(rule.Exprs, sources[0], true), "Rule should contain source CIDR matcher for %s", sources[0])
+		} else {
+			assert.True(t, containsCIDRMatcher(rule.Exprs, sources[0], false), "Rule should contain destination CIDR matcher for %s", sources[0])
+		}
+	}
+
+	if direction == firewall.RuleDirectionIN {
+		assert.True(t, containsCIDRMatcher(rule.Exprs, destination, false), "Rule should contain destination CIDR matcher for %s", destination)
+	} else {
+		assert.True(t, containsCIDRMatcher(rule.Exprs, destination, true), "Rule should contain source CIDR matcher for %s", destination)
+	}
+
+	// Verify protocol
+	if proto != firewall.ProtocolALL {
+		assert.True(t, containsProtocol(rule.Exprs, proto), "Rule should contain protocol matcher for %s", proto)
+	}
+
+	// Verify ports
+	if sPort != nil {
+		assert.True(t, containsPort(rule.Exprs, sPort, true), "Rule should contain source port matcher for %v", sPort)
+	}
+	if dPort != nil {
+		assert.True(t, containsPort(rule.Exprs, dPort, false), "Rule should contain destination port matcher for %v", dPort)
+	}
+
+	// Verify action
+	assert.True(t, containsAction(rule.Exprs, action), "Rule should contain correct action: %s", action)
+}
+
+func containsSetLookup(exprs []expr.Any) bool {
+	for _, e := range exprs {
+		if _, ok := e.(*expr.Lookup); ok {
+			return true
+		}
+	}
+	return false
+}
+
+func containsCIDRMatcher(exprs []expr.Any, prefix netip.Prefix, isSource bool) bool {
+	var offset uint32
+	if isSource {
+		offset = 12 // src offset
+	} else {
+		offset = 16 // dst offset
+	}
+
+	var payloadFound, bitwiseFound, cmpFound bool
+	for _, e := range exprs {
+		switch ex := e.(type) {
+		case *expr.Payload:
+			if ex.Base == expr.PayloadBaseNetworkHeader && ex.Offset == offset && ex.Len == 4 {
+				payloadFound = true
+			}
+		case *expr.Bitwise:
+			if ex.Len == 4 && len(ex.Mask) == 4 && len(ex.Xor) == 4 {
+				bitwiseFound = true
+			}
+		case *expr.Cmp:
+			if ex.Op == expr.CmpOpEq && len(ex.Data) == 4 {
+				cmpFound = true
+			}
+		}
+	}
+	return (payloadFound && bitwiseFound && cmpFound) || prefix.Bits() == 0
+}
+
+func containsPort(exprs []expr.Any, port *firewall.Port, isSource bool) bool {
+	var offset uint32 = 2 // Default offset for destination port
+	if isSource {
+		offset = 0 // Offset for source port
+	}
+
+	var payloadFound, portMatchFound bool
+	for _, e := range exprs {
+		switch ex := e.(type) {
+		case *expr.Payload:
+			if ex.Base == expr.PayloadBaseTransportHeader && ex.Offset == offset && ex.Len == 2 {
+				payloadFound = true
+			}
+		case *expr.Cmp:
+			if port.IsRange {
+				if ex.Op == expr.CmpOpGte || ex.Op == expr.CmpOpLte {
+					portMatchFound = true
+				}
+			} else {
+				if ex.Op == expr.CmpOpEq && len(ex.Data) == 2 {
+					portValue := binary.BigEndian.Uint16(ex.Data)
+					for _, p := range port.Values {
+						if uint16(p) == portValue {
+							portMatchFound = true
+							break
+						}
 					}
 				}
 			}
-		})
+		}
+		if payloadFound && portMatchFound {
+			return true
+		}
 	}
+	return false
+}
+
+func containsProtocol(exprs []expr.Any, proto firewall.Protocol) bool {
+	var metaFound, cmpFound bool
+	expectedProto, _ := protoToInt(proto)
+	for _, e := range exprs {
+		switch ex := e.(type) {
+		case *expr.Meta:
+			if ex.Key == expr.MetaKeyL4PROTO {
+				metaFound = true
+			}
+		case *expr.Cmp:
+			if ex.Op == expr.CmpOpEq && len(ex.Data) == 1 && ex.Data[0] == expectedProto {
+				cmpFound = true
+			}
+		}
+	}
+	return metaFound && cmpFound
+}
+
+func containsAction(exprs []expr.Any, action firewall.Action) bool {
+	for _, e := range exprs {
+		if verdict, ok := e.(*expr.Verdict); ok {
+			switch action {
+			case firewall.ActionAccept:
+				return verdict.Kind == expr.VerdictAccept
+			case firewall.ActionDrop:
+				return verdict.Kind == expr.VerdictDrop
+			}
+		}
+	}
+	return false
 }
 
 // check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found.
@@ -250,12 +686,12 @@ func createWorkTable() (*nftables.Table, error) {
 	}
 
 	for _, t := range tables {
-		if t.Name == tableName {
+		if t.Name == tableNameNetbird {
 			sConn.DelTable(t)
 		}
 	}
 
-	table := sConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4})
+	table := sConn.AddTable(&nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4})
 	err = sConn.Flush()
 
 	return table, err
@@ -273,7 +709,7 @@ func deleteWorkTable() {
 	}
 
 	for _, t := range tables {
-		if t.Name == tableName {
+		if t.Name == tableNameNetbird {
 			sConn.DelTable(t)
 		}
 	}

client/firewall/nftables/state_linux.go

@@ -0,0 +1,47 @@
+package nftables
+
+import (
+	"fmt"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "nftables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	nft, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create nftables manager: %w", err)
+	}
+
+	if err := nft.Reset(nil); err != nil {
+		return fmt.Errorf("reset nftables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/test/cases_linux.go

@@ -1,8 +1,10 @@
-//go:build !android
-
 package test
 
-import firewall "github.com/netbirdio/netbird/client/firewall/manager"
+import (
+	"net/netip"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+)
 
 var (
 	InsertRuleTestCases = []struct {
@@ -13,8 +15,8 @@ var (
 			Name: "Insert Forwarding IPV4 Rule",
 			InputPair: firewall.RouterPair{
 				ID:          "zxa",
-				Source:      "100.100.100.1/32",
-				Destination: "100.100.200.0/24",
+				Source:      netip.MustParsePrefix("100.100.100.1/32"),
+				Destination: netip.MustParsePrefix("100.100.200.0/24"),
 				Masquerade:  false,
 			},
 		},
@@ -22,8 +24,8 @@ var (
 			Name: "Insert Forwarding And Nat IPV4 Rules",
 			InputPair: firewall.RouterPair{
 				ID:          "zxa",
-				Source:      "100.100.100.1/32",
-				Destination: "100.100.200.0/24",
+				Source:      netip.MustParsePrefix("100.100.100.1/32"),
+				Destination: netip.MustParsePrefix("100.100.200.0/24"),
 				Masquerade:  true,
 			},
 		},
@@ -38,8 +40,8 @@ var (
 			Name: "Remove Forwarding And Nat IPV4 Rules",
 			InputPair: firewall.RouterPair{
 				ID:          "zxa",
-				Source:      "100.100.100.1/32",
-				Destination: "100.100.200.0/24",
+				Source:      netip.MustParsePrefix("100.100.100.1/32"),
+				Destination: netip.MustParsePrefix("100.100.200.0/24"),
 				Masquerade:  true,
 			},
 		},

client/firewall/uspfilter/allow_netbird.go

@@ -2,8 +2,10 @@
 
 package uspfilter
 
+import "github.com/netbirdio/netbird/client/internal/statemanager"
+
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
@@ -11,7 +13,7 @@ func (m *Manager) Reset() error {
 	m.incomingRules = make(map[string]RuleSet)
 
 	if m.nativeFirewall != nil {
-		return m.nativeFirewall.Reset()
+		return m.nativeFirewall.Reset(stateManager)
 	}
 	return nil
 }

client/firewall/uspfilter/allow_netbird_windows.go

@@ -6,6 +6,8 @@ import (
 	"syscall"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 type action string
@@ -17,7 +19,7 @@ const (
 )
 
 // Reset firewall to the default state
-func (m *Manager) Reset() error {
+func (m *Manager) Reset(*statemanager.Manager) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 

client/firewall/uspfilter/uspfilter.go

@@ -3,6 +3,7 @@ package uspfilter
 import (
 	"fmt"
 	"net"
+	"net/netip"
 	"sync"
 
 	"github.com/google/gopacket"
@@ -11,7 +12,9 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const layerTypeAll = 0
@@ -22,7 +25,7 @@ var (
 
 // IFaceMapper defines subset methods of interface required for manager
 type IFaceMapper interface {
-	SetFilter(iface.PacketFilter) error
+	SetFilter(device.PacketFilter) error
 	Address() iface.WGAddress
 }
 
@@ -95,6 +98,10 @@ func create(iface IFaceMapper) (*Manager, error) {
 	return m, nil
 }
 
+func (m *Manager) Init(*statemanager.Manager) error {
+	return nil
+}
+
 func (m *Manager) IsServerRouteSupported() bool {
 	if m.nativeFirewall == nil {
 		return false
@@ -103,26 +110,26 @@ func (m *Manager) IsServerRouteSupported() bool {
 	}
 }
 
-func (m *Manager) InsertRoutingRules(pair firewall.RouterPair) error {
+func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
 	if m.nativeFirewall == nil {
 		return errRouteNotSupported
 	}
-	return m.nativeFirewall.InsertRoutingRules(pair)
+	return m.nativeFirewall.AddNatRule(pair)
 }
 
-// RemoveRoutingRules removes a routing firewall rule
-func (m *Manager) RemoveRoutingRules(pair firewall.RouterPair) error {
+// RemoveNatRule removes a routing firewall rule
+func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
 	if m.nativeFirewall == nil {
 		return errRouteNotSupported
 	}
-	return m.nativeFirewall.RemoveRoutingRules(pair)
+	return m.nativeFirewall.RemoveNatRule(pair)
 }
 
-// AddFiltering rule to the firewall
+// AddPeerFiltering rule to the firewall
 //
 // If comment argument is empty firewall manager should set
 // rule ID as comment for the rule
-func (m *Manager) AddFiltering(
+func (m *Manager) AddPeerFiltering(
 	ip net.IP,
 	proto firewall.Protocol,
 	sPort *firewall.Port,
@@ -188,8 +195,22 @@ func (m *Manager) AddFiltering(
 	return []firewall.Rule{&r}, nil
 }
 
-// DeleteRule from the firewall by rule definition
-func (m *Manager) DeleteRule(rule firewall.Rule) error {
+func (m *Manager) AddRouteFiltering(sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action) (firewall.Rule, error) {
+	if m.nativeFirewall == nil {
+		return nil, errRouteNotSupported
+	}
+	return m.nativeFirewall.AddRouteFiltering(sources, destination, proto, sPort, dPort, action)
+}
+
+func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
+	if m.nativeFirewall == nil {
+		return errRouteNotSupported
+	}
+	return m.nativeFirewall.DeleteRouteRule(rule)
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
@@ -215,6 +236,14 @@ func (m *Manager) DeleteRule(rule firewall.Rule) error {
 	return nil
 }
 
+// SetLegacyManagement doesn't need to be implemented for this manager
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	if m.nativeFirewall == nil {
+		return nil
+	}
+	return m.nativeFirewall.SetLegacyManagement(isLegacy)
+}
+
 // Flush doesn't need to be implemented for this manager
 func (m *Manager) Flush() error { return nil }
 
@@ -337,7 +366,6 @@ func validateRule(ip net.IP, packetData []byte, rules map[string]Rule, d *decode
 			if rule.dPort != 0 && rule.dPort == uint16(d.udp.DstPort) {
 				return rule.drop, true
 			}
-			return rule.drop, true
 		case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6:
 			return rule.drop, true
 		}
@@ -396,7 +424,7 @@ func (m *Manager) RemovePacketHook(hookID string) error {
 		for _, r := range arr {
 			if r.id == hookID {
 				rule := r
-				return m.DeleteRule(&rule)
+				return m.DeletePeerRule(&rule)
 			}
 		}
 	}
@@ -404,7 +432,7 @@ func (m *Manager) RemovePacketHook(hookID string) error {
 		for _, r := range arr {
 			if r.id == hookID {
 				rule := r
-				return m.DeleteRule(&rule)
+				return m.DeletePeerRule(&rule)
 			}
 		}
 	}

client/firewall/uspfilter/uspfilter_test.go

@@ -11,15 +11,16 @@ import (
 	"github.com/stretchr/testify/require"
 
 	fw "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
 )
 
 type IFaceMock struct {
-	SetFilterFunc func(iface.PacketFilter) error
+	SetFilterFunc func(device.PacketFilter) error
 	AddressFunc   func() iface.WGAddress
 }
 
-func (i *IFaceMock) SetFilter(iface iface.PacketFilter) error {
+func (i *IFaceMock) SetFilter(iface device.PacketFilter) error {
 	if i.SetFilterFunc == nil {
 		return fmt.Errorf("not implemented")
 	}
@@ -35,7 +36,7 @@ func (i *IFaceMock) Address() iface.WGAddress {
 
 func TestManagerCreate(t *testing.T) {
 	ifaceMock := &IFaceMock{
-		SetFilterFunc: func(iface.PacketFilter) error { return nil },
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
 	}
 
 	m, err := Create(ifaceMock)
@@ -49,10 +50,10 @@ func TestManagerCreate(t *testing.T) {
 	}
 }
 
-func TestManagerAddFiltering(t *testing.T) {
+func TestManagerAddPeerFiltering(t *testing.T) {
 	isSetFilterCalled := false
 	ifaceMock := &IFaceMock{
-		SetFilterFunc: func(iface.PacketFilter) error {
+		SetFilterFunc: func(device.PacketFilter) error {
 			isSetFilterCalled = true
 			return nil
 		},
@@ -71,7 +72,7 @@ func TestManagerAddFiltering(t *testing.T) {
 	action := fw.ActionDrop
 	comment := "Test rule"
 
-	rule, err := m.AddFiltering(ip, proto, nil, port, direction, action, "", comment)
+	rule, err := m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
 	if err != nil {
 		t.Errorf("failed to add filtering: %v", err)
 		return
@@ -90,7 +91,7 @@ func TestManagerAddFiltering(t *testing.T) {
 
 func TestManagerDeleteRule(t *testing.T) {
 	ifaceMock := &IFaceMock{
-		SetFilterFunc: func(iface.PacketFilter) error { return nil },
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
 	}
 
 	m, err := Create(ifaceMock)
@@ -106,7 +107,7 @@ func TestManagerDeleteRule(t *testing.T) {
 	action := fw.ActionDrop
 	comment := "Test rule"
 
-	rule, err := m.AddFiltering(ip, proto, nil, port, direction, action, "", comment)
+	rule, err := m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
 	if err != nil {
 		t.Errorf("failed to add filtering: %v", err)
 		return
@@ -119,14 +120,14 @@ func TestManagerDeleteRule(t *testing.T) {
 	action = fw.ActionDrop
 	comment = "Test rule 2"
 
-	rule2, err := m.AddFiltering(ip, proto, nil, port, direction, action, "", comment)
+	rule2, err := m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
 	if err != nil {
 		t.Errorf("failed to add filtering: %v", err)
 		return
 	}
 
 	for _, r := range rule {
-		err = m.DeleteRule(r)
+		err = m.DeletePeerRule(r)
 		if err != nil {
 			t.Errorf("failed to delete rule: %v", err)
 			return
@@ -140,7 +141,7 @@ func TestManagerDeleteRule(t *testing.T) {
 	}
 
 	for _, r := range rule2 {
-		err = m.DeleteRule(r)
+		err = m.DeletePeerRule(r)
 		if err != nil {
 			t.Errorf("failed to delete rule: %v", err)
 			return
@@ -236,7 +237,7 @@ func TestAddUDPPacketHook(t *testing.T) {
 
 func TestManagerReset(t *testing.T) {
 	ifaceMock := &IFaceMock{
-		SetFilterFunc: func(iface.PacketFilter) error { return nil },
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
 	}
 
 	m, err := Create(ifaceMock)
@@ -252,13 +253,13 @@ func TestManagerReset(t *testing.T) {
 	action := fw.ActionDrop
 	comment := "Test rule"
 
-	_, err = m.AddFiltering(ip, proto, nil, port, direction, action, "", comment)
+	_, err = m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
 	if err != nil {
 		t.Errorf("failed to add filtering: %v", err)
 		return
 	}
 
-	err = m.Reset()
+	err = m.Reset(nil)
 	if err != nil {
 		t.Errorf("failed to reset Manager: %v", err)
 		return
@@ -271,7 +272,7 @@ func TestManagerReset(t *testing.T) {
 
 func TestNotMatchByIP(t *testing.T) {
 	ifaceMock := &IFaceMock{
-		SetFilterFunc: func(iface.PacketFilter) error { return nil },
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
 	}
 
 	m, err := Create(ifaceMock)
@@ -290,7 +291,7 @@ func TestNotMatchByIP(t *testing.T) {
 	action := fw.ActionAccept
 	comment := "Test rule"
 
-	_, err = m.AddFiltering(ip, proto, nil, nil, direction, action, "", comment)
+	_, err = m.AddPeerFiltering(ip, proto, nil, nil, direction, action, "", comment)
 	if err != nil {
 		t.Errorf("failed to add filtering: %v", err)
 		return
@@ -329,7 +330,7 @@ func TestNotMatchByIP(t *testing.T) {
 		return
 	}
 
-	if err = m.Reset(); err != nil {
+	if err = m.Reset(nil); err != nil {
 		t.Errorf("failed to reset Manager: %v", err)
 		return
 	}
@@ -339,7 +340,7 @@ func TestNotMatchByIP(t *testing.T) {
 func TestRemovePacketHook(t *testing.T) {
 	// creating mock iface
 	iface := &IFaceMock{
-		SetFilterFunc: func(iface.PacketFilter) error { return nil },
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
 	}
 
 	// creating manager instance
@@ -388,14 +389,14 @@ func TestUSPFilterCreatePerformance(t *testing.T) {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
 			ifaceMock := &IFaceMock{
-				SetFilterFunc: func(iface.PacketFilter) error { return nil },
+				SetFilterFunc: func(device.PacketFilter) error { return nil },
 			}
 			manager, err := Create(ifaceMock)
 			require.NoError(t, err)
 			time.Sleep(time.Second)
 
 			defer func() {
-				if err := manager.Reset(); err != nil {
+				if err := manager.Reset(nil); err != nil {
 					t.Errorf("clear the manager state: %v", err)
 				}
 				time.Sleep(time.Second)
@@ -406,9 +407,9 @@ func TestUSPFilterCreatePerformance(t *testing.T) {
 			for i := 0; i < testMax; i++ {
 				port := &fw.Port{Values: []int{1000 + i}}
 				if i%2 == 0 {
-					_, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
 				} else {
-					_, err = manager.AddFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
 				}
 
 				require.NoError(t, err, "failed to add rule")

client/iface/bind/control_android.go

@@ -0,0 +1,12 @@
+package bind
+
+import (
+	wireguard "golang.zx2c4.com/wireguard/conn"
+
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func init() {
+	// ControlFns is not thread safe and should only be modified during init.
+	*wireguard.ControlFns = append(*wireguard.ControlFns, nbnet.ControlProtectSocket)
+}

client/iface/bind/endpoint.go

@@ -0,0 +1,5 @@
+package bind
+
+import wgConn "golang.zx2c4.com/wireguard/conn"
+
+type Endpoint = wgConn.StdNetEndpoint

client/iface/bind/ice_bind.go

@@ -0,0 +1,303 @@
+package bind
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/pion/stun/v2"
+	"github.com/pion/transport/v3"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+	wgConn "golang.zx2c4.com/wireguard/conn"
+)
+
+type RecvMessage struct {
+	Endpoint *Endpoint
+	Buffer   []byte
+}
+
+type receiverCreator struct {
+	iceBind *ICEBind
+}
+
+func (rc receiverCreator) CreateIPv4ReceiverFn(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool, msgPool *sync.Pool) wgConn.ReceiveFunc {
+	return rc.iceBind.createIPv4ReceiverFn(pc, conn, rxOffload, msgPool)
+}
+
+// ICEBind is a bind implementation with two main features:
+// 1. filter out STUN messages and handle them
+// 2. forward the received packets to the WireGuard interface from the relayed connection
+//
+// ICEBind.endpoints var is a map that stores the connection for each relayed peer. Fake address is just an IP address
+// without port, in the format of 127.1.x.x where x.x is the last two octets of the peer address. We try to avoid to
+// use the port because in the Send function the wgConn.Endpoint the port info is not exported.
+type ICEBind struct {
+	*wgConn.StdNetBind
+	RecvChan chan RecvMessage
+
+	transportNet transport.Net
+	filterFn     FilterFn
+	endpoints    map[netip.Addr]net.Conn
+	endpointsMu  sync.Mutex
+	// every time when Close() is called (i.e. BindUpdate()) we need to close exit from the receiveRelayed and create a
+	// new closed channel. With the closedChanMu we can safely close the channel and create a new one
+	closedChan   chan struct{}
+	closedChanMu sync.RWMutex // protect the closeChan recreation from reading from it.
+	closed       bool
+
+	muUDPMux sync.Mutex
+	udpMux   *UniversalUDPMuxDefault
+}
+
+func NewICEBind(transportNet transport.Net, filterFn FilterFn) *ICEBind {
+	b, _ := wgConn.NewStdNetBind().(*wgConn.StdNetBind)
+	ib := &ICEBind{
+		StdNetBind:   b,
+		RecvChan:     make(chan RecvMessage, 1),
+		transportNet: transportNet,
+		filterFn:     filterFn,
+		endpoints:    make(map[netip.Addr]net.Conn),
+		closedChan:   make(chan struct{}),
+		closed:       true,
+	}
+
+	rc := receiverCreator{
+		ib,
+	}
+	ib.StdNetBind = wgConn.NewStdNetBindWithReceiverCreator(rc)
+	return ib
+}
+
+func (s *ICEBind) Open(uport uint16) ([]wgConn.ReceiveFunc, uint16, error) {
+	s.closed = false
+	s.closedChanMu.Lock()
+	s.closedChan = make(chan struct{})
+	s.closedChanMu.Unlock()
+	fns, port, err := s.StdNetBind.Open(uport)
+	if err != nil {
+		return nil, 0, err
+	}
+	fns = append(fns, s.receiveRelayed)
+	return fns, port, nil
+}
+
+func (s *ICEBind) Close() error {
+	if s.closed {
+		return nil
+	}
+	s.closed = true
+
+	close(s.closedChan)
+
+	return s.StdNetBind.Close()
+}
+
+// GetICEMux returns the ICE UDPMux that was created and used by ICEBind
+func (s *ICEBind) GetICEMux() (*UniversalUDPMuxDefault, error) {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+	if s.udpMux == nil {
+		return nil, fmt.Errorf("ICEBind has not been initialized yet")
+	}
+
+	return s.udpMux, nil
+}
+
+func (b *ICEBind) SetEndpoint(peerAddress *net.UDPAddr, conn net.Conn) (*net.UDPAddr, error) {
+	fakeUDPAddr, err := fakeAddress(peerAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// force IPv4
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		return nil, fmt.Errorf("failed to convert IP to netip.Addr")
+	}
+
+	b.endpointsMu.Lock()
+	b.endpoints[fakeAddr] = conn
+	b.endpointsMu.Unlock()
+
+	return fakeUDPAddr, nil
+}
+
+func (b *ICEBind) RemoveEndpoint(fakeUDPAddr *net.UDPAddr) {
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		log.Warnf("failed to convert IP to netip.Addr")
+		return
+	}
+
+	b.endpointsMu.Lock()
+	defer b.endpointsMu.Unlock()
+	delete(b.endpoints, fakeAddr)
+}
+
+func (b *ICEBind) Send(bufs [][]byte, ep wgConn.Endpoint) error {
+	b.endpointsMu.Lock()
+	conn, ok := b.endpoints[ep.DstIP()]
+	b.endpointsMu.Unlock()
+	if !ok {
+		return b.StdNetBind.Send(bufs, ep)
+	}
+
+	for _, buf := range bufs {
+		if _, err := conn.Write(buf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *ICEBind) createIPv4ReceiverFn(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool, msgsPool *sync.Pool) wgConn.ReceiveFunc {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+
+	s.udpMux = NewUniversalUDPMuxDefault(
+		UniversalUDPMuxParams{
+			UDPConn:  conn,
+			Net:      s.transportNet,
+			FilterFn: s.filterFn,
+		},
+	)
+	return func(bufs [][]byte, sizes []int, eps []wgConn.Endpoint) (n int, err error) {
+		msgs := getMessages(msgsPool)
+		for i := range bufs {
+			(*msgs)[i].Buffers[0] = bufs[i]
+			(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
+		}
+		defer putMessages(msgs, msgsPool)
+		var numMsgs int
+		if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+			if rxOffload {
+				readAt := len(*msgs) - (wgConn.IdealBatchSize / wgConn.UdpSegmentMaxDatagrams)
+				//nolint
+				numMsgs, err = pc.ReadBatch((*msgs)[readAt:], 0)
+				if err != nil {
+					return 0, err
+				}
+				numMsgs, err = wgConn.SplitCoalescedMessages(*msgs, readAt, wgConn.GetGSOSize)
+				if err != nil {
+					return 0, err
+				}
+			} else {
+				numMsgs, err = pc.ReadBatch(*msgs, 0)
+				if err != nil {
+					return 0, err
+				}
+			}
+		} else {
+			msg := &(*msgs)[0]
+			msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
+			if err != nil {
+				return 0, err
+			}
+			numMsgs = 1
+		}
+		for i := 0; i < numMsgs; i++ {
+			msg := &(*msgs)[i]
+
+			// todo: handle err
+			ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr)
+			if ok {
+				continue
+			}
+			sizes[i] = msg.N
+			if sizes[i] == 0 {
+				continue
+			}
+			addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
+			ep := &wgConn.StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
+			wgConn.GetSrcFromControl(msg.OOB[:msg.NN], ep)
+			eps[i] = ep
+		}
+		return numMsgs, nil
+	}
+}
+
+func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) {
+	for i := range buffers {
+		if !stun.IsMessage(buffers[i]) {
+			continue
+		}
+
+		msg, err := s.parseSTUNMessage(buffers[i][:n])
+		if err != nil {
+			buffers[i] = []byte{}
+			return true, err
+		}
+
+		muxErr := s.udpMux.HandleSTUNMessage(msg, addr)
+		if muxErr != nil {
+			log.Warnf("failed to handle STUN packet")
+		}
+
+		buffers[i] = []byte{}
+		return true, nil
+	}
+	return false, nil
+}
+
+func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) {
+	msg := &stun.Message{
+		Raw: raw,
+	}
+	if err := msg.Decode(); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}
+
+// receiveRelayed is a receive function that is used to receive packets from the relayed connection and forward to the
+// WireGuard. Critical part is do not block if the Closed() has been called.
+func (c *ICEBind) receiveRelayed(buffs [][]byte, sizes []int, eps []wgConn.Endpoint) (int, error) {
+	c.closedChanMu.RLock()
+	defer c.closedChanMu.RUnlock()
+
+	select {
+	case <-c.closedChan:
+		return 0, net.ErrClosed
+	case msg, ok := <-c.RecvChan:
+		if !ok {
+			return 0, net.ErrClosed
+		}
+		copy(buffs[0], msg.Buffer)
+		sizes[0] = len(msg.Buffer)
+		eps[0] = wgConn.Endpoint(msg.Endpoint)
+		return 1, nil
+	}
+}
+
+// fakeAddress returns a fake address that is used to as an identifier for the peer.
+// The fake address is in the format of 127.1.x.x where x.x is the last two octets of the peer address.
+func fakeAddress(peerAddress *net.UDPAddr) (*net.UDPAddr, error) {
+	octets := strings.Split(peerAddress.IP.String(), ".")
+	if len(octets) != 4 {
+		return nil, fmt.Errorf("invalid IP format")
+	}
+
+	newAddr := &net.UDPAddr{
+		IP:   net.ParseIP(fmt.Sprintf("127.1.%s.%s", octets[2], octets[3])),
+		Port: peerAddress.Port,
+	}
+	return newAddr, nil
+}
+
+func getMessages(msgsPool *sync.Pool) *[]ipv6.Message {
+	return msgsPool.Get().(*[]ipv6.Message)
+}
+
+func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) {
+	for i := range *msgs {
+		(*msgs)[i].OOB = (*msgs)[i].OOB[:0]
+		(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
+	}
+	msgsPool.Put(msgs)
+}

client/iface/bind/udp_mux.go

@@ -162,12 +162,13 @@ func NewUDPMuxDefault(params UDPMuxParams) *UDPMuxDefault {
 		params.Logger.Warn("UDPMuxDefault should not listening on unspecified address, use NewMultiUDPMuxFromPort instead")
 		var networks []ice.NetworkType
 		switch {
-		case addr.IP.To4() != nil:
-			networks = []ice.NetworkType{ice.NetworkTypeUDP4}
 
 		case addr.IP.To16() != nil:
 			networks = []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}
 
+		case addr.IP.To4() != nil:
+			networks = []ice.NetworkType{ice.NetworkTypeUDP4}
+
 		default:
 			params.Logger.Errorf("LocalAddr expected IPV4 or IPV6, got %T", params.UDPConn.LocalAddr())
 		}

client/iface/configurer/err.go

@@ -0,0 +1,5 @@
+package configurer
+
+import "errors"
+
+var ErrPeerNotFound = errors.New("peer not found")

client/iface/configurer/kernel_unix.go

@@ -1,6 +1,6 @@
 //go:build (linux && !android) || freebsd
 
-package iface
+package configurer
 
 import (
 	"fmt"
@@ -12,18 +12,17 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
-type wgKernelConfigurer struct {
+type KernelConfigurer struct {
 	deviceName string
 }
 
-func newWGConfigurer(deviceName string) wgConfigurer {
-	wgc := &wgKernelConfigurer{
+func NewKernelConfigurer(deviceName string) *KernelConfigurer {
+	return &KernelConfigurer{
 		deviceName: deviceName,
 	}
-	return wgc
 }
 
-func (c *wgKernelConfigurer) configureInterface(privateKey string, port int) error {
+func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error {
 	log.Debugf("adding Wireguard private key")
 	key, err := wgtypes.ParseKey(privateKey)
 	if err != nil {
@@ -44,7 +43,7 @@ func (c *wgKernelConfigurer) configureInterface(privateKey string, port int) err
 	return nil
 }
 
-func (c *wgKernelConfigurer) updatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
+func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
 	// parse allowed ips
 	_, ipNet, err := net.ParseCIDR(allowedIps)
 	if err != nil {
@@ -56,8 +55,9 @@ func (c *wgKernelConfigurer) updatePeer(peerKey string, allowedIps string, keepA
 		return err
 	}
 	peer := wgtypes.PeerConfig{
-		PublicKey:                   peerKeyParsed,
-		ReplaceAllowedIPs:           true,
+		PublicKey:         peerKeyParsed,
+		ReplaceAllowedIPs: false,
+		// don't replace allowed ips, wg will handle duplicated peer IP
 		AllowedIPs:                  []net.IPNet{*ipNet},
 		PersistentKeepaliveInterval: &keepAlive,
 		Endpoint:                    endpoint,
@@ -74,7 +74,7 @@ func (c *wgKernelConfigurer) updatePeer(peerKey string, allowedIps string, keepA
 	return nil
 }
 
-func (c *wgKernelConfigurer) removePeer(peerKey string) error {
+func (c *KernelConfigurer) RemovePeer(peerKey string) error {
 	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
 	if err != nil {
 		return err
@@ -95,7 +95,7 @@ func (c *wgKernelConfigurer) removePeer(peerKey string) error {
 	return nil
 }
 
-func (c *wgKernelConfigurer) addAllowedIP(peerKey string, allowedIP string) error {
+func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP string) error {
 	_, ipNet, err := net.ParseCIDR(allowedIP)
 	if err != nil {
 		return err
@@ -122,7 +122,7 @@ func (c *wgKernelConfigurer) addAllowedIP(peerKey string, allowedIP string) erro
 	return nil
 }
 
-func (c *wgKernelConfigurer) removeAllowedIP(peerKey string, allowedIP string) error {
+func (c *KernelConfigurer) RemoveAllowedIP(peerKey string, allowedIP string) error {
 	_, ipNet, err := net.ParseCIDR(allowedIP)
 	if err != nil {
 		return fmt.Errorf("parse allowed IP: %w", err)
@@ -164,7 +164,7 @@ func (c *wgKernelConfigurer) removeAllowedIP(peerKey string, allowedIP string) e
 	return nil
 }
 
-func (c *wgKernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
+func (c *KernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
 	wg, err := wgctrl.New()
 	if err != nil {
 		return wgtypes.Peer{}, fmt.Errorf("wgctl: %w", err)
@@ -188,7 +188,7 @@ func (c *wgKernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer
 	return wgtypes.Peer{}, ErrPeerNotFound
 }
 
-func (c *wgKernelConfigurer) configure(config wgtypes.Config) error {
+func (c *KernelConfigurer) configure(config wgtypes.Config) error {
 	wg, err := wgctrl.New()
 	if err != nil {
 		return err
@@ -204,10 +204,10 @@ func (c *wgKernelConfigurer) configure(config wgtypes.Config) error {
 	return wg.ConfigureDevice(c.deviceName, config)
 }
 
-func (c *wgKernelConfigurer) close() {
+func (c *KernelConfigurer) Close() {
 }
 
-func (c *wgKernelConfigurer) getStats(peerKey string) (WGStats, error) {
+func (c *KernelConfigurer) GetStats(peerKey string) (WGStats, error) {
 	peer, err := c.getPeer(c.deviceName, peerKey)
 	if err != nil {
 		return WGStats{}, fmt.Errorf("get wireguard stats: %w", err)

client/iface/configurer/name.go

@@ -1,6 +1,6 @@
 //go:build linux || windows || freebsd
 
-package iface
+package configurer
 
 // WgInterfaceDefault is a default interface name of Wiretrustee
 const WgInterfaceDefault = "wt0"

client/iface/configurer/name_darwin.go

@@ -1,6 +1,6 @@
 //go:build darwin
 
-package iface
+package configurer
 
 // WgInterfaceDefault is a default interface name of Wiretrustee
 const WgInterfaceDefault = "utun100"

client/iface/configurer/uapi.go

@@ -1,6 +1,6 @@
 //go:build !windows
 
-package iface
+package configurer
 
 import (
 	"net"

client/iface/configurer/uapi_windows.go

@@ -1,4 +1,4 @@
-package iface
+package configurer
 
 import (
 	"net"

client/iface/configurer/usp.go

@@ -1,4 +1,4 @@
-package iface
+package configurer
 
 import (
 	"encoding/hex"
@@ -19,15 +19,15 @@ import (
 
 var ErrAllowedIPNotFound = fmt.Errorf("allowed IP not found")
 
-type wgUSPConfigurer struct {
+type WGUSPConfigurer struct {
 	device     *device.Device
 	deviceName string
 
 	uapiListener net.Listener
 }
 
-func newWGUSPConfigurer(device *device.Device, deviceName string) wgConfigurer {
-	wgCfg := &wgUSPConfigurer{
+func NewUSPConfigurer(device *device.Device, deviceName string) *WGUSPConfigurer {
+	wgCfg := &WGUSPConfigurer{
 		device:     device,
 		deviceName: deviceName,
 	}
@@ -35,7 +35,7 @@ func newWGUSPConfigurer(device *device.Device, deviceName string) wgConfigurer {
 	return wgCfg
 }
 
-func (c *wgUSPConfigurer) configureInterface(privateKey string, port int) error {
+func (c *WGUSPConfigurer) ConfigureInterface(privateKey string, port int) error {
 	log.Debugf("adding Wireguard private key")
 	key, err := wgtypes.ParseKey(privateKey)
 	if err != nil {
@@ -52,7 +52,7 @@ func (c *wgUSPConfigurer) configureInterface(privateKey string, port int) error
 	return c.device.IpcSet(toWgUserspaceString(config))
 }
 
-func (c *wgUSPConfigurer) updatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
+func (c *WGUSPConfigurer) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
 	// parse allowed ips
 	_, ipNet, err := net.ParseCIDR(allowedIps)
 	if err != nil {
@@ -64,8 +64,9 @@ func (c *wgUSPConfigurer) updatePeer(peerKey string, allowedIps string, keepAliv
 		return err
 	}
 	peer := wgtypes.PeerConfig{
-		PublicKey:                   peerKeyParsed,
-		ReplaceAllowedIPs:           true,
+		PublicKey:         peerKeyParsed,
+		ReplaceAllowedIPs: false,
+		// don't replace allowed ips, wg will handle duplicated peer IP
 		AllowedIPs:                  []net.IPNet{*ipNet},
 		PersistentKeepaliveInterval: &keepAlive,
 		PresharedKey:                preSharedKey,
@@ -79,7 +80,7 @@ func (c *wgUSPConfigurer) updatePeer(peerKey string, allowedIps string, keepAliv
 	return c.device.IpcSet(toWgUserspaceString(config))
 }
 
-func (c *wgUSPConfigurer) removePeer(peerKey string) error {
+func (c *WGUSPConfigurer) RemovePeer(peerKey string) error {
 	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
 	if err != nil {
 		return err
@@ -96,7 +97,7 @@ func (c *wgUSPConfigurer) removePeer(peerKey string) error {
 	return c.device.IpcSet(toWgUserspaceString(config))
 }
 
-func (c *wgUSPConfigurer) addAllowedIP(peerKey string, allowedIP string) error {
+func (c *WGUSPConfigurer) AddAllowedIP(peerKey string, allowedIP string) error {
 	_, ipNet, err := net.ParseCIDR(allowedIP)
 	if err != nil {
 		return err
@@ -120,7 +121,7 @@ func (c *wgUSPConfigurer) addAllowedIP(peerKey string, allowedIP string) error {
 	return c.device.IpcSet(toWgUserspaceString(config))
 }
 
-func (c *wgUSPConfigurer) removeAllowedIP(peerKey string, ip string) error {
+func (c *WGUSPConfigurer) RemoveAllowedIP(peerKey string, ip string) error {
 	ipc, err := c.device.IpcGet()
 	if err != nil {
 		return err
@@ -184,7 +185,7 @@ func (c *wgUSPConfigurer) removeAllowedIP(peerKey string, ip string) error {
 }
 
 // startUAPI starts the UAPI listener for managing the WireGuard interface via external tool
-func (t *wgUSPConfigurer) startUAPI() {
+func (t *WGUSPConfigurer) startUAPI() {
 	var err error
 	t.uapiListener, err = openUAPI(t.deviceName)
 	if err != nil {
@@ -206,7 +207,7 @@ func (t *wgUSPConfigurer) startUAPI() {
 	}(t.uapiListener)
 }
 
-func (t *wgUSPConfigurer) close() {
+func (t *WGUSPConfigurer) Close() {
 	if t.uapiListener != nil {
 		err := t.uapiListener.Close()
 		if err != nil {
@@ -222,7 +223,7 @@ func (t *wgUSPConfigurer) close() {
 	}
 }
 
-func (t *wgUSPConfigurer) getStats(peerKey string) (WGStats, error) {
+func (t *WGUSPConfigurer) GetStats(peerKey string) (WGStats, error) {
 	ipc, err := t.device.IpcGet()
 	if err != nil {
 		return WGStats{}, fmt.Errorf("ipc get: %w", err)

client/iface/configurer/usp_test.go

@@ -1,4 +1,4 @@
-package iface
+package configurer
 
 import (
 	"encoding/hex"

client/iface/configurer/wgstats.go

@@ -0,0 +1,9 @@
+package configurer
+
+import "time"
+
+type WGStats struct {
+	LastHandshake time.Time
+	TxBytes       int64
+	RxBytes       int64
+}

client/iface/device.go

@@ -0,0 +1,18 @@
+//go:build !android
+
+package iface
+
+import (
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type WGTunDevice interface {
+	Create() (device.WGConfigurer, error)
+	Up() (*bind.UniversalUDPMuxDefault, error)
+	UpdateAddr(address WGAddress) error
+	WgAddress() WGAddress
+	DeviceName() string
+	Close() error
+	FilteredDevice() *device.FilteredDevice
+}

client/iface/device/adapter.go

@@ -1,4 +1,4 @@
-package iface
+package device
 
 // TunAdapter is an interface for create tun device from external service
 type TunAdapter interface {

client/iface/device/address.go

@@ -1,18 +1,18 @@
-package iface
+package device
 
 import (
 	"fmt"
 	"net"
 )
 
-// WGAddress Wireguard parsed address
+// WGAddress WireGuard parsed address
 type WGAddress struct {
 	IP      net.IP
 	Network *net.IPNet
 }
 
-// parseWGAddress parse a string ("1.2.3.4/24") address to WG Address
-func parseWGAddress(address string) (WGAddress, error) {
+// ParseWGAddress parse a string ("1.2.3.4/24") address to WG Address
+func ParseWGAddress(address string) (WGAddress, error) {
 	ip, network, err := net.ParseCIDR(address)
 	if err != nil {
 		return WGAddress{}, err

client/iface/device/args.go

@@ -1,4 +1,4 @@
-package iface
+package device
 
 type MobileIFaceArguments struct {
 	TunAdapter TunAdapter // only for Android

client/iface/device/device_android.go

@@ -1,22 +1,21 @@
 //go:build android
-// +build android
 
-package iface
+package device
 
 import (
 	"strings"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 
-	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
 )
 
-// ignore the wgTunDevice interface on Android because the creation of the tun device is different on this platform
-type wgTunDevice struct {
+// WGTunDevice ignore the WGTunDevice interface on Android because the creation of the tun device is different on this platform
+type WGTunDevice struct {
 	address    WGAddress
 	port       int
 	key        string
@@ -24,25 +23,25 @@ type wgTunDevice struct {
 	iceBind    *bind.ICEBind
 	tunAdapter TunAdapter
 
-	name       string
-	device     *device.Device
-	wrapper    *DeviceWrapper
-	udpMux     *bind.UniversalUDPMuxDefault
-	configurer wgConfigurer
+	name           string
+	device         *device.Device
+	filteredDevice *FilteredDevice
+	udpMux         *bind.UniversalUDPMuxDefault
+	configurer     WGConfigurer
 }
 
-func newTunDevice(address WGAddress, port int, key string, mtu int, transportNet transport.Net, tunAdapter TunAdapter, filterFn bind.FilterFn) wgTunDevice {
-	return wgTunDevice{
+func NewTunDevice(address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind, tunAdapter TunAdapter) *WGTunDevice {
+	return &WGTunDevice{
 		address:    address,
 		port:       port,
 		key:        key,
 		mtu:        mtu,
-		iceBind:    bind.NewICEBind(transportNet, filterFn),
+		iceBind:    iceBind,
 		tunAdapter: tunAdapter,
 	}
 }
 
-func (t *wgTunDevice) Create(routes []string, dns string, searchDomains []string) (wgConfigurer, error) {
+func (t *WGTunDevice) Create(routes []string, dns string, searchDomains []string) (WGConfigurer, error) {
 	log.Info("create tun interface")
 
 	routesString := routesToString(routes)
@@ -61,24 +60,24 @@ func (t *wgTunDevice) Create(routes []string, dns string, searchDomains []string
 		return nil, err
 	}
 	t.name = name
-	t.wrapper = newDeviceWrapper(tunDevice)
+	t.filteredDevice = newDeviceFilter(tunDevice)
 
 	log.Debugf("attaching to interface %v", name)
-	t.device = device.NewDevice(t.wrapper, t.iceBind, device.NewLogger(device.LogLevelSilent, "[wiretrustee] "))
+	t.device = device.NewDevice(t.filteredDevice, t.iceBind, device.NewLogger(wgLogLevel(), "[wiretrustee] "))
 	// without this property mobile devices can discover remote endpoints if the configured one was wrong.
 	// this helps with support for the older NetBird clients that had a hardcoded direct mode
 	// t.device.DisableSomeRoamingForBrokenMobileSemantics()
 
-	t.configurer = newWGUSPConfigurer(t.device, t.name)
-	err = t.configurer.configureInterface(t.key, t.port)
+	t.configurer = configurer.NewUSPConfigurer(t.device, t.name)
+	err = t.configurer.ConfigureInterface(t.key, t.port)
 	if err != nil {
 		t.device.Close()
-		t.configurer.close()
+		t.configurer.Close()
 		return nil, err
 	}
 	return t.configurer, nil
 }
-func (t *wgTunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+func (t *WGTunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	err := t.device.Up()
 	if err != nil {
 		return nil, err
@@ -93,14 +92,14 @@ func (t *wgTunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	return udpMux, nil
 }
 
-func (t *wgTunDevice) UpdateAddr(addr WGAddress) error {
+func (t *WGTunDevice) UpdateAddr(addr WGAddress) error {
 	// todo implement
 	return nil
 }
 
-func (t *wgTunDevice) Close() error {
+func (t *WGTunDevice) Close() error {
 	if t.configurer != nil {
-		t.configurer.close()
+		t.configurer.Close()
 	}
 
 	if t.device != nil {
@@ -115,20 +114,20 @@ func (t *wgTunDevice) Close() error {
 	return nil
 }
 
-func (t *wgTunDevice) Device() *device.Device {
+func (t *WGTunDevice) Device() *device.Device {
 	return t.device
 }
 
-func (t *wgTunDevice) DeviceName() string {
+func (t *WGTunDevice) DeviceName() string {
 	return t.name
 }
 
-func (t *wgTunDevice) WgAddress() WGAddress {
+func (t *WGTunDevice) WgAddress() WGAddress {
 	return t.address
 }
 
-func (t *wgTunDevice) Wrapper() *DeviceWrapper {
-	return t.wrapper
+func (t *WGTunDevice) FilteredDevice() *FilteredDevice {
+	return t.filteredDevice
 }
 
 func routesToString(routes []string) string {

client/iface/device/device_darwin.go

@@ -1,19 +1,20 @@
 //go:build !ios
 
-package iface
+package device
 
 import (
+	"fmt"
 	"os/exec"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 
-	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
 )
 
-type tunDevice struct {
+type TunDevice struct {
 	name    string
 	address WGAddress
 	port    int
@@ -21,54 +22,54 @@ type tunDevice struct {
 	mtu     int
 	iceBind *bind.ICEBind
 
-	device     *device.Device
-	wrapper    *DeviceWrapper
-	udpMux     *bind.UniversalUDPMuxDefault
-	configurer wgConfigurer
+	device         *device.Device
+	filteredDevice *FilteredDevice
+	udpMux         *bind.UniversalUDPMuxDefault
+	configurer     WGConfigurer
 }
 
-func newTunDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) wgTunDevice {
-	return &tunDevice{
+func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *TunDevice {
+	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 
-func (t *tunDevice) Create() (wgConfigurer, error) {
+func (t *TunDevice) Create() (WGConfigurer, error) {
 	tunDevice, err := tun.CreateTUN(t.name, t.mtu)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error creating tun device: %s", err)
 	}
-	t.wrapper = newDeviceWrapper(tunDevice)
+	t.filteredDevice = newDeviceFilter(tunDevice)
 
 	// We need to create a wireguard-go device and listen to configuration requests
 	t.device = device.NewDevice(
-		t.wrapper,
+		t.filteredDevice,
 		t.iceBind,
-		device.NewLogger(device.LogLevelSilent, "[netbird] "),
+		device.NewLogger(wgLogLevel(), "[netbird] "),
 	)
 
 	err = t.assignAddr()
 	if err != nil {
 		t.device.Close()
-		return nil, err
+		return nil, fmt.Errorf("error assigning ip: %s", err)
 	}
 
-	t.configurer = newWGUSPConfigurer(t.device, t.name)
-	err = t.configurer.configureInterface(t.key, t.port)
+	t.configurer = configurer.NewUSPConfigurer(t.device, t.name)
+	err = t.configurer.ConfigureInterface(t.key, t.port)
 	if err != nil {
 		t.device.Close()
-		t.configurer.close()
-		return nil, err
+		t.configurer.Close()
+		return nil, fmt.Errorf("error configuring interface: %s", err)
 	}
 	return t.configurer, nil
 }
 
-func (t *tunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+func (t *TunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	err := t.device.Up()
 	if err != nil {
 		return nil, err
@@ -83,14 +84,14 @@ func (t *tunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	return udpMux, nil
 }
 
-func (t *tunDevice) UpdateAddr(address WGAddress) error {
+func (t *TunDevice) UpdateAddr(address WGAddress) error {
 	t.address = address
 	return t.assignAddr()
 }
 
-func (t *tunDevice) Close() error {
+func (t *TunDevice) Close() error {
 	if t.configurer != nil {
-		t.configurer.close()
+		t.configurer.Close()
 	}
 
 	if t.device != nil {
@@ -104,20 +105,20 @@ func (t *tunDevice) Close() error {
 	return nil
 }
 
-func (t *tunDevice) WgAddress() WGAddress {
+func (t *TunDevice) WgAddress() WGAddress {
 	return t.address
 }
 
-func (t *tunDevice) DeviceName() string {
+func (t *TunDevice) DeviceName() string {
 	return t.name
 }
 
-func (t *tunDevice) Wrapper() *DeviceWrapper {
-	return t.wrapper
+func (t *TunDevice) FilteredDevice() *FilteredDevice {
+	return t.filteredDevice
 }
 
 // assignAddr Adds IP address to the tunnel interface and network route based on the range provided
-func (t *tunDevice) assignAddr() error {
+func (t *TunDevice) assignAddr() error {
 	cmd := exec.Command("ifconfig", t.name, "inet", t.address.IP.String(), t.address.IP.String())
 	if out, err := cmd.CombinedOutput(); err != nil {
 		log.Errorf("adding address command '%v' failed with output: %s", cmd.String(), out)

client/iface/device/device_filter.go

@@ -1,4 +1,4 @@
-package iface
+package device
 
 import (
 	"net"
@@ -28,22 +28,23 @@ type PacketFilter interface {
 	SetNetwork(*net.IPNet)
 }
 
-// DeviceWrapper to override Read or Write of packets
-type DeviceWrapper struct {
+// FilteredDevice to override Read or Write of packets
+type FilteredDevice struct {
 	tun.Device
+
 	filter PacketFilter
 	mutex  sync.RWMutex
 }
 
-// newDeviceWrapper constructor function
-func newDeviceWrapper(device tun.Device) *DeviceWrapper {
-	return &DeviceWrapper{
+// newDeviceFilter constructor function
+func newDeviceFilter(device tun.Device) *FilteredDevice {
+	return &FilteredDevice{
 		Device: device,
 	}
 }
 
 // Read wraps read method with filtering feature
-func (d *DeviceWrapper) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
+func (d *FilteredDevice) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) {
 	if n, err = d.Device.Read(bufs, sizes, offset); err != nil {
 		return 0, err
 	}
@@ -68,7 +69,7 @@ func (d *DeviceWrapper) Read(bufs [][]byte, sizes []int, offset int) (n int, err
 }
 
 // Write wraps write method with filtering feature
-func (d *DeviceWrapper) Write(bufs [][]byte, offset int) (int, error) {
+func (d *FilteredDevice) Write(bufs [][]byte, offset int) (int, error) {
 	d.mutex.RLock()
 	filter := d.filter
 	d.mutex.RUnlock()
@@ -92,7 +93,7 @@ func (d *DeviceWrapper) Write(bufs [][]byte, offset int) (int, error) {
 }
 
 // SetFilter sets packet filter to device
-func (d *DeviceWrapper) SetFilter(filter PacketFilter) {
+func (d *FilteredDevice) SetFilter(filter PacketFilter) {
 	d.mutex.Lock()
 	d.filter = filter
 	d.mutex.Unlock()

client/iface/device/device_filter_test.go

@@ -1,4 +1,4 @@
-package iface
+package device
 
 import (
 	"net"
@@ -7,7 +7,8 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
-	mocks "github.com/netbirdio/netbird/iface/mocks"
+
+	mocks "github.com/netbirdio/netbird/client/iface/mocks"
 )
 
 func TestDeviceWrapperRead(t *testing.T) {
@@ -51,7 +52,7 @@ func TestDeviceWrapperRead(t *testing.T) {
 				return 1, nil
 			})
 
-		wrapped := newDeviceWrapper(tun)
+		wrapped := newDeviceFilter(tun)
 
 		bufs := [][]byte{{}}
 		sizes := []int{0}
@@ -99,7 +100,7 @@ func TestDeviceWrapperRead(t *testing.T) {
 		tun := mocks.NewMockDevice(ctrl)
 		tun.EXPECT().Write(mockBufs, 0).Return(1, nil)
 
-		wrapped := newDeviceWrapper(tun)
+		wrapped := newDeviceFilter(tun)
 
 		bufs := [][]byte{buffer.Bytes()}
 
@@ -147,7 +148,7 @@ func TestDeviceWrapperRead(t *testing.T) {
 		filter := mocks.NewMockPacketFilter(ctrl)
 		filter.EXPECT().DropIncoming(gomock.Any()).Return(true)
 
-		wrapped := newDeviceWrapper(tun)
+		wrapped := newDeviceFilter(tun)
 		wrapped.filter = filter
 
 		bufs := [][]byte{buffer.Bytes()}
@@ -202,7 +203,7 @@ func TestDeviceWrapperRead(t *testing.T) {
 		filter := mocks.NewMockPacketFilter(ctrl)
 		filter.EXPECT().DropOutgoing(gomock.Any()).Return(true)
 
-		wrapped := newDeviceWrapper(tun)
+		wrapped := newDeviceFilter(tun)
 		wrapped.filter = filter
 
 		bufs := [][]byte{{}}

client/iface/device/device_ios.go

@@ -1,21 +1,21 @@
 //go:build ios
 // +build ios
 
-package iface
+package device
 
 import (
 	"os"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 
-	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
 )
 
-type tunDevice struct {
+type TunDevice struct {
 	name    string
 	address WGAddress
 	port    int
@@ -23,24 +23,24 @@ type tunDevice struct {
 	iceBind *bind.ICEBind
 	tunFd   int
 
-	device     *device.Device
-	wrapper    *DeviceWrapper
-	udpMux     *bind.UniversalUDPMuxDefault
-	configurer wgConfigurer
+	device         *device.Device
+	filteredDevice *FilteredDevice
+	udpMux         *bind.UniversalUDPMuxDefault
+	configurer     WGConfigurer
 }
 
-func newTunDevice(name string, address WGAddress, port int, key string, transportNet transport.Net, tunFd int, filterFn bind.FilterFn) *tunDevice {
-	return &tunDevice{
+func NewTunDevice(name string, address WGAddress, port int, key string, iceBind *bind.ICEBind, tunFd int) *TunDevice {
+	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 		tunFd:   tunFd,
 	}
 }
 
-func (t *tunDevice) Create() (wgConfigurer, error) {
+func (t *TunDevice) Create() (WGConfigurer, error) {
 	log.Infof("create tun interface")
 
 	dupTunFd, err := unix.Dup(t.tunFd)
@@ -62,24 +62,24 @@ func (t *tunDevice) Create() (wgConfigurer, error) {
 		return nil, err
 	}
 
-	t.wrapper = newDeviceWrapper(tunDevice)
+	t.filteredDevice = newDeviceFilter(tunDevice)
 	log.Debug("Attaching to interface")
-	t.device = device.NewDevice(t.wrapper, t.iceBind, device.NewLogger(device.LogLevelSilent, "[wiretrustee] "))
+	t.device = device.NewDevice(t.filteredDevice, t.iceBind, device.NewLogger(wgLogLevel(), "[wiretrustee] "))
 	// without this property mobile devices can discover remote endpoints if the configured one was wrong.
 	// this helps with support for the older NetBird clients that had a hardcoded direct mode
 	// t.device.DisableSomeRoamingForBrokenMobileSemantics()
 
-	t.configurer = newWGUSPConfigurer(t.device, t.name)
-	err = t.configurer.configureInterface(t.key, t.port)
+	t.configurer = configurer.NewUSPConfigurer(t.device, t.name)
+	err = t.configurer.ConfigureInterface(t.key, t.port)
 	if err != nil {
 		t.device.Close()
-		t.configurer.close()
+		t.configurer.Close()
 		return nil, err
 	}
 	return t.configurer, nil
 }
 
-func (t *tunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+func (t *TunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	err := t.device.Up()
 	if err != nil {
 		return nil, err
@@ -94,17 +94,17 @@ func (t *tunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	return udpMux, nil
 }
 
-func (t *tunDevice) Device() *device.Device {
+func (t *TunDevice) Device() *device.Device {
 	return t.device
 }
 
-func (t *tunDevice) DeviceName() string {
+func (t *TunDevice) DeviceName() string {
 	return t.name
 }
 
-func (t *tunDevice) Close() error {
+func (t *TunDevice) Close() error {
 	if t.configurer != nil {
-		t.configurer.close()
+		t.configurer.Close()
 	}
 
 	if t.device != nil {
@@ -119,15 +119,15 @@ func (t *tunDevice) Close() error {
 	return nil
 }
 
-func (t *tunDevice) WgAddress() WGAddress {
+func (t *TunDevice) WgAddress() WGAddress {
 	return t.address
 }
 
-func (t *tunDevice) UpdateAddr(addr WGAddress) error {
+func (t *TunDevice) UpdateAddr(addr WGAddress) error {
 	// todo implement
 	return nil
 }
 
-func (t *tunDevice) Wrapper() *DeviceWrapper {
-	return t.wrapper
+func (t *TunDevice) FilteredDevice() *FilteredDevice {
+	return t.filteredDevice
 }

client/iface/device/device_kernel_unix.go

@@ -1,6 +1,6 @@
 //go:build (linux && !android) || freebsd
 
-package iface
+package device
 
 import (
 	"context"
@@ -10,11 +10,12 @@ import (
 	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
 	"github.com/netbirdio/netbird/sharedsock"
 )
 
-type tunKernelDevice struct {
+type TunKernelDevice struct {
 	name         string
 	address      WGAddress
 	wgPort       int
@@ -31,11 +32,11 @@ type tunKernelDevice struct {
 	filterFn bind.FilterFn
 }
 
-func newTunDevice(name string, address WGAddress, wgPort int, key string, mtu int, transportNet transport.Net) wgTunDevice {
+func NewKernelDevice(name string, address WGAddress, wgPort int, key string, mtu int, transportNet transport.Net) *TunKernelDevice {
 	checkUser()
 
 	ctx, cancel := context.WithCancel(context.Background())
-	return &tunKernelDevice{
+	return &TunKernelDevice{
 		ctx:          ctx,
 		ctxCancel:    cancel,
 		name:         name,
@@ -47,7 +48,7 @@ func newTunDevice(name string, address WGAddress, wgPort int, key string, mtu in
 	}
 }
 
-func (t *tunKernelDevice) Create() (wgConfigurer, error) {
+func (t *TunKernelDevice) Create() (WGConfigurer, error) {
 	link := newWGLink(t.name)
 
 	if err := link.recreate(); err != nil {
@@ -67,16 +68,16 @@ func (t *tunKernelDevice) Create() (wgConfigurer, error) {
 		return nil, fmt.Errorf("set mtu: %w", err)
 	}
 
-	configurer := newWGConfigurer(t.name)
+	configurer := configurer.NewKernelConfigurer(t.name)
 
-	if err := configurer.configureInterface(t.key, t.wgPort); err != nil {
-		return nil, err
+	if err := configurer.ConfigureInterface(t.key, t.wgPort); err != nil {
+		return nil, fmt.Errorf("error configuring interface: %s", err)
 	}
 
 	return configurer, nil
 }
 
-func (t *tunKernelDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+func (t *TunKernelDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	if t.udpMux != nil {
 		return t.udpMux, nil
 	}
@@ -111,12 +112,12 @@ func (t *tunKernelDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	return t.udpMux, nil
 }
 
-func (t *tunKernelDevice) UpdateAddr(address WGAddress) error {
+func (t *TunKernelDevice) UpdateAddr(address WGAddress) error {
 	t.address = address
 	return t.assignAddr()
 }
 
-func (t *tunKernelDevice) Close() error {
+func (t *TunKernelDevice) Close() error {
 	if t.link == nil {
 		return nil
 	}
@@ -144,19 +145,19 @@ func (t *tunKernelDevice) Close() error {
 	return closErr
 }
 
-func (t *tunKernelDevice) WgAddress() WGAddress {
+func (t *TunKernelDevice) WgAddress() WGAddress {
 	return t.address
 }
 
-func (t *tunKernelDevice) DeviceName() string {
+func (t *TunKernelDevice) DeviceName() string {
 	return t.name
 }
 
-func (t *tunKernelDevice) Wrapper() *DeviceWrapper {
+func (t *TunKernelDevice) FilteredDevice() *FilteredDevice {
 	return nil
 }
 
 // assignAddr Adds IP address to the tunnel interface
-func (t *tunKernelDevice) assignAddr() error {
+func (t *TunKernelDevice) assignAddr() error {
 	return t.link.assignAddr(t.address)
 }

client/iface/device/device_netstack.go

@@ -0,0 +1,119 @@
+//go:build !android
+// +build !android
+
+package device
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"golang.zx2c4.com/wireguard/device"
+
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
+	"github.com/netbirdio/netbird/client/iface/netstack"
+)
+
+type TunNetstackDevice struct {
+	name          string
+	address       WGAddress
+	port          int
+	key           string
+	mtu           int
+	listenAddress string
+	iceBind       *bind.ICEBind
+
+	device         *device.Device
+	filteredDevice *FilteredDevice
+	nsTun          *netstack.NetStackTun
+	udpMux         *bind.UniversalUDPMuxDefault
+	configurer     WGConfigurer
+}
+
+func NewNetstackDevice(name string, address WGAddress, wgPort int, key string, mtu int, iceBind *bind.ICEBind, listenAddress string) *TunNetstackDevice {
+	return &TunNetstackDevice{
+		name:          name,
+		address:       address,
+		port:          wgPort,
+		key:           key,
+		mtu:           mtu,
+		listenAddress: listenAddress,
+		iceBind:       iceBind,
+	}
+}
+
+func (t *TunNetstackDevice) Create() (WGConfigurer, error) {
+	log.Info("create netstack tun interface")
+	t.nsTun = netstack.NewNetStackTun(t.listenAddress, t.address.IP.String(), t.mtu)
+	tunIface, err := t.nsTun.Create()
+	if err != nil {
+		return nil, fmt.Errorf("error creating tun device: %s", err)
+	}
+	t.filteredDevice = newDeviceFilter(tunIface)
+
+	t.device = device.NewDevice(
+		t.filteredDevice,
+		t.iceBind,
+		device.NewLogger(wgLogLevel(), "[netbird] "),
+	)
+
+	t.configurer = configurer.NewUSPConfigurer(t.device, t.name)
+	err = t.configurer.ConfigureInterface(t.key, t.port)
+	if err != nil {
+		_ = tunIface.Close()
+		return nil, fmt.Errorf("error configuring interface: %s", err)
+	}
+
+	log.Debugf("device has been created: %s", t.name)
+	return t.configurer, nil
+}
+
+func (t *TunNetstackDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+	if t.device == nil {
+		return nil, fmt.Errorf("device is not ready yet")
+	}
+
+	err := t.device.Up()
+	if err != nil {
+		return nil, err
+	}
+
+	udpMux, err := t.iceBind.GetICEMux()
+	if err != nil {
+		return nil, err
+	}
+	t.udpMux = udpMux
+	log.Debugf("netstack device is ready to use")
+	return udpMux, nil
+}
+
+func (t *TunNetstackDevice) UpdateAddr(WGAddress) error {
+	return nil
+}
+
+func (t *TunNetstackDevice) Close() error {
+	if t.configurer != nil {
+		t.configurer.Close()
+	}
+
+	if t.device != nil {
+		t.device.Close()
+	}
+
+	if t.udpMux != nil {
+		return t.udpMux.Close()
+	}
+	return nil
+}
+
+func (t *TunNetstackDevice) WgAddress() WGAddress {
+	return t.address
+}
+
+func (t *TunNetstackDevice) DeviceName() string {
+	return t.name
+}
+
+func (t *TunNetstackDevice) FilteredDevice() *FilteredDevice {
+	return t.filteredDevice
+}

client/iface/device/device_usp_unix.go

@@ -1,21 +1,21 @@
 //go:build (linux && !android) || freebsd
 
-package iface
+package device
 
 import (
 	"fmt"
 	"os"
 	"runtime"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 
-	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
 )
 
-type tunUSPDevice struct {
+type USPDevice struct {
 	name    string
 	address WGAddress
 	port    int
@@ -23,60 +23,60 @@ type tunUSPDevice struct {
 	mtu     int
 	iceBind *bind.ICEBind
 
-	device     *device.Device
-	wrapper    *DeviceWrapper
-	udpMux     *bind.UniversalUDPMuxDefault
-	configurer wgConfigurer
+	device         *device.Device
+	filteredDevice *FilteredDevice
+	udpMux         *bind.UniversalUDPMuxDefault
+	configurer     WGConfigurer
 }
 
-func newTunUSPDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) wgTunDevice {
+func NewUSPDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *USPDevice {
 	log.Infof("using userspace bind mode")
 
 	checkUser()
 
-	return &tunUSPDevice{
+	return &USPDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 
-func (t *tunUSPDevice) Create() (wgConfigurer, error) {
+func (t *USPDevice) Create() (WGConfigurer, error) {
 	log.Info("create tun interface")
 	tunIface, err := tun.CreateTUN(t.name, t.mtu)
 	if err != nil {
-		log.Debugf("failed to create tun unterface (%s, %d): %s", t.name, t.mtu, err)
-		return nil, err
+		log.Debugf("failed to create tun interface (%s, %d): %s", t.name, t.mtu, err)
+		return nil, fmt.Errorf("error creating tun device: %s", err)
 	}
-	t.wrapper = newDeviceWrapper(tunIface)
+	t.filteredDevice = newDeviceFilter(tunIface)
 
 	// We need to create a wireguard-go device and listen to configuration requests
 	t.device = device.NewDevice(
-		t.wrapper,
+		t.filteredDevice,
 		t.iceBind,
-		device.NewLogger(device.LogLevelSilent, "[netbird] "),
+		device.NewLogger(wgLogLevel(), "[netbird] "),
 	)
 
 	err = t.assignAddr()
 	if err != nil {
 		t.device.Close()
-		return nil, err
+		return nil, fmt.Errorf("error assigning ip: %s", err)
 	}
 
-	t.configurer = newWGUSPConfigurer(t.device, t.name)
-	err = t.configurer.configureInterface(t.key, t.port)
+	t.configurer = configurer.NewUSPConfigurer(t.device, t.name)
+	err = t.configurer.ConfigureInterface(t.key, t.port)
 	if err != nil {
 		t.device.Close()
-		t.configurer.close()
-		return nil, err
+		t.configurer.Close()
+		return nil, fmt.Errorf("error configuring interface: %s", err)
 	}
 	return t.configurer, nil
 }
 
-func (t *tunUSPDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+func (t *USPDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	if t.device == nil {
 		return nil, fmt.Errorf("device is not ready yet")
 	}
@@ -96,14 +96,14 @@ func (t *tunUSPDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	return udpMux, nil
 }
 
-func (t *tunUSPDevice) UpdateAddr(address WGAddress) error {
+func (t *USPDevice) UpdateAddr(address WGAddress) error {
 	t.address = address
 	return t.assignAddr()
 }
 
-func (t *tunUSPDevice) Close() error {
+func (t *USPDevice) Close() error {
 	if t.configurer != nil {
-		t.configurer.close()
+		t.configurer.Close()
 	}
 
 	if t.device != nil {
@@ -116,20 +116,20 @@ func (t *tunUSPDevice) Close() error {
 	return nil
 }
 
-func (t *tunUSPDevice) WgAddress() WGAddress {
+func (t *USPDevice) WgAddress() WGAddress {
 	return t.address
 }
 
-func (t *tunUSPDevice) DeviceName() string {
+func (t *USPDevice) DeviceName() string {
 	return t.name
 }
 
-func (t *tunUSPDevice) Wrapper() *DeviceWrapper {
-	return t.wrapper
+func (t *USPDevice) FilteredDevice() *FilteredDevice {
+	return t.filteredDevice
 }
 
 // assignAddr Adds IP address to the tunnel interface
-func (t *tunUSPDevice) assignAddr() error {
+func (t *USPDevice) assignAddr() error {
 	link := newWGLink(t.name)
 
 	return link.assignAddr(t.address)

client/iface/device/device_windows.go

@@ -1,20 +1,22 @@
-package iface
+package device
 
 import (
 	"fmt"
 	"net/netip"
 
-	"github.com/pion/transport/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 
-	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/bind"
+	"github.com/netbirdio/netbird/client/iface/configurer"
 )
 
-type tunDevice struct {
+const defaultWindowsGUIDSTring = "{f2f29e61-d91f-4d76-8151-119b20c4bdeb}"
+
+type TunDevice struct {
 	name    string
 	address WGAddress
 	port    int
@@ -24,35 +26,49 @@ type tunDevice struct {
 
 	device          *device.Device
 	nativeTunDevice *tun.NativeTun
-	wrapper         *DeviceWrapper
+	filteredDevice  *FilteredDevice
 	udpMux          *bind.UniversalUDPMuxDefault
-	configurer      wgConfigurer
+	configurer      WGConfigurer
 }
 
-func newTunDevice(name string, address WGAddress, port int, key string, mtu int, transportNet transport.Net, filterFn bind.FilterFn) wgTunDevice {
-	return &tunDevice{
+func NewTunDevice(name string, address WGAddress, port int, key string, mtu int, iceBind *bind.ICEBind) *TunDevice {
+	return &TunDevice{
 		name:    name,
 		address: address,
 		port:    port,
 		key:     key,
 		mtu:     mtu,
-		iceBind: bind.NewICEBind(transportNet, filterFn),
+		iceBind: iceBind,
 	}
 }
 
-func (t *tunDevice) Create() (wgConfigurer, error) {
-	tunDevice, err := tun.CreateTUN(t.name, t.mtu)
+func getGUID() (windows.GUID, error) {
+	guidString := defaultWindowsGUIDSTring
+	if CustomWindowsGUIDString != "" {
+		guidString = CustomWindowsGUIDString
+	}
+	return windows.GUIDFromString(guidString)
+}
+
+func (t *TunDevice) Create() (WGConfigurer, error) {
+	guid, err := getGUID()
 	if err != nil {
+		log.Errorf("failed to get GUID: %s", err)
 		return nil, err
 	}
+	log.Info("create tun interface")
+	tunDevice, err := tun.CreateTUNWithRequestedGUID(t.name, &guid, t.mtu)
+	if err != nil {
+		return nil, fmt.Errorf("error creating tun device: %s", err)
+	}
 	t.nativeTunDevice = tunDevice.(*tun.NativeTun)
-	t.wrapper = newDeviceWrapper(tunDevice)
+	t.filteredDevice = newDeviceFilter(tunDevice)
 
 	// We need to create a wireguard-go device and listen to configuration requests
 	t.device = device.NewDevice(
-		t.wrapper,
+		t.filteredDevice,
 		t.iceBind,
-		device.NewLogger(device.LogLevelSilent, "[netbird] "),
+		device.NewLogger(wgLogLevel(), "[netbird] "),
 	)
 
 	luid := winipcfg.LUID(t.nativeTunDevice.LUID())
@@ -73,20 +89,20 @@ func (t *tunDevice) Create() (wgConfigurer, error) {
 	err = t.assignAddr()
 	if err != nil {
 		t.device.Close()
-		return nil, err
+		return nil, fmt.Errorf("error assigning ip: %s", err)
 	}
 
-	t.configurer = newWGUSPConfigurer(t.device, t.name)
-	err = t.configurer.configureInterface(t.key, t.port)
+	t.configurer = configurer.NewUSPConfigurer(t.device, t.name)
+	err = t.configurer.ConfigureInterface(t.key, t.port)
 	if err != nil {
 		t.device.Close()
-		t.configurer.close()
-		return nil, err
+		t.configurer.Close()
+		return nil, fmt.Errorf("error configuring interface: %s", err)
 	}
 	return t.configurer, nil
 }
 
-func (t *tunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
+func (t *TunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	err := t.device.Up()
 	if err != nil {
 		return nil, err
@@ -101,14 +117,14 @@ func (t *tunDevice) Up() (*bind.UniversalUDPMuxDefault, error) {
 	return udpMux, nil
 }
 
-func (t *tunDevice) UpdateAddr(address WGAddress) error {
+func (t *TunDevice) UpdateAddr(address WGAddress) error {
 	t.address = address
 	return t.assignAddr()
 }
 
-func (t *tunDevice) Close() error {
+func (t *TunDevice) Close() error {
 	if t.configurer != nil {
-		t.configurer.close()
+		t.configurer.Close()
 	}
 
 	if t.device != nil {
@@ -122,19 +138,19 @@ func (t *tunDevice) Close() error {
 	}
 	return nil
 }
-func (t *tunDevice) WgAddress() WGAddress {
+func (t *TunDevice) WgAddress() WGAddress {
 	return t.address
 }
 
-func (t *tunDevice) DeviceName() string {
+func (t *TunDevice) DeviceName() string {
 	return t.name
 }
 
-func (t *tunDevice) Wrapper() *DeviceWrapper {
-	return t.wrapper
+func (t *TunDevice) FilteredDevice() *FilteredDevice {
+	return t.filteredDevice
 }
 
-func (t *tunDevice) getInterfaceGUIDString() (string, error) {
+func (t *TunDevice) GetInterfaceGUIDString() (string, error) {
 	if t.nativeTunDevice == nil {
 		return "", fmt.Errorf("interface has not been initialized yet")
 	}
@@ -148,7 +164,7 @@ func (t *tunDevice) getInterfaceGUIDString() (string, error) {
 }
 
 // assignAddr Adds IP address to the tunnel interface and network route based on the range provided
-func (t *tunDevice) assignAddr() error {
+func (t *TunDevice) assignAddr() error {
 	luid := winipcfg.LUID(t.nativeTunDevice.LUID())
 	log.Debugf("adding address %s to interface: %s", t.address.IP, t.name)
 	return luid.SetIPAddresses([]netip.Prefix{netip.MustParsePrefix(t.address.String())})

client/iface/device/interface.go

@@ -0,0 +1,20 @@
+package device
+
+import (
+	"net"
+	"time"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	"github.com/netbirdio/netbird/client/iface/configurer"
+)
+
+type WGConfigurer interface {
+	ConfigureInterface(privateKey string, port int) error
+	UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
+	RemovePeer(peerKey string) error
+	AddAllowedIP(peerKey string, allowedIP string) error
+	RemoveAllowedIP(peerKey string, allowedIP string) error
+	Close()
+	GetStats(peerKey string) (configurer.WGStats, error)
+}

client/iface/device/kernel_module.go

@@ -1,6 +1,6 @@
 //go:build (!linux && !freebsd) || android
 
-package iface
+package device
 
 // WireGuardModuleIsLoaded check if we can load WireGuard mod (linux only)
 func WireGuardModuleIsLoaded() bool {

client/iface/device/kernel_module_freebsd.go

@@ -1,4 +1,4 @@
-package iface
+package device
 
 // WireGuardModuleIsLoaded check if kernel support wireguard
 func WireGuardModuleIsLoaded() bool {
@@ -10,8 +10,8 @@ func WireGuardModuleIsLoaded() bool {
 	return false
 }
 
-// tunModuleIsLoaded check if tun module exist, if is not attempt to load it
-func tunModuleIsLoaded() bool {
+// ModuleTunIsLoaded check if tun module exist, if is not attempt to load it
+func ModuleTunIsLoaded() bool {
 	// Assume tun supported by freebsd kernel by default
 	// TODO: implement check for module loaded in kernel or build-it
 	return true

client/iface/device/kernel_module_linux.go

@@ -1,7 +1,7 @@
 //go:build linux && !android
 
 // Package iface provides wireguard network interface creation and management
-package iface
+package device
 
 import (
 	"bufio"
@@ -66,8 +66,8 @@ func getModuleRoot() string {
 	return filepath.Join(moduleLibDir, string(uname.Release[:i]))
 }
 
-// tunModuleIsLoaded check if tun module exist, if is not attempt to load it
-func tunModuleIsLoaded() bool {
+// ModuleTunIsLoaded check if tun module exist, if is not attempt to load it
+func ModuleTunIsLoaded() bool {
 	_, err := os.Stat("/dev/net/tun")
 	if err == nil {
 		return true

client/iface/device/kernel_module_linux_test.go

@@ -1,4 +1,6 @@
-package iface
+//go:build linux && !android
+
+package device
 
 import (
 	"bufio"
@@ -132,7 +134,7 @@ func resetGlobals() {
 }
 
 func createFiles(t *testing.T) (string, []module) {
-    t.Helper()
+	t.Helper()
 	writeFile := func(path, text string) {
 		if err := os.WriteFile(path, []byte(text), 0644); err != nil {
 			t.Fatal(err)
@@ -168,7 +170,7 @@ func createFiles(t *testing.T) (string, []module) {
 }
 
 func getRandomLoadedModule(t *testing.T) (string, error) {
-    t.Helper()
+	t.Helper()
 	f, err := os.Open("/proc/modules")
 	if err != nil {
 		return "", err

client/iface/device/wg_link_freebsd.go

@@ -1,10 +1,11 @@
-package iface
+package device
 
 import (
 	"fmt"
 
-	"github.com/netbirdio/netbird/iface/freebsd"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/iface/freebsd"
 )
 
 type wgLink struct {

client/iface/device/wg_link_linux.go

@@ -1,6 +1,6 @@
 //go:build linux && !android
 
-package iface
+package device
 
 import (
 	"fmt"