From 6cba9c0818c718d678ee67db5be47b010c54d8f1 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Tue, 19 Mar 2024 12:32:07 +0100 Subject: [PATCH 1/3] Remove context niling (#1729) --- client/internal/routemanager/manager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index fde943757..b624d8c34 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -71,7 +71,6 @@ func (m *DefaultManager) Stop() { if m.serverRouter != nil { m.serverRouter.cleanUp() } - m.ctx = nil } // UpdateRoutes compares received routes with existing routes and remove, update or add them to the client and server maps From 846871913dccc7d80ddfa9b0fa90c2c225b94b0b Mon Sep 17 00:00:00 2001 From: pascal-fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:18:34 +0100 Subject: [PATCH 2/3] Add latency checks to peer connection and status output (#1725) * adding peer healthcheck * generate proto file * fix return in udp mux and replace with continue * use ice agent for latency checks * fix status output * remove some logs * fix status test * revert bind and ebpf code * fix error handling on binding response callback * extend error handling on binding response callback --------- Co-authored-by: Maycon Santos --- client/cmd/status.go | 6 +- client/cmd/status_test.go | 11 ++ client/internal/peer/conn.go | 18 ++ client/internal/peer/status.go | 17 ++ client/proto/daemon.pb.go | 320 +++++++++++++++++---------------- client/proto/daemon.proto | 2 + client/server/server.go | 3 + go.mod | 18 +- go.sum | 42 ++--- 9 files changed, 252 insertions(+), 185 deletions(-) diff --git a/client/cmd/status.go b/client/cmd/status.go index 4c7218fde..2840cc6c9 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -34,6 +34,7 @@ type peerStateDetailOutput struct { LastWireguardHandshake time.Time `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"` TransferReceived int64 `json:"transferReceived" yaml:"transferReceived"` TransferSent int64 `json:"transferSent" yaml:"transferSent"` + Latency time.Duration `json:"latency" yaml:"latency"` RosenpassEnabled bool `json:"quantumResistance" yaml:"quantumResistance"` Routes []string `json:"routes" yaml:"routes"` } @@ -376,6 +377,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { LastWireguardHandshake: lastHandshake, TransferReceived: transferReceived, TransferSent: transferSent, + Latency: pbPeerState.GetLatency().AsDuration(), RosenpassEnabled: pbPeerState.GetRosenpassEnabled(), Routes: pbPeerState.GetRoutes(), } @@ -638,7 +640,8 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo " Last WireGuard handshake: %s\n"+ " Transfer status (received/sent) %s/%s\n"+ " Quantum resistance: %s\n"+ - " Routes: %s\n", + " Routes: %s\n"+ + " Latency: %s\n", peerState.FQDN, peerState.IP, peerState.PubKey, @@ -655,6 +658,7 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo toIEC(peerState.TransferSent), rosenpassEnabledStatus, routes, + peerState.Latency.String(), ) peersString += peerString diff --git a/client/cmd/status_test.go b/client/cmd/status_test.go index ea6980c3d..cc0cce134 100644 --- a/client/cmd/status_test.go +++ b/client/cmd/status_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" "github.com/netbirdio/netbird/client/proto" @@ -45,6 +46,7 @@ var resp = &proto.StatusResponse{ Routes: []string{ "10.1.0.0/24", }, + Latency: durationpb.New(time.Duration(10000000)), }, { IP: "192.168.178.102", @@ -61,6 +63,7 @@ var resp = &proto.StatusResponse{ LastWireguardHandshake: timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 3, 0, time.UTC)), BytesRx: 2000, BytesTx: 1000, + Latency: durationpb.New(time.Duration(10000000)), }, }, ManagementState: &proto.ManagementState{ @@ -147,6 +150,7 @@ var overview = statusOutputOverview{ Routes: []string{ "10.1.0.0/24", }, + Latency: time.Duration(10000000), }, { IP: "192.168.178.102", @@ -167,6 +171,7 @@ var overview = statusOutputOverview{ LastWireguardHandshake: time.Date(2002, 2, 2, 2, 2, 3, 0, time.UTC), TransferReceived: 2000, TransferSent: 1000, + Latency: time.Duration(10000000), }, }, }, @@ -288,6 +293,7 @@ func TestParsingToJSON(t *testing.T) { "lastWireguardHandshake": "2001-01-01T01:01:02Z", "transferReceived": 200, "transferSent": 100, + "latency": 10000000, "quantumResistance": false, "routes": [ "10.1.0.0/24" @@ -312,6 +318,7 @@ func TestParsingToJSON(t *testing.T) { "lastWireguardHandshake": "2002-02-02T02:02:03Z", "transferReceived": 2000, "transferSent": 1000, + "latency": 10000000, "quantumResistance": false, "routes": null } @@ -409,6 +416,7 @@ func TestParsingToYAML(t *testing.T) { lastWireguardHandshake: 2001-01-01T01:01:02Z transferReceived: 200 transferSent: 100 + latency: 10ms quantumResistance: false routes: - 10.1.0.0/24 @@ -428,6 +436,7 @@ func TestParsingToYAML(t *testing.T) { lastWireguardHandshake: 2002-02-02T02:02:03Z transferReceived: 2000 transferSent: 1000 + latency: 10ms quantumResistance: false routes: [] cliVersion: development @@ -496,6 +505,7 @@ func TestParsingToDetail(t *testing.T) { Transfer status (received/sent) 200 B/100 B Quantum resistance: false Routes: 10.1.0.0/24 + Latency: 10ms peer-2.awesome-domain.com: NetBird IP: 192.168.178.102 @@ -511,6 +521,7 @@ func TestParsingToDetail(t *testing.T) { Transfer status (received/sent) 2.0 KiB/1000 B Quantum resistance: false Routes: - + Latency: 10ms Daemon version: 0.14.1 CLI version: development diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b7db310e6..c180e8f03 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -133,6 +133,9 @@ type Conn struct { adapter iface.TunAdapter iFaceDiscover stdnet.ExternalIFaceDiscover sentExtraSrflx bool + + remoteEndpoint *net.UDPAddr + remoteConn *ice.Conn } // meta holds meta information about a connection @@ -234,6 +237,17 @@ func (conn *Conn) reCreateAgent() error { return err } + err = conn.agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { + err := conn.statusRecorder.UpdateLatency(conn.config.Key, p.Latency()) + if err != nil { + log.Debugf("failed to update latency for peer %s: %s", conn.config.Key, err) + return + } + }) + if err != nil { + return fmt.Errorf("failed setting binding response callback: %w", err) + } + return nil } @@ -348,6 +362,9 @@ func (conn *Conn) Open() error { if remoteOfferAnswer.WgListenPort != 0 { remoteWgPort = remoteOfferAnswer.WgListenPort } + + conn.remoteConn = remoteConn + // the ice connection has been established successfully so we are ready to start the proxy remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort, remoteOfferAnswer.RosenpassPubKey, remoteOfferAnswer.RosenpassAddr) @@ -397,6 +414,7 @@ func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, rem } endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.remoteEndpoint = endpointUdpAddr err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) if err != nil { diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 1e252c5dd..ca97c3ea4 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -28,6 +28,7 @@ type State struct { LastWireguardHandshake time.Time BytesTx int64 BytesRx int64 + Latency time.Duration RosenpassEnabled bool Routes map[string]struct{} } @@ -410,6 +411,22 @@ func (d *Status) GetManagementState() ManagementState { } } +func (d *Status) UpdateLatency(pubKey string, latency time.Duration) error { + if latency <= 0 { + return nil + } + + d.mux.Lock() + defer d.mux.Unlock() + peerState, ok := d.peers[pubKey] + if !ok { + return errors.New("peer doesn't exist") + } + peerState.Latency = latency + d.peers[pubKey] = peerState + return nil +} + // IsLoginRequired determines if a peer's login has expired. func (d *Status) IsLoginRequired() bool { d.mux.Lock() diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index 869eceee5..81998b115 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -1,16 +1,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v4.24.3 // source: daemon.proto package proto import ( - _ "github.com/golang/protobuf/protoc-gen-go/descriptor" - timestamp "github.com/golang/protobuf/ptypes/timestamp" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/descriptorpb" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -757,22 +758,23 @@ type PeerState struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - IP string `protobuf:"bytes,1,opt,name=IP,proto3" json:"IP,omitempty"` - PubKey string `protobuf:"bytes,2,opt,name=pubKey,proto3" json:"pubKey,omitempty"` - ConnStatus string `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"` - ConnStatusUpdate *timestamp.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"` - Relayed bool `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"` - Direct bool `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"` - LocalIceCandidateType string `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"` - RemoteIceCandidateType string `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"` - Fqdn string `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"` - LocalIceCandidateEndpoint string `protobuf:"bytes,10,opt,name=localIceCandidateEndpoint,proto3" json:"localIceCandidateEndpoint,omitempty"` - RemoteIceCandidateEndpoint string `protobuf:"bytes,11,opt,name=remoteIceCandidateEndpoint,proto3" json:"remoteIceCandidateEndpoint,omitempty"` - LastWireguardHandshake *timestamp.Timestamp `protobuf:"bytes,12,opt,name=lastWireguardHandshake,proto3" json:"lastWireguardHandshake,omitempty"` - BytesRx int64 `protobuf:"varint,13,opt,name=bytesRx,proto3" json:"bytesRx,omitempty"` - BytesTx int64 `protobuf:"varint,14,opt,name=bytesTx,proto3" json:"bytesTx,omitempty"` - RosenpassEnabled bool `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` - Routes []string `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"` + IP string `protobuf:"bytes,1,opt,name=IP,proto3" json:"IP,omitempty"` + PubKey string `protobuf:"bytes,2,opt,name=pubKey,proto3" json:"pubKey,omitempty"` + ConnStatus string `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"` + ConnStatusUpdate *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"` + Relayed bool `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"` + Direct bool `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"` + LocalIceCandidateType string `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"` + RemoteIceCandidateType string `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"` + Fqdn string `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"` + LocalIceCandidateEndpoint string `protobuf:"bytes,10,opt,name=localIceCandidateEndpoint,proto3" json:"localIceCandidateEndpoint,omitempty"` + RemoteIceCandidateEndpoint string `protobuf:"bytes,11,opt,name=remoteIceCandidateEndpoint,proto3" json:"remoteIceCandidateEndpoint,omitempty"` + LastWireguardHandshake *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=lastWireguardHandshake,proto3" json:"lastWireguardHandshake,omitempty"` + BytesRx int64 `protobuf:"varint,13,opt,name=bytesRx,proto3" json:"bytesRx,omitempty"` + BytesTx int64 `protobuf:"varint,14,opt,name=bytesTx,proto3" json:"bytesTx,omitempty"` + RosenpassEnabled bool `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` + Routes []string `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"` + Latency *durationpb.Duration `protobuf:"bytes,17,opt,name=latency,proto3" json:"latency,omitempty"` } func (x *PeerState) Reset() { @@ -828,7 +830,7 @@ func (x *PeerState) GetConnStatus() string { return "" } -func (x *PeerState) GetConnStatusUpdate() *timestamp.Timestamp { +func (x *PeerState) GetConnStatusUpdate() *timestamppb.Timestamp { if x != nil { return x.ConnStatusUpdate } @@ -884,7 +886,7 @@ func (x *PeerState) GetRemoteIceCandidateEndpoint() string { return "" } -func (x *PeerState) GetLastWireguardHandshake() *timestamp.Timestamp { +func (x *PeerState) GetLastWireguardHandshake() *timestamppb.Timestamp { if x != nil { return x.LastWireguardHandshake } @@ -919,6 +921,13 @@ func (x *PeerState) GetRoutes() []string { return nil } +func (x *PeerState) GetLatency() *durationpb.Duration { + if x != nil { + return x.Latency + } + return nil +} + // LocalPeerState contains the latest state of the local peer type LocalPeerState struct { state protoimpl.MessageState @@ -1374,7 +1383,9 @@ var file_daemon_proto_rawDesc = []byte{ 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xdd, 0x06, 0x0a, 0x0c, 0x4c, 0x6f, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xdd, 0x06, 0x0a, 0x0c, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x53, 0x68, 0x61, @@ -1475,7 +1486,7 @@ var file_daemon_proto_rawDesc = []byte{ 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x55, 0x52, 0x4c, - 0x22, 0x99, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, + 0x22, 0xce, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, @@ -1516,92 +1527,95 @@ var file_daemon_proto_rawDesc = []byte{ 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x10, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0xec, 0x01, 0x0a, - 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, - 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, - 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, - 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, - 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, - 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, - 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, - 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, - 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, - 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, - 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, - 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, - 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, - 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, - 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, - 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, - 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x32, 0xf7, 0x02, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, - 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, - 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, - 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, - 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, - 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, - 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, - 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, - 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, + 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, + 0x79, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, + 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, + 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, + 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, + 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, + 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, + 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, + 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, + 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, + 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, + 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, + 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, + 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x32, 0xf7, 0x02, 0x0a, 0x0d, + 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, + 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, + 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, + 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, + 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, + 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, + 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1618,54 +1632,56 @@ func file_daemon_proto_rawDescGZIP() []byte { var file_daemon_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_daemon_proto_goTypes = []interface{}{ - (*LoginRequest)(nil), // 0: daemon.LoginRequest - (*LoginResponse)(nil), // 1: daemon.LoginResponse - (*WaitSSOLoginRequest)(nil), // 2: daemon.WaitSSOLoginRequest - (*WaitSSOLoginResponse)(nil), // 3: daemon.WaitSSOLoginResponse - (*UpRequest)(nil), // 4: daemon.UpRequest - (*UpResponse)(nil), // 5: daemon.UpResponse - (*StatusRequest)(nil), // 6: daemon.StatusRequest - (*StatusResponse)(nil), // 7: daemon.StatusResponse - (*DownRequest)(nil), // 8: daemon.DownRequest - (*DownResponse)(nil), // 9: daemon.DownResponse - (*GetConfigRequest)(nil), // 10: daemon.GetConfigRequest - (*GetConfigResponse)(nil), // 11: daemon.GetConfigResponse - (*PeerState)(nil), // 12: daemon.PeerState - (*LocalPeerState)(nil), // 13: daemon.LocalPeerState - (*SignalState)(nil), // 14: daemon.SignalState - (*ManagementState)(nil), // 15: daemon.ManagementState - (*RelayState)(nil), // 16: daemon.RelayState - (*NSGroupState)(nil), // 17: daemon.NSGroupState - (*FullStatus)(nil), // 18: daemon.FullStatus - (*timestamp.Timestamp)(nil), // 19: google.protobuf.Timestamp + (*LoginRequest)(nil), // 0: daemon.LoginRequest + (*LoginResponse)(nil), // 1: daemon.LoginResponse + (*WaitSSOLoginRequest)(nil), // 2: daemon.WaitSSOLoginRequest + (*WaitSSOLoginResponse)(nil), // 3: daemon.WaitSSOLoginResponse + (*UpRequest)(nil), // 4: daemon.UpRequest + (*UpResponse)(nil), // 5: daemon.UpResponse + (*StatusRequest)(nil), // 6: daemon.StatusRequest + (*StatusResponse)(nil), // 7: daemon.StatusResponse + (*DownRequest)(nil), // 8: daemon.DownRequest + (*DownResponse)(nil), // 9: daemon.DownResponse + (*GetConfigRequest)(nil), // 10: daemon.GetConfigRequest + (*GetConfigResponse)(nil), // 11: daemon.GetConfigResponse + (*PeerState)(nil), // 12: daemon.PeerState + (*LocalPeerState)(nil), // 13: daemon.LocalPeerState + (*SignalState)(nil), // 14: daemon.SignalState + (*ManagementState)(nil), // 15: daemon.ManagementState + (*RelayState)(nil), // 16: daemon.RelayState + (*NSGroupState)(nil), // 17: daemon.NSGroupState + (*FullStatus)(nil), // 18: daemon.FullStatus + (*timestamppb.Timestamp)(nil), // 19: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 20: google.protobuf.Duration } var file_daemon_proto_depIdxs = []int32{ 18, // 0: daemon.StatusResponse.fullStatus:type_name -> daemon.FullStatus 19, // 1: daemon.PeerState.connStatusUpdate:type_name -> google.protobuf.Timestamp 19, // 2: daemon.PeerState.lastWireguardHandshake:type_name -> google.protobuf.Timestamp - 15, // 3: daemon.FullStatus.managementState:type_name -> daemon.ManagementState - 14, // 4: daemon.FullStatus.signalState:type_name -> daemon.SignalState - 13, // 5: daemon.FullStatus.localPeerState:type_name -> daemon.LocalPeerState - 12, // 6: daemon.FullStatus.peers:type_name -> daemon.PeerState - 16, // 7: daemon.FullStatus.relays:type_name -> daemon.RelayState - 17, // 8: daemon.FullStatus.dns_servers:type_name -> daemon.NSGroupState - 0, // 9: daemon.DaemonService.Login:input_type -> daemon.LoginRequest - 2, // 10: daemon.DaemonService.WaitSSOLogin:input_type -> daemon.WaitSSOLoginRequest - 4, // 11: daemon.DaemonService.Up:input_type -> daemon.UpRequest - 6, // 12: daemon.DaemonService.Status:input_type -> daemon.StatusRequest - 8, // 13: daemon.DaemonService.Down:input_type -> daemon.DownRequest - 10, // 14: daemon.DaemonService.GetConfig:input_type -> daemon.GetConfigRequest - 1, // 15: daemon.DaemonService.Login:output_type -> daemon.LoginResponse - 3, // 16: daemon.DaemonService.WaitSSOLogin:output_type -> daemon.WaitSSOLoginResponse - 5, // 17: daemon.DaemonService.Up:output_type -> daemon.UpResponse - 7, // 18: daemon.DaemonService.Status:output_type -> daemon.StatusResponse - 9, // 19: daemon.DaemonService.Down:output_type -> daemon.DownResponse - 11, // 20: daemon.DaemonService.GetConfig:output_type -> daemon.GetConfigResponse - 15, // [15:21] is the sub-list for method output_type - 9, // [9:15] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 20, // 3: daemon.PeerState.latency:type_name -> google.protobuf.Duration + 15, // 4: daemon.FullStatus.managementState:type_name -> daemon.ManagementState + 14, // 5: daemon.FullStatus.signalState:type_name -> daemon.SignalState + 13, // 6: daemon.FullStatus.localPeerState:type_name -> daemon.LocalPeerState + 12, // 7: daemon.FullStatus.peers:type_name -> daemon.PeerState + 16, // 8: daemon.FullStatus.relays:type_name -> daemon.RelayState + 17, // 9: daemon.FullStatus.dns_servers:type_name -> daemon.NSGroupState + 0, // 10: daemon.DaemonService.Login:input_type -> daemon.LoginRequest + 2, // 11: daemon.DaemonService.WaitSSOLogin:input_type -> daemon.WaitSSOLoginRequest + 4, // 12: daemon.DaemonService.Up:input_type -> daemon.UpRequest + 6, // 13: daemon.DaemonService.Status:input_type -> daemon.StatusRequest + 8, // 14: daemon.DaemonService.Down:input_type -> daemon.DownRequest + 10, // 15: daemon.DaemonService.GetConfig:input_type -> daemon.GetConfigRequest + 1, // 16: daemon.DaemonService.Login:output_type -> daemon.LoginResponse + 3, // 17: daemon.DaemonService.WaitSSOLogin:output_type -> daemon.WaitSSOLoginResponse + 5, // 18: daemon.DaemonService.Up:output_type -> daemon.UpResponse + 7, // 19: daemon.DaemonService.Status:output_type -> daemon.StatusResponse + 9, // 20: daemon.DaemonService.Down:output_type -> daemon.DownResponse + 11, // 21: daemon.DaemonService.GetConfig:output_type -> daemon.GetConfigResponse + 16, // [16:22] is the sub-list for method output_type + 10, // [10:16] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_daemon_proto_init() } diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index bdb1cb83e..8f9148d68 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -2,6 +2,7 @@ syntax = "proto3"; import "google/protobuf/descriptor.proto"; import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; option go_package = "/proto"; @@ -142,6 +143,7 @@ message PeerState { int64 bytesTx = 14; bool rosenpassEnabled = 15; repeated string routes = 16; + google.protobuf.Duration latency = 17; } // LocalPeerState contains the latest state of the local peer diff --git a/client/server/server.go b/client/server/server.go index 5f1bf0100..481ef0f7c 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -13,6 +13,8 @@ import ( "github.com/cenkalti/backoff/v4" "golang.org/x/exp/maps" + "google.golang.org/protobuf/types/known/durationpb" + "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/system" @@ -711,6 +713,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { BytesTx: peerState.BytesTx, RosenpassEnabled: peerState.RosenpassEnabled, Routes: maps.Keys(peerState.Routes), + Latency: durationpb.New(peerState.Latency), } pbFullStatus.Peers = append(pbFullStatus.Peers, pbPeerState) } diff --git a/go.mod b/go.mod index 6aba599f8..ce3da619e 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/cloudflare/circl v1.3.3 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.3 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 github.com/onsi/ginkgo v1.16.5 @@ -21,8 +21,8 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 - golang.org/x/crypto v0.17.0 - golang.org/x/sys v0.15.0 + golang.org/x/crypto v0.18.0 + golang.org/x/sys v0.16.0 golang.zx2c4.com/wireguard v0.0.0-20230704135630-469159ecf7d1 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 golang.zx2c4.com/wireguard/windows v0.5.3 @@ -81,10 +81,10 @@ require ( goauthentik.io/api/v3 v3.2023051.3 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 - golang.org/x/net v0.17.0 + golang.org/x/net v0.20.0 golang.org/x/oauth2 v0.8.0 golang.org/x/sync v0.3.0 - golang.org/x/term v0.15.0 + golang.org/x/term v0.16.0 google.golang.org/api v0.126.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/sqlite v1.5.3 @@ -137,10 +137,10 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect github.com/pegasus-kv/thrift v0.13.0 // indirect - github.com/pion/dtls/v2 v2.2.7 // indirect - github.com/pion/mdns v0.0.9 // indirect + github.com/pion/dtls/v2 v2.2.10 // indirect + github.com/pion/mdns v0.0.12 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/transport/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect @@ -175,3 +175,5 @@ replace github.com/getlantern/systray => github.com/netbirdio/systray v0.0.0-202 replace golang.zx2c4.com/wireguard => github.com/netbirdio/wireguard-go v0.0.0-20240105182236-6c340dd55aed replace github.com/cloudflare/circl => github.com/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6 + +replace github.com/pion/ice/v3 => github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e diff --git a/go.sum b/go.sum index ca10cd553..e304e3191 100644 --- a/go.sum +++ b/go.sum @@ -271,8 +271,8 @@ github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -380,6 +380,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nadoo/ipset v0.5.0 h1:5GJUAuZ7ITQQQGne5J96AmFjRtI8Avlbk6CabzYWVUc= github.com/nadoo/ipset v0.5.0/go.mod h1:rYF5DQLRGGoQ8ZSWeK+6eX5amAuPqwFkWjhQlEITGJQ= +github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e h1:PURA50S8u4mF6RrkYYCAvvPCixhqqEiEy3Ej6avh04c= +github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e/go.mod h1:YMLU7qbKfVjmEv7EoZPIVEI+kNYxWCdPK3VS0BU+U4Q= github.com/netbirdio/management-integrations/additions v0.0.0-20240212121739-8ea8c89a4552 h1:yzcQKizAK9YufCHMMCIsr467Dw/OU/4xyHbWizGb1E4= github.com/netbirdio/management-integrations/additions v0.0.0-20240212121739-8ea8c89a4552/go.mod h1:31FhBNvQ+riHEIu6LSTmqr8IeuSIsGfQffqV4LFmbwA= github.com/netbirdio/management-integrations/integrations v0.0.0-20240212121739-8ea8c89a4552 h1:OFlzVZtkXCoJsfDKrMigFpuad8ZXTm8epq6x27K0irA= @@ -423,20 +425,20 @@ github.com/pegasus-kv/thrift v0.13.0 h1:4ESwaNoHImfbHa9RUGJiJZ4hrxorihZHk5aarYwY github.com/pegasus-kv/thrift v0.13.0/go.mod h1:Gl9NT/WHG6ABm6NsrbfE8LiJN0sAyneCrvB4qN4NPqQ= github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v3 v3.0.2 h1:dNQnKsjLvOWz+PaI4tw1VnLYTp9adihC1HIASFGajmI= -github.com/pion/ice/v3 v3.0.2/go.mod h1:q3BDzTsxbqP0ySMSHrFuw2MYGUx/AC3WQfRGC5F/0Is= +github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= +github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= -github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= +github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= +github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= -github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/turn/v3 v3.0.1 h1:wLi7BTQr6/Q20R0vt/lHbjv6y4GChFtC33nkYbasoT8= @@ -580,10 +582,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -671,9 +671,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -765,20 +764,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -792,7 +787,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 2475473227f02bf9a362e23ed9a289c6ca97bc7b Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Thu, 21 Mar 2024 16:49:28 +0100 Subject: [PATCH 3/3] Support client default routes for Linux (#1667) All routes are now installed in a custom netbird routing table. Management and wireguard traffic is now marked with a custom fwmark. When the mark is present the traffic is routed via the main routing table, bypassing the VPN. When the mark is absent the traffic is routed via the netbird routing table, if: - there's no match in the main routing table - it would match the default route in the routing table IPv6 traffic is blocked when a default route IPv4 route is configured to avoid leakage. --- .github/workflows/golang-test-linux.yml | 16 +- .github/workflows/golangci-lint.yml | 2 +- client/internal/engine.go | 22 +- client/internal/relay/relay.go | 24 +- client/internal/routemanager/client.go | 63 ++- client/internal/routemanager/manager.go | 52 +- client/internal/routemanager/manager_test.go | 28 +- client/internal/routemanager/mock.go | 4 + .../routemanager/server_nonandroid.go | 51 +- .../routemanager/systemops_android.go | 4 +- client/internal/routemanager/systemops_bsd.go | 1 - .../routemanager/systemops_bsd_nonios.go | 13 + client/internal/routemanager/systemops_ios.go | 6 +- .../internal/routemanager/systemops_linux.go | 473 ++++++++++++++---- .../routemanager/systemops_linux_test.go | 469 +++++++++++++++++ .../routemanager/systemops_nonandroid.go | 148 +++--- .../routemanager/systemops_nonandroid_test.go | 142 +++--- .../routemanager/systemops_nonlinux.go | 27 +- .../routemanager/systemops_nonlinux_test.go | 80 +++ .../routemanager/systemops_windows.go | 15 +- client/internal/stdnet/dialer.go | 24 + client/internal/stdnet/listener.go | 20 + client/internal/wgproxy/portlookup.go | 6 +- client/internal/wgproxy/proxy_ebpf.go | 44 +- client/internal/wgproxy/proxy_userspace.go | 4 +- go.mod | 4 +- go.sum | 4 +- iface/address.go | 18 + iface/wg_configurer_kernel.go | 4 +- iface/wg_configurer_usp.go | 11 +- management/client/grpc.go | 2 + sharedsock/sock_linux.go | 59 ++- signal/client/grpc.go | 2 + util/grpc/dialer_generic.go | 9 + util/grpc/dialer_linux.go | 18 + util/net/dialer_generic.go | 19 + util/net/dialer_linux.go | 60 +++ util/net/listener_generic.go | 13 + util/net/listener_linux.go | 30 ++ util/net/net.go | 6 + util/net/net_linux.go | 35 ++ 41 files changed, 1656 insertions(+), 376 deletions(-) create mode 100644 client/internal/routemanager/systemops_bsd_nonios.go create mode 100644 client/internal/routemanager/systemops_linux_test.go create mode 100644 client/internal/routemanager/systemops_nonlinux_test.go create mode 100644 client/internal/stdnet/dialer.go create mode 100644 client/internal/stdnet/listener.go create mode 100644 util/grpc/dialer_generic.go create mode 100644 util/grpc/dialer_linux.go create mode 100644 util/net/dialer_generic.go create mode 100644 util/net/dialer_linux.go create mode 100644 util/net/listener_generic.go create mode 100644 util/net/listener_linux.go create mode 100644 util/net/net.go create mode 100644 util/net/net_linux.go diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index f6fab80c5..42f740e9b 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -14,8 +14,8 @@ jobs: test: strategy: matrix: - arch: ['386','amd64'] - store: ['jsonfile', 'sqlite'] + arch: [ '386','amd64' ] + store: [ 'jsonfile', 'sqlite' ] runs-on: ubuntu-latest steps: - name: Install Go @@ -36,7 +36,11 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib + run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev + + - name: Install 32-bit libpcap + if: matrix.arch == '386' + run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386 - name: Install modules run: go mod tidy @@ -67,7 +71,7 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib + run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - name: Install modules run: go mod tidy @@ -82,7 +86,7 @@ jobs: run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock - name: Generate RouteManager Test bin - run: CGO_ENABLED=0 go test -c -o routemanager-testing.bin ./client/internal/routemanager/... + run: CGO_ENABLED=1 go test -c -o routemanager-testing.bin -tags netgo -ldflags '-w -extldflags "-static -ldbus-1 -lpcap"' ./client/internal/routemanager/... - name: Generate nftables Manager Test bin run: CGO_ENABLED=0 go test -c -o nftablesmanager-testing.bin ./client/firewall/nftables/... @@ -109,7 +113,7 @@ jobs: - name: Run Engine tests in docker with file store run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="jsonfile" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin -test.timeout 5m -test.parallel 1 - + - name: Run Engine tests in docker with sqlite store run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="sqlite" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin -test.timeout 5m -test.parallel 1 diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 9f543c74c..13228250d 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -40,7 +40,7 @@ jobs: cache: false - name: Install dependencies if: matrix.os == 'ubuntu-latest' - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev + run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev libpcap-dev - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: diff --git a/client/internal/engine.go b/client/internal/engine.go index 78d26f0b8..7f7b5ef55 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -230,8 +230,8 @@ func (e *Engine) Start() error { wgIface, err := e.newWgIface() if err != nil { - log.Errorf("failed creating wireguard interface instance %s: [%s]", e.config.WgIfaceName, err.Error()) - return err + log.Errorf("failed creating wireguard interface instance %s: [%s]", e.config.WgIfaceName, err) + return fmt.Errorf("new wg interface: %w", err) } e.wgInterface = wgIface @@ -244,29 +244,33 @@ func (e *Engine) Start() error { } e.rpManager, err = rosenpass.NewManager(e.config.PreSharedKey, e.config.WgIfaceName) if err != nil { - return err + return fmt.Errorf("create rosenpass manager: %w", err) } err := e.rpManager.Run() if err != nil { - return err + return fmt.Errorf("run rosenpass manager: %w", err) } } initialRoutes, dnsServer, err := e.newDnsServer() if err != nil { e.close() - return err + return fmt.Errorf("create dns server: %w", err) } e.dnsServer = dnsServer e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.wgInterface, e.statusRecorder, initialRoutes) + if err := e.routeManager.Init(); err != nil { + e.close() + return fmt.Errorf("init route manager: %w", err) + } e.routeManager.SetRouteChangeListener(e.mobileDep.NetworkChangeListener) err = e.wgInterfaceCreate() if err != nil { log.Errorf("failed creating tunnel interface %s: [%s]", e.config.WgIfaceName, err.Error()) e.close() - return err + return fmt.Errorf("create wg interface: %w", err) } e.firewall, err = firewall.NewFirewall(e.ctx, e.wgInterface) @@ -278,7 +282,7 @@ func (e *Engine) Start() error { err = e.routeManager.EnableServerRouter(e.firewall) if err != nil { e.close() - return err + return fmt.Errorf("enable server router: %w", err) } } @@ -286,7 +290,7 @@ func (e *Engine) Start() error { if err != nil { log.Errorf("failed to pull up wgInterface [%s]: %s", e.wgInterface.Name(), err.Error()) e.close() - return err + return fmt.Errorf("up wg interface: %w", err) } if e.firewall != nil { @@ -296,7 +300,7 @@ func (e *Engine) Start() error { err = e.dnsServer.Initialize() if err != nil { e.close() - return err + return fmt.Errorf("initialize dns server: %w", err) } e.receiveSignalEvents() diff --git a/client/internal/relay/relay.go b/client/internal/relay/relay.go index 1d8e6846d..84fd72e49 100644 --- a/client/internal/relay/relay.go +++ b/client/internal/relay/relay.go @@ -10,6 +10,9 @@ import ( "github.com/pion/stun/v2" "github.com/pion/turn/v3" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/stdnet" + nbnet "github.com/netbirdio/netbird/util/net" ) // ProbeResult holds the info about the result of a relay probe request @@ -27,7 +30,15 @@ func ProbeSTUN(ctx context.Context, uri *stun.URI) (addr string, probeErr error) } }() - client, err := stun.DialURI(uri, &stun.DialConfig{}) + net, err := stdnet.NewNet(nil) + if err != nil { + probeErr = fmt.Errorf("new net: %w", err) + return + } + + client, err := stun.DialURI(uri, &stun.DialConfig{ + Net: net, + }) if err != nil { probeErr = fmt.Errorf("dial: %w", err) return @@ -85,14 +96,13 @@ func ProbeTURN(ctx context.Context, uri *stun.URI) (addr string, probeErr error) switch uri.Proto { case stun.ProtoTypeUDP: var err error - conn, err = net.ListenPacket("udp", "") + conn, err = nbnet.NewListener().ListenPacket(ctx, "udp", "") if err != nil { probeErr = fmt.Errorf("listen: %w", err) return } case stun.ProtoTypeTCP: - dialer := net.Dialer{} - tcpConn, err := dialer.DialContext(ctx, "tcp", turnServerAddr) + tcpConn, err := nbnet.NewDialer().DialContext(ctx, "tcp", turnServerAddr) if err != nil { probeErr = fmt.Errorf("dial: %w", err) return @@ -109,12 +119,18 @@ func ProbeTURN(ctx context.Context, uri *stun.URI) (addr string, probeErr error) } }() + net, err := stdnet.NewNet(nil) + if err != nil { + probeErr = fmt.Errorf("new net: %w", err) + return + } cfg := &turn.ClientConfig{ STUNServerAddr: turnServerAddr, TURNServerAddr: turnServerAddr, Conn: conn, Username: uri.Username, Password: uri.Password, + Net: net, } client, err := turn.NewClient(cfg) if err != nil { diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index f7ead5827..b2dff7f08 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -41,6 +41,7 @@ type clientNetwork struct { func newClientNetworkWatcher(ctx context.Context, wgInterface *iface.WGIface, statusRecorder *peer.Status, network netip.Prefix) *clientNetwork { ctx, cancel := context.WithCancel(ctx) + client := &clientNetwork{ ctx: ctx, stop: cancel, @@ -72,6 +73,18 @@ func (c *clientNetwork) getRouterPeerStatuses() map[string]routerPeerStatus { return routePeerStatuses } +// getBestRouteFromStatuses determines the most optimal route from the available routes +// within a clientNetwork, taking into account peer connection status, route metrics, and +// preference for non-relayed and direct connections. +// +// It follows these prioritization rules: +// * Connected peers: Only routes with connected peers are considered. +// * Metric: Routes with lower metrics (better) are prioritized. +// * Non-relayed: Routes without relays are preferred. +// * Direct connections: Routes with direct peer connections are favored. +// * Stability: In case of equal scores, the currently active route (if any) is maintained. +// +// It returns the ID of the selected optimal route. func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[string]routerPeerStatus) string { chosen := "" chosenScore := 0 @@ -158,7 +171,7 @@ func (c *clientNetwork) startPeersStatusChangeWatcher() { func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error { state, err := c.statusRecorder.GetPeer(peerKey) if err != nil { - return err + return fmt.Errorf("get peer state: %v", err) } delete(state.Routes, c.network.String()) @@ -172,7 +185,7 @@ func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error { err = c.wgInterface.RemoveAllowedIP(peerKey, c.network.String()) if err != nil { - return fmt.Errorf("couldn't remove allowed IP %s removed for peer %s, err: %v", + return fmt.Errorf("remove allowed IP %s removed for peer %s, err: %v", c.network, c.chosenRoute.Peer, err) } return nil @@ -180,30 +193,26 @@ func (c *clientNetwork) removeRouteFromWireguardPeer(peerKey string) error { func (c *clientNetwork) removeRouteFromPeerAndSystem() error { if c.chosenRoute != nil { - err := c.removeRouteFromWireguardPeer(c.chosenRoute.Peer) - if err != nil { - return err + if err := removeFromRouteTableIfNonSystem(c.network, c.wgInterface.Address().IP.String(), c.wgInterface.Name()); err != nil { + return fmt.Errorf("remove route %s from system, err: %v", c.network, err) } - err = removeFromRouteTableIfNonSystem(c.network, c.wgInterface.Address().IP.String()) - if err != nil { - return fmt.Errorf("couldn't remove route %s from system, err: %v", - c.network, err) + + if err := c.removeRouteFromWireguardPeer(c.chosenRoute.Peer); err != nil { + return fmt.Errorf("remove route: %v", err) } } return nil } func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error { - - var err error - routerPeerStatuses := c.getRouterPeerStatuses() chosen := c.getBestRouteFromStatuses(routerPeerStatuses) + + // If no route is chosen, remove the route from the peer and system if chosen == "" { - err = c.removeRouteFromPeerAndSystem() - if err != nil { - return err + if err := c.removeRouteFromPeerAndSystem(); err != nil { + return fmt.Errorf("remove route from peer and system: %v", err) } c.chosenRoute = nil @@ -211,6 +220,7 @@ func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error { return nil } + // If the chosen route is the same as the current route, do nothing if c.chosenRoute != nil && c.chosenRoute.ID == chosen { if c.chosenRoute.IsEqual(c.routes[chosen]) { return nil @@ -218,13 +228,13 @@ func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error { } if c.chosenRoute != nil { - err = c.removeRouteFromWireguardPeer(c.chosenRoute.Peer) - if err != nil { - return err + // If a previous route exists, remove it from the peer + if err := c.removeRouteFromWireguardPeer(c.chosenRoute.Peer); err != nil { + return fmt.Errorf("remove route from peer: %v", err) } } else { - err = addToRouteTableIfNoExists(c.network, c.wgInterface.Address().IP.String()) - if err != nil { + // otherwise add the route to the system + if err := addToRouteTableIfNoExists(c.network, c.wgInterface.Address().IP.String(), c.wgInterface.Name()); err != nil { return fmt.Errorf("route %s couldn't be added for peer %s, err: %v", c.network.String(), c.wgInterface.Address().IP.String(), err) } @@ -245,8 +255,7 @@ func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem() error { } } - err = c.wgInterface.AddAllowedIP(c.chosenRoute.Peer, c.network.String()) - if err != nil { + if err := c.wgInterface.AddAllowedIP(c.chosenRoute.Peer, c.network.String()); err != nil { log.Errorf("couldn't add allowed IP %s added for peer %s, err: %v", c.network, c.chosenRoute.Peer, err) } @@ -287,21 +296,21 @@ func (c *clientNetwork) peersStateAndUpdateWatcher() { log.Debugf("stopping watcher for network %s", c.network) err := c.removeRouteFromPeerAndSystem() if err != nil { - log.Error(err) + log.Errorf("Couldn't remove route from peer and system for network %s: %v", c.network, err) } return case <-c.peerStateUpdate: err := c.recalculateRouteAndUpdatePeerAndSystem() if err != nil { - log.Error(err) + log.Errorf("Couldn't recalculate route and update peer and system: %v", err) } case update := <-c.routeUpdate: if update.updateSerial < c.updateSerial { - log.Warnf("received a routes update with smaller serial number, ignoring it") + log.Warnf("Received a routes update with smaller serial number, ignoring it") continue } - log.Debugf("received a new client network route update for %s", c.network) + log.Debugf("Received a new client network route update for %s", c.network) c.handleUpdate(update) @@ -309,7 +318,7 @@ func (c *clientNetwork) peersStateAndUpdateWatcher() { err := c.recalculateRouteAndUpdatePeerAndSystem() if err != nil { - log.Error(err) + log.Errorf("Couldn't recalculate route and update peer and system for network %s: %v", c.network, err) } c.startPeersStatusChangeWatcher() diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index b624d8c34..6a0d954da 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -2,6 +2,8 @@ package routemanager import ( "context" + "fmt" + "net/netip" "runtime" "sync" @@ -15,8 +17,14 @@ import ( "github.com/netbirdio/netbird/version" ) +var defaultv4 = netip.PrefixFrom(netip.IPv4Unspecified(), 0) + +// nolint:unused +var defaultv6 = netip.PrefixFrom(netip.IPv6Unspecified(), 0) + // Manager is a route manager interface type Manager interface { + Init() error UpdateRoutes(updateSerial uint64, newRoutes []*route.Route) error SetRouteChangeListener(listener listener.NetworkChangeListener) InitialRouteRange() []string @@ -56,6 +64,19 @@ func NewManager(ctx context.Context, pubKey string, wgInterface *iface.WGIface, return dm } +// Init sets up the routing +func (m *DefaultManager) Init() error { + if err := cleanupRouting(); err != nil { + log.Warnf("Failed cleaning up routing: %v", err) + } + + if err := setupRouting(); err != nil { + return fmt.Errorf("setup routing: %w", err) + } + log.Info("Routing setup complete") + return nil +} + func (m *DefaultManager) EnableServerRouter(firewall firewall.Manager) error { var err error m.serverRouter, err = newServerRouter(m.ctx, m.wgInterface, firewall, m.statusRecorder) @@ -71,9 +92,15 @@ func (m *DefaultManager) Stop() { if m.serverRouter != nil { m.serverRouter.cleanUp() } + if err := cleanupRouting(); err != nil { + log.Errorf("Error cleaning up routing: %v", err) + } else { + log.Info("Routing cleanup complete") + } + m.ctx = nil } -// UpdateRoutes compares received routes with existing routes and remove, update or add them to the client and server maps +// UpdateRoutes compares received routes with existing routes and removes, updates or adds them to the client and server maps func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Route) error { select { case <-m.ctx.Done(): @@ -91,7 +118,7 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro if m.serverRouter != nil { err := m.serverRouter.updateRoutes(newServerRoutesMap) if err != nil { - return err + return fmt.Errorf("update routes: %w", err) } } @@ -156,11 +183,7 @@ func (m *DefaultManager) classifiesRoutes(newRoutes []*route.Route) (map[string] for _, newRoute := range newRoutes { networkID := route.GetHAUniqueID(newRoute) if !ownNetworkIDs[networkID] { - // if prefix is too small, lets assume is a possible default route which is not yet supported - // we skip this route management - if newRoute.Network.Bits() < minRangeBits { - log.Errorf("this agent version: %s, doesn't support default routes, received %s, skipping this route", - version.NetbirdVersion(), newRoute.Network) + if !isPrefixSupported(newRoute.Network) { continue } newClientRoutesIDMap[networkID] = append(newClientRoutesIDMap[networkID], newRoute) @@ -178,3 +201,18 @@ func (m *DefaultManager) clientRoutes(initialRoutes []*route.Route) []*route.Rou } return rs } + +func isPrefixSupported(prefix netip.Prefix) bool { + if runtime.GOOS == "linux" { + return true + } + + // If prefix is too small, lets assume it is a possible default prefix which is not yet supported + // we skip this prefix management + if prefix.Bits() < minRangeBits { + log.Warnf("This agent version: %s, doesn't support default routes, received %s, skipping this prefix", + version.NetbirdVersion(), prefix) + return false + } + return true +} diff --git a/client/internal/routemanager/manager_test.go b/client/internal/routemanager/manager_test.go index 2e5cf6649..9d92bf90d 100644 --- a/client/internal/routemanager/manager_test.go +++ b/client/internal/routemanager/manager_test.go @@ -28,13 +28,14 @@ const remotePeerKey2 = "remote1" func TestManagerUpdateRoutes(t *testing.T) { testCases := []struct { - name string - inputInitRoutes []*route.Route - inputRoutes []*route.Route - inputSerial uint64 - removeSrvRouter bool - serverRoutesExpected int - clientNetworkWatchersExpected int + name string + inputInitRoutes []*route.Route + inputRoutes []*route.Route + inputSerial uint64 + removeSrvRouter bool + serverRoutesExpected int + clientNetworkWatchersExpected int + clientNetworkWatchersExpectedLinux int }{ { name: "Should create 2 client networks", @@ -200,8 +201,9 @@ func TestManagerUpdateRoutes(t *testing.T) { Enabled: true, }, }, - inputSerial: 1, - clientNetworkWatchersExpected: 0, + inputSerial: 1, + clientNetworkWatchersExpected: 0, + clientNetworkWatchersExpectedLinux: 1, }, { name: "Remove 1 Client Route", @@ -415,6 +417,8 @@ func TestManagerUpdateRoutes(t *testing.T) { statusRecorder := peer.NewRecorder("https://mgm") ctx := context.TODO() routeManager := NewManager(ctx, localPeerKey, wgInterface, statusRecorder, nil) + err = routeManager.Init() + require.NoError(t, err, "should init route manager") defer routeManager.Stop() if testCase.removeSrvRouter { @@ -429,7 +433,11 @@ func TestManagerUpdateRoutes(t *testing.T) { err = routeManager.UpdateRoutes(testCase.inputSerial+uint64(len(testCase.inputInitRoutes)), testCase.inputRoutes) require.NoError(t, err, "should update routes") - require.Len(t, routeManager.clientNetworks, testCase.clientNetworkWatchersExpected, "client networks size should match") + expectedWatchers := testCase.clientNetworkWatchersExpected + if runtime.GOOS == "linux" && testCase.clientNetworkWatchersExpectedLinux != 0 { + expectedWatchers = testCase.clientNetworkWatchersExpectedLinux + } + require.Len(t, routeManager.clientNetworks, expectedWatchers, "client networks size should match") if runtime.GOOS == "linux" && routeManager.serverRouter != nil { sr := routeManager.serverRouter.(*defaultServerRouter) diff --git a/client/internal/routemanager/mock.go b/client/internal/routemanager/mock.go index a1214cbb9..e812b3a85 100644 --- a/client/internal/routemanager/mock.go +++ b/client/internal/routemanager/mock.go @@ -16,6 +16,10 @@ type MockManager struct { StopFunc func() } +func (m *MockManager) Init() error { + return nil +} + // InitialRouteRange mock implementation of InitialRouteRange from Manager interface func (m *MockManager) InitialRouteRange() []string { return nil diff --git a/client/internal/routemanager/server_nonandroid.go b/client/internal/routemanager/server_nonandroid.go index 192367877..00df735fb 100644 --- a/client/internal/routemanager/server_nonandroid.go +++ b/client/internal/routemanager/server_nonandroid.go @@ -4,6 +4,7 @@ package routemanager import ( "context" + "fmt" "net/netip" "sync" @@ -48,7 +49,7 @@ func (m *defaultServerRouter) updateRoutes(routesMap map[string]*route.Route) er oldRoute := m.routes[routeID] err := m.removeFromServerNetwork(oldRoute) if err != nil { - log.Errorf("unable to remove route id: %s, network %s, from server, got: %v", + log.Errorf("Unable to remove route id: %s, network %s, from server, got: %v", oldRoute.ID, oldRoute.Network, err) } delete(m.routes, routeID) @@ -62,7 +63,7 @@ func (m *defaultServerRouter) updateRoutes(routesMap map[string]*route.Route) er err := m.addToServerNetwork(newRoute) if err != nil { - log.Errorf("unable to add route %s from server, got: %v", newRoute.ID, err) + log.Errorf("Unable to add route %s from server, got: %v", newRoute.ID, err) continue } m.routes[id] = newRoute @@ -81,15 +82,22 @@ func (m *defaultServerRouter) updateRoutes(routesMap map[string]*route.Route) er func (m *defaultServerRouter) removeFromServerNetwork(route *route.Route) error { select { case <-m.ctx.Done(): - log.Infof("not removing from server network because context is done") + log.Infof("Not removing from server network because context is done") return m.ctx.Err() default: m.mux.Lock() defer m.mux.Unlock() - err := m.firewall.RemoveRoutingRules(routeToRouterPair(m.wgInterface.Address().String(), route)) + + routerPair, err := routeToRouterPair(m.wgInterface.Address().Masked().String(), route) if err != nil { - return err + return fmt.Errorf("parse prefix: %w", err) } + + err = m.firewall.RemoveRoutingRules(routerPair) + if err != nil { + return fmt.Errorf("remove routing rules: %w", err) + } + delete(m.routes, route.ID) state := m.statusRecorder.GetLocalPeerState() @@ -103,15 +111,22 @@ func (m *defaultServerRouter) removeFromServerNetwork(route *route.Route) error func (m *defaultServerRouter) addToServerNetwork(route *route.Route) error { select { case <-m.ctx.Done(): - log.Infof("not adding to server network because context is done") + log.Infof("Not adding to server network because context is done") return m.ctx.Err() default: m.mux.Lock() defer m.mux.Unlock() - err := m.firewall.InsertRoutingRules(routeToRouterPair(m.wgInterface.Address().String(), route)) + + routerPair, err := routeToRouterPair(m.wgInterface.Address().Masked().String(), route) if err != nil { - return err + return fmt.Errorf("parse prefix: %w", err) } + + err = m.firewall.InsertRoutingRules(routerPair) + if err != nil { + return fmt.Errorf("insert routing rules: %w", err) + } + m.routes[route.ID] = route state := m.statusRecorder.GetLocalPeerState() @@ -129,9 +144,15 @@ func (m *defaultServerRouter) cleanUp() { m.mux.Lock() defer m.mux.Unlock() for _, r := range m.routes { - err := m.firewall.RemoveRoutingRules(routeToRouterPair(m.wgInterface.Address().String(), r)) + routerPair, err := routeToRouterPair(m.wgInterface.Address().Masked().String(), r) if err != nil { - log.Warnf("failed to remove clean up route: %s", r.ID) + log.Errorf("Failed to convert route to router pair: %v", err) + continue + } + + err = m.firewall.RemoveRoutingRules(routerPair) + if err != nil { + log.Errorf("Failed to remove cleanup route: %v", err) } state := m.statusRecorder.GetLocalPeerState() @@ -139,13 +160,15 @@ func (m *defaultServerRouter) cleanUp() { m.statusRecorder.UpdateLocalPeerState(state) } } - -func routeToRouterPair(source string, route *route.Route) firewall.RouterPair { - parsed := netip.MustParsePrefix(source).Masked() +func routeToRouterPair(source string, route *route.Route) (firewall.RouterPair, error) { + parsed, err := netip.ParsePrefix(source) + if err != nil { + return firewall.RouterPair{}, err + } return firewall.RouterPair{ ID: route.ID, Source: parsed.String(), Destination: route.Network.Masked().String(), Masquerade: route.Masquerade, - } + }, nil } diff --git a/client/internal/routemanager/systemops_android.go b/client/internal/routemanager/systemops_android.go index 950a26843..291826780 100644 --- a/client/internal/routemanager/systemops_android.go +++ b/client/internal/routemanager/systemops_android.go @@ -4,10 +4,10 @@ import ( "net/netip" ) -func addToRouteTableIfNoExists(prefix netip.Prefix, addr string) error { +func addToRouteTableIfNoExists(prefix netip.Prefix, addr, intf string) error { return nil } -func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr string) error { +func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr, intf string) error { return nil } diff --git a/client/internal/routemanager/systemops_bsd.go b/client/internal/routemanager/systemops_bsd.go index b2da8075c..173e7c0e8 100644 --- a/client/internal/routemanager/systemops_bsd.go +++ b/client/internal/routemanager/systemops_bsd.go @@ -1,5 +1,4 @@ //go:build darwin || dragonfly || freebsd || netbsd || openbsd -// +build darwin dragonfly freebsd netbsd openbsd package routemanager diff --git a/client/internal/routemanager/systemops_bsd_nonios.go b/client/internal/routemanager/systemops_bsd_nonios.go new file mode 100644 index 000000000..f60c7afc3 --- /dev/null +++ b/client/internal/routemanager/systemops_bsd_nonios.go @@ -0,0 +1,13 @@ +//go:build (darwin || dragonfly || freebsd || netbsd || openbsd) && !ios + +package routemanager + +import "net/netip" + +func addToRouteTableIfNoExists(prefix netip.Prefix, addr string, intf string) error { + return genericAddToRouteTableIfNoExists(prefix, addr, intf) +} + +func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr string, intf string) error { + return genericRemoveFromRouteTableIfNonSystem(prefix, addr, intf) +} diff --git a/client/internal/routemanager/systemops_ios.go b/client/internal/routemanager/systemops_ios.go index aae0f8dc8..291826780 100644 --- a/client/internal/routemanager/systemops_ios.go +++ b/client/internal/routemanager/systemops_ios.go @@ -1,15 +1,13 @@ -//go:build ios - package routemanager import ( "net/netip" ) -func addToRouteTableIfNoExists(prefix netip.Prefix, addr string) error { +func addToRouteTableIfNoExists(prefix netip.Prefix, addr, intf string) error { return nil } -func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr string) error { +func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr, intf string) error { return nil } diff --git a/client/internal/routemanager/systemops_linux.go b/client/internal/routemanager/systemops_linux.go index 0562826a5..192509992 100644 --- a/client/internal/routemanager/systemops_linux.go +++ b/client/internal/routemanager/systemops_linux.go @@ -3,142 +3,298 @@ package routemanager import ( + "bufio" + "errors" + "fmt" "net" "net/netip" "os" "syscall" - "unsafe" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + + nbnet "github.com/netbirdio/netbird/util/net" ) -// Pulled from http://man7.org/linux/man-pages/man7/rtnetlink.7.html -// See the section on RTM_NEWROUTE, specifically 'struct rtmsg'. -type routeInfoInMemory struct { - Family byte - DstLen byte - SrcLen byte - TOS byte +const ( + // NetbirdVPNTableID is the ID of the custom routing table used by Netbird. + NetbirdVPNTableID = 0x1BD0 + // NetbirdVPNTableName is the name of the custom routing table used by Netbird. + NetbirdVPNTableName = "netbird" - Table byte - Protocol byte - Scope byte - Type byte + // rtTablesPath is the path to the file containing the routing table names. + rtTablesPath = "/etc/iproute2/rt_tables" - Flags uint32 + // ipv4ForwardingPath is the path to the file containing the IP forwarding setting. + ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward" +) + +var ErrTableIDExists = errors.New("ID exists with different name") + +type ruleParams struct { + fwmark int + tableID int + family int + priority int + invert bool + suppressPrefix int + description string } -const ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward" +func getSetupRules() []ruleParams { + return []ruleParams{ + {nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V4, -1, true, -1, "add rule v4 netbird"}, + {nbnet.NetbirdFwmark, NetbirdVPNTableID, netlink.FAMILY_V6, -1, true, -1, "add rule v6 netbird"}, + {-1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V4, -1, false, 0, "add rule with suppress prefixlen v4"}, + {-1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V6, -1, false, 0, "add rule with suppress prefixlen v6"}, + } +} -func addToRouteTable(prefix netip.Prefix, addr string) error { - _, ipNet, err := net.ParseCIDR(prefix.String()) - if err != nil { - return err +// setupRouting establishes the routing configuration for the VPN, including essential rules +// to ensure proper traffic flow for management, locally configured routes, and VPN traffic. +// +// Rule 1 (Main Route Precedence): Safeguards locally installed routes by giving them precedence over +// potential routes received and configured for the VPN. This rule is skipped for the default route and routes +// that are not in the main table. +// +// Rule 2 (VPN Traffic Routing): Directs all remaining traffic to the 'NetbirdVPNTableID' custom routing table. +// This table is where a default route or other specific routes received from the management server are configured, +// enabling VPN connectivity. +// +// The rules are inserted in reverse order, as rules are added from the bottom up in the rule list. +func setupRouting() (err error) { + if err = addRoutingTableName(); err != nil { + log.Errorf("Error adding routing table name: %v", err) } - addrMask := "/32" - if prefix.Addr().Unmap().Is6() { - addrMask = "/128" - } + defer func() { + if err != nil { + if cleanErr := cleanupRouting(); cleanErr != nil { + log.Errorf("Error cleaning up routing: %v", cleanErr) + } + } + }() - ip, _, err := net.ParseCIDR(addr + addrMask) - if err != nil { - return err - } - - route := &netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - Dst: ipNet, - Gw: ip, - } - - err = netlink.RouteAdd(route) - if err != nil { - return err + rules := getSetupRules() + for _, rule := range rules { + if err := addRule(rule); err != nil { + return fmt.Errorf("%s: %w", rule.description, err) + } } return nil } -func removeFromRouteTable(prefix netip.Prefix, addr string) error { - _, ipNet, err := net.ParseCIDR(prefix.String()) - if err != nil { - return err +// cleanupRouting performs a thorough cleanup of the routing configuration established by 'setupRouting'. +// It systematically removes the three rules and any associated routing table entries to ensure a clean state. +// The function uses error aggregation to report any errors encountered during the cleanup process. +func cleanupRouting() error { + var result *multierror.Error + + if err := flushRoutes(NetbirdVPNTableID, netlink.FAMILY_V4); err != nil { + result = multierror.Append(result, fmt.Errorf("flush routes v4: %w", err)) + } + if err := flushRoutes(NetbirdVPNTableID, netlink.FAMILY_V6); err != nil { + result = multierror.Append(result, fmt.Errorf("flush routes v6: %w", err)) } - addrMask := "/32" - if prefix.Addr().Unmap().Is6() { - addrMask = "/128" + rules := getSetupRules() + for _, rule := range rules { + if err := removeAllRules(rule); err != nil { + result = multierror.Append(result, fmt.Errorf("%s: %w", rule.description, err)) + } } - ip, _, err := net.ParseCIDR(addr + addrMask) - if err != nil { - return err - } + return result.ErrorOrNil() +} - route := &netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - Dst: ipNet, - Gw: ip, - } +func addToRouteTableIfNoExists(prefix netip.Prefix, _ string, intf string) error { + // No need to check if routes exist as main table takes precedence over the VPN table via Rule 2 - err = netlink.RouteDel(route) - if err != nil { - return err + // TODO remove this once we have ipv6 support + if prefix == defaultv4 { + if err := addUnreachableRoute(&defaultv6, NetbirdVPNTableID, netlink.FAMILY_V6); err != nil { + return fmt.Errorf("add blackhole: %w", err) + } } + if err := addRoute(&prefix, nil, &intf, NetbirdVPNTableID, netlink.FAMILY_V4); err != nil { + return fmt.Errorf("add route: %w", err) + } + return nil +} +func removeFromRouteTableIfNonSystem(prefix netip.Prefix, _ string, intf string) error { + // TODO remove this once we have ipv6 support + if prefix == defaultv4 { + if err := removeUnreachableRoute(&defaultv6, NetbirdVPNTableID, netlink.FAMILY_V6); err != nil { + return fmt.Errorf("remove unreachable route: %w", err) + } + } + if err := removeRoute(&prefix, nil, &intf, NetbirdVPNTableID, netlink.FAMILY_V4); err != nil { + return fmt.Errorf("remove route: %w", err) + } return nil } func getRoutesFromTable() ([]netip.Prefix, error) { - tab, err := syscall.NetlinkRIB(syscall.RTM_GETROUTE, syscall.AF_UNSPEC) - if err != nil { - return nil, err + return getRoutes(NetbirdVPNTableID, netlink.FAMILY_V4) +} + +// addRoute adds a route to a specific routing table identified by tableID. +func addRoute(prefix *netip.Prefix, addr, intf *string, tableID, family int) error { + route := &netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + Table: tableID, + Family: family, } - msgs, err := syscall.ParseNetlinkMessage(tab) - if err != nil { - return nil, err + + if prefix != nil { + _, ipNet, err := net.ParseCIDR(prefix.String()) + if err != nil { + return fmt.Errorf("parse prefix %s: %w", prefix, err) + } + route.Dst = ipNet } - var prefixList []netip.Prefix -loop: - for _, m := range msgs { - switch m.Header.Type { - case syscall.NLMSG_DONE: - break loop - case syscall.RTM_NEWROUTE: - rt := (*routeInfoInMemory)(unsafe.Pointer(&m.Data[0])) - msg := m - attrs, err := syscall.ParseNetlinkRouteAttr(&msg) - if err != nil { - return nil, err + + if err := addNextHop(addr, intf, route); err != nil { + return fmt.Errorf("add gateway and device: %w", err) + } + + if err := netlink.RouteAdd(route); err != nil && !errors.Is(err, syscall.EEXIST) { + return fmt.Errorf("netlink add route: %w", err) + } + + return nil +} + +// addUnreachableRoute adds an unreachable route for the specified IP family and routing table. +// ipFamily should be netlink.FAMILY_V4 for IPv4 or netlink.FAMILY_V6 for IPv6. +// tableID specifies the routing table to which the unreachable route will be added. +func addUnreachableRoute(prefix *netip.Prefix, tableID, ipFamily int) error { + _, ipNet, err := net.ParseCIDR(prefix.String()) + if err != nil { + return fmt.Errorf("parse prefix %s: %w", prefix, err) + } + + route := &netlink.Route{ + Type: syscall.RTN_UNREACHABLE, + Table: tableID, + Family: ipFamily, + Dst: ipNet, + } + + if err := netlink.RouteAdd(route); err != nil && !errors.Is(err, syscall.EEXIST) { + return fmt.Errorf("netlink add unreachable route: %w", err) + } + + return nil +} + +func removeUnreachableRoute(prefix *netip.Prefix, tableID, ipFamily int) error { + _, ipNet, err := net.ParseCIDR(prefix.String()) + if err != nil { + return fmt.Errorf("parse prefix %s: %w", prefix, err) + } + + route := &netlink.Route{ + Type: syscall.RTN_UNREACHABLE, + Table: tableID, + Family: ipFamily, + Dst: ipNet, + } + + if err := netlink.RouteDel(route); err != nil && !errors.Is(err, syscall.ESRCH) { + return fmt.Errorf("netlink remove unreachable route: %w", err) + } + + return nil + +} + +// removeRoute removes a route from a specific routing table identified by tableID. +func removeRoute(prefix *netip.Prefix, addr, intf *string, tableID, family int) error { + _, ipNet, err := net.ParseCIDR(prefix.String()) + if err != nil { + return fmt.Errorf("parse prefix %s: %w", prefix, err) + } + + route := &netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + Table: tableID, + Family: family, + Dst: ipNet, + } + + if err := addNextHop(addr, intf, route); err != nil { + return fmt.Errorf("add gateway and device: %w", err) + } + + if err := netlink.RouteDel(route); err != nil && !errors.Is(err, syscall.ESRCH) { + return fmt.Errorf("netlink remove route: %w", err) + } + + return nil +} + +func flushRoutes(tableID, family int) error { + routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE) + if err != nil { + return fmt.Errorf("list routes from table %d: %w", tableID, err) + } + + var result *multierror.Error + for i := range routes { + route := routes[i] + // unreachable default routes don't come back with Dst set + if route.Gw == nil && route.Src == nil && route.Dst == nil { + if family == netlink.FAMILY_V4 { + routes[i].Dst = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)} + } else { + routes[i].Dst = &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)} } - if rt.Family != syscall.AF_INET { - continue loop + } + if err := netlink.RouteDel(&routes[i]); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to delete route %v from table %d: %w", routes[i], tableID, err)) + } + } + + return result.ErrorOrNil() +} + +// getRoutes fetches routes from a specific routing table identified by tableID. +func getRoutes(tableID, family int) ([]netip.Prefix, error) { + var prefixList []netip.Prefix + + routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE) + if err != nil { + return nil, fmt.Errorf("list routes from table %d: %v", tableID, err) + } + + for _, route := range routes { + if route.Dst != nil { + addr, ok := netip.AddrFromSlice(route.Dst.IP) + if !ok { + return nil, fmt.Errorf("parse route destination IP: %v", route.Dst.IP) } - for _, attr := range attrs { - if attr.Attr.Type == syscall.RTA_DST { - addr, ok := netip.AddrFromSlice(attr.Value) - if !ok { - continue - } - mask := net.CIDRMask(int(rt.DstLen), len(attr.Value)*8) - cidr, _ := mask.Size() - routePrefix := netip.PrefixFrom(addr, cidr) - if routePrefix.IsValid() && routePrefix.Addr().Is4() { - prefixList = append(prefixList, routePrefix) - } - } + ones, _ := route.Dst.Mask.Size() + + prefix := netip.PrefixFrom(addr, ones) + if prefix.IsValid() { + prefixList = append(prefixList, prefix) } } } + return prefixList, nil } func enableIPForwarding() error { bytes, err := os.ReadFile(ipv4ForwardingPath) if err != nil { - return err + return fmt.Errorf("read file %s: %w", ipv4ForwardingPath, err) } // check if it is already enabled @@ -147,5 +303,142 @@ func enableIPForwarding() error { return nil } - return os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644) //nolint:gosec + //nolint:gosec + if err := os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644); err != nil { + return fmt.Errorf("write file %s: %w", ipv4ForwardingPath, err) + } + return nil +} + +// entryExists checks if the specified ID or name already exists in the rt_tables file +// and verifies if existing names start with "netbird_". +func entryExists(file *os.File, id int) (bool, error) { + if _, err := file.Seek(0, 0); err != nil { + return false, fmt.Errorf("seek rt_tables: %w", err) + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + var existingID int + var existingName string + if _, err := fmt.Sscanf(line, "%d %s\n", &existingID, &existingName); err == nil { + if existingID == id { + if existingName != NetbirdVPNTableName { + return true, ErrTableIDExists + } + return true, nil + } + } + } + if err := scanner.Err(); err != nil { + return false, fmt.Errorf("scan rt_tables: %w", err) + } + return false, nil +} + +// addRoutingTableName adds human-readable names for custom routing tables. +func addRoutingTableName() error { + file, err := os.Open(rtTablesPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("open rt_tables: %w", err) + } + defer func() { + if err := file.Close(); err != nil { + log.Errorf("Error closing rt_tables: %v", err) + } + }() + + exists, err := entryExists(file, NetbirdVPNTableID) + if err != nil { + return fmt.Errorf("verify entry %d, %s: %w", NetbirdVPNTableID, NetbirdVPNTableName, err) + } + if exists { + return nil + } + + // Reopen the file in append mode to add new entries + if err := file.Close(); err != nil { + log.Errorf("Error closing rt_tables before appending: %v", err) + } + file, err = os.OpenFile(rtTablesPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if err != nil { + return fmt.Errorf("open rt_tables for appending: %w", err) + } + + if _, err := file.WriteString(fmt.Sprintf("\n%d\t%s\n", NetbirdVPNTableID, NetbirdVPNTableName)); err != nil { + return fmt.Errorf("append entry to rt_tables: %w", err) + } + + return nil +} + +// addRule adds a routing rule to a specific routing table identified by tableID. +func addRule(params ruleParams) error { + rule := netlink.NewRule() + rule.Table = params.tableID + rule.Mark = params.fwmark + rule.Family = params.family + rule.Priority = params.priority + rule.Invert = params.invert + rule.SuppressPrefixlen = params.suppressPrefix + + if err := netlink.RuleAdd(rule); err != nil { + return fmt.Errorf("add routing rule: %w", err) + } + + return nil +} + +// removeRule removes a routing rule from a specific routing table identified by tableID. +func removeRule(params ruleParams) error { + rule := netlink.NewRule() + rule.Table = params.tableID + rule.Mark = params.fwmark + rule.Family = params.family + rule.Invert = params.invert + rule.Priority = params.priority + rule.SuppressPrefixlen = params.suppressPrefix + + if err := netlink.RuleDel(rule); err != nil { + return fmt.Errorf("remove routing rule: %w", err) + } + + return nil +} + +func removeAllRules(params ruleParams) error { + for { + if err := removeRule(params); err != nil { + if errors.Is(err, syscall.ENOENT) { + break + } + return err + } + } + return nil +} + +// addNextHop adds the gateway and device to the route. +func addNextHop(addr *string, intf *string, route *netlink.Route) error { + if addr != nil { + ip := net.ParseIP(*addr) + if ip == nil { + return fmt.Errorf("parsing address %s failed", *addr) + } + + route.Gw = ip + } + + if intf != nil { + link, err := netlink.LinkByName(*intf) + if err != nil { + return fmt.Errorf("set interface %s: %w", *intf, err) + } + route.LinkIndex = link.Attrs().Index + } + + return nil } diff --git a/client/internal/routemanager/systemops_linux_test.go b/client/internal/routemanager/systemops_linux_test.go new file mode 100644 index 000000000..96e43d20f --- /dev/null +++ b/client/internal/routemanager/systemops_linux_test.go @@ -0,0 +1,469 @@ +//go:build !android + +package routemanager + +import ( + "errors" + "fmt" + "net" + "net/netip" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/gopacket/gopacket" + "github.com/gopacket/gopacket/layers" + "github.com/gopacket/gopacket/pcap" + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/iface" + nbnet "github.com/netbirdio/netbird/util/net" +) + +type PacketExpectation struct { + SrcIP net.IP + DstIP net.IP + SrcPort int + DstPort int + UDP bool + TCP bool +} + +func TestEntryExists(t *testing.T) { + tempDir := t.TempDir() + tempFilePath := fmt.Sprintf("%s/rt_tables", tempDir) + + content := []string{ + "1000 reserved", + fmt.Sprintf("%d %s", NetbirdVPNTableID, NetbirdVPNTableName), + "9999 other_table", + } + require.NoError(t, os.WriteFile(tempFilePath, []byte(strings.Join(content, "\n")), 0644)) + + file, err := os.Open(tempFilePath) + require.NoError(t, err) + defer func() { + assert.NoError(t, file.Close()) + }() + + tests := []struct { + name string + id int + shouldExist bool + err error + }{ + { + name: "ExistsWithNetbirdPrefix", + id: 7120, + shouldExist: true, + err: nil, + }, + { + name: "ExistsWithDifferentName", + id: 1000, + shouldExist: true, + err: ErrTableIDExists, + }, + { + name: "DoesNotExist", + id: 1234, + shouldExist: false, + err: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + exists, err := entryExists(file, tc.id) + if tc.err != nil { + assert.ErrorIs(t, err, tc.err) + } else { + assert.NoError(t, err) + } + assert.Equal(t, tc.shouldExist, exists) + }) + } +} + +func TestRoutingWithTables(t *testing.T) { + testCases := []struct { + name string + destination string + captureInterface string + dialer *net.Dialer + packetExpectation PacketExpectation + }{ + { + name: "To external host without fwmark via vpn", + destination: "192.0.2.1:53", + captureInterface: "wgtest0", + dialer: &net.Dialer{}, + packetExpectation: createPacketExpectation("100.64.0.1", 12345, "192.0.2.1", 53), + }, + { + name: "To external host with fwmark via physical interface", + destination: "192.0.2.1:53", + captureInterface: "dummyext0", + dialer: nbnet.NewDialer(), + packetExpectation: createPacketExpectation("192.168.0.1", 12345, "192.0.2.1", 53), + }, + + { + name: "To duplicate internal route with fwmark via physical interface", + destination: "10.0.0.1:53", + captureInterface: "dummyint0", + dialer: nbnet.NewDialer(), + packetExpectation: createPacketExpectation("192.168.1.1", 12345, "10.0.0.1", 53), + }, + { + name: "To duplicate internal route without fwmark via physical interface", // local route takes precedence + destination: "10.0.0.1:53", + captureInterface: "dummyint0", + dialer: &net.Dialer{}, + packetExpectation: createPacketExpectation("192.168.1.1", 12345, "10.0.0.1", 53), + }, + + { + name: "To unique vpn route with fwmark via physical interface", + destination: "172.16.0.1:53", + captureInterface: "dummyext0", + dialer: nbnet.NewDialer(), + packetExpectation: createPacketExpectation("192.168.0.1", 12345, "172.16.0.1", 53), + }, + { + name: "To unique vpn route without fwmark via vpn", + destination: "172.16.0.1:53", + captureInterface: "wgtest0", + dialer: &net.Dialer{}, + packetExpectation: createPacketExpectation("100.64.0.1", 12345, "172.16.0.1", 53), + }, + + { + name: "To more specific route without fwmark via vpn interface", + destination: "10.10.0.1:53", + captureInterface: "dummyint0", + dialer: &net.Dialer{}, + packetExpectation: createPacketExpectation("192.168.1.1", 12345, "10.10.0.1", 53), + }, + + { + name: "To more specific route (local) without fwmark via physical interface", + destination: "127.0.10.1:53", + captureInterface: "lo", + dialer: &net.Dialer{}, + packetExpectation: createPacketExpectation("127.0.0.1", 12345, "127.0.10.1", 53), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wgIface, _, _ := setupTestEnv(t) + + // default route exists in main table and vpn table + err := addToRouteTableIfNoExists(netip.MustParsePrefix("0.0.0.0/0"), wgIface.Address().IP.String(), wgIface.Name()) + require.NoError(t, err, "addToRouteTableIfNoExists should not return err") + + // 10.0.0.0/8 route exists in main table and vpn table + err = addToRouteTableIfNoExists(netip.MustParsePrefix("10.0.0.0/8"), wgIface.Address().IP.String(), wgIface.Name()) + require.NoError(t, err, "addToRouteTableIfNoExists should not return err") + + // 10.10.0.0/24 more specific route exists in vpn table + err = addToRouteTableIfNoExists(netip.MustParsePrefix("10.10.0.0/24"), wgIface.Address().IP.String(), wgIface.Name()) + require.NoError(t, err, "addToRouteTableIfNoExists should not return err") + + // 127.0.10.0/24 more specific route exists in vpn table + err = addToRouteTableIfNoExists(netip.MustParsePrefix("127.0.10.0/24"), wgIface.Address().IP.String(), wgIface.Name()) + require.NoError(t, err, "addToRouteTableIfNoExists should not return err") + + // unique route in vpn table + err = addToRouteTableIfNoExists(netip.MustParsePrefix("172.16.0.0/16"), wgIface.Address().IP.String(), wgIface.Name()) + require.NoError(t, err, "addToRouteTableIfNoExists should not return err") + + filter := createBPFFilter(tc.destination) + handle := startPacketCapture(t, tc.captureInterface, filter) + + sendTestPacket(t, tc.destination, tc.packetExpectation.SrcPort, tc.dialer) + + packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) + packet, err := packetSource.NextPacket() + require.NoError(t, err) + + verifyPacket(t, packet, tc.packetExpectation) + }) + } +} + +func verifyPacket(t *testing.T, packet gopacket.Packet, exp PacketExpectation) { + t.Helper() + + ipLayer := packet.Layer(layers.LayerTypeIPv4) + require.NotNil(t, ipLayer, "Expected IPv4 layer not found in packet") + + ip, ok := ipLayer.(*layers.IPv4) + require.True(t, ok, "Failed to cast to IPv4 layer") + + // Convert both source and destination IP addresses to 16-byte representation + expectedSrcIP := exp.SrcIP.To16() + actualSrcIP := ip.SrcIP.To16() + assert.Equal(t, expectedSrcIP, actualSrcIP, "Source IP mismatch") + + expectedDstIP := exp.DstIP.To16() + actualDstIP := ip.DstIP.To16() + assert.Equal(t, expectedDstIP, actualDstIP, "Destination IP mismatch") + + if exp.UDP { + udpLayer := packet.Layer(layers.LayerTypeUDP) + require.NotNil(t, udpLayer, "Expected UDP layer not found in packet") + + udp, ok := udpLayer.(*layers.UDP) + require.True(t, ok, "Failed to cast to UDP layer") + + assert.Equal(t, layers.UDPPort(exp.SrcPort), udp.SrcPort, "UDP source port mismatch") + assert.Equal(t, layers.UDPPort(exp.DstPort), udp.DstPort, "UDP destination port mismatch") + } + + if exp.TCP { + tcpLayer := packet.Layer(layers.LayerTypeTCP) + require.NotNil(t, tcpLayer, "Expected TCP layer not found in packet") + + tcp, ok := tcpLayer.(*layers.TCP) + require.True(t, ok, "Failed to cast to TCP layer") + + assert.Equal(t, layers.TCPPort(exp.SrcPort), tcp.SrcPort, "TCP source port mismatch") + assert.Equal(t, layers.TCPPort(exp.DstPort), tcp.DstPort, "TCP destination port mismatch") + } + +} + +func createAndSetupDummyInterface(t *testing.T, interfaceName, ipAddressCIDR string) *netlink.Dummy { + t.Helper() + + dummy := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: interfaceName}} + err := netlink.LinkDel(dummy) + if err != nil && !errors.Is(err, syscall.EINVAL) { + t.Logf("Failed to delete dummy interface: %v", err) + } + + err = netlink.LinkAdd(dummy) + require.NoError(t, err) + + err = netlink.LinkSetUp(dummy) + require.NoError(t, err) + + if ipAddressCIDR != "" { + addr, err := netlink.ParseAddr(ipAddressCIDR) + require.NoError(t, err) + err = netlink.AddrAdd(dummy, addr) + require.NoError(t, err) + } + + return dummy +} + +func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, linkIndex int) { + t.Helper() + + _, dstIPNet, err := net.ParseCIDR(dstCIDR) + require.NoError(t, err) + + if dstIPNet.String() == "0.0.0.0/0" { + gw, linkIndex, err := fetchOriginalGateway(netlink.FAMILY_V4) + if err != nil { + t.Logf("Failed to fetch original gateway: %v", err) + } + + // Handle existing routes with metric 0 + err = netlink.RouteDel(&netlink.Route{Dst: dstIPNet, Priority: 0}) + if err == nil { + t.Cleanup(func() { + err := netlink.RouteAdd(&netlink.Route{Dst: dstIPNet, Gw: gw, LinkIndex: linkIndex, Priority: 0}) + if err != nil && !errors.Is(err, syscall.EEXIST) { + t.Fatalf("Failed to add route: %v", err) + } + }) + } else if !errors.Is(err, syscall.ESRCH) { + t.Logf("Failed to delete route: %v", err) + } + } + + route := &netlink.Route{ + Dst: dstIPNet, + Gw: gw, + LinkIndex: linkIndex, + } + err = netlink.RouteDel(route) + if err != nil && !errors.Is(err, syscall.ESRCH) { + t.Logf("Failed to delete route: %v", err) + } + + err = netlink.RouteAdd(route) + if err != nil && !errors.Is(err, syscall.EEXIST) { + t.Fatalf("Failed to add route: %v", err) + } +} + +// fetchOriginalGateway returns the original gateway IP address and the interface index. +func fetchOriginalGateway(family int) (net.IP, int, error) { + routes, err := netlink.RouteList(nil, family) + if err != nil { + return nil, 0, err + } + + for _, route := range routes { + if route.Dst == nil { + return route.Gw, route.LinkIndex, nil + } + } + + return nil, 0, fmt.Errorf("default route not found") +} + +func setupDummyInterfacesAndRoutes(t *testing.T) (string, string) { + t.Helper() + + defaultDummy := createAndSetupDummyInterface(t, "dummyext0", "192.168.0.1/24") + addDummyRoute(t, "0.0.0.0/0", net.IPv4(192, 168, 0, 1), defaultDummy.Attrs().Index) + + otherDummy := createAndSetupDummyInterface(t, "dummyint0", "192.168.1.1/24") + addDummyRoute(t, "10.0.0.0/8", nil, otherDummy.Attrs().Index) + + t.Cleanup(func() { + err := netlink.LinkDel(defaultDummy) + assert.NoError(t, err) + err = netlink.LinkDel(otherDummy) + assert.NoError(t, err) + }) + + return defaultDummy.Name, otherDummy.Name +} + +func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, listenPort int) *iface.WGIface { + t.Helper() + + peerPrivateKey, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + newNet, err := stdnet.NewNet(nil) + require.NoError(t, err) + + wgInterface, err := iface.NewWGIFace(interfaceName, ipAddressCIDR, listenPort, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil) + require.NoError(t, err, "should create testing WireGuard interface") + + err = wgInterface.Create() + require.NoError(t, err, "should create testing WireGuard interface") + + t.Cleanup(func() { + wgInterface.Close() + }) + + return wgInterface +} + +func setupTestEnv(t *testing.T) (*iface.WGIface, string, string) { + t.Helper() + + defaultDummy, otherDummy := setupDummyInterfacesAndRoutes(t) + + wgIface := createWGInterface(t, "wgtest0", "100.64.0.1/24", 51820) + t.Cleanup(func() { + assert.NoError(t, wgIface.Close()) + }) + + err := setupRouting() + require.NoError(t, err, "setupRouting should not return err") + t.Cleanup(func() { + assert.NoError(t, cleanupRouting()) + }) + + return wgIface, defaultDummy, otherDummy +} + +func startPacketCapture(t *testing.T, intf, filter string) *pcap.Handle { + t.Helper() + + inactive, err := pcap.NewInactiveHandle(intf) + require.NoError(t, err, "Failed to create inactive pcap handle") + defer inactive.CleanUp() + + err = inactive.SetSnapLen(1600) + require.NoError(t, err, "Failed to set snap length on inactive handle") + + err = inactive.SetTimeout(time.Second * 10) + require.NoError(t, err, "Failed to set timeout on inactive handle") + + err = inactive.SetImmediateMode(true) + require.NoError(t, err, "Failed to set immediate mode on inactive handle") + + handle, err := inactive.Activate() + require.NoError(t, err, "Failed to activate pcap handle") + t.Cleanup(handle.Close) + + err = handle.SetBPFFilter(filter) + require.NoError(t, err, "Failed to set BPF filter") + + return handle +} + +func sendTestPacket(t *testing.T, destination string, sourcePort int, dialer *net.Dialer) { + t.Helper() + + if dialer == nil { + dialer = &net.Dialer{} + } + + if sourcePort != 0 { + localUDPAddr := &net.UDPAddr{ + IP: net.IPv4zero, + Port: sourcePort, + } + dialer.LocalAddr = localUDPAddr + } + + msg := new(dns.Msg) + msg.Id = dns.Id() + msg.RecursionDesired = true + msg.Question = []dns.Question{ + {Name: "example.com.", Qtype: dns.TypeA, Qclass: dns.ClassINET}, + } + + conn, err := dialer.Dial("udp", destination) + require.NoError(t, err, "Failed to dial UDP") + defer conn.Close() + + data, err := msg.Pack() + require.NoError(t, err, "Failed to pack DNS message") + + _, err = conn.Write(data) + if err != nil { + if strings.Contains(err.Error(), "required key not available") { + t.Logf("Ignoring WireGuard key error: %v", err) + return + } + t.Fatalf("Failed to send DNS query: %v", err) + } +} + +func createBPFFilter(destination string) string { + host, port, err := net.SplitHostPort(destination) + if err != nil { + return fmt.Sprintf("udp and dst host %s and dst port %s", host, port) + } + return "udp" +} + +func createPacketExpectation(srcIP string, srcPort int, dstIP string, dstPort int) PacketExpectation { + return PacketExpectation{ + SrcIP: net.ParseIP(srcIP), + DstIP: net.ParseIP(dstIP), + SrcPort: srcPort, + DstPort: dstPort, + UDP: true, + } +} diff --git a/client/internal/routemanager/systemops_nonandroid.go b/client/internal/routemanager/systemops_nonandroid.go index 11247c7dc..65f670ace 100644 --- a/client/internal/routemanager/systemops_nonandroid.go +++ b/client/internal/routemanager/systemops_nonandroid.go @@ -1,11 +1,15 @@ -//go:build !android && !ios +//go:build !android +//nolint:unused package routemanager import ( + "errors" "fmt" "net" "net/netip" + "os/exec" + "runtime" "github.com/libp2p/go-netroute" log "github.com/sirupsen/logrus" @@ -13,41 +17,16 @@ import ( var errRouteNotFound = fmt.Errorf("route not found") -func addToRouteTableIfNoExists(prefix netip.Prefix, addr string) error { - ok, err := existsInRouteTable(prefix) - if err != nil { - return err - } - if ok { - log.Warnf("skipping adding a new route for network %s because it already exists", prefix) - return nil - } - - ok, err = isSubRange(prefix) - if err != nil { - return err - } - - if ok { - err := addRouteForCurrentDefaultGateway(prefix) - if err != nil { - log.Warnf("unable to add route for current default gateway route. Will proceed without it. error: %s", err) - } - } - - return addToRouteTable(prefix, addr) -} - -func addRouteForCurrentDefaultGateway(prefix netip.Prefix) error { - defaultGateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0")) - if err != nil && err != errRouteNotFound { - return err +func genericAddRouteForCurrentDefaultGateway(prefix netip.Prefix) error { + defaultGateway, err := getExistingRIBRouteGateway(defaultv4) + if err != nil && !errors.Is(err, errRouteNotFound) { + return fmt.Errorf("get existing route gateway: %s", err) } addr := netip.MustParseAddr(defaultGateway.String()) if !prefix.Contains(addr) { - log.Debugf("skipping adding a new route for gateway %s because it is not in the network %s", addr, prefix) + log.Debugf("Skipping adding a new route for gateway %s because it is not in the network %s", addr, prefix) return nil } @@ -59,22 +38,93 @@ func addRouteForCurrentDefaultGateway(prefix netip.Prefix) error { } if ok { - log.Debugf("skipping adding a new route for gateway %s because it already exists", gatewayPrefix) + log.Debugf("Skipping adding a new route for gateway %s because it already exists", gatewayPrefix) return nil } gatewayHop, err := getExistingRIBRouteGateway(gatewayPrefix) - if err != nil && err != errRouteNotFound { + if err != nil && !errors.Is(err, errRouteNotFound) { return fmt.Errorf("unable to get the next hop for the default gateway address. error: %s", err) } - log.Debugf("adding a new route for gateway %s with next hop %s", gatewayPrefix, gatewayHop) - return addToRouteTable(gatewayPrefix, gatewayHop.String()) + log.Debugf("Adding a new route for gateway %s with next hop %s", gatewayPrefix, gatewayHop) + return genericAddToRouteTable(gatewayPrefix, gatewayHop.String(), "") +} + +func genericAddToRouteTableIfNoExists(prefix netip.Prefix, addr string, intf string) error { + ok, err := existsInRouteTable(prefix) + if err != nil { + return fmt.Errorf("exists in route table: %w", err) + } + if ok { + log.Warnf("Skipping adding a new route for network %s because it already exists", prefix) + return nil + } + + ok, err = isSubRange(prefix) + if err != nil { + return fmt.Errorf("sub range: %w", err) + } + + if ok { + err := genericAddRouteForCurrentDefaultGateway(prefix) + if err != nil { + log.Warnf("Unable to add route for current default gateway route. Will proceed without it. error: %s", err) + } + } + + return genericAddToRouteTable(prefix, addr, intf) +} + +func genericRemoveFromRouteTableIfNonSystem(prefix netip.Prefix, addr string, intf string) error { + return genericRemoveFromRouteTable(prefix, addr, intf) +} + +func genericAddToRouteTable(prefix netip.Prefix, addr, _ string) error { + cmd := exec.Command("route", "add", prefix.String(), addr) + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("add route: %w", err) + } + log.Debugf(string(out)) + return nil +} + +func genericRemoveFromRouteTable(prefix netip.Prefix, addr, _ string) error { + args := []string{"delete", prefix.String()} + if runtime.GOOS == "darwin" { + args = append(args, addr) + } + cmd := exec.Command("route", args...) + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("remove route: %w", err) + } + log.Debugf(string(out)) + return nil +} + +func getExistingRIBRouteGateway(prefix netip.Prefix) (net.IP, error) { + r, err := netroute.New() + if err != nil { + return nil, fmt.Errorf("new netroute: %w", err) + } + _, gateway, preferredSrc, err := r.Route(prefix.Addr().AsSlice()) + if err != nil { + log.Errorf("Getting routes returned an error: %v", err) + return nil, errRouteNotFound + } + + if gateway == nil { + return preferredSrc, nil + } + + return gateway, nil } func existsInRouteTable(prefix netip.Prefix) (bool, error) { routes, err := getRoutesFromTable() if err != nil { - return false, err + return false, fmt.Errorf("get routes from table: %w", err) } for _, tableRoute := range routes { if tableRoute == prefix { @@ -87,34 +137,12 @@ func existsInRouteTable(prefix netip.Prefix) (bool, error) { func isSubRange(prefix netip.Prefix) (bool, error) { routes, err := getRoutesFromTable() if err != nil { - return false, err + return false, fmt.Errorf("get routes from table: %w", err) } for _, tableRoute := range routes { - if tableRoute.Bits() > minRangeBits && tableRoute.Contains(prefix.Addr()) && tableRoute.Bits() < prefix.Bits() { + if isPrefixSupported(tableRoute) && tableRoute.Contains(prefix.Addr()) && tableRoute.Bits() < prefix.Bits() { return true, nil } } return false, nil } - -func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr string) error { - return removeFromRouteTable(prefix, addr) -} - -func getExistingRIBRouteGateway(prefix netip.Prefix) (net.IP, error) { - r, err := netroute.New() - if err != nil { - return nil, err - } - _, gateway, preferredSrc, err := r.Route(prefix.Addr().AsSlice()) - if err != nil { - log.Errorf("getting routes returned an error: %v", err) - return nil, errRouteNotFound - } - - if gateway == nil { - return preferredSrc, nil - } - - return gateway, nil -} diff --git a/client/internal/routemanager/systemops_nonandroid_test.go b/client/internal/routemanager/systemops_nonandroid_test.go index 6f32d9634..aae5e5faa 100644 --- a/client/internal/routemanager/systemops_nonandroid_test.go +++ b/client/internal/routemanager/systemops_nonandroid_test.go @@ -8,17 +8,63 @@ import ( "net" "net/netip" "os" + "os/exec" + "runtime" "strings" "testing" "github.com/pion/transport/v3/stdnet" log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/iface" ) +func assertWGOutInterface(t *testing.T, prefix netip.Prefix, wgIface *iface.WGIface, invert bool) { + t.Helper() + + if runtime.GOOS == "linux" { + outIntf, err := getOutgoingInterfaceLinux(prefix.Addr().String()) + require.NoError(t, err, "getOutgoingInterfaceLinux should not return error") + if invert { + require.NotEqual(t, wgIface.Name(), outIntf, "outgoing interface should not be the wireguard interface") + } else { + require.Equal(t, wgIface.Name(), outIntf, "outgoing interface should be the wireguard interface") + } + return + } + + prefixGateway, err := getExistingRIBRouteGateway(prefix) + require.NoError(t, err, "getExistingRIBRouteGateway should not return err") + if invert { + assert.NotEqual(t, wgIface.Address().IP.String(), prefixGateway.String(), "route should not point to wireguard interface IP") + } else { + assert.Equal(t, wgIface.Address().IP.String(), prefixGateway.String(), "route should point to wireguard interface IP") + } +} + +func getOutgoingInterfaceLinux(destination string) (string, error) { + cmd := exec.Command("ip", "route", "get", destination) + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("executing ip route get: %w", err) + } + + return parseOutgoingInterface(string(output)), nil +} + +func parseOutgoingInterface(routeGetOutput string) string { + fields := strings.Fields(routeGetOutput) + for i, field := range fields { + if field == "dev" && i+1 < len(fields) { + return fields[i+1] + } + } + return "" +} + func TestAddRemoveRoutes(t *testing.T) { testCases := []struct { name string @@ -54,23 +100,26 @@ func TestAddRemoveRoutes(t *testing.T) { err = wgInterface.Create() require.NoError(t, err, "should create testing wireguard interface") - err = addToRouteTableIfNoExists(testCase.prefix, wgInterface.Address().IP.String()) + require.NoError(t, setupRouting()) + t.Cleanup(func() { + assert.NoError(t, cleanupRouting()) + }) + + err = addToRouteTableIfNoExists(testCase.prefix, wgInterface.Address().IP.String(), wgInterface.Name()) require.NoError(t, err, "addToRouteTableIfNoExists should not return err") - prefixGateway, err := getExistingRIBRouteGateway(testCase.prefix) - require.NoError(t, err, "getExistingRIBRouteGateway should not return err") if testCase.shouldRouteToWireguard { - require.Equal(t, wgInterface.Address().IP.String(), prefixGateway.String(), "route should point to wireguard interface IP") + assertWGOutInterface(t, testCase.prefix, wgInterface, false) } else { - require.NotEqual(t, wgInterface.Address().IP.String(), prefixGateway.String(), "route should point to a different interface") + assertWGOutInterface(t, testCase.prefix, wgInterface, true) } exists, err := existsInRouteTable(testCase.prefix) require.NoError(t, err, "existsInRouteTable should not return err") if exists && testCase.shouldRouteToWireguard { - err = removeFromRouteTableIfNonSystem(testCase.prefix, wgInterface.Address().IP.String()) + err = removeFromRouteTableIfNonSystem(testCase.prefix, wgInterface.Address().IP.String(), wgInterface.Name()) require.NoError(t, err, "removeFromRouteTableIfNonSystem should not return err") - prefixGateway, err = getExistingRIBRouteGateway(testCase.prefix) + prefixGateway, err := getExistingRIBRouteGateway(testCase.prefix) require.NoError(t, err, "getExistingRIBRouteGateway should not return err") internetGateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0")) @@ -189,16 +238,21 @@ func TestAddExistAndRemoveRouteNonAndroid(t *testing.T) { err = wgInterface.Create() require.NoError(t, err, "should create testing wireguard interface") + require.NoError(t, setupRouting()) + t.Cleanup(func() { + assert.NoError(t, cleanupRouting()) + }) + MockAddr := wgInterface.Address().IP.String() // Prepare the environment if testCase.preExistingPrefix.IsValid() { - err := addToRouteTableIfNoExists(testCase.preExistingPrefix, MockAddr) + err := addToRouteTableIfNoExists(testCase.preExistingPrefix, MockAddr, wgInterface.Name()) require.NoError(t, err, "should not return err when adding pre-existing route") } // Add the route - err = addToRouteTableIfNoExists(testCase.prefix, MockAddr) + err = addToRouteTableIfNoExists(testCase.prefix, MockAddr, wgInterface.Name()) require.NoError(t, err, "should not return err when adding route") if testCase.shouldAddRoute { @@ -208,7 +262,7 @@ func TestAddExistAndRemoveRouteNonAndroid(t *testing.T) { require.True(t, ok, "route should exist") // remove route again if added - err = removeFromRouteTableIfNonSystem(testCase.prefix, MockAddr) + err = removeFromRouteTableIfNonSystem(testCase.prefix, MockAddr, wgInterface.Name()) require.NoError(t, err, "should not return err") } @@ -217,72 +271,12 @@ func TestAddExistAndRemoveRouteNonAndroid(t *testing.T) { ok, err := existsInRouteTable(testCase.prefix) t.Log("Buffer string: ", buf.String()) require.NoError(t, err, "should not return err") - if !strings.Contains(buf.String(), "because it already exists") { + + // Linux uses a separate routing table, so the route can exist in both tables. + // The main routing table takes precedence over the wireguard routing table. + if !strings.Contains(buf.String(), "because it already exists") && runtime.GOOS != "linux" { require.False(t, ok, "route should not exist") } }) } } - -func TestExistsInRouteTable(t *testing.T) { - addresses, err := net.InterfaceAddrs() - if err != nil { - t.Fatal("shouldn't return error when fetching interface addresses: ", err) - } - - var addressPrefixes []netip.Prefix - for _, address := range addresses { - p := netip.MustParsePrefix(address.String()) - if p.Addr().Is4() { - addressPrefixes = append(addressPrefixes, p.Masked()) - } - } - - for _, prefix := range addressPrefixes { - exists, err := existsInRouteTable(prefix) - if err != nil { - t.Fatal("shouldn't return error when checking if address exists in route table: ", err) - } - if !exists { - t.Fatalf("address %s should exist in route table", prefix) - } - } -} - -func TestIsSubRange(t *testing.T) { - addresses, err := net.InterfaceAddrs() - if err != nil { - t.Fatal("shouldn't return error when fetching interface addresses: ", err) - } - - var subRangeAddressPrefixes []netip.Prefix - var nonSubRangeAddressPrefixes []netip.Prefix - for _, address := range addresses { - p := netip.MustParsePrefix(address.String()) - if !p.Addr().IsLoopback() && p.Addr().Is4() && p.Bits() < 32 { - p2 := netip.PrefixFrom(p.Masked().Addr(), p.Bits()+1) - subRangeAddressPrefixes = append(subRangeAddressPrefixes, p2) - nonSubRangeAddressPrefixes = append(nonSubRangeAddressPrefixes, p.Masked()) - } - } - - for _, prefix := range subRangeAddressPrefixes { - isSubRangePrefix, err := isSubRange(prefix) - if err != nil { - t.Fatal("shouldn't return error when checking if address is sub-range: ", err) - } - if !isSubRangePrefix { - t.Fatalf("address %s should be sub-range of an existing route in the table", prefix) - } - } - - for _, prefix := range nonSubRangeAddressPrefixes { - isSubRangePrefix, err := isSubRange(prefix) - if err != nil { - t.Fatal("shouldn't return error when checking if address is sub-range: ", err) - } - if isSubRangePrefix { - t.Fatalf("address %s should not be sub-range of an existing route in the table", prefix) - } - } -} diff --git a/client/internal/routemanager/systemops_nonlinux.go b/client/internal/routemanager/systemops_nonlinux.go index 47bd60eb0..d793f0fbd 100644 --- a/client/internal/routemanager/systemops_nonlinux.go +++ b/client/internal/routemanager/systemops_nonlinux.go @@ -1,41 +1,22 @@ -//go:build !linux -// +build !linux +//go:build !linux || android package routemanager import ( - "net/netip" - "os/exec" "runtime" log "github.com/sirupsen/logrus" ) -func addToRouteTable(prefix netip.Prefix, addr string) error { - cmd := exec.Command("route", "add", prefix.String(), addr) - out, err := cmd.Output() - if err != nil { - return err - } - log.Debugf(string(out)) +func setupRouting() error { return nil } -func removeFromRouteTable(prefix netip.Prefix, addr string) error { - args := []string{"delete", prefix.String()} - if runtime.GOOS == "darwin" { - args = append(args, addr) - } - cmd := exec.Command("route", args...) - out, err := cmd.Output() - if err != nil { - return err - } - log.Debugf(string(out)) +func cleanupRouting() error { return nil } func enableIPForwarding() error { - log.Infof("enable IP forwarding is not implemented on %s", runtime.GOOS) + log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS) return nil } diff --git a/client/internal/routemanager/systemops_nonlinux_test.go b/client/internal/routemanager/systemops_nonlinux_test.go new file mode 100644 index 000000000..afaf5ba77 --- /dev/null +++ b/client/internal/routemanager/systemops_nonlinux_test.go @@ -0,0 +1,80 @@ +//go:build !linux || android + +package routemanager + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsSubRange(t *testing.T) { + addresses, err := net.InterfaceAddrs() + if err != nil { + t.Fatal("shouldn't return error when fetching interface addresses: ", err) + } + + var subRangeAddressPrefixes []netip.Prefix + var nonSubRangeAddressPrefixes []netip.Prefix + for _, address := range addresses { + p := netip.MustParsePrefix(address.String()) + if !p.Addr().IsLoopback() && p.Addr().Is4() && p.Bits() < 32 { + p2 := netip.PrefixFrom(p.Masked().Addr(), p.Bits()+1) + subRangeAddressPrefixes = append(subRangeAddressPrefixes, p2) + nonSubRangeAddressPrefixes = append(nonSubRangeAddressPrefixes, p.Masked()) + } + } + + for _, prefix := range subRangeAddressPrefixes { + isSubRangePrefix, err := isSubRange(prefix) + if err != nil { + t.Fatal("shouldn't return error when checking if address is sub-range: ", err) + } + if !isSubRangePrefix { + t.Fatalf("address %s should be sub-range of an existing route in the table", prefix) + } + } + + for _, prefix := range nonSubRangeAddressPrefixes { + isSubRangePrefix, err := isSubRange(prefix) + if err != nil { + t.Fatal("shouldn't return error when checking if address is sub-range: ", err) + } + if isSubRangePrefix { + t.Fatalf("address %s should not be sub-range of an existing route in the table", prefix) + } + } +} + +func TestExistsInRouteTable(t *testing.T) { + require.NoError(t, setupRouting()) + t.Cleanup(func() { + assert.NoError(t, cleanupRouting()) + }) + + addresses, err := net.InterfaceAddrs() + if err != nil { + t.Fatal("shouldn't return error when fetching interface addresses: ", err) + } + + var addressPrefixes []netip.Prefix + for _, address := range addresses { + p := netip.MustParsePrefix(address.String()) + if p.Addr().Is4() { + addressPrefixes = append(addressPrefixes, p.Masked()) + } + } + + for _, prefix := range addressPrefixes { + exists, err := existsInRouteTable(prefix) + if err != nil { + t.Fatal("shouldn't return error when checking if address exists in route table: ", err) + } + if !exists { + t.Fatalf("address %s should exist in route table", prefix) + } + } +} diff --git a/client/internal/routemanager/systemops_windows.go b/client/internal/routemanager/systemops_windows.go index 309c184b9..c009ce66b 100644 --- a/client/internal/routemanager/systemops_windows.go +++ b/client/internal/routemanager/systemops_windows.go @@ -1,12 +1,13 @@ //go:build windows -// +build windows package routemanager import ( + "fmt" "net" "net/netip" + log "github.com/sirupsen/logrus" "github.com/yusufpapurcu/wmi" ) @@ -21,17 +22,19 @@ func getRoutesFromTable() ([]netip.Prefix, error) { err := wmi.Query(query, &routes) if err != nil { - return nil, err + return nil, fmt.Errorf("get routes: %w", err) } var prefixList []netip.Prefix for _, route := range routes { addr, err := netip.ParseAddr(route.Destination) if err != nil { + log.Warnf("Unable to parse route destination %s: %v", route.Destination, err) continue } maskSlice := net.ParseIP(route.Mask).To4() if maskSlice == nil { + log.Warnf("Unable to parse route mask %s", route.Mask) continue } mask := net.IPv4Mask(maskSlice[0], maskSlice[1], maskSlice[2], maskSlice[3]) @@ -44,3 +47,11 @@ func getRoutesFromTable() ([]netip.Prefix, error) { } return prefixList, nil } + +func addToRouteTableIfNoExists(prefix netip.Prefix, addr string, intf string) error { + return genericAddToRouteTableIfNoExists(prefix, addr, intf) +} + +func removeFromRouteTableIfNonSystem(prefix netip.Prefix, addr string, intf string) error { + return genericRemoveFromRouteTableIfNonSystem(prefix, addr, intf) +} diff --git a/client/internal/stdnet/dialer.go b/client/internal/stdnet/dialer.go new file mode 100644 index 000000000..e80adb42b --- /dev/null +++ b/client/internal/stdnet/dialer.go @@ -0,0 +1,24 @@ +package stdnet + +import ( + "net" + + "github.com/pion/transport/v3" + + nbnet "github.com/netbirdio/netbird/util/net" +) + +// Dial connects to the address on the named network. +func (n *Net) Dial(network, address string) (net.Conn, error) { + return nbnet.NewDialer().Dial(network, address) +} + +// DialUDP connects to the address on the named UDP network. +func (n *Net) DialUDP(network string, laddr, raddr *net.UDPAddr) (transport.UDPConn, error) { + return nbnet.DialUDP(network, laddr, raddr) +} + +// DialTCP connects to the address on the named TCP network. +func (n *Net) DialTCP(network string, laddr, raddr *net.TCPAddr) (transport.TCPConn, error) { + return nbnet.DialTCP(network, laddr, raddr) +} diff --git a/client/internal/stdnet/listener.go b/client/internal/stdnet/listener.go new file mode 100644 index 000000000..9ce0a5556 --- /dev/null +++ b/client/internal/stdnet/listener.go @@ -0,0 +1,20 @@ +package stdnet + +import ( + "context" + "net" + + "github.com/pion/transport/v3" + + nbnet "github.com/netbirdio/netbird/util/net" +) + +// ListenPacket listens for incoming packets on the given network and address. +func (n *Net) ListenPacket(network, address string) (net.PacketConn, error) { + return nbnet.NewListener().ListenPacket(context.Background(), network, address) +} + +// ListenUDP acts like ListenPacket for UDP networks. +func (n *Net) ListenUDP(network string, locAddr *net.UDPAddr) (transport.UDPConn, error) { + return nbnet.ListenUDP(network, locAddr) +} diff --git a/client/internal/wgproxy/portlookup.go b/client/internal/wgproxy/portlookup.go index 6f3d33487..6ede4b83f 100644 --- a/client/internal/wgproxy/portlookup.go +++ b/client/internal/wgproxy/portlookup.go @@ -1,8 +1,10 @@ package wgproxy import ( + "context" "fmt" - "net" + + nbnet "github.com/netbirdio/netbird/util/net" ) const ( @@ -23,7 +25,7 @@ func (pl portLookup) searchFreePort() (int, error) { } func (pl portLookup) tryToBind(port int) error { - l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) + l, err := nbnet.NewListener().ListenPacket(context.Background(), "udp", fmt.Sprintf(":%d", port)) if err != nil { return err } diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index 6ca19c973..b91cd7b43 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -16,6 +16,7 @@ import ( "github.com/netbirdio/netbird/client/internal/ebpf" ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" + nbnet "github.com/netbirdio/netbird/util/net" ) // WGEBPFProxy definition for proxy with EBPF support @@ -66,7 +67,7 @@ func (p *WGEBPFProxy) Listen() error { IP: net.ParseIP("127.0.0.1"), } - p.conn, err = net.ListenUDP("udp", &addr) + p.conn, err = nbnet.ListenUDP("udp", &addr) if err != nil { cErr := p.Free() if cErr != nil { @@ -208,20 +209,41 @@ generatePort: } func (p *WGEBPFProxy) prepareSenderRawSocket() (net.PacketConn, error) { + // Create a raw socket. fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW) if err != nil { - return nil, err - } - err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1) - if err != nil { - return nil, err - } - err = syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, "lo") - if err != nil { - return nil, err + return nil, fmt.Errorf("creating raw socket failed: %w", err) } - return net.FilePacketConn(os.NewFile(uintptr(fd), fmt.Sprintf("fd %d", fd))) + // Set the IP_HDRINCL option on the socket to tell the kernel that headers are included in the packet. + err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1) + if err != nil { + return nil, fmt.Errorf("setting IP_HDRINCL failed: %w", err) + } + + // Bind the socket to the "lo" interface. + err = syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, "lo") + if err != nil { + return nil, fmt.Errorf("binding to lo interface failed: %w", err) + } + + // Set the fwmark on the socket. + err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, nbnet.NetbirdFwmark) + if err != nil { + return nil, fmt.Errorf("setting fwmark failed: %w", err) + } + + // Convert the file descriptor to a PacketConn. + file := os.NewFile(uintptr(fd), fmt.Sprintf("fd %d", fd)) + if file == nil { + return nil, fmt.Errorf("converting fd to file failed") + } + packetConn, err := net.FilePacketConn(file) + if err != nil { + return nil, fmt.Errorf("converting file to packet conn failed: %w", err) + } + + return packetConn, nil } func (p *WGEBPFProxy) sendPkg(data []byte, port uint16) error { diff --git a/client/internal/wgproxy/proxy_userspace.go b/client/internal/wgproxy/proxy_userspace.go index b692ea708..17ebfbc49 100644 --- a/client/internal/wgproxy/proxy_userspace.go +++ b/client/internal/wgproxy/proxy_userspace.go @@ -6,6 +6,8 @@ import ( "net" log "github.com/sirupsen/logrus" + + nbnet "github.com/netbirdio/netbird/util/net" ) // WGUserSpaceProxy proxies @@ -33,7 +35,7 @@ func (p *WGUserSpaceProxy) AddTurnConn(remoteConn net.Conn) (net.Addr, error) { p.remoteConn = remoteConn var err error - p.localConn, err = net.Dial("udp", fmt.Sprintf(":%d", p.localWGListenPort)) + p.localConn, err = nbnet.NewDialer().Dial("udp", fmt.Sprintf(":%d", p.localWGListenPort)) if err != nil { log.Errorf("failed dialing to local Wireguard port %s", err) return nil, err diff --git a/go.mod b/go.mod index ce3da619e..67ec9c42e 100644 --- a/go.mod +++ b/go.mod @@ -47,8 +47,9 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/gopacket v1.1.19 github.com/google/nftables v0.0.0-20220808154552-2eca00135732 + github.com/gopacket/gopacket v1.1.1 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357 - github.com/hashicorp/go-multierror v1.1.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 github.com/hashicorp/go-version v1.6.0 github.com/libp2p/go-netroute v0.2.0 @@ -123,7 +124,6 @@ require ( github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.10.0 // indirect - github.com/gopacket/gopacket v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index e304e3191..c36b8aff3 100644 --- a/go.sum +++ b/go.sum @@ -291,8 +291,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f2 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 h1:ET4pqyjiGmY09R5y+rSd70J2w45CtbWDNvGqWp/R3Ng= github.com/hashicorp/go-secure-stdlib/base62 v0.1.2/go.mod h1:EdWO6czbmthiwZ3/PUsDV+UD1D5IRU4ActiaWGwt0Yw= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= diff --git a/iface/address.go b/iface/address.go index 5ff4fbc06..2920d009f 100644 --- a/iface/address.go +++ b/iface/address.go @@ -23,6 +23,24 @@ func parseWGAddress(address string) (WGAddress, error) { }, nil } +// Masked returns the WGAddress with the IP address part masked according to its network mask. +func (addr WGAddress) Masked() WGAddress { + ip := addr.IP.To4() + if ip == nil { + ip = addr.IP.To16() + } + + maskedIP := make(net.IP, len(ip)) + for i := range ip { + maskedIP[i] = ip[i] & addr.Network.Mask[i] + } + + return WGAddress{ + IP: maskedIP, + Network: addr.Network, + } +} + func (addr WGAddress) String() string { maskSize, _ := addr.Network.Mask.Size() return fmt.Sprintf("%s/%d", addr.IP.String(), maskSize) diff --git a/iface/wg_configurer_kernel.go b/iface/wg_configurer_kernel.go index 36fd13cc2..9fe987cee 100644 --- a/iface/wg_configurer_kernel.go +++ b/iface/wg_configurer_kernel.go @@ -10,6 +10,8 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + nbnet "github.com/netbirdio/netbird/util/net" ) type wgKernelConfigurer struct { @@ -29,7 +31,7 @@ func (c *wgKernelConfigurer) configureInterface(privateKey string, port int) err if err != nil { return err } - fwmark := 0 + fwmark := nbnet.NetbirdFwmark config := wgtypes.Config{ PrivateKey: &key, ReplacePeers: true, diff --git a/iface/wg_configurer_usp.go b/iface/wg_configurer_usp.go index 200bfbc96..24dfadf14 100644 --- a/iface/wg_configurer_usp.go +++ b/iface/wg_configurer_usp.go @@ -13,6 +13,8 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + nbnet "github.com/netbirdio/netbird/util/net" ) type wgUSPConfigurer struct { @@ -37,7 +39,7 @@ func (c *wgUSPConfigurer) configureInterface(privateKey string, port int) error if err != nil { return err } - fwmark := 0 + fwmark := getFwmark() config := wgtypes.Config{ PrivateKey: &key, ReplacePeers: true, @@ -345,3 +347,10 @@ func toWgUserspaceString(wgCfg wgtypes.Config) string { } return sb.String() } + +func getFwmark() int { + if runtime.GOOS == "linux" { + return nbnet.NetbirdFwmark + } + return 0 +} diff --git a/management/client/grpc.go b/management/client/grpc.go index 0234f866c..0b1804906 100644 --- a/management/client/grpc.go +++ b/management/client/grpc.go @@ -24,6 +24,7 @@ import ( "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/management/proto" + nbgrpc "github.com/netbirdio/netbird/util/grpc" ) const ConnectTimeout = 10 * time.Second @@ -57,6 +58,7 @@ func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsE mgmCtx, addr, transportOption, + nbgrpc.WithCustomDialer(), grpc.WithBlock(), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, diff --git a/sharedsock/sock_linux.go b/sharedsock/sock_linux.go index 656fdc8ca..74ac6c163 100644 --- a/sharedsock/sock_linux.go +++ b/sharedsock/sock_linux.go @@ -21,6 +21,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "golang.org/x/sys/unix" + + nbnet "github.com/netbirdio/netbird/util/net" ) // ErrSharedSockStopped indicates that shared socket has been stopped @@ -55,8 +57,7 @@ var writeSerializerOptions = gopacket.SerializeOptions{ } // Listen creates an IPv4 and IPv6 raw sockets, starts a reader and routing table routines -func Listen(port int, filter BPFFilter) (net.PacketConn, error) { - var err error +func Listen(port int, filter BPFFilter) (_ net.PacketConn, err error) { ctx, cancel := context.WithCancel(context.Background()) rawSock := &SharedSocket{ ctx: ctx, @@ -65,37 +66,51 @@ func Listen(port int, filter BPFFilter) (net.PacketConn, error) { packetDemux: make(chan rcvdPacket), } + defer func() { + if err != nil { + if closeErr := rawSock.Close(); closeErr != nil { + log.Errorf("Failed to close raw socket: %v", closeErr) + } + } + }() + rawSock.router, err = netroute.New() if err != nil { - return nil, fmt.Errorf("failed to create raw socket router: %v", err) + return nil, fmt.Errorf("failed to create raw socket router: %w", err) } rawSock.conn4, err = socket.Socket(unix.AF_INET, unix.SOCK_RAW, unix.IPPROTO_UDP, "raw_udp4", nil) if err != nil { - return nil, fmt.Errorf("failed to create ipv4 raw socket: %v", err) + return nil, fmt.Errorf("failed to create ipv4 raw socket: %w", err) } - rawSock.conn6, err = socket.Socket(unix.AF_INET6, unix.SOCK_RAW, unix.IPPROTO_UDP, "raw_udp6", nil) - if err != nil { - log.Errorf("failed to create ipv6 raw socket: %v", err) + if err = nbnet.SetSocketMark(rawSock.conn4); err != nil { + return nil, fmt.Errorf("failed to set SO_MARK on ipv4 socket: %w", err) + } + + var sockErr error + rawSock.conn6, sockErr = socket.Socket(unix.AF_INET6, unix.SOCK_RAW, unix.IPPROTO_UDP, "raw_udp6", nil) + if sockErr != nil { + log.Errorf("Failed to create ipv6 raw socket: %v", err) + } else { + if err = nbnet.SetSocketMark(rawSock.conn6); err != nil { + return nil, fmt.Errorf("failed to set SO_MARK on ipv6 socket: %w", err) + } } ipv4Instructions, ipv6Instructions, err := filter.GetInstructions(uint32(rawSock.port)) if err != nil { - _ = rawSock.Close() - return nil, fmt.Errorf("getBPFInstructions failed with: %rawSock", err) + return nil, fmt.Errorf("getBPFInstructions failed with: %w", err) } err = rawSock.conn4.SetBPF(ipv4Instructions) if err != nil { - _ = rawSock.Close() - return nil, fmt.Errorf("socket4.SetBPF failed with: %rawSock", err) + return nil, fmt.Errorf("socket4.SetBPF failed with: %w", err) } if rawSock.conn6 != nil { err = rawSock.conn6.SetBPF(ipv6Instructions) if err != nil { - _ = rawSock.Close() - return nil, fmt.Errorf("socket6.SetBPF failed with: %rawSock", err) + return nil, fmt.Errorf("socket6.SetBPF failed with: %w", err) } } @@ -121,7 +136,7 @@ func (s *SharedSocket) updateRouter() { case <-ticker.C: router, err := netroute.New() if err != nil { - log.Errorf("failed to create and update packet router for stunListener: %s", err) + log.Errorf("Failed to create and update packet router for stunListener: %s", err) continue } s.routerMux.Lock() @@ -144,7 +159,7 @@ func (s *SharedSocket) LocalAddr() net.Addr { func (s *SharedSocket) SetDeadline(t time.Time) error { err := s.conn4.SetDeadline(t) if err != nil { - return fmt.Errorf("s.conn4.SetDeadline error: %s", err) + return fmt.Errorf("s.conn4.SetDeadline error: %w", err) } if s.conn6 == nil { return nil @@ -152,7 +167,7 @@ func (s *SharedSocket) SetDeadline(t time.Time) error { err = s.conn6.SetDeadline(t) if err != nil { - return fmt.Errorf("s.conn6.SetDeadline error: %s", err) + return fmt.Errorf("s.conn6.SetDeadline error: %w", err) } return nil } @@ -161,7 +176,7 @@ func (s *SharedSocket) SetDeadline(t time.Time) error { func (s *SharedSocket) SetReadDeadline(t time.Time) error { err := s.conn4.SetReadDeadline(t) if err != nil { - return fmt.Errorf("s.conn4.SetReadDeadline error: %s", err) + return fmt.Errorf("s.conn4.SetReadDeadline error: %w", err) } if s.conn6 == nil { return nil @@ -169,7 +184,7 @@ func (s *SharedSocket) SetReadDeadline(t time.Time) error { err = s.conn6.SetReadDeadline(t) if err != nil { - return fmt.Errorf("s.conn6.SetReadDeadline error: %s", err) + return fmt.Errorf("s.conn6.SetReadDeadline error: %w", err) } return nil } @@ -178,7 +193,7 @@ func (s *SharedSocket) SetReadDeadline(t time.Time) error { func (s *SharedSocket) SetWriteDeadline(t time.Time) error { err := s.conn4.SetWriteDeadline(t) if err != nil { - return fmt.Errorf("s.conn4.SetWriteDeadline error: %s", err) + return fmt.Errorf("s.conn4.SetWriteDeadline error: %w", err) } if s.conn6 == nil { return nil @@ -186,7 +201,7 @@ func (s *SharedSocket) SetWriteDeadline(t time.Time) error { err = s.conn6.SetWriteDeadline(t) if err != nil { - return fmt.Errorf("s.conn6.SetWriteDeadline error: %s", err) + return fmt.Errorf("s.conn6.SetWriteDeadline error: %w", err) } return nil } @@ -282,7 +297,7 @@ func (s *SharedSocket) WriteTo(buf []byte, rAddr net.Addr) (n int, err error) { _, _, src, err := s.router.Route(rUDPAddr.IP) if err != nil { - return 0, fmt.Errorf("got an error while checking route, err: %s", err) + return 0, fmt.Errorf("got an error while checking route, err: %w", err) } rSockAddr, conn, nwLayer := s.getWriterObjects(src, rUDPAddr.IP) @@ -292,7 +307,7 @@ func (s *SharedSocket) WriteTo(buf []byte, rAddr net.Addr) (n int, err error) { } if err := gopacket.SerializeLayers(buffer, writeSerializerOptions, udp, payload); err != nil { - return -1, fmt.Errorf("failed serialize rcvdPacket: %s", err) + return -1, fmt.Errorf("failed serialize rcvdPacket: %w", err) } bufser := buffer.Bytes() diff --git a/signal/client/grpc.go b/signal/client/grpc.go index 7531608c3..7c4535e28 100644 --- a/signal/client/grpc.go +++ b/signal/client/grpc.go @@ -23,6 +23,7 @@ import ( "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/management/client" "github.com/netbirdio/netbird/signal/proto" + nbgrpc "github.com/netbirdio/netbird/util/grpc" ) // ConnStateNotifier is a wrapper interface of the status recorder @@ -76,6 +77,7 @@ func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled boo sigCtx, addr, transportOption, + nbgrpc.WithCustomDialer(), grpc.WithBlock(), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, diff --git a/util/grpc/dialer_generic.go b/util/grpc/dialer_generic.go new file mode 100644 index 000000000..1c2285b14 --- /dev/null +++ b/util/grpc/dialer_generic.go @@ -0,0 +1,9 @@ +//go:build !linux || android + +package grpc + +import "google.golang.org/grpc" + +func WithCustomDialer() grpc.DialOption { + return grpc.EmptyDialOption{} +} diff --git a/util/grpc/dialer_linux.go b/util/grpc/dialer_linux.go new file mode 100644 index 000000000..b29ee4b29 --- /dev/null +++ b/util/grpc/dialer_linux.go @@ -0,0 +1,18 @@ +//go:build !android + +package grpc + +import ( + "context" + "net" + + "google.golang.org/grpc" + + nbnet "github.com/netbirdio/netbird/util/net" +) + +func WithCustomDialer() grpc.DialOption { + return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return nbnet.NewDialer().DialContext(ctx, "tcp", addr) + }) +} diff --git a/util/net/dialer_generic.go b/util/net/dialer_generic.go new file mode 100644 index 000000000..a3c3ad67c --- /dev/null +++ b/util/net/dialer_generic.go @@ -0,0 +1,19 @@ +//go:build !linux || android + +package net + +import ( + "net" +) + +func NewDialer() *net.Dialer { + return &net.Dialer{} +} + +func DialUDP(network string, laddr, raddr *net.UDPAddr) (*net.UDPConn, error) { + return net.DialUDP(network, laddr, raddr) +} + +func DialTCP(network string, laddr, raddr *net.TCPAddr) (*net.TCPConn, error) { + return net.DialTCP(network, laddr, raddr) +} diff --git a/util/net/dialer_linux.go b/util/net/dialer_linux.go new file mode 100644 index 000000000..d559490c5 --- /dev/null +++ b/util/net/dialer_linux.go @@ -0,0 +1,60 @@ +//go:build !android + +package net + +import ( + "context" + "fmt" + "net" + "syscall" + + log "github.com/sirupsen/logrus" +) + +func NewDialer() *net.Dialer { + return &net.Dialer{ + Control: func(network, address string, c syscall.RawConn) error { + return SetRawSocketMark(c) + }, + } +} + +func DialUDP(network string, laddr, raddr *net.UDPAddr) (*net.UDPConn, error) { + dialer := NewDialer() + dialer.LocalAddr = laddr + + conn, err := dialer.DialContext(context.Background(), network, raddr.String()) + if err != nil { + return nil, fmt.Errorf("dialing UDP %s: %w", raddr.String(), err) + } + + udpConn, ok := conn.(*net.UDPConn) + if !ok { + if err := conn.Close(); err != nil { + log.Errorf("Failed to close connection: %v", err) + } + return nil, fmt.Errorf("expected UDP connection, got different type") + } + + return udpConn, nil +} + +func DialTCP(network string, laddr, raddr *net.TCPAddr) (*net.TCPConn, error) { + dialer := NewDialer() + dialer.LocalAddr = laddr + + conn, err := dialer.DialContext(context.Background(), network, raddr.String()) + if err != nil { + return nil, fmt.Errorf("dialing TCP %s: %w", raddr.String(), err) + } + + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + if err := conn.Close(); err != nil { + log.Errorf("Failed to close connection: %v", err) + } + return nil, fmt.Errorf("expected TCP connection, got different type") + } + + return tcpConn, nil +} diff --git a/util/net/listener_generic.go b/util/net/listener_generic.go new file mode 100644 index 000000000..241c744e5 --- /dev/null +++ b/util/net/listener_generic.go @@ -0,0 +1,13 @@ +//go:build !linux || android + +package net + +import "net" + +func NewListener() *net.ListenConfig { + return &net.ListenConfig{} +} + +func ListenUDP(network string, locAddr *net.UDPAddr) (*net.UDPConn, error) { + return net.ListenUDP(network, locAddr) +} diff --git a/util/net/listener_linux.go b/util/net/listener_linux.go new file mode 100644 index 000000000..7b9bda97c --- /dev/null +++ b/util/net/listener_linux.go @@ -0,0 +1,30 @@ +//go:build !android + +package net + +import ( + "context" + "fmt" + "net" + "syscall" +) + +func NewListener() *net.ListenConfig { + return &net.ListenConfig{ + Control: func(network, address string, c syscall.RawConn) error { + return SetRawSocketMark(c) + }, + } +} + +func ListenUDP(network string, laddr *net.UDPAddr) (*net.UDPConn, error) { + pc, err := NewListener().ListenPacket(context.Background(), network, laddr.String()) + if err != nil { + return nil, fmt.Errorf("listening on %s:%s with fwmark: %w", network, laddr, err) + } + udpConn, ok := pc.(*net.UDPConn) + if !ok { + return nil, fmt.Errorf("packetConn is not a *net.UDPConn") + } + return udpConn, nil +} diff --git a/util/net/net.go b/util/net/net.go new file mode 100644 index 000000000..5714e5229 --- /dev/null +++ b/util/net/net.go @@ -0,0 +1,6 @@ +package net + +const ( + // NetbirdFwmark is the fwmark value used by Netbird via wireguard + NetbirdFwmark = 0x1BD00 +) diff --git a/util/net/net_linux.go b/util/net/net_linux.go new file mode 100644 index 000000000..821417500 --- /dev/null +++ b/util/net/net_linux.go @@ -0,0 +1,35 @@ +//go:build !android + +package net + +import ( + "fmt" + "syscall" +) + +// SetSocketMark sets the SO_MARK option on the given socket connection +func SetSocketMark(conn syscall.Conn) error { + sysconn, err := conn.SyscallConn() + if err != nil { + return fmt.Errorf("get raw conn: %w", err) + } + + return SetRawSocketMark(sysconn) +} + +func SetRawSocketMark(conn syscall.RawConn) error { + var setErr error + + err := conn.Control(func(fd uintptr) { + setErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark) + }) + if err != nil { + return fmt.Errorf("control: %w", err) + } + + if setErr != nil { + return fmt.Errorf("set SO_MARK: %w", setErr) + } + + return nil +}