From 2c85d94c6c3e848bb084730c087a7fdcbfe94079 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 21:47:41 +0200 Subject: [PATCH 01/11] hookup validated peers --- .../network_map/controller/controller.go | 4 +-- .../internals/network_map_db/db_store.go | 33 ++++++++++++++++++- management/internals/server/boot.go | 6 ++-- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/management/internals/controllers/network_map/controller/controller.go b/management/internals/controllers/network_map/controller/controller.go index 91957bae5..dac3f6990 100644 --- a/management/internals/controllers/network_map/controller/controller.go +++ b/management/internals/controllers/network_map/controller/controller.go @@ -65,7 +65,7 @@ type Controller struct { perAccountServerSupportedSyncMessageVersions map[string]sharedgrpc.SyncMessageVersion - nmdataStore networkmapdb.NetworkMapDBStore + nmdataStore *networkmapdb.NetworkMapDBStoreImpl } type bufferUpdate struct { @@ -83,7 +83,7 @@ type bufferAffectedUpdate struct { var _ network_map.Controller = (*Controller)(nil) -func NewController(ctx context.Context, store store.Store, metrics telemetry.AppMetrics, peersUpdateManager network_map.PeersUpdateManager, requestBuffer account.RequestBuffer, integratedPeerValidator integrated_validator.IntegratedValidator, settingsManager settings.Manager, dnsDomain string, proxyController port_forwarding.Controller, ephemeralPeersManager ephemeral.Manager, config *config.Config, nmdataStore networkmapdb.NetworkMapDBStore) *Controller { +func NewController(ctx context.Context, store store.Store, metrics telemetry.AppMetrics, peersUpdateManager network_map.PeersUpdateManager, requestBuffer account.RequestBuffer, integratedPeerValidator integrated_validator.IntegratedValidator, settingsManager settings.Manager, dnsDomain string, proxyController port_forwarding.Controller, ephemeralPeersManager ephemeral.Manager, config *config.Config, nmdataStore *networkmapdb.NetworkMapDBStoreImpl) *Controller { nMetrics, err := newMetrics(metrics.UpdateChannelMetrics()) if err != nil { log.Fatal(fmt.Errorf("error creating metrics: %w", err)) diff --git a/management/internals/network_map_db/db_store.go b/management/internals/network_map_db/db_store.go index c8ba94b80..73282cdfb 100644 --- a/management/internals/network_map_db/db_store.go +++ b/management/internals/network_map_db/db_store.go @@ -8,10 +8,13 @@ import ( "reflect" "strings" + "github.com/rs/xid" + "golang.org/x/exp/maps" + + "github.com/netbirdio/management-integrations/integrations" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator" "github.com/netbirdio/netbird/shared/management/networkmap" "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" - "github.com/rs/xid" ) const ( @@ -40,6 +43,34 @@ type NetworkMapDBStore interface { type NetworkMapDBStoreImpl struct { store NetworkMapDBStore integratedPeerValidator integrated_validator.IntegratedValidator + extraSettingsManager integrations.Manager +} + +func NewNetworkMapDBStoreImpl(store NetworkMapDBStore, integratedPeerValidator integrated_validator.IntegratedValidator, extraSettingsManager integrations.Manager) *NetworkMapDBStoreImpl { + return &NetworkMapDBStoreImpl{ + store: store, + integratedPeerValidator: integratedPeerValidator, + extraSettingsManager: extraSettingsManager, + } +} + +func (s *NetworkMapDBStoreImpl) GetNetworkMapData(ctx context.Context, accountId string) (*networkmap.NetworkMapData, error) { + nmdata, err := s.store.GetNetworkMapData(ctx, accountId) + if err != nil { + return nil, err + } + + extraSettings, err := s.extraSettingsManager.GetExtraSettings(ctx, accountId) + if err != nil { + return nil, err + } + + nmdata.ValidatedPeers, err = s.integratedPeerValidator.GetValidatedPeers(ctx, accountId, maps.Values(nmdata.Groups), maps.Values(nmdata.Peers), extraSettings) + if err != nil { + return nil, err + } + + return nmdata, nil } func FromSqlTypesToSharedTypes(src reflect.Value, dst reflect.Value) error { diff --git a/management/internals/server/boot.go b/management/internals/server/boot.go index 5bc755754..ed4d22a8d 100644 --- a/management/internals/server/boot.go +++ b/management/internals/server/boot.go @@ -102,8 +102,8 @@ func (s *BaseServer) Store() store.Store { }) } -func (s *BaseServer) NetworkMapStore() networkmapdb.NetworkMapDBStore { - return Create(s, func() networkmapdb.NetworkMapDBStore { +func (s *BaseServer) NetworkMapStore() *networkmapdb.NetworkMapDBStoreImpl { + return Create(s, func() *networkmapdb.NetworkMapDBStoreImpl { dsn := os.Getenv("NETBIRD_NMAP_STORE_DSN") // Todo: this needs to be hoocked up properly if dsn == "" { return nil @@ -114,7 +114,7 @@ func (s *BaseServer) NetworkMapStore() networkmapdb.NetworkMapDBStore { log.Fatalf("failed to create network map store: %v", err) } - return store + return networkmapdb.NewNetworkMapDBStoreImpl(store, s.IntegratedValidator(), s.SettingsManager()) }) } From 07d0440e345450ffb4c749f43a8166dc78e15e91 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 21:51:48 +0200 Subject: [PATCH 02/11] fix slice initialization to avoid nil pointer dereference --- management/internals/network_map_db/pgsql/network_map_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/internals/network_map_db/pgsql/network_map_data.go b/management/internals/network_map_db/pgsql/network_map_data.go index a52022c53..715529631 100644 --- a/management/internals/network_map_db/pgsql/network_map_data.go +++ b/management/internals/network_map_db/pgsql/network_map_data.go @@ -152,7 +152,7 @@ func toMap[T any](all []T, id func(t T) string) map[string]*T { } func toSliceOfPtrs[T any](all []T) []*T { - toret := make([]*T, len(all)) + toret := make([]*T, 0, len(all)) for _, t := range all { toret = append(toret, &t) } From 02ec1f5dcb59322de4eff0514aebdd61bd43a95a Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 22:24:11 +0200 Subject: [PATCH 03/11] fix equivalence test --- .../types/legacynmap/equivalence_test.go | 81 ++++++++++++++----- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/management/server/types/legacynmap/equivalence_test.go b/management/server/types/legacynmap/equivalence_test.go index 9b6eab651..f80734165 100644 --- a/management/server/types/legacynmap/equivalence_test.go +++ b/management/server/types/legacynmap/equivalence_test.go @@ -3,10 +3,14 @@ // Main-vs-branch equivalence check. For every peer of every account in a real // Postgres copy it computes the client-facing proto.NetworkMap twice: // -// - legacy path: main's Account → NetworkMapComponents → Calculate → proto +// - legacy path: main's Account → NetworkMapComponents → Calculate → proto // (the frozen copy in this package) -// - new path: this branch's Account → NetworkMapData → components → -// Calculate → ToSyncResponse → proto +// - store path: the pgsql nmdata store's NetworkMapData → components → +// Calculate → ToSyncResponse → proto (no Account involved) +// - account path: Account → toNetworkMapData twins → components → Calculate +// → ToSyncResponse → proto (the in-memory builder, no store queries) +// +// Both new paths are checked against the legacy proto. // // proto.NetworkMap is generated code identical in both trees, which is what // makes it the one usable comparison surface — the intermediate Go types differ @@ -47,10 +51,12 @@ import ( nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/management/internals/controllers/network_map/controller/cache" + networkmap_pgsql "github.com/netbirdio/netbird/management/internals/network_map_db/pgsql" mgmtgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc" "github.com/netbirdio/netbird/management/server/store" "github.com/netbirdio/netbird/management/server/types" "github.com/netbirdio/netbird/management/server/types/legacynmap" + "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" "github.com/netbirdio/netbird/shared/management/proto" ) @@ -62,7 +68,6 @@ const ( type equivStats struct { accounts int peersChecked int - skippedNilNM int } func TestNetworkMapProtoEquivalence(t *testing.T) { @@ -81,6 +86,10 @@ func TestNetworkMapProtoEquivalence(t *testing.T) { require.NoError(t, err, "connect to postgres") t.Cleanup(func() { testStore.Close(ctx) }) + nmStore, err := networkmap_pgsql.NewPostgresqlStore(ctx, dsn) + require.NoError(t, err, "connect nmdata store") + t.Cleanup(func() { nmStore.Pool.Close() }) + accountIDs := equivAccountIDs(t, dsn) require.NotEmpty(t, accountIDs, "no accounts selected") @@ -94,7 +103,7 @@ func TestNetworkMapProtoEquivalence(t *testing.T) { continue } - checkAccount(ctx, t, account, maxPeers, stats) + checkAccount(ctx, t, nmStore, account, maxPeers, stats) account = nil debug.FreeOSMemory() @@ -106,19 +115,22 @@ func TestNetworkMapProtoEquivalence(t *testing.T) { } } - t.Logf("equivalence: accounts=%d peers_checked=%d skipped_nil_nm=%d — no divergence", - stats.accounts, stats.peersChecked, stats.skippedNilNM) + t.Logf("equivalence: accounts=%d peers_checked=%d — no divergence", + stats.accounts, stats.peersChecked) } // checkAccount compares both paths for every peer of one account. Nothing is // retained across peers, so memory stays flat within an account. -func checkAccount(ctx context.Context, t *testing.T, account *types.Account, maxPeers int, stats *equivStats) { +func checkAccount(ctx context.Context, t *testing.T, nmStore *networkmap_pgsql.PgStore, account *types.Account, maxPeers int, stats *equivStats) { t.Helper() if len(account.Peers) == 0 { return } + nmData, err := nmStore.GetNetworkMapData(ctx, account.Id) + require.NoError(t, err, "account %s: nmdata store load", account.Id) + validated := make(map[string]struct{}, len(account.Peers)) peerIDs := make([]string, 0, len(account.Peers)) for peerID := range account.Peers { @@ -130,6 +142,15 @@ func checkAccount(ctx context.Context, t *testing.T, account *types.Account, max peerIDs = peerIDs[:maxPeers] } + // Production fills ValidatedPeers via the integrated-validator wrapper; here + // every peer counts as validated, matching the legacy side's map. + nmData.ValidatedPeers = validated + // The legacy side receives no account zones (main sourced them from the + // external zones manager), so the DB-sourced zone candidates must be dropped + // to keep the comparison surface identical. + nmData.AppliedZoneCandidates = nil + nmData.PrivateServiceCandidates = nil + resourcePolicies := account.GetResourcePoliciesMap() routers := account.GetResourceRoutersMap() groupUsers := account.GetActiveGroupUsers() @@ -144,18 +165,32 @@ func checkAccount(ctx context.Context, t *testing.T, account *types.Account, max if peer == nil { continue } + dataPeer := nmData.Peers[peerID] + if dataPeer == nil { + t.Fatalf("after %d peers: account=%s peer=%s present in account store, missing in nmdata store", stats.peersChecked, account.Id, peerID) + } - // NEW PATH — this branch, through the production conversion. - newNM := account.GetPeerNetworkMapFromComponents( + // STORE PATH — nmdata store through the production computation, mirroring + // the controller's networkMapFromData. + components := nmData.GetPeerNetworkMapComponents(peerID, nmdata.CustomZone{}) + storeNM := &types.NetworkMap{Network: components.Network} + if !components.IsEmpty() { + storeNM = types.CalculateNetworkMapFromComponents(ctx, components) + } + // A separate cache per side: sharing one would let the first path + // populate entries the second then reuses, which can mask a real diff. + storeProto := mgmtgrpc.ToSyncResponse( + ctx, nil, nil, nil, dataPeer, nil, nil, storeNM, equivDNSName, nil, + &cache.DNSConfigCache{}, nmData.AccountSettings, settings.Extra, nil, 0, + ).NetworkMap + + // ACCOUNT PATH — Account → toNetworkMapData twins → components. + acctNM := account.GetPeerNetworkMapFromComponents( ctx, peerID, nbdns.CustomZone{}, nil, validated, resourcePolicies, routers, nil, groupUsers, ) - if newNM == nil { - stats.skippedNilNM++ - continue - } - newProto := mgmtgrpc.ToSyncResponse( - ctx, nil, nil, nil, peer, nil, nil, newNM, equivDNSName, nil, - &cache.DNSConfigCache{}, settings, settings.Extra, nil, 0, + acctProto := mgmtgrpc.ToSyncResponse( + ctx, nil, nil, nil, types.TwinPeer(peer), nil, nil, acctNM, equivDNSName, nil, + &cache.DNSConfigCache{}, types.TwinAccountSettings(settings), settings.Extra, nil, 0, ).NetworkMap // LEGACY PATH — main's frozen copy. @@ -165,18 +200,20 @@ func checkAccount(ctx context.Context, t *testing.T, account *types.Account, max if legacyNM == nil { t.Fatalf("after %d peers: account=%s peer=%s legacy NetworkMap nil, new non-nil", stats.peersChecked, account.Id, peerID) } - // A separate cache per side: sharing one would let the first path - // populate entries the second then reuses, which can mask a real diff. legacyProto := legacynmap.ToProtoNetworkMap( ctx, peer, legacyNM, equivDNSName, settings, nil, &cache.DNSConfigCache{}, 0, ) canonicalize(legacyProto) - canonicalize(newProto) + canonicalize(storeProto) + canonicalize(acctProto) stats.peersChecked++ - if !goproto.Equal(legacyProto, newProto) { - t.Fatalf("after %d peers: %s", stats.peersChecked, describeDivergence(legacyProto, newProto, account.Id, peerID)) + if !goproto.Equal(legacyProto, storeProto) { + t.Fatalf("after %d peers: store path: %s", stats.peersChecked, describeDivergence(legacyProto, storeProto, account.Id, peerID)) + } + if !goproto.Equal(legacyProto, acctProto) { + t.Fatalf("after %d peers: account path: %s", stats.peersChecked, describeDivergence(legacyProto, acctProto, account.Id, peerID)) } } } From 02138dffdd896a2dc45c6e8f6a5b7721c30ac07b Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 22:24:28 +0200 Subject: [PATCH 04/11] fix group id mapping --- management/internals/network_map_db/pgsql/network_map_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/internals/network_map_db/pgsql/network_map_data.go b/management/internals/network_map_db/pgsql/network_map_data.go index 715529631..41be6cb22 100644 --- a/management/internals/network_map_db/pgsql/network_map_data.go +++ b/management/internals/network_map_db/pgsql/network_map_data.go @@ -117,7 +117,7 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne DNSSettings: &dnsSettings, Network: &network, Peers: toMap(peers, func(p nmdata.Peer) string { return p.ID }), - Groups: toMap(groups, func(g nmdata.Group) string { return g.PublicID }), + Groups: toMap(groups, func(g nmdata.Group) string { return g.ID }), Policies: toSliceOfPtrs(policies), ResourcePolicies: resourcePolicies, Routes: toSliceOfPtrs(routes), From a00c5164a8d1a19a0631e9f08a7d680674ed9c73 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 22:29:47 +0200 Subject: [PATCH 05/11] continue on resource policy loop + exclude disabled policies form index --- .../network_map_db/pgsql/network_map_data.go | 2 +- .../internals/network_map_db/pgsql/policy.go | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/management/internals/network_map_db/pgsql/network_map_data.go b/management/internals/network_map_db/pgsql/network_map_data.go index 41be6cb22..e94444908 100644 --- a/management/internals/network_map_db/pgsql/network_map_data.go +++ b/management/internals/network_map_db/pgsql/network_map_data.go @@ -94,7 +94,7 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne } if _, ok := policyToDestinationResourceIdx[policy.ID][resource.ID]; ok { resourcePolicies[resource.ID] = append(resourcePolicies[resource.ID], &policy) // TODO (dmitri) maybe use public id? - break + continue } if groupIds, ok := policyToDestinationGroupIdx[policy.ID]; ok { for networkResourceGroup := range networkResourceGroups { diff --git a/management/internals/network_map_db/pgsql/policy.go b/management/internals/network_map_db/pgsql/policy.go index ce7a8f9d0..72aecaed8 100644 --- a/management/internals/network_map_db/pgsql/policy.go +++ b/management/internals/network_map_db/pgsql/policy.go @@ -86,11 +86,13 @@ func GetPoliciesViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId s return toret, nil, nil, err } - for _, dst := range pr().Destinations { - if _, ok := policyToDestinationGroupIdx[p.ID]; !ok { - policyToDestinationGroupIdx[p.ID] = make(map[string]any) + if p.RuleEnabled.Valid && p.RuleEnabled.Bool { + for _, dst := range pr().Destinations { + if _, ok := policyToDestinationGroupIdx[p.ID]; !ok { + policyToDestinationGroupIdx[p.ID] = make(map[string]any) + } + policyToDestinationGroupIdx[p.ID][dst] = struct{}{} } - policyToDestinationGroupIdx[p.ID][dst] = struct{}{} } } if len(p.SourceResource) > 0 { @@ -105,10 +107,12 @@ func GetPoliciesViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId s return toret, nil, nil, err } - if _, ok := policyToDestinationResourceIdx[p.ID]; !ok { - policyToDestinationResourceIdx[p.ID] = make(map[string]any) + if p.RuleEnabled.Valid && p.RuleEnabled.Bool { + if _, ok := policyToDestinationResourceIdx[p.ID]; !ok { + policyToDestinationResourceIdx[p.ID] = make(map[string]any) + } + policyToDestinationResourceIdx[p.ID][pr().DestinationResource.ID] = struct{}{} } - policyToDestinationResourceIdx[p.ID][pr().DestinationResource.ID] = struct{}{} } if len(p.Ports) > 0 { err := json.Unmarshal([]byte(p.Ports), &pr().Ports) From 23579e4dd42b1e144e464bb79d948eb1bd3af6ad Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 22:38:12 +0200 Subject: [PATCH 06/11] Skip the left-join NULL row instead of failing the account load + drop unsupported record types silently to match buildAppliedZoneCandidates --- .../internals/network_map_db/pgsql/dns.go | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/management/internals/network_map_db/pgsql/dns.go b/management/internals/network_map_db/pgsql/dns.go index 182aeeeca..05d3e75d2 100644 --- a/management/internals/network_map_db/pgsql/dns.go +++ b/management/internals/network_map_db/pgsql/dns.go @@ -49,6 +49,10 @@ func GetAppliedZoneCandidatesViaPgxConnection(ctx context.Context, conn *pgx.Con toret := make([]networkmap.AppliedZoneCandidate, 0, len(zones)) currentZoneId := "" for _, z := range zones { + if !z.RecordType.Valid { + continue + } + zone := nmdata.CustomZone{} err := networkmapdb.FromSqlTypesToSharedTypes( reflect.ValueOf(&z), reflect.ValueOf(&zone)) @@ -61,33 +65,28 @@ func GetAppliedZoneCandidatesViaPgxConnection(ctx context.Context, conn *pgx.Con return nil, err } + if z.Id != currentZoneId { + zone.Records = []nmdata.SimpleRecord{} + toret = append(toret, appliedZoneCandidateFromZone(zone, distributionGroups)) + currentZoneId = z.Id + } + rtype, rdata, err := recordTypeAndRdata(z.RecordType.String, z.RecordRData.String) if err != nil { + if errors.Is(err, DnsUnsupportedRecordTypeError) { + continue + } return nil, err } - record := nmdata.SimpleRecord{ + + lastZone := &toret[len(toret)-1] + lastZone.Zone.Records = append(lastZone.Zone.Records, nmdata.SimpleRecord{ Name: z.RecordName.String, Class: z.RecordClass.String, TTL: int(z.RecordTTL.Int64), RData: rdata, Type: rtype, - } - zone.Records = []nmdata.SimpleRecord{record} - - if len(toret) == 0 { - toret = append(toret, appliedZoneCandidateFromZone(zone, distributionGroups)) - currentZoneId = z.Id - continue - } - - if z.Id == currentZoneId { - lastZone := &toret[len(toret)-1] - lastZone.Zone.Records = append(lastZone.Zone.Records, record) - continue - } - - toret = append(toret, appliedZoneCandidateFromZone(zone, distributionGroups)) - currentZoneId = z.Id + }) } return toret, nil } From d5ac70d8067e64fd61ba48b9b021bf84d3c9f459 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 22:50:06 +0200 Subject: [PATCH 07/11] fix missing ForceRoutingPeerDNSResolution on the nmdata store path --- .../network_map_db/pgsql/network_map_data.go | 6 ++ .../internals/network_map_db/pgsql/service.go | 26 +++++++ management/server/types/account.go | 28 -------- management/server/types/account_components.go | 6 +- .../server/types/account_networkmapdata.go | 1 + .../networkmap/networkmapcompute.go | 68 +++++++++++++------ .../management/networkmap/networkmapdata.go | 5 ++ 7 files changed, 88 insertions(+), 52 deletions(-) diff --git a/management/internals/network_map_db/pgsql/network_map_data.go b/management/internals/network_map_db/pgsql/network_map_data.go index e94444908..6d150ca30 100644 --- a/management/internals/network_map_db/pgsql/network_map_data.go +++ b/management/internals/network_map_db/pgsql/network_map_data.go @@ -81,6 +81,10 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne if err != nil { return rollbackAndReturnError(ctx, tx, err) } + proxyTargetedDomainResourceIDs, err := GetProxyTargetedDomainResourceIDsViaPgxConnection(ctx, tx.Conn(), accountId) + if err != nil { + return rollbackAndReturnError(ctx, tx, fmt.Errorf("failed to get proxy targeted domain resources: %w", err)) + } resourcePolicies := make(map[string][]*nmdata.Policy) for _, resource := range networkResources { @@ -131,6 +135,8 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne AppliedZoneCandidates: dnsZones, PrivateServiceCandidates: buildPrivateServiceCandidates(services, domains, proxyPeers), PostureCheckXIDToPublicID: postureCheckXIDToPublicID, + + ProxyTargetedDomainResourceIDs: proxyTargetedDomainResourceIDs, } return &toret, nil diff --git a/management/internals/network_map_db/pgsql/service.go b/management/internals/network_map_db/pgsql/service.go index d4ad31219..add69a208 100644 --- a/management/internals/network_map_db/pgsql/service.go +++ b/management/internals/network_map_db/pgsql/service.go @@ -13,6 +13,14 @@ const ( from services where account_id=$1 ` + + GetProxyTargetedDomainResourcesQuery = ` + select t.target_id + from targets as t + join services as s on s.id = t.service_id + where s.account_id=$1 and s.enabled and not coalesce(s.terminated, false) + and t.enabled and t.target_type='domain' and t.target_id is not null + ` ) func (pg *PgStore) GetPrivateServices(ctx context.Context, accountId string) ([]service, error) { @@ -32,6 +40,24 @@ func GetPrivateServicesViaPgxConnection(ctx context.Context, conn *pgx.Conn, acc return pgx.CollectRows(rows, pgx.RowToStructByName[service]) } +func GetProxyTargetedDomainResourceIDsViaPgxConnection(ctx context.Context, conn *pgx.Conn, accountId string) (map[string]struct{}, error) { + rows, err := conn.Query(ctx, GetProxyTargetedDomainResourcesQuery, accountId) + if err != nil { + return nil, err + } + + ids, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, err + } + + toret := make(map[string]struct{}, len(ids)) + for _, id := range ids { + toret[id] = struct{}{} + } + return toret, nil +} + type service struct { Enabled sql.NullBool Private sql.NullBool diff --git a/management/server/types/account.go b/management/server/types/account.go index 54f1f21b0..4d726acbc 100644 --- a/management/server/types/account.go +++ b/management/server/types/account.go @@ -978,34 +978,6 @@ func (a *Account) GetPeerConnectionResources(ctx context.Context, peer *nbpeer.P return peers, fwRules, authorizedUsers, sshEnabled } -// forcesRoutingPeerDNSResolution reports whether the given peer must run -// routing-peer DNS resolution regardless of the account-global -// RoutingPeerDNSResolutionEnabled setting. It returns true when the peer is a -// router for a domain network resource that is targeted by an enabled -// reverse-proxy service, so the peer's DNS forwarder starts and can resolve -// the target for the embedded proxy peers. Embedded proxy peers themselves are -// handled at PeerConfig build time. -func (a *Account) forcesRoutingPeerDNSResolution(peerID string, routers map[string]map[string]*routerTypes.NetworkRouter) bool { - targeted := a.proxyTargetedDomainResourceIDs() - if len(targeted) == 0 { - return false - } - - for _, resource := range a.NetworkResources { - if resource == nil || !resource.Enabled || resource.Type != resourceTypes.Domain { - continue - } - if _, ok := targeted[resource.ID]; !ok { - continue - } - if _, isRouter := routers[resource.NetworkID][peerID]; isRouter { - return true - } - } - - return false -} - // proxyTargetedDomainResourceIDs returns the set of domain network resource IDs // targeted by an enabled, non-terminated reverse-proxy service. func (a *Account) proxyTargetedDomainResourceIDs() map[string]struct{} { diff --git a/management/server/types/account_components.go b/management/server/types/account_components.go index 1a4f87ef0..f95ad817d 100644 --- a/management/server/types/account_components.go +++ b/management/server/types/account_components.go @@ -104,9 +104,5 @@ func (a *Account) GetPeerNetworkMapComponents( groupIDToUserIDs map[string][]string, ) *NetworkMapComponents { nmd := a.toNetworkMapData(accountZones, validatedPeersMap, resourcePolicies, routers, groupIDToUserIDs) - components := nmd.GetPeerNetworkMapComponents(peerID, TwinCustomZone(peersCustomZone)) - if components != nil { - components.ForceRoutingPeerDNSResolution = a.forcesRoutingPeerDNSResolution(peerID, routers) - } - return components + return nmd.GetPeerNetworkMapComponents(peerID, TwinCustomZone(peersCustomZone)) } diff --git a/management/server/types/account_networkmapdata.go b/management/server/types/account_networkmapdata.go index 1613628da..5ec15aecd 100644 --- a/management/server/types/account_networkmapdata.go +++ b/management/server/types/account_networkmapdata.go @@ -108,6 +108,7 @@ func (a *Account) toNetworkMapData( nmd.Routers[networkID] = twinInner } + nmd.ProxyTargetedDomainResourceIDs = a.proxyTargetedDomainResourceIDs() nmd.AppliedZoneCandidates = buildAppliedZoneCandidates(accountZones) nmd.PrivateServiceCandidates = a.buildPrivateServiceCandidates() diff --git a/shared/management/networkmap/networkmapcompute.go b/shared/management/networkmap/networkmapcompute.go index c5f126228..0d4cf4b6f 100644 --- a/shared/management/networkmap/networkmapcompute.go +++ b/shared/management/networkmap/networkmapcompute.go @@ -17,37 +17,42 @@ type sshRequirements struct { // exactly, operating on nmdata twins throughout — no Account reference and no // twin↔real conversion, since the produced components hold twins. func (nmd *NetworkMapData) GetPeerNetworkMapComponents(peerID string, peersCustomZone nmdata.CustomZone) *types.NetworkMapComponents { + forceRoutingPeerDNS := nmd.forcesRoutingPeerDNSResolution(peerID) + peer := nmd.Peers[peerID] if peer == nil { return types.EmptyNetworkMapComponents(&types.NetworkMapComponents{ - PeerID: peerID, - Network: nmd.Network, - Peers: map[string]*nmdata.Peer{peerID: peer}, + PeerID: peerID, + Network: nmd.Network, + Peers: map[string]*nmdata.Peer{peerID: peer}, + ForceRoutingPeerDNSResolution: forceRoutingPeerDNS, }) } if _, ok := nmd.ValidatedPeers[peerID]; !ok { return types.EmptyNetworkMapComponents(&types.NetworkMapComponents{ - PeerID: peerID, - Network: nmd.Network, - Peers: map[string]*nmdata.Peer{peerID: peer}, + PeerID: peerID, + Network: nmd.Network, + Peers: map[string]*nmdata.Peer{peerID: peer}, + ForceRoutingPeerDNSResolution: forceRoutingPeerDNS, }) } components := &types.NetworkMapComponents{ - PeerID: peerID, - Network: nmd.Network, - AccountSettings: nmd.AccountSettings, - DNSSettings: nmd.DNSSettings, - CustomZoneDomain: peersCustomZone.Domain, - NameServerGroups: make([]*nmdata.NameServerGroup, 0), - ResourcePoliciesMap: make(map[string][]*nmdata.Policy), - RoutersMap: make(map[string]map[string]*nmdata.NetworkRouter), - NetworkResources: make([]*nmdata.NetworkResource, 0), - PostureFailedPeers: make(map[string]map[string]struct{}, len(nmd.PostureChecks)), - RouterPeers: make(map[string]*nmdata.Peer), - NetworkXIDToPublicID: nmd.NetworkXIDToPublicID, - PostureCheckXIDToPublicID: nmd.PostureCheckXIDToPublicID, + PeerID: peerID, + Network: nmd.Network, + AccountSettings: nmd.AccountSettings, + DNSSettings: nmd.DNSSettings, + CustomZoneDomain: peersCustomZone.Domain, + NameServerGroups: make([]*nmdata.NameServerGroup, 0), + ResourcePoliciesMap: make(map[string][]*nmdata.Policy), + RoutersMap: make(map[string]map[string]*nmdata.NetworkRouter), + NetworkResources: make([]*nmdata.NetworkResource, 0), + PostureFailedPeers: make(map[string]map[string]struct{}, len(nmd.PostureChecks)), + RouterPeers: make(map[string]*nmdata.Peer), + NetworkXIDToPublicID: nmd.NetworkXIDToPublicID, + PostureCheckXIDToPublicID: nmd.PostureCheckXIDToPublicID, + ForceRoutingPeerDNSResolution: forceRoutingPeerDNS, } relevantPeers, relevantGroups, relevantPolicies, relevantRoutes, sshReqs := nmd.getPeersGroupsPoliciesRoutes(peerID, peer.SSHEnabled, &components.PostureFailedPeers) @@ -473,6 +478,31 @@ func (nmd *NetworkMapData) getPostureValidPeersSaveFailed(inputPeers []string, p return dest } +// forcesRoutingPeerDNSResolution reports whether the given peer must run +// routing-peer DNS resolution regardless of the account-global +// RoutingPeerDNSResolutionEnabled setting: true when the peer routes a domain +// network resource targeted by an enabled reverse-proxy service, so the peer's +// DNS forwarder starts and can resolve the target for the embedded proxy peers. +func (nmd *NetworkMapData) forcesRoutingPeerDNSResolution(peerID string) bool { + if len(nmd.ProxyTargetedDomainResourceIDs) == 0 { + return false + } + + for _, resource := range nmd.NetworkResources { + if resource == nil || !resource.Enabled || resource.Type != string(types.ResourceTypeDomain) { + continue + } + if _, ok := nmd.ProxyTargetedDomainResourceIDs[resource.ID]; !ok { + continue + } + if _, isRouter := nmd.Routers[resource.NetworkID][peerID]; isRouter { + return true + } + } + + return false +} + func (nmd *NetworkMapData) GetPeerGroups(peerID string) map[string]struct{} { groups := make(map[string]struct{}) for groupID, group := range nmd.Groups { diff --git a/shared/management/networkmap/networkmapdata.go b/shared/management/networkmap/networkmapdata.go index 9193daa41..272f6152a 100644 --- a/shared/management/networkmap/networkmapdata.go +++ b/shared/management/networkmap/networkmapdata.go @@ -32,6 +32,11 @@ type NetworkMapData struct { GroupIDToUserIDs map[string][]string DNSDomain string + // ProxyTargetedDomainResourceIDs is the account-level half of + // forcesRoutingPeerDNSResolution: domain network resources targeted by an + // enabled reverse-proxy service. + ProxyTargetedDomainResourceIDs map[string]struct{} + AppliedZoneCandidates []AppliedZoneCandidate PrivateServiceCandidates []PrivateServiceCandidate } From d4e1c8978e20f7eb99377284fd3b037e626826c7 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 23:08:43 +0200 Subject: [PATCH 08/11] add all group to user group lookup --- .../internals/network_map_db/pgsql/user.go | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/management/internals/network_map_db/pgsql/user.go b/management/internals/network_map_db/pgsql/user.go index ede46ec96..b75fe9cb6 100644 --- a/management/internals/network_map_db/pgsql/user.go +++ b/management/internals/network_map_db/pgsql/user.go @@ -8,10 +8,15 @@ import ( const ( GetAllowedUserIdsQuery = ` - select id, array (select json_array_elements_text(auto_groups::json)) as auto_groups + select id, array (select json_array_elements_text(auto_groups::json)) as auto_groups from users where account_id=$1 and not blocked and not is_service_user ` + + GetAllGroupIdQuery = ` + select id from groups + where account_id=$1 and name='All' + ` ) func (pg *PgStore) GetAllowedUsers(ctx context.Context, accountId string) (map[string]struct{}, map[string][]string, error) { @@ -33,6 +38,19 @@ func GetAllowedUsersViaPgxConnection(ctx context.Context, con *pgx.Conn, account return nil, nil, err } + rows, err = con.Query(ctx, GetAllGroupIdQuery, accountId) + if err != nil { + return nil, nil, err + } + allGroupIds, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, nil, err + } + allGroupId := "" + if len(allGroupIds) > 0 { + allGroupId = allGroupIds[0] + } + userIdIdx := make(map[string]struct{}) groupIdToUserIds := make(map[string][]string) for _, user := range users { @@ -40,6 +58,9 @@ func GetAllowedUsersViaPgxConnection(ctx context.Context, con *pgx.Conn, account for _, groupId := range user.AutoGroups { groupIdToUserIds[groupId] = append(groupIdToUserIds[groupId], user.ID) } + if allGroupId != "" { + groupIdToUserIds[allGroupId] = append(groupIdToUserIds[allGroupId], user.ID) + } } return userIdIdx, groupIdToUserIds, nil From 5ed38569f78c258684569592eb999c1f776abcf7 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 23:19:55 +0200 Subject: [PATCH 09/11] fix PrivateServiceCandidates nil assumption --- management/server/types/legacynmap/equivalence_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/management/server/types/legacynmap/equivalence_test.go b/management/server/types/legacynmap/equivalence_test.go index f80734165..a7785d330 100644 --- a/management/server/types/legacynmap/equivalence_test.go +++ b/management/server/types/legacynmap/equivalence_test.go @@ -146,10 +146,10 @@ func checkAccount(ctx context.Context, t *testing.T, nmStore *networkmap_pgsql.P // every peer counts as validated, matching the legacy side's map. nmData.ValidatedPeers = validated // The legacy side receives no account zones (main sourced them from the - // external zones manager), so the DB-sourced zone candidates must be dropped - // to keep the comparison surface identical. + // external zones manager), so the DB-sourced applied-zone candidates must be + // dropped to keep the comparison surface identical. PrivateServiceCandidates + // stay: all paths derive them from account/DB data. nmData.AppliedZoneCandidates = nil - nmData.PrivateServiceCandidates = nil resourcePolicies := account.GetResourcePoliciesMap() routers := account.GetResourceRoutersMap() From ef0032685b0aa39283fb33c9a60edd6a32f8f1e7 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 23:54:22 +0200 Subject: [PATCH 10/11] fix nmdata store shipping zones for disabled/public services --- management/internals/network_map_db/pgsql/network_map_data.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/management/internals/network_map_db/pgsql/network_map_data.go b/management/internals/network_map_db/pgsql/network_map_data.go index 6d150ca30..3f4b89eee 100644 --- a/management/internals/network_map_db/pgsql/network_map_data.go +++ b/management/internals/network_map_db/pgsql/network_map_data.go @@ -198,6 +198,9 @@ func buildPrivateServiceCandidates(svcs []service, domains []domain, proxyPeersB } for _, svc := range svcs { + if !svc.Enabled.Bool || !svc.Private.Bool { + continue + } if len(svc.AccessGroups) == 0 { continue } From 6fadf8f24a8d533ab98fd2e41e8de4b7c7d0f3c3 Mon Sep 17 00:00:00 2001 From: pascal Date: Mon, 3 Aug 2026 23:58:47 +0200 Subject: [PATCH 11/11] fix nmdata store expanding router peer groups alongside static peer --- .../network_map_db/pgsql/network_router.go | 1 + .../types/legacynmap/equivalence_test.go | 48 ++++++++++++++++--- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/management/internals/network_map_db/pgsql/network_router.go b/management/internals/network_map_db/pgsql/network_router.go index 8d6b594fc..82cad224b 100644 --- a/management/internals/network_map_db/pgsql/network_router.go +++ b/management/internals/network_map_db/pgsql/network_router.go @@ -65,6 +65,7 @@ func GetNetworkRoutersViaPgxConnection(ctx context.Context, con *pgx.Conn, accou } if router.Peer.String != "" { toret[networkId][router.Peer.String] = &nmdatarouter + continue } for _, peerId := range router.PeersViaGroups { toret[networkId][peerId] = &nmdatarouter diff --git a/management/server/types/legacynmap/equivalence_test.go b/management/server/types/legacynmap/equivalence_test.go index a7785d330..79eafd784 100644 --- a/management/server/types/legacynmap/equivalence_test.go +++ b/management/server/types/legacynmap/equivalence_test.go @@ -545,17 +545,18 @@ func describeDivergence(legacy, updated *proto.NetworkMap, accountID, peerID str lens := []struct { field string a, b int + diff func() string }{ - {"RemotePeers", len(legacy.RemotePeers), len(updated.RemotePeers)}, - {"OfflinePeers", len(legacy.OfflinePeers), len(updated.OfflinePeers)}, - {"Routes", len(legacy.Routes), len(updated.Routes)}, - {"FirewallRules", len(legacy.FirewallRules), len(updated.FirewallRules)}, - {"RoutesFirewallRules", len(legacy.RoutesFirewallRules), len(updated.RoutesFirewallRules)}, - {"ForwardingRules", len(legacy.ForwardingRules), len(updated.ForwardingRules)}, + {"RemotePeers", len(legacy.RemotePeers), len(updated.RemotePeers), func() string { return diffLists(legacy.RemotePeers, updated.RemotePeers) }}, + {"OfflinePeers", len(legacy.OfflinePeers), len(updated.OfflinePeers), func() string { return diffLists(legacy.OfflinePeers, updated.OfflinePeers) }}, + {"Routes", len(legacy.Routes), len(updated.Routes), func() string { return diffLists(legacy.Routes, updated.Routes) }}, + {"FirewallRules", len(legacy.FirewallRules), len(updated.FirewallRules), func() string { return diffLists(legacy.FirewallRules, updated.FirewallRules) }}, + {"RoutesFirewallRules", len(legacy.RoutesFirewallRules), len(updated.RoutesFirewallRules), func() string { return diffLists(legacy.RoutesFirewallRules, updated.RoutesFirewallRules) }}, + {"ForwardingRules", len(legacy.ForwardingRules), len(updated.ForwardingRules), func() string { return diffLists(legacy.ForwardingRules, updated.ForwardingRules) }}, } for _, l := range lens { if l.a != l.b { - return prefix + " field=" + l.field + " legacy_len=" + strconv.Itoa(l.a) + " new_len=" + strconv.Itoa(l.b) + return prefix + " field=" + l.field + " legacy_len=" + strconv.Itoa(l.a) + " new_len=" + strconv.Itoa(l.b) + l.diff() } } @@ -594,6 +595,39 @@ func describeDivergence(legacy, updated *proto.NetworkMap, accountID, peerID str return prefix + " (repeated fields equal element-wise — scalar/oneof mismatch)" } +// diffLists reports the multiset difference of two repeated proto fields, so a +// length mismatch shows which elements each side is missing. +func diffLists[M goproto.Message](legacy, updated []M) string { + counts := make(map[string]int) + for _, m := range legacy { + counts[prototext.MarshalOptions{}.Format(m)]++ + } + for _, m := range updated { + counts[prototext.MarshalOptions{}.Format(m)]-- + } + + var onlyLegacy, onlyNew []string + for k, c := range counts { + for ; c > 0; c-- { + onlyLegacy = append(onlyLegacy, k) + } + for ; c < 0; c++ { + onlyNew = append(onlyNew, k) + } + } + slices.Sort(onlyLegacy) + slices.Sort(onlyNew) + + var b strings.Builder + for _, k := range onlyLegacy { + b.WriteString("\n only_legacy: " + k) + } + for _, k := range onlyNew { + b.WriteString("\n only_new: " + k) + } + return b.String() +} + func protoStr(m goproto.Message) string { if m == nil { return ""