From 206bb1676b49c1de0807dc1920642570823d6ca6 Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Mon, 3 Aug 2026 10:53:54 +0200 Subject: [PATCH] build resourcePolicies map Signed-off-by: Dmitri Dolguikh --- .../internals/network_map_db/db_store.go | 8 ++-- .../internals/network_map_db/pgsql/group.go | 19 ++++++--- .../network_map_db/pgsql/group_test.go | 6 ++- .../network_map_db/pgsql/network_map_data.go | 30 ++++++++++++- .../internals/network_map_db/pgsql/policy.go | 42 ++++++++++++------- 5 files changed, 78 insertions(+), 27 deletions(-) diff --git a/management/internals/network_map_db/db_store.go b/management/internals/network_map_db/db_store.go index e32a2008c..ffbc63b0e 100644 --- a/management/internals/network_map_db/db_store.go +++ b/management/internals/network_map_db/db_store.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/netbirdio/netbird/management/server/integrations/integrated_validator" "github.com/netbirdio/netbird/shared/management/networkmap" "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" ) @@ -19,9 +20,9 @@ const ( ) type NetworkMapDBStore interface { - GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, map[string][]*nmdata.Group, error) + GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, map[string]map[string]any, error) GetPeers(ctx context.Context, accountId string) ([]nmdata.Peer, error) - GetPolicies(ctx context.Context, accountId string) ([]nmdata.Policy, error) + GetPolicies(ctx context.Context, accountId string) ([]nmdata.Policy, map[string]map[string]any, map[string]map[string]any, error) GetRoutes(ctx context.Context, accountId string) ([]nmdata.Route, error) GetNameServerGroups(ctx context.Context, accountId string) ([]nmdata.NameServerGroup, error) GetNetworkResources(ctx context.Context, accountId string) ([]nmdata.NetworkResource, error) @@ -34,7 +35,8 @@ type NetworkMapDBStore interface { } type NetworkMapDBStoreImpl struct { - store NetworkMapDBStore + store NetworkMapDBStore + integratedPeerValidator integrated_validator.IntegratedValidator } func FromSqlTypesToSharedTypes(src reflect.Value, dst reflect.Value) error { diff --git a/management/internals/network_map_db/pgsql/group.go b/management/internals/network_map_db/pgsql/group.go index f33de66b9..eda43dd82 100644 --- a/management/internals/network_map_db/pgsql/group.go +++ b/management/internals/network_map_db/pgsql/group.go @@ -13,7 +13,7 @@ import ( const ( GetGroupsQuery = ` - select name, public_id, resources, + select id, name, public_id, resources, ( select array_agg(group_peers.peer_id) from group_peers @@ -23,7 +23,10 @@ const ( ` ) -func (pg *PgStore) GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, map[string][]*nmdata.Group, error) { +// we also return a resource-to-group index. +// an alternative is to add json indexes, query this directly. Not sure how expensive +// json indexes are. TODO (dmitri) verify and maybe change the implementation here. +func (pg *PgStore) GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, map[string]map[string]any, error) { c, err := pg.Pool.Acquire(ctx) if err != nil { return nil, nil, err @@ -31,7 +34,7 @@ func (pg *PgStore) GetGroups(ctx context.Context, accountId string) ([]nmdata.Gr return GetGroupsViaPgxConnection(ctx, c.Conn(), accountId) } -func GetGroupsViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId string) ([]nmdata.Group, map[string][]*nmdata.Group, error) { +func GetGroupsViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId string) ([]nmdata.Group, map[string]map[string]any, error) { rows, err := con.Query(ctx, GetGroupsQuery, accountId) if err != nil { return nil, nil, err @@ -39,7 +42,7 @@ func GetGroupsViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId str groups, err := pgx.CollectRows(rows, pgx.RowToStructByName[group]) toret := make([]nmdata.Group, 0, len(groups)) - toretidx := make(map[string][]*nmdata.Group) + resourceToGroupIdx := make(map[string]map[string]any) for _, g := range groups { dg := nmdata.Group{} @@ -50,14 +53,18 @@ func GetGroupsViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId str } toret = append(toret, dg) for _, resource := range dg.Resources { - toretidx[resource.ID] = append(toretidx[resource.ID], &dg) + if _, ok := resourceToGroupIdx[resource.ID]; !ok { + resourceToGroupIdx[resource.ID] = make(map[string]any) + } + resourceToGroupIdx[resource.ID][g.Id] = struct{}{} } } - return toret, toretidx, err + return toret, resourceToGroupIdx, err } type group struct { + Id string `nmap:"skip"` Name sql.NullString PublicID sql.NullString Resources json.RawMessage diff --git a/management/internals/network_map_db/pgsql/group_test.go b/management/internals/network_map_db/pgsql/group_test.go index 3bc75c10c..e5f3bf0f8 100644 --- a/management/internals/network_map_db/pgsql/group_test.go +++ b/management/internals/network_map_db/pgsql/group_test.go @@ -25,7 +25,7 @@ func TestGetGroups(t *testing.T) { _, err = s.Pool.Query(ctx, "insert into groups (id, account_id, name, resources, public_id) VALUES('test-group-id-1','ck7bnf2t2r9s739pkug0','test-group-1', '[{\"ID\":\"cui7q2jl0ubs73d8qpi0\",\"Type\":\"host\"}]','public-id-1')") assert.NoError(t, err) - groups, err := s.GetGroups(ctx, "ck7bnf2t2r9s739pkug0") //"ckd7ee2fic3c73dtendg") + groups, _, err := s.GetGroups(ctx, "ck7bnf2t2r9s739pkug0") //"ckd7ee2fic3c73dtendg") assert.NoError(t, err) assert.Contains(t, groups, @@ -67,8 +67,10 @@ func TestGetPolocies(t *testing.T) { // err = loadSQL(ctx, s.pool, initDb) //assert.NoError(t, err) - peers, err := s.GetPolicies(ctx, "ck7bnf2t2r9s739pkug0") //"ckd7ee2fic3c73dtendg") + peers, idx1, idx2, err := s.GetPolicies(ctx, "ck7bnf2t2r9s739pkug0") //"ckd7ee2fic3c73dtendg") assert.NoError(t, err) + assert.NotEmpty(t, idx1) + assert.NotEmpty(t, idx2) fmt.Print(peers) // assert.Contains(t, diff --git a/management/internals/network_map_db/pgsql/network_map_data.go b/management/internals/network_map_db/pgsql/network_map_data.go index 896937323..d5a785b47 100644 --- a/management/internals/network_map_db/pgsql/network_map_data.go +++ b/management/internals/network_map_db/pgsql/network_map_data.go @@ -22,7 +22,7 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne // if err != nil { // return rollbackAndReturnError(ctx, tx, err) // } - groups, netResourceToGroups, err := GetGroupsViaPgxConnection(ctx, tx.Conn(), accountId) + groups, resourceToGroupIdx, err := GetGroupsViaPgxConnection(ctx, tx.Conn(), accountId) if err != nil { return rollbackAndReturnError(ctx, tx, err) } @@ -46,7 +46,7 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne if err != nil { return rollbackAndReturnError(ctx, tx, err) } - policies, err := GetPoliciesViaPgxConnection(ctx, tx.Conn(), accountId) + policies, policyToDestinationResourceIdx, policyToDestinationGroupIdx, err := GetPoliciesViaPgxConnection(ctx, tx.Conn(), accountId) if err != nil { return rollbackAndReturnError(ctx, tx, err) } @@ -63,6 +63,31 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne return rollbackAndReturnError(ctx, tx, err) } + resourcePolicies := make(map[string][]*nmdata.Policy) + for _, resource := range networkResources { + if !resource.Enabled { + continue + } + networkResourceGroups := resourceToGroupIdx[resource.ID] + for _, policy := range policies { + if !policy.Enabled { + continue + } + if _, ok := policyToDestinationResourceIdx[policy.ID][resource.ID]; ok { + resourcePolicies[resource.ID] = append(resourcePolicies[resource.ID], &policy) // TODO (dmitri) maybe use public id? + break + } + if groupIds, ok := policyToDestinationGroupIdx[policy.ID]; ok { + for networkResourceGroup := range networkResourceGroups { + if _, ok := groupIds[networkResourceGroup]; ok { + resourcePolicies[resource.ID] = append(resourcePolicies[resource.ID], &policy) + break + } + } + } + } + } + err = tx.Commit(ctx) if err != nil { // TODO log and ignore? @@ -73,6 +98,7 @@ func (pg *PgStore) GetNetworkMapData(ctx context.Context, accountId string) (*ne Peers: toMap(peers, func(p nmdata.Peer) string { return p.ID }), Groups: toMap(groups, func(g nmdata.Group) string { return g.PublicID }), Policies: toSliceOfPtrs(policies), + ResourcePolicies: resourcePolicies, Routes: toSliceOfPtrs(routes), Routers: routers, NameServerGroups: toSliceOfPtrs(nsGroups), diff --git a/management/internals/network_map_db/pgsql/policy.go b/management/internals/network_map_db/pgsql/policy.go index 81e889d15..ce7a8f9d0 100644 --- a/management/internals/network_map_db/pgsql/policy.go +++ b/management/internals/network_map_db/pgsql/policy.go @@ -22,32 +22,34 @@ const ( ` ) -func (pg *PgStore) GetPolicies(ctx context.Context, accountId string) ([]nmdata.Policy, error) { +func (pg *PgStore) GetPolicies(ctx context.Context, accountId string) ([]nmdata.Policy, map[string]map[string]any, map[string]map[string]any, error) { c, err := pg.Pool.Acquire(ctx) if err != nil { - return nil, err + return nil, nil, nil, err } return GetPoliciesViaPgxConnection(ctx, c.Conn(), accountId) } -func GetPoliciesViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId string) ([]nmdata.Policy, error) { +func GetPoliciesViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId string) ([]nmdata.Policy, map[string]map[string]any, map[string]map[string]any, error) { rows, err := con.Query(ctx, GetPoliciesQuery, accountId) if err != nil { - return nil, err + return nil, nil, nil, err } policies, err := pgx.CollectRows(rows, pgx.RowToStructByName[policy]) if err != nil { - return nil, err + return nil, nil, nil, err } toret := make([]nmdata.Policy, 0, len(policies)) + policyToDestinationResourceIdx := make(map[string]map[string]any) // policy id to destination resource id + policyToDestinationGroupIdx := make(map[string]map[string]any) // policy id to destination group id for _, p := range policies { policy := nmdata.Policy{} err := networkmapdb.FromSqlTypesToSharedTypes( reflect.ValueOf(&p), reflect.ValueOf(&policy)) if err != nil { - return nil, err + return nil, nil, nil, err } var policyRule *nmdata.PolicyRule @@ -75,43 +77,55 @@ func GetPoliciesViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId s if len(p.Sources) > 0 { err := json.Unmarshal([]byte(p.Sources), &pr().Sources) if err != nil { - return toret, err + return toret, nil, nil, err } } if len(p.Destinations) > 0 { err := json.Unmarshal([]byte(p.Destinations), &pr().Destinations) if err != nil { - return toret, err + return toret, nil, nil, err + } + + for _, dst := range pr().Destinations { + if _, ok := policyToDestinationGroupIdx[p.ID]; !ok { + policyToDestinationGroupIdx[p.ID] = make(map[string]any) + } + policyToDestinationGroupIdx[p.ID][dst] = struct{}{} } } if len(p.SourceResource) > 0 { err := json.Unmarshal([]byte(p.SourceResource), &pr().SourceResource) if err != nil { - return toret, err + return toret, nil, nil, err } } if len(p.DestinationResource) > 0 { err := json.Unmarshal([]byte(p.DestinationResource), &pr().DestinationResource) if err != nil { - return toret, err + return toret, nil, nil, err } + + if _, ok := policyToDestinationResourceIdx[p.ID]; !ok { + policyToDestinationResourceIdx[p.ID] = make(map[string]any) + } + policyToDestinationResourceIdx[p.ID][pr().DestinationResource.ID] = struct{}{} } if len(p.Ports) > 0 { err := json.Unmarshal([]byte(p.Ports), &pr().Ports) if err != nil { - return toret, err + return toret, nil, nil, err } } if len(p.PortRanges) > 0 { err := json.Unmarshal([]byte(p.PortRanges), &pr().PortRanges) if err != nil { - return toret, err + return toret, nil, nil, err } } if len(p.AuthorizedGroups) > 0 { err := json.Unmarshal([]byte(p.AuthorizedGroups), &pr().AuthorizedGroups) if err != nil { - return toret, err + return toret, nil, nil, err } } if p.AuthorizedUser.Valid { @@ -127,7 +141,7 @@ func GetPoliciesViaPgxConnection(ctx context.Context, con *pgx.Conn, accountId s toret = append(toret, policy) } - return toret, err + return toret, policyToDestinationResourceIdx, policyToDestinationGroupIdx, err } type policy struct {