From 15003258d2ab080a098a1df911a4c5400e612c5c Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Tue, 28 Jul 2026 16:46:45 +0200 Subject: [PATCH] networkmap read-only interface for pgsql Signed-off-by: Dmitri Dolguikh --- .../internals/network_map_db/db_store.go | 15 ++ management/internals/network_map_db/group.go | 11 ++ .../internals/network_map_db/pgsql/group.go | 53 +++++++ .../network_map_db/pgsql/group_test.go | 55 +++++++ .../internals/network_map_db/pgsql/peer.go | 135 ++++++++++++++++++ .../network_map_db/pgsql/pg_store.go | 56 ++++++++ .../internals/network_map_db/pgsql/policy.go | 0 .../network_map_db/pgsql/test_db.sql | 8 ++ 8 files changed, 333 insertions(+) create mode 100644 management/internals/network_map_db/db_store.go create mode 100644 management/internals/network_map_db/group.go create mode 100644 management/internals/network_map_db/pgsql/group.go create mode 100644 management/internals/network_map_db/pgsql/group_test.go create mode 100644 management/internals/network_map_db/pgsql/peer.go create mode 100644 management/internals/network_map_db/pgsql/pg_store.go create mode 100644 management/internals/network_map_db/pgsql/policy.go create mode 100644 management/internals/network_map_db/pgsql/test_db.sql diff --git a/management/internals/network_map_db/db_store.go b/management/internals/network_map_db/db_store.go new file mode 100644 index 000000000..ad3609e87 --- /dev/null +++ b/management/internals/network_map_db/db_store.go @@ -0,0 +1,15 @@ +package networkmapdb + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" +) + +type NetworkMapDBStore interface { + GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, error) +} + +type NetworkMapDBStoreImpl struct { + store NetworkMapDBStore +} diff --git a/management/internals/network_map_db/group.go b/management/internals/network_map_db/group.go new file mode 100644 index 000000000..c307039f0 --- /dev/null +++ b/management/internals/network_map_db/group.go @@ -0,0 +1,11 @@ +package networkmapdb + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" +) + +func (db *NetworkMapDBStoreImpl) GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, error) { + return db.store.GetGroups(ctx, accountId) +} diff --git a/management/internals/network_map_db/pgsql/group.go b/management/internals/network_map_db/pgsql/group.go new file mode 100644 index 000000000..9e2ce5a34 --- /dev/null +++ b/management/internals/network_map_db/pgsql/group.go @@ -0,0 +1,53 @@ +package networkmap_pgsql + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/jackc/pgx/v5" + "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" +) + +const ( + GetGroupsQuery = ` + select name, public_id, resources from groups where account_id=$1 + ` +) + +func (pg *PgStore) GetGroups(ctx context.Context, accountId string) ([]nmdata.Group, error) { + rows, err := pg.pool.Query(ctx, GetGroupsQuery, accountId) + if err != nil { + return nil, err + } + + var ( + group nmdata.Group + name, publicId, resources sql.NullString + ) + + groups, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (nmdata.Group, error) { + err := row.Scan(&name, &publicId, &resources) + if err != nil { + return group, err + } + + if name.Valid { + group.Name = name.String + } + if publicId.Valid { + group.PublicID = publicId.String + } + if resources.Valid { + groupResources := make([]nmdata.Resource, 0) + err := json.Unmarshal([]byte(resources.String), &groupResources) + if err != nil { + return group, err + } + group.Resources = groupResources + } + return group, nil + }) + + return groups, err +} diff --git a/management/internals/network_map_db/pgsql/group_test.go b/management/internals/network_map_db/pgsql/group_test.go new file mode 100644 index 000000000..5c1c6ae1e --- /dev/null +++ b/management/internals/network_map_db/pgsql/group_test.go @@ -0,0 +1,55 @@ +package networkmap_pgsql + +import ( + "context" + "strings" + "testing" + + _ "embed" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" + "github.com/stretchr/testify/assert" +) + +//go:embed test_db.sql +var initDb string + +func TestXxx(t *testing.T) { + ctx := context.TODO() + + s, err := NewPostgresqlStore(ctx, "postgresql://root:netbird@localhost:5432/netbird") + assert.NoError(t, err) + // err = loadSQL(ctx, s.pool, initDb) + //assert.NoError(t, err) + + _, err = s.pool.Query(ctx, "insert into groups (id, account_id, name, resources, public_id) VALUES('test-group-id-1','ck7bnf2t2r9s739pkug0','test-group-1', '[{\"ID\":\"cui7q2jl0ubs73d8qpi0\",\"Type\":\"host\"}]','public-id-1')") + assert.NoError(t, err) + + groups, err := s.GetGroups(ctx, "ck7bnf2t2r9s739pkug0") + assert.NoError(t, err) + assert.Contains(t, + groups, + nmdata.Group{Name: "test-group-1", PublicID: "public-id-1", Resources: []nmdata.Resource{{ID: "cui7q2jl0ubs73d8qpi0", Type: "host"}}}, + ) + assert.Contains(t, + groups, + nmdata.Group{Name: "All", PublicID: "d9aejspvcsu517nkh4a0", Resources: []nmdata.Resource{{ID: "cui7olrl0ubs73d8qpe0", Type: "subnet"}, {ID: "cui7q2jl0ubs73d8qpi0", Type: "host"}}}, + ) +} + +func loadSQL(ctx context.Context, pool *pgxpool.Pool, initdb string) error { + queries := strings.Split(string(initdb), ";") + + for _, query := range queries { + query = strings.TrimSpace(query) + if query != "" { + _, err := pool.Query(ctx, query) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/management/internals/network_map_db/pgsql/peer.go b/management/internals/network_map_db/pgsql/peer.go new file mode 100644 index 000000000..e3b4a8d73 --- /dev/null +++ b/management/internals/network_map_db/pgsql/peer.go @@ -0,0 +1,135 @@ +package networkmap_pgsql + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/jackc/pgx/v5" + "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" +) + +const ( + GetPeersQuery = ` + select id, key, ssh_key, dns_label, user_id, ssh_enabled, login_expiration_enabled, last_login, ip, ipv6, + meta_wt_version, meta_go_os, meta_os_version, meta_kernel_version, meta_network_addresses, meta_files, meta_capabilities, meta_flags, + location_country_code, location_city_name, location_connection_ip + from peers + where account_id = $1 + ` +) + +func (pg *PgStore) GetPeers(ctx context.Context, accountId string) ([]nmdata.Peer, error) { + rows, err := pg.pool.Query(ctx, GetPeersQuery, accountId) + if err != nil { + return nil, err + } + + var ( + id, key, sshKey, dnsLabel, userId sql.NullString + lastLogin sql.NullTime + sshEnabled, loginExpirationEnabled sql.NullBool + ip, ipv6, locationConnectionIp []byte + metaFiles, metaCapabilities, metaFlags, metaNetworkAddresses []byte + metaWtVersion, metaGoOS, metaOSVersion, metaKernelVersion sql.NullString + locationCountryCode, locationCityName sql.NullString + ) + + peers, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (nmdata.Peer, error) { + var peer nmdata.Peer + err := row.Scan(&id, &key, &sshKey, &dnsLabel, &userId, &sshEnabled, &loginExpirationEnabled, &lastLogin, &ip, &ipv6, + &metaWtVersion, &metaGoOS, &metaOSVersion, &metaKernelVersion, &metaNetworkAddresses, &metaFiles, &metaCapabilities, &metaFlags, + &locationCountryCode, &locationCityName, &locationConnectionIp) + if err != nil { + return peer, err + } + + if id.Valid { + peer.ID = id.String + } + if key.Valid { + peer.Key = key.String + } + if sshKey.Valid { + peer.SSHKey = sshKey.String + } + if dnsLabel.Valid { + peer.DNSLabel = dnsLabel.String + } + if userId.Valid { + peer.UserID = userId.String + } + if lastLogin.Valid { + peer.LastLogin = &lastLogin.Time + } + if sshEnabled.Valid { + peer.SSHEnabled = sshEnabled.Bool + } + if loginExpirationEnabled.Valid { + peer.LoginExpirationEnabled = loginExpirationEnabled.Bool + } + if metaWtVersion.Valid { + peer.Meta.WtVersion = metaWtVersion.String + } + if metaGoOS.Valid { + peer.Meta.GoOS = metaGoOS.String + } + if metaOSVersion.Valid { + peer.Meta.OSVersion = metaOSVersion.String + } + if metaKernelVersion.Valid { + peer.Meta.KernelVersion = metaKernelVersion.String + } + if locationCountryCode.Valid { + peer.Location.CountryCode = locationCountryCode.String + } + if locationCityName.Valid { + peer.Location.CityName = locationCityName.String + } + if ip != nil { + err := json.Unmarshal(ip, &peer.IP) + if err != nil { + return peer, err + } + } + if ipv6 != nil { + err := json.Unmarshal(ipv6, &peer.IPv6) + if err != nil { + return peer, err + } + } + if locationConnectionIp != nil { + err := json.Unmarshal(locationConnectionIp, &peer.Location.ConnectionIP) + if err != nil { + return peer, err + } + } + if metaFiles != nil { + err := json.Unmarshal(metaFiles, &peer.Meta.Files) + if err != nil { + return peer, err + } + } + if metaCapabilities != nil { + err := json.Unmarshal(metaCapabilities, &peer.Meta.Capabilities) + if err != nil { + return peer, err + } + } + if metaFlags != nil { + err := json.Unmarshal(metaFlags, &peer.Meta.Flags) + if err != nil { + return peer, err + } + } + if metaNetworkAddresses != nil { + err := json.Unmarshal(metaNetworkAddresses, &peer.Meta.NetworkAddresses) + if err != nil { + return peer, err + } + } + return peer, nil + }) + + return peers, err +} diff --git a/management/internals/network_map_db/pgsql/pg_store.go b/management/internals/network_map_db/pgsql/pg_store.go new file mode 100644 index 000000000..b7a3ffc43 --- /dev/null +++ b/management/internals/network_map_db/pgsql/pg_store.go @@ -0,0 +1,56 @@ +package networkmap_pgsql + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + networkmapdb "github.com/netbirdio/netbird/management/internals/network_map_db" +) + +const ( + pgMaxConnections = 30 + pgMinConnections = 1 + pgMaxConnLifetime = 60 * time.Minute + pgHealthCheckPeriod = 1 * time.Minute +) + +var _ networkmapdb.NetworkMapDBStore = &PgStore{} + +type PgStore struct { + pool *pgxpool.Pool +} + +func NewPostgresqlStore(ctx context.Context, dsn string) (*PgStore, error) { + pool, err := connectToPgDb(context.Background(), dsn) + if err != nil { + return nil, err + } + + return &PgStore{pool: pool}, nil +} + +func connectToPgDb(ctx context.Context, dsn string) (*pgxpool.Pool, error) { + config, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("unable to parse database config: %w", err) + } + + config.MaxConns = pgMaxConnections + config.MinConns = pgMinConnections + config.MaxConnLifetime = pgMaxConnLifetime + config.HealthCheckPeriod = pgHealthCheckPeriod + + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + return nil, fmt.Errorf("unable to create connection pool: %w", err) + } + + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("unable to ping database: %w", err) + } + + return pool, nil +} diff --git a/management/internals/network_map_db/pgsql/policy.go b/management/internals/network_map_db/pgsql/policy.go new file mode 100644 index 000000000..e69de29bb diff --git a/management/internals/network_map_db/pgsql/test_db.sql b/management/internals/network_map_db/pgsql/test_db.sql new file mode 100644 index 000000000..5a8072ec5 --- /dev/null +++ b/management/internals/network_map_db/pgsql/test_db.sql @@ -0,0 +1,8 @@ +CREATE TABLE `accounts` (`id` text,`created_by` text,`created_at` datetime,`domain` text,`domain_category` text,`is_domain_primary_account` numeric,`network_identifier` text,`network_net` text,`network_dns` text,`network_serial` integer,`dns_settings_disabled_management_groups` text,`settings_peer_login_expiration_enabled` numeric,`settings_peer_login_expiration` integer,`settings_regular_users_view_blocked` numeric,`settings_groups_propagation_enabled` numeric,`settings_jwt_groups_enabled` numeric,`settings_jwt_groups_claim_name` text,`settings_jwt_allow_groups` text,`settings_extra_peer_approval_enabled` numeric,`settings_extra_integrated_validator_groups` text,PRIMARY KEY (`id`)); +CREATE TABLE `peers` (`id` text,`account_id` text,`key` text,`setup_key` text,`ip` text,`meta_hostname` text,`meta_go_os` text,`meta_kernel` text,`meta_core` text,`meta_platform` text,`meta_os` text,`meta_os_version` text,`meta_wt_version` text,`meta_ui_version` text,`meta_kernel_version` text,`meta_network_addresses` text,`meta_system_serial_number` text,`meta_system_product_name` text,`meta_system_manufacturer` text,`meta_environment` text,`meta_files` text,`name` text,`dns_label` text,`peer_status_last_seen` datetime,`peer_status_connected` numeric,`peer_status_login_expired` numeric,`peer_status_requires_approval` numeric,`user_id` text,`ssh_key` text,`ssh_enabled` numeric,`login_expiration_enabled` numeric,`last_login` datetime,`created_at` datetime,`ephemeral` numeric,`location_connection_ip` text,`location_country_code` text,`location_city_name` text,`location_geo_name_id` integer,PRIMARY KEY (`id`),CONSTRAINT `fk_accounts_peers_g` FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`)); +CREATE TABLE `groups` (`id` text,`account_id` text,`name` text,`issued` text,`peers` text,`integration_ref_id` integer,`integration_ref_integration_type` text,PRIMARY KEY (`id`),CONSTRAINT `fk_accounts_groups_g` FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`)); + +INSERT INTO accounts VALUES('bf1c8084-ba50-4ce7-9439-34653001fc3b','edafee4e-63fb-11ec-90d6-0242ac120003','2024-10-02 16:01:38.210000+02:00','test.com','private',1,'af1c8024-ha40-4ce2-9418-34653101fc3c','{"IP":"100.64.0.0","Mask":"//8AAA=="}','',0,'[]',0,86400000000000,0,0,0,'',NULL,NULL,NULL); +INSERT INTO "groups" VALUES('cfefqs706sqkneg59g4g','bf1c8084-ba50-4ce7-9439-34653001fc3b','All','api','[]',0,''); +INSERT INTO "groups" VALUES('cfefqs706sqkneg59g3g','bf1c8084-ba50-4ce7-9439-34653001fc3b','AwesomeGroup1','api','[]',0,''); +INSERT INTO "groups" VALUES('cfefqs706sqkneg59g2g','bf1c8084-ba50-4ce7-9439-34653001fc3b','AwesomeGroup2','api','[]',0,'');